From 958461d9ea9cefaa656e652fb9f881b57436038b Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 23 Sep 2026 19:51:56 -0700 Subject: [PATCH 001/107] docs(calibration): findings notebook for bias, staleness, decay and sqrt transform --- R/calibration/hub_data.R | 26 +- notes/CALIBRATION.md | 416 +++++----- notes/calibration-runbook.md | 16 +- .../calibration/calibration_findings_flu.Rmd | 713 ++++++++++++++++++ .../calibration/calibration_qt_covid.Rmd | 10 +- .../calibration/calibration_qt_flu.Rmd | 10 +- .../calibration_qt_gallery_covid.Rmd | 15 +- .../calibration_qt_gallery_flu.Rmd | 10 +- .../calibration_qt_seasons_covid.Rmd | 15 +- .../calibration_qt_seasons_flu.Rmd | 10 +- .../calibration/calibration_export_series.R | 16 +- scripts/calibration/calibration_harness.R | 2 +- .../calibration/calibration_ili_backfill.R | 15 +- scripts/calibration/calibration_ws_backfill.R | 4 +- .../calibration/calibration_ws_experiments.R | 26 +- uv.lock | 6 +- 16 files changed, 1050 insertions(+), 260 deletions(-) create mode 100644 reports/writeups/calibration/calibration_findings_flu.Rmd diff --git a/R/calibration/hub_data.R b/R/calibration/hub_data.R index 1d7fa7cf..46f23eec 100644 --- a/R/calibration/hub_data.R +++ b/R/calibration/hub_data.R @@ -67,8 +67,8 @@ hub_level_index <- function(x, levels = HUB_QUANTILE_LEVELS, tol = 1e-8) { #' Read one model's quantile forecasts for one target out of a hub checkout. #' #' @param hub_dir local hub checkout. For flu, use `HUB_FLU_DIR` -#' (cdcepi/FluSight-forecast-hub); for covid, use `HUB_COVID_DIR` -#' (cdcepi/covid19-forecast-hub). Only the model-output directory is needed. +#' (cdcepi/FluSight-forecast-hub); for covid, pass a checkout of +#' cdcepi/covid19-forecast-hub. Only the model-output directory is needed. #' @param model model-output subdirectory. #' @param target hub target string. 2023-24 files also carry #' `wk flu hosp rate change` pmf rows, so both `target` and @@ -136,6 +136,28 @@ hub_read_forecasts <- function( } +#' Latest-vintage NHSN admissions as calibration truth, on hub coordinates. +#' +#' NHSN `time_value` is the Saturday week end, the same grid as the hub's +#' `target_end_date`. +#' @param archive an NHSN `epi_archive`, by default [get_nhsn_data_archive()]. +#' @return tibble with `target_end_date`, `location` (FIPS), `truth`. +#' @export +nhsn_read_truth <- function(disease = "flu", archive = get_nhsn_data_archive(disease)) { + archive %>% + epix_as_of(archive$versions_end) %>% + as_tibble() %>% + mutate(geo_value = ifelse(.data$geo_value == "usa", "us", .data$geo_value)) %>% + inner_join( + get_population_data() %>% distinct(.data$state_id, .keep_all = TRUE) %>% select("state_id", location = "state_code"), + by = c("geo_value" = "state_id") + ) %>% + select(target_end_date = "time_value", "location", truth = "value") %>% + filter(!is.na(.data$truth)) %>% + arrange(.data$location, .data$target_end_date) +} + + #' Convert an internal-format forecast tibble to hub forecast format. #' #' The internal format uses geo_value (state abbreviation), forecast_date diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index 1cf9af8a..ec213b12 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -15,10 +15,12 @@ projection. Full repro commands live in `notes/calibration-runbook.md`. on the `delphi-fixes` branch of `~/repos/delphi/multiQT` (their published code had several bugs we fixed and reported; see that branch's commit message). Oracle fixtures in `tests/testthat/test-qt.R`, plus a real-series - cross-check (`scripts/calibration_export_series.R`). + cross-check (`scripts/calibration/calibration_export_series.R`). - `R/calibration/hub_data.R` — FluSight hub adapters: `hub_read_forecasts()` (CMU-TimeSeries submissions from the sparse hub checkout), - `hub_read_truth()` (hub finalized truth CSV), `hub_label_seasons()`. + `nhsn_read_truth()` (latest NHSN vintage from `get_nhsn_data_archive()`, on + the hub's Saturday grid; replaced the hub target-data CSV on 2026-09-23), + `hub_label_seasons()`. - `R/calibration/calibrate.R` — `calibrate_hub_forecasts()` driver (burn-in, season policy, ~5 s for all 265 series) and metrics: `hub_coverage()`, `hub_coverage_summary()`, `hub_quantile_loss()`, `hub_rolling_tradeoff()`. @@ -28,16 +30,17 @@ projection. Full repro commands live in `notes/calibration-runbook.md`. two-term offset, below), `off_after`, `lr_seasonal`. (`lr_geo_pool`, the geo-pooled eta in the sweep table below, was removed 2026-09-17: it never beat the baseline.) -- `scripts/calibration_ws_backfill.R` — replays flu prod's `windowed_seasonal` +- `scripts/calibration/calibration_ws_backfill.R` — replays flu prod's `windowed_seasonal` over the NHSN seasons (hub schema, cached to - `cache/calibration/ws_pseudo_hub_forecasts.parquet`); `scripts/calibration_ws_experiments.R` + `cache/calibration/ws_pseudo_hub_forecasts.parquet`); `scripts/calibration/calibration_ws_experiments.R` runs the one-forecaster-across-eras comparison (below). -- `scripts/calibration_ili_backfill.R` — runs flu prod's `windowed_seasonal` +- `scripts/calibration/calibration_ili_backfill.R` — runs flu prod's `windowed_seasonal` forecaster on the ILI+ state history (2010–2024, Wednesday labels, snapshot truncated one week to mimic reporting lag) and writes a pseudo-hub `forecasts`/`truth` pair in hub schema to `cache/calibration/ili_pseudo_hub_*.parquet` - (~30 min). Multi-season burn-in material for the slow term. -- `scripts/calibration_harness.R` — targets-based harness for calibrating our + (~30 min). Rebuilt 2026-09-23 after the latency fix described in "The ILI+ + replay was broken" below; ILI+ results older than that are invalid. +- `scripts/calibration/calibration_harness.R` — targets-based harness for calibrating our own forecasters outside the hub path (covid `windowed_seasonal_extra_sources`, aheads 0:4, `covid_hosp_evaluation` store). WIP. @@ -58,9 +61,11 @@ As-of NHSN vintages for the gallery come from vintages exist before 2024-11-19 anywhere). Note: the S3 `nhsn_data_archive.parquet` object stalled at version 2026-01-30 — the polling job appears to have stopped; refresh the copy from a newer oracle -capture if one exists. +capture if one exists. Since 2026-09-23 `get_nhsn_data_archive("flu")` (the +cast API, versions 2024-11-19 onward) serves both truth (`nhsn_read_truth()`) +and vintages live; the findings notebook uses it. -## Findings (factorial sweep, `scripts/reports/calibration_qt_flu.Rmd`) +## Findings (factorial sweep, `calibration_qt_flu.Rmd`; count space, pre-fix) Sweep: `season_policy` {carry, reset} × `lr_window` {8, 20, 50, Inf} × `lr_mult` {0.3, 0.1, 0.03, 0.01}; results in `cache/calibration/sweep.rds`. @@ -90,18 +95,35 @@ validated grid point and the window is immaterial). At this multiplier the tracker needs most of a season to warm up, which makes burn-in/state-carrying the central integration design issue. -**Current operating point (2026-09-16): `transform = "sqrt"`, `lr_mult = 0.03`, +**Current operating point: `transform = "sqrt"`, `lr_mult = 0.03`, `floor = 1e-3`, `lr_window = 20`, carry, `slow_init = "burn_in_quantile"`, -`lr_slow = list(mult = 0.003)`, `fast_decay = 0.1`.** WIS-positive at every -horizon in both live seasons (+9.0/+5.3/+1.9/+0.8/+0.5 at h−1…3) with -calibration error 0.044–0.072 against 0.095–0.118 uncalibrated. See "Warm -start" below for the table. +`lr_slow = list(mult = 0.003)`, `fast_decay = 0.1`.** Re-measured 2026-09-23 +(after the correctness fixes, NHSN truth, all 53 hub locations): WIS change ++8.9/+5.4/+2.6/+1.9/+1.9 at h−1…3 over both live seasons, calibration error +0.040–0.073 against 0.098–0.118 uncalibrated. Current numbers for every +variant are in `reports/writeups/calibration/calibration_findings_flu.Rmd` +(section "Current numbers" below). + +**Numbers dated before 2026-09-17 predate the two correctness fixes** +("Correctness fixes" below) and used the hub target-data CSV as truth. The +sqrt-space tables from 2026-09-16 were also hit by the unclamped sqrt +inverse, so they have been replaced; the count-space and 2026-08-27 tables are +kept as a record of how the design was reached, not as current numbers. ## Notebooks -- `scripts/reports/calibration_qt_flu.Rmd` — the parameter-sweep EDA. Frozen as - the sweep record; rendered HTML alongside. -- `scripts/reports/calibration_qt_seasons_flu.Rmd` — whole-season views of four +All in `reports/writeups/calibration/`, rendered into `rendered_reports/`. +Each takes a `hub_dir` param (the hub checkout, relative to the repo root; +defaults `../FluSight-forecast-hub` and `../covid19-forecast-hub`) and scores +against the latest NHSN vintage via `nhsn_read_truth()`. + +- `calibration_findings_flu.Rmd` — the current summary: base bias on NHSN, + the same forecaster's bias on ILI+, staleness (lagged correlation of played vs needed + shift), the leak by month, count vs sqrt, ensemble vs `windowed_seasonal`. + All numbers are post-fix. +- `calibration_qt_flu.Rmd` — the parameter-sweep EDA. Frozen as + the sweep record. +- `calibration_qt_seasons_flu.Rmd` — whole-season views of four states × two live seasons, one panel per (state, season): every-other-round fans (80% band) for h 0–3, the NHSN vintage each round saw painted over its fortnight, finalized truth, and an eta strip; plus collapsible internals @@ -115,7 +137,7 @@ start" below for the table. WIS/calibration-error comparison at the top, plus a by-month breakdown of the baseline (WIS change, share of base WIS, median shift) that shows where in the season the method gains and loses. -- `scripts/reports/calibration_qt_gallery_flu.Rmd` — fixed operating point. Slim +- `calibration_qt_gallery_flu.Rmd` — fixed operating point. Slim headline table (WIS + calibration error per horizon per season), then a ranked per-forecast gallery: top-N (location, round) panels ordered by mean absolute quantile displacement relative to the base median. Each panel: @@ -125,10 +147,11 @@ start" below for the table. 2025-26 only, flu only. A dynamic/paginated app version was considered and rejected for now in favor of static top-N HTML. -## Method variants (`calibrate_hub_forecasts()` options; 2026-08-27) +## Method variants (`calibrate_hub_forecasts()` options; 2026-08-27, pre-fix) -All at the operating point above; WIS change vs base, both live seasons, by -horizon −1/0/1/2/3: +All count space at the original operating point; WIS change vs base, both +live seasons, by horizon −1/0/1/2/3. Pre-fix numbers (see above); the +transform rows measured on 2026-09-16 were removed, see "Current numbers": | variant | option | h−1 | h0 | h1 | h2 | h3 | |---|---|---|---|---|---|---| @@ -140,11 +163,6 @@ horizon −1/0/1/2/3: | geo-pooled eta (option since removed) | `lr_geo_pool = ` | +4.4 | +2.0 | −2.2 | −5.5 | −7.7 | | seasonal eta window (carry) | `lr_window = 10, lr_seasonal = list(half_width_weeks = 5)` | +2.9 | +0.1 | −3.0 | −5.5 | −7.5 | | seasonal eta window (reset) | same + `season_policy = "reset"` | +3.2 | +1.1 | −2.1 | −4.7 | −7.0 | -| log-space tracker | `transform = "log1p", lr_args = list(mult = 0.03, floor = 1e-3)` | +10.8 | +1.8 | −5.7 | −9.0 | −8.8 | -| log-space, mult 0.1 | `transform = "log1p", lr_args = list(mult = 0.1, floor = 1e-3)` | +5.3 | −5.8 | −21.8 | −30.9 | −37.8 | -| sqrt-space | `transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)` | +8.0 | +4.1 | −0.4 | −3.1 | −3.4 | -| fourth-root space | `transform = "quartic_root", lr_args = list(mult = 0.03, floor = 1e-3)` | +9.6 | +4.1 | −2.2 | −4.9 | −5.0 | -| fourth-root, mult 0.1 | same, `mult = 0.1` | +12.3 | +0.9 | −9.8 | −17.8 | −17.6 | - Eta was never pooled across horizons: each (location, horizon) series has its own tracker and its own eta (pooled over the 23 levels and the window). @@ -169,35 +187,27 @@ horizon −1/0/1/2/3: Reset instead of carry recovers some of that (h2 −4.7) but still trails baseline; 2024-25 is fine (+5.7/+2.8/+0.1 at h−1/0/1), 2025-26 is uniformly worse. The notebook's seasonal section now runs the reset variant. -- **Log-space tracking** (offsets relative instead of in counts; 2026-09-16) - is worse at every horizon except h−1 and degrades fast with the multiplier - (mult 0.3: −50 to −270%). Calibration error does fall with a larger - multiplier (0.011–0.030 at mult 0.1 vs 0.022–0.035 baseline), so it buys - coverage, but relative residuals at trough counts (base 1–5) are large and - noisy and the staleness problem is untouched. Not in the notebook. If - scale-awareness is retried, normalize by a smooth scale proxy (trailing - 4-week truth, floored) rather than a log. -- **Power transforms** (2026-09-16; `transform = "sqrt"` / `"quartic_root"`, - the latter being the forecasters' whitening map `(x + 0.01)^0.25`). The - bookkeeping: Y and all 23 base quantiles are mapped, residuals/eta/offsets/ - PAVA all live in the mapped space, the played quantiles are mapped back and - clamped, and scoring is on counts. Coverage indicators are invariant under - a monotone map, so the gradient is identical to count space; only the - step geometry changes. Multipliers are not comparable across spaces, so - compare at matched calibration error. `sqrt` at mult 0.03 matches the - baseline's calibration error (0.029–0.043 vs 0.022–0.035) with better WIS - at every horizon, and beats the baseline in the Nov–Feb core at every - horizon (+7.8/+4.5/+0.5/−1.5/−1.2 vs +5.3/+1.9/−1.4/−3.2/−3.1); the spring - tail is still lost at h2/h3 (−19/−33% vs −26/−55%). Fourth root at 0.03 - under-steps (calibration error 0.037–0.049) and at 0.1 over-pays like every - other aggressive setting. Staleness is untouched by any transform; the - transform only bounds how badly a stale offset is sized. Window 20 vs 50 - is again immaterial. Worth a small sweep over power × mult. - -## Where the WIS moves (2026-09-16, baseline, both live seasons pooled) - -By calendar month of the reference date; scratch numbers, reproduced live in -the seasons notebook's by-month tables. +- **Transforms** (`transform = "log1p"` / `"sqrt"` / `"quartic_root"`, the + last being the forecasters' whitening map `(x + 0.01)^0.25`). Y and all 23 + base quantiles are mapped, residuals/eta/offsets/PAVA all live in the mapped + space, the played quantiles are mapped back and clamped at zero, and scoring + is on counts. Coverage indicators are invariant under a monotone map, so the + gradient is identical to count space; only the step geometry changes, and + multipliers are not comparable across spaces. The 2026-09-16 measurements + of these variants predate the correctness fixes (the sqrt ones were also hit + by the unclamped inverse) and were removed. Re-measured since: sqrt vs + count only, in "Current numbers" below. Log space and fourth root have not + been re-run; before the fixes log space was worse than count space at h ≥ 1 + and fourth root at mult 0.03 under-stepped, but neither result should be + relied on. If scale-awareness is retried beyond sqrt, normalize by a smooth + scale proxy (trailing 4-week truth, floored) rather than a log. + +## Where the WIS moves (2026-09-16, count-space baseline, pre-fix) + +By calendar month of the reference date, both live seasons pooled. These are +pre-fix count-space numbers; the mechanism (the spring and October losses, +the stale wrong-signed offset) is re-confirmed post-fix for the sqrt variants +in the findings notebook, with the numbers in "Current numbers" below. - **The mass is in December–February.** At h3, Dec + Jan carry ~75% of the season's base WIS and Mar–May ~7%. Calibration is −2% in Dec/Jan at h2/h3 @@ -240,33 +250,21 @@ and small multiplier, never decays, ignores the season policy, and may train through burn-in (`burn_in_learns_slow`). Without `lr_slow` and with `fast_decay = 0` the output is bit-identical to before (tests pin this). -All on `transform = "sqrt"`, mult 0.03, floor 1e-3, window 20, carry, hub -2023-24 burn-in. WIS change vs base by horizon −1…3, then calibration error -range, then Mar–May WIS change at h2/h3: - -| variant | h−1 | h0 | h1 | h2 | h3 | cal err | spring h2/h3 | -|---|---|---|---|---|---|---|---| -| sqrt, single term | +8.0 | +4.1 | −0.4 | −3.1 | −3.4 | 0.029–0.043 | −19 / −33 | -| + fast_decay 0.1 | +8.2 | +4.3 | +1.3 | +0.1 | −0.8 | 0.074–0.089 | −1 / −9 | -| + fast_decay 0.2 | +6.5 | +3.2 | +1.1 | +0.3 | −0.1 | 0.084–0.101 | 0 / −5 | -| + slow 0.01 (no decay) | +8.9 | +4.2 | −1.3 | −3.5 | −4.8 | 0.024–0.034 | −20 / −44 | -| + slow 0.01, decay 0.1 | +9.9 | +5.1 | +0.8 | −0.9 | −2.2 | 0.056–0.066 | −7 / −21 | -| + slow 0.01, decay 0.1, slow trains in burn-in | +10.1 | +5.7 | +1.1 | −0.8 | −1.6 | 0.052–0.066 | −8 / −24 | -| + slow 0.03, decay 0.2, slow trains in burn-in | +11.6 | +5.7 | −0.1 | −2.7 | −3.0 | 0.034–0.041 | −20 / −48 | - -- **The leak is what fixes the spring and h2/h3.** Decay 0.1–0.2 alone makes - the tracker WIS-neutral-or-better at every horizon and removes most of the - spring loss, but it gives back most of the coverage gain (calibration error - doubles): a correction that must be continually refreshed cannot hold - coverage through a 5-round reveal lag. -- **The slow term alone does nothing visible** at mult 0.01 over one burn-in - season plus one live season: it has not had time to move. At mult 0.03 it - is no longer slow and behaves like a second fast term (spring loss is back). -- **Slow + leaky fast is the balance point**: WIS-positive at h ≤ 1, - −1 to −2 at h2/h3, calibration error about half-way between the single-term - tracker and uncalibrated. Letting the slow term train through burn-in - helps a little with one season; the whole point of the ILI+ backfill is to - give it thirteen. +The 2026-09-16 comparison table for this section (sqrt space) predated the +correctness fixes and was removed; the post-fix numbers for the variants that +were re-run (single term, leak 0.1, and the warm-start combinations) are in +"Current numbers" below. What still holds: + +- **The leak is what fixes the spring and h2/h3.** Re-measured: decay 0.1 + alone takes h3 from −1.4% to +0.5% and cuts the h3 March/October losses from + about −47% to −17% and −48% to −14%, but gives back most of the coverage + gain (calibration error 0.022–0.043 single-term vs 0.073–0.089 leaky): a + correction that must be continually refreshed cannot hold coverage through + a 5-round reveal lag. +- **A slow gradient term on its own** (no warm start) was measured only + before the fixes: at mult 0.01 it barely moved over one burn-in plus one + live season, and at mult 0.03 it acted like a second fast term. Not re-run; + the warm start below replaced it as the way to hold the persistent level. ### Is a learned seasonal offset viable? @@ -310,30 +308,21 @@ weeks if so. conformal offset implied by the burn-in rounds pooled over locations: the `level`-quantile of `Y − Yhat[level]` in working units. The tracker then runs on top of it. Same setup as the two-term table (sqrt, mult 0.03, window 20, -carry, hub 2023-24 burn-in); uncalibrated calibration error is 0.095–0.118. - -| variant | h−1 | h0 | h1 | h2 | h3 | cal err | 2024-25 h1/h2/h3 | 2025-26 h1/h2/h3 | -|---|---|---|---|---|---|---|---|---| -| sqrt, single term | +8.0 | +4.1 | −0.4 | −3.1 | −3.4 | 0.029–0.043 | +0.5/−1.8/−3.6 | −2.3/−5.4/−3.1 | -| warm start only, no tracking | +0.5 | +1.2 | +0.9 | +1.1 | +2.0 | 0.062–0.104 | +1.2/+1.7/+3.0 | +0.3/+0.1/+0.1 | -| warm start + fast, no decay | +8.1 | +4.7 | +0.1 | −2.2 | −1.9 | 0.016–0.038 | +1.4/−0.3/−0.9 | −2.5/−5.6/−3.6 | -| warm start + fast, decay 0.1 | +8.4 | +5.0 | +1.9 | +0.9 | +0.8 | 0.048–0.079 | +2.1/+1.0/+1.0 | +1.6/+0.9/+0.5 | -| warm start + slow 0.003 + fast decay 0.1 | +9.0 | +5.3 | +1.9 | +0.8 | +0.5 | 0.044–0.072 | +2.0/+0.7/+0.6 | +1.5/+0.8/+0.3 | -| warm start + slow 0.01 + fast decay 0.1 | +10.1 | +5.7 | +1.3 | −0.1 | −0.6 | 0.036–0.058 | +1.9/+0.1/−0.4 | +0.1/−0.5/−0.8 | -| warm start + slow 0.01 + fast decay 0.2 | +9.1 | +5.4 | +1.5 | +0.5 | +0.4 | 0.040–0.064 | +2.2/+0.8/+0.7 | 0.0/+0.1/0.0 | +carry, hub 2023-24 burn-in). The 2026-09-16 table was pre-fix and was removed; +re-measured numbers are in "Current numbers" below. - **One season of batch conformal offsets is already WIS-positive at every - horizon** (+0.5 to +2.0) while removing a third of the coverage error: the - persistent low bias is real and cheap to correct. -- **Warm start + leaky fast tracker is the first configuration that is - WIS-positive at every horizon in *both* seasons** and it halves the - coverage error. The leak is still what keeps the spring in check - (h3 spring: −19% with decay 0.1 vs −45% without); the warm start is what - lets the leak be affordable, since coverage no longer depends on the fast - term holding a level. -- Adding a slow gradient term on top of the warm start is a small dial - between WIS (mult 0.003) and coverage (mult 0.01); the chosen point is - 0.003 with decay 0.1. + horizon** (+0.4 to +1.9, re-measured) while removing a third of the + coverage error: the persistent low bias is real and cheap to correct. +- **Warm start + leaky fast tracker is WIS-positive at every horizon** and + halves the coverage error (+8.3/+5.1/+2.6/+2.0/+2.1, calibration error + 0.046–0.080 vs 0.098–0.118 base). Its h3 gain comes from December–January + (+4% each, the months holding ~75% of the h3 base WIS); its spring losses + are only partly contained by the leak (h3 March −28%), because the warm + start lives in the slow term, which does not decay. +- Adding a slow gradient term on top of the warm start was, before the fixes, + a small dial between WIS (mult 0.003) and coverage (mult 0.01); the chosen + point is 0.003 with decay 0.1. Post-fix only 0.003 was re-run. - The h3 warm-start offsets at the outer levels are large (0.95 level: +2.9 scaled-sqrt units for CA) because the 2023-24 ensemble was badly under-dispersed at long horizons; it is the correct conformal answer for @@ -341,46 +330,22 @@ carry, hub 2023-24 burn-in); uncalibrated calibration error is 0.095–0.118. season's base forecaster. In prod the warm start would come from the previous live season(s) of the *current* forecaster. -### ILI+ burn-in (2026-09-16): negative result - -The prod flu forecasters already train on ILI+ (percent positive × percent -ILI, state level, 2010–2024) as faux-versioned augmentation rows folded into -`nhsn_prod_archive`. For calibration the analogous move is to *forecast* the -ILI+ seasons with the same `windowed_seasonal` forecaster and use those -residuals as burn-in: 13 usable seasons (2020/21 and 2021/22 dropped as -non-seasons; 2023/24 dropped as overlapping the hub's own burn-in) instead -of one. The pseudo-hub tables are on the ILI+ percent scale, so the tracker -must run in scaled units: `scales` divides each location's rounds by an -era-specific divisor (90th percentile of in-season truth: ILI+ seasons for the -ILI+ era, the hub 2023-24 season for the NHSN era) before the sqrt transform, -so offsets and eta learned on ILI+ carry over as fractions of a typical -season-peak level. Caveats: (1) the pseudo-hub base is one component -(`windowed_seasonal`), the hub base is the submitted ensemble; (2) ILI+ has -no revisions, so the pseudo-hub residuals lack the as-of under-reporting -that drives part of the live bias; (3) the h−1 pseudo-forecast is a model -forecast of an already-observed week, as in prod. - -Result: caveat (1) is fatal for this use. In-season, in scaled sqrt units: - -| era | base | truth > median | median scaled residual (h0 … h3) | -|---|---|---|---| -| ILI+ 2010–20 | windowed_seasonal on ILI+ | 0.52–0.54 | 0.014 … 0.031 | -| hub 2023-24 | submitted ensemble | 0.61–0.67 | 0.030 … 0.112 | -| hub 2024-26 | submitted ensemble | 0.68–0.75 | 0.058 … 0.209 | - -The pseudo-base is essentially unbiased in the median, and its by-season bias -swings from −213% (2015-16) to +66% with no stable component, whereas the -submitted ensemble runs low in every hub season. So ILI+ residuals carry no -information about the live base's bias. Worse, thirteen seasons of near-zero -drift (slightly more under- than over-coverage, and a residual scale 2–3× -NHSN's inflating eta) still summed to a slow term of ~+0.1 scaled-sqrt units -at the first live round, which is +25% at peak and ~3× at trough: WIS −130 to -−500%. Eta warm-up from ILI+ alone (no slow term) changed only h−1, where the -hub burn-in has almost no rounds, and made it worse. Kept: -`scripts/calibration_ili_backfill.R` and the `scales` machinery, which would -be the right tools if the live base were the `windowed_seasonal` component -itself (calibrating per forecaster rather than post-ensemble, roadmap item 3), -since then the pseudo-base and the live base are the same model. +### ILI+ burn-in (2026-09-16): invalid, measured on the broken replay + +The idea: the prod flu forecasters already train on ILI+ (percent positive × +percent ILI, state level, 2010–2024) as faux-versioned augmentation rows, so +*forecast* the ILI+ seasons with the same `windowed_seasonal` forecaster and +use those residuals as a multi-season burn-in (13 usable seasons instead of +one). The pseudo-hub tables are on the ILI+ percent scale, so the tracker runs +in scaled units: `scales` divides each location's rounds by an era-specific +divisor (90th percentile of in-season truth) before the sqrt transform, so +offsets and eta learned on ILI+ carry over as fractions of a typical +season-peak level. + +The result recorded here on 2026-09-16 (ILI+ residuals carry no information +about the live bias; an ILI+-trained slow term costs 130–500% WIS) was +measured on a broken replay (see "The ILI+ replay was broken" below) and has +been removed. The `scales` machinery is fine and stays. The useful implication is the opposite one: the hub's own 2023-24 burn-in *does* share the live bias, so the slow term should be warm-started from it @@ -425,18 +390,111 @@ own `mult` before); `off_after` and `fast_decay` abort when combined (the leak would run through the off window, so "frozen offset" would be false); the hub reader aborts on duplicate (round, level) rows; the accidentally committed `covid_hosp_prod/workspaces/` binaries were dropped from history and the -pattern is now ignored. The notebooks have not been re-rendered since. +pattern is now ignored. The older notebooks have not been re-rendered since; +`calibration_findings_flu.Rmd` has the post-fix numbers. + +### The ILI+ replay was broken (fixed 2026-09-23) + +Symptom: `windowed_seasonal`'s ILI+ pseudo-hub forecasts did not follow their +input. Log-log slope of median on truth was 0.40 at h0 (0.29 at h3). Indiana +on 2018-01-03 had a last observed value of 2.28 and an h0 median of 0.14. + +Cause: `adjust_latency = "extend_lags"` (the `default_args_list()` default) +computes one latency per column as `forecast_date - min over geos of that +geo's last time_value`, i.e. the *worst* state, and adds it to every lag. The +ILI+ archive has states whose series end years before the forecast date (DC +stops 2015-04-08; `hhs > 1e-4` also drops trailing zero weeks). On 2018-01-03 +the latency was 1001 days, so every predictor was about three years old. +Every round from 2011 on was affected, at every horizon, with latencies from +weeks to about nine years. The NHSN replay was not affected: all NHSN states +are current, and the ILI+/flusurv augmentation rows are in `keys_to_ignore`. +The same trap applies to any archive in which one geo stops reporting, NHSN +included (a single late state would stretch every state's lags). + +Fix: `ili_forecast_one()` passes the states whose last observation predates +the snapshot's latest week as `keys_to_ignore = list(list("geo_value", ...))`, +so they are left out of the latency check. They still contribute training +rows; they get no forecast that round (forecast locations per round are +unchanged: min 26, median 45, max 48). Indiana's h0 median is now 2.44. + +Checks on the rebuilt replay (all seasons): log-log slope 0.73/0.86/0.78/ +0.69/0.59 at h−1..h3. The overall slope is pulled down by the 43% of points +near the trough (truth under 10% of the location's 90th percentile, where +the zero clip and whitening flatten the median); above that the slope is +0.92/0.98/0.90/0.81/0.70. + +h−1 is not an echo of the last observed week, in this replay or in prod. +`filter_minus_one_ahead()` drops the week at `as_of + ahead`, because +otherwise the target column equals the extended lag-7 predictor and +`quantile_reg` freezes. With that week gone, the latency is 14 days and h−1 +is a one-week forecast from the week before. + +Still invalid, not re-run on the fixed replay: the ILI+ burn-in result above +and setup C in "One forecaster across eras" below. The bias on ILI+ is in +section 1.2 of the findings notebook: about level (43–46% of truth above the +median; summed level within ±4% at every horizon), with the same peak +shortfall at h3 as NHSN. So the "bias flips sign with the data source" +reading does not hold. On ILI+ the base is about level on average, not +biased high. + +### Current numbers (2026-09-23) + +From `reports/writeups/calibration/calibration_findings_flu.Rmd`: post-fix, +truth from `nhsn_read_truth()`, all 53 hub locations, the submitted ensemble, +sqrt unless stated, mult 0.03, floor 1e-3, window 20, carry, 2023-24 burn-in, +both live seasons pooled. WIS change % vs base, then calibration error +(base 0.118/0.105/0.095/0.098/0.102): + +| variant | h−1 | h0 | h1 | h2 | h3 | cal err h−1…h3 | +|---|---|---|---|---|---|---| +| count space, single term (the paper) | +4.2 | +2.4 | −0.8 | −3.4 | −5.6 | 0.033/0.020/0.025/0.027/0.029 | +| count space, leak 0.1 | +6.0 | +2.9 | +0.6 | +0.3 | −0.8 | 0.079/0.062/0.061/0.066/0.065 | +| sqrt, single term | +7.9 | +4.4 | +0.7 | −0.5 | −1.4 | 0.043/0.027/0.022/0.023/0.022 | +| sqrt, leak 0.1 | +8.1 | +4.4 | +2.0 | +1.1 | +0.5 | 0.089/0.078/0.073/0.078/0.080 | +| warm start only, no tracking | +0.4 | +1.1 | +0.8 | +1.1 | +1.9 | 0.105/0.076/0.063/0.066/0.062 | +| warm start + leak 0.1 | +8.3 | +5.1 | +2.6 | +2.0 | +2.1 | 0.080/0.057/0.049/0.050/0.046 | +| operating point (warm start + slow 0.003 + leak 0.1) | +8.9 | +5.4 | +2.6 | +1.9 | +1.9 | 0.073/0.051/0.042/0.044/0.040 | + +h3 WIS change % by month of the reference date, and each month's share of the +h3 base WIS: + +| variant | Oct | Nov | Dec | Jan | Feb | Mar | Apr | May | +|---|---|---|---|---|---|---|---|---| +| sqrt, single term | −48.4 | +6.5 | +1.9 | +0.6 | −1.9 | −47.3 | −36.2 | +20.1 | +| sqrt, leak 0.1 | −13.5 | +5.2 | +0.6 | +1.2 | +1.3 | −17.2 | +8.5 | +6.8 | +| warm start + leak 0.1 | −26.9 | +5.7 | +4.1 | +4.1 | +0.6 | −27.5 | −4.7 | +3.3 | +| operating point | −28.9 | +6.4 | +4.3 | +4.2 | 0.0 | −33.2 | −6.6 | +5.4 | +| share of base WIS % | 0.2 | 6.1 | 41.5 | 33.5 | 11.8 | 4.7 | 1.1 | 1.1 | + +Staleness, measured directly: correlate the played median shift at round `t` +with the needed shift (`sqrt(truth) − sqrt(base median)`) at round `t − k`, +per series, averaged over locations (single-term sqrt tracker). At `k = 0` the +correlation is negative from h0 up (−0.22/−0.25/−0.31/−0.35 at h0…h3): the +offset currently played pushes the wrong way on average. It peaks at +`k = 4/6/7/9` for h0…h3, i.e. the reveal lag `h + 2` plus two to four rounds of +build-up. By direction alone (share of >1% moves with the right sign), no +tracker beats the static warm start at h0–h3 (67–69%); the single-term count +tracker is worst (56–62%). + +Base bias on NHSN (share of forecasts with truth above the median, h−1…h3): +ensemble 69/65/63/63/63, `windowed_seasonal` 58/63/64/67/70; summed median vs +summed truth −11…−38% and −8…−46%. At h3 the bias concentrates at the peak +(median 68–69% low when truth is above the location's 90th-percentile level); +at h0 it is a fairly even 5–20% low. + +Count vs sqrt: the ratio of the relative median shift at the trough bin to +that at the peak bin is 32–38 in count space and 5–10 in sqrt. ### One forecaster across eras (2026-09-17) Question: are the hub-season gains an artifact of three seasons of one ensemble, and does the tracker behave the same on a decade of the *same* -forecaster? `scripts/calibration_ws_backfill.R` replays flu prod's +forecaster? `scripts/calibration/calibration_ws_backfill.R` replays flu prod's `windowed_seasonal` (prod archive, seed, substitutions) over the NHSN seasons in hub schema: 2024-25 and 2025-26 as honest as-of replays, 2023-24 with the `"cheating"` policy (NHSN vintages start 2024-11-19). With the ILI+ pseudo-hub -(`scripts/calibration_ili_backfill.R`) this is one forecaster from 2011 to -2026. `scripts/calibration_ws_experiments.R` runs three setups, all sqrt, +(`scripts/calibration/calibration_ili_backfill.R`) this is one forecaster from 2011 to +2026. `scripts/calibration/calibration_ws_experiments.R` runs three setups, all sqrt, mult 0.03, window 20, carry, on the 50 locations common to all sources and on the hub's submission rounds only: @@ -445,10 +503,13 @@ on the hub's submission rounds only: - **C** `windowed_seasonal` on the ILI+ decade (2011-12 burn-in, 2012-2020 and 2022-23 tracked live) then NHSN 2023-26, whitened per location and era by the 90th percentile of in-season truth (`scales`), offsets carried across - the source switch. + the source switch. **C is invalid**: its ILI+ half comes from the broken + replay (above). Its rows and the ILI+ numbers were removed. Variants: `single` (paper's tracker), `leaky` (`fast_decay = 0.1`), -`operating` (warm start + slow 0.003 + leaky fast). WIS change % vs base: +`operating` (warm start + slow 0.003 + leaky fast). WIS change % vs base +(post-fix, hub target-data truth, 50 locations; the findings notebook re-runs +A and B on NHSN truth, with the same pattern): | setup / variant | 2024-25 h−1…h3 | 2025-26 h−1…h3 | |---|---|---| @@ -457,13 +518,8 @@ Variants: `single` (paper's tracker), `leaky` (`fast_decay = 0.1`), | B single | +0.7 / +2.0 / +0.3 / −0.7 / −1.5 | −1.4 / +2.3 / −0.5 / −0.7 / −0.3 | | B leaky | +1.4 / +2.1 / +0.8 / +0.3 / −0.3 | −0.3 / +1.7 / +0.4 / +0.8 / +1.3 | | B operating | +2.1 / +2.7 / +1.1 / +0.7 / +0.8 | −0.1 / +2.3 / +0.4 / +0.5 / +0.6 | -| C single | −6.0 / −0.3 / −0.3 / +0.4 / +1.3 | −11.4 / −3.8 / −4.0 / −3.4 / −2.9 | -| C leaky | +1.5 / +1.9 / +0.4 / −0.5 / −1.2 | −0.3 / +1.7 / +0.4 / +0.8 / +1.3 | -| C operating | −0.7 / +5.1 / +2.9 / +3.3 / +4.1 | −10.1 / −1.1 / −2.0 / −1.3 / −0.6 | -C on the first NHSN season (2023-24, live there): single −42 / −41 / −27 / -−22 / −19, operating −24 / −23 / −15 / −10 / −6, leaky +1.4 / +0.5 / +0.4 / -+0.7 / +0.6. Calibration error (mean over levels; base first) for B: +Calibration error (mean over levels; base first) for B: 2024-25 base 0.074–0.168, single 0.053–0.088, operating 0.048–0.093; 2025-26 base 0.044–0.088, single 0.007–0.016, operating 0.016–0.042. @@ -473,13 +529,6 @@ Fraction of truth above the base median (the sign of the bias): |---|---|---|---| | ensemble | 0.56 / 0.55 / 0.55 / 0.53 / 0.52 | 0.83 / 0.74 / 0.73 / 0.74 / 0.76 | 0.58 / 0.63 / 0.59 / 0.60 / 0.60 | | windowed_seasonal on NHSN | 0.55 / 0.54 / 0.58 / 0.62 / 0.68 | 0.59 / 0.68 / 0.68 / 0.72 / 0.76 | 0.58 / 0.66 / 0.65 / 0.65 / 0.65 | -| windowed_seasonal on ILI+ (2014-2019 seasons) | 0.25–0.45 | | | - -`windowed_seasonal` calibrated on ILI+ alone (2011-12 burn-in, ten seasons): -`single` improves WIS in 8 of 10 seasons (+5 to +26%, losses in 2012-13 and -2022-23) and cuts calibration error at h ≥ 0 from 0.019–0.036 to 0.013–0.017; -`operating` is negative in 6 of 10 seasons and triples calibration error -(0.046–0.053). Same numbers appear as the ILI+ rows of setup C. Reading: @@ -490,27 +539,17 @@ Reading: is less miscalibrated to begin with (base cal err 0.07–0.17 vs the ensemble's 0.13–0.19 in 2024-25) and because the ensemble's h−1 is a different, worse model (climate_linear). -- **The same forecaster's bias flips sign with the data source.** On ILI+ - it over-predicts (truth above median 25–45%); on NHSN it under-predicts. - Carrying ILI-era offsets into the NHSN era (C) therefore costs 20–40% WIS - in the first NHSN season, and the warm start learned on the ILI decade - keeps hurting through 2025-26. The ILI+ history is not a proxy for the - NHSN regime even with the same model, so the earlier negative result on - ILI+ burn-in was not a plumbing bug (date and geo alignment were checked - and hold) but a regime difference. -- **What transfers across seasons is the plain tracker, not the warm start.** - Across ten ILI+ seasons the single-term tracker is reliably positive; the - warm start + slow term, chosen on two hub seasons, is reliably negative - there. On NHSN with one forecaster (B) the two are close, with `operating` - slightly ahead on WIS and `single` far ahead on coverage. The leaky - variant is the only one indifferent to history (identical in B and C): - small, consistent WIS gains, weakest coverage gains. -- **Overfitting verdict.** The tracker itself is not overfit to the three - hub seasons; the operating point's extra machinery is fit to the ensemble's - persistent low bias and should not be assumed to carry over to a new base - or a new data source. A prod design should warm-start from the *current* - forecaster's most recent NHSN season only, or use the leaky tracker with no - warm start when that season is unavailable. +- **On NHSN with one forecaster (B)** the single-term and operating variants + are close, with `operating` slightly ahead on WIS and `single` far ahead on + coverage in 2025-26; the leaky variant gives small, consistent WIS gains + with the weakest coverage gains. +- **Overfitting verdict (NHSN only).** The tracker is not an artifact of the + ensemble: it helps `windowed_seasonal` the same way, with smaller gains. + Whether the warm start transfers to a different data source is open (the + ILI+ evidence for "it does not" came from the broken replay). A prod design + should warm-start from the *current* forecaster's most recent NHSN season, + or use the leaky tracker with no warm start when that season is + unavailable. ### Directions this points to, ranked @@ -526,12 +565,15 @@ Reading: (all horizons under-predict in the ramp). Feeding the fresh h−1/h0 gradient into h2/h3's update (a cross-horizon proxy signal) is the only idea on the list that changes *when* the offset turns rather than how fast. + The lagged-correlation measurement in "Current numbers" is the baseline to + beat: a fix should move the best-matching lag toward `k = 0`. 3. **Phase-gate updates on data, not the calendar.** The Feb-15 cutoff works because both live seasons peaked around the turn of the year; a late-peaking season would break it. Gating on the base median's trailing trend (post-peak ⇒ stop updating, or shrink) expresses the same rule without a hard-coded date. -4. **Scale-aware offsets via a smooth proxy**, given the log-space failure. +4. **Scale-aware offsets via a smooth proxy**, if sqrt is not enough (the + log-space result is pre-fix and was not re-run). 5. **Eta variants** (seasonal, per-level, pooled): parked. None can fix a wrong-signed step. @@ -563,13 +605,15 @@ Fix: `group_by(geo_value, time_value) %>% slice_max(version, n = 1L, with_ties = Possible next steps, roughly ordered: +0. **Re-run the ILI+ experiments on the fixed replay**: the ILI+ burn-in and + setup C (does a warm start learned on ILI+ transfer to NHSN?). 1. **Review the ranked gallery** — does the WIS cost concentrate at turning points (the `h + 2` staleness prediction)? Are the biggest offsets fixing real miscalibration or chasing data-revision artifacts? 2. **Send the collaborator email** (sweep findings; h −1 free win; carry vs reset a wash). 3. **`windowed_seasonal_extra_sources` retrospective** via - `scripts/calibration_harness.R` (covid, evaluation store) — does + `scripts/calibration/calibration_harness.R` (covid, evaluation store) — does calibration help our best component forecaster, not just the submitted ensemble? 4. **Integration design** for prod: where the tracker lives (per-forecaster diff --git a/notes/calibration-runbook.md b/notes/calibration-runbook.md index 0568cf14..2d35081f 100644 --- a/notes/calibration-runbook.md +++ b/notes/calibration-runbook.md @@ -34,10 +34,8 @@ cd ~/allHail/delphi/multiQT && git checkout delphi-fixes # Refresh the hub submissions (83 CMU-TimeSeries rounds, 2023-10-14 .. 2026-05-30): cd ~/allHail/delphi/FluSight-forecast-hub && git pull --ff-only -# The hub's truth file is downloaded and cached on first use by hub_read_truth(); -# to force a refresh: -cd ~/allHail/delphi/exploration-tooling -rm -f cache/calibration/target-hospital-admissions.csv +# Truth is the latest NHSN vintage (nhsn_read_truth()), fetched from the +# Delphi API; nothing to download by hand. ``` The hub checkout is sparse (`hub-config`, `model-output/CMU-TimeSeries`, @@ -104,7 +102,7 @@ cd ~/allHail/delphi/exploration-tooling distrobox enter rocker -- Rscript -e ' suppressPackageStartupMessages(source("R/load_all.R")) fc <- hub_read_forecasts() -tr <- hub_read_truth() +tr <- nhsn_read_truth("flu") cal <- calibrate_hub_forecasts(fc, tr, burn_in_seasons = "2023-2024", season_policy = "carry", # or "reset" / "shrink" @@ -126,15 +124,15 @@ Takes ~5 s for all 265 (location, horizon) series. Other entry points: ## 3b. COVID calibration The COVID hub checkout at `~/allHail/delphi/covid19-forecast-hub` carries all -model-output for CMU-TimeSeries (89 rounds, 2024-11-23 … present) and a local -truth CSV — no HTTP download required. +model-output for CMU-TimeSeries (89 rounds, 2024-11-23 … present); truth is +the latest NHSN covid vintage. ```sh cd ~/allHail/delphi/exploration-tooling distrobox enter rocker -- Rscript -e ' suppressPackageStartupMessages(source("R/load_all.R")) -fc <- hub_read_forecasts(hub_dir = HUB_COVID_DIR, target = HUB_COVID_TARGET) -tr <- hub_read_covid_truth() +fc <- hub_read_forecasts(hub_dir = "../covid19-forecast-hub", target = HUB_COVID_TARGET) +tr <- nhsn_read_truth("covid") cal <- calibrate_hub_forecasts(fc, tr, burn_in_seasons = character(0), # no off-season in covid data yet season_policy = "carry", diff --git a/reports/writeups/calibration/calibration_findings_flu.Rmd b/reports/writeups/calibration/calibration_findings_flu.Rmd new file mode 100644 index 00000000..2a09d417 --- /dev/null +++ b/reports/writeups/calibration/calibration_findings_flu.Rmd @@ -0,0 +1,713 @@ +--- +title: Online calibration of flu forecasts — what we are seeing +date: "`r format(Sys.time(), '%d %B %Y')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + hub_dir: "../FluSight-forecast-hub" + rep_min_pop: 5000000 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +# cache = TRUE chunks are keyed on chunk code, NOT on functions in R/ -- after +# changing calibration code, clear cache/calibration/knitr_cache_findings_flu/. +knitr::opts_chunk$set( + fig.align = "center", fig.width = 10, fig.height = 6, + message = FALSE, warning = FALSE, cache = FALSE, + cache.path = here::here("cache/calibration/knitr_cache_findings_flu/") +) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw()) +``` + +This notebook collects the main findings of the MultiQT calibration work +(`notes/CALIBRATION.md` has the full record and the method). In short: per +(location, horizon) series, an additive offset over the 23 hub quantile levels +is learned online from past coverage misses and added to the base forecast. +The questions: + +1. **Which way is the base biased?** On NHSN, low, in every season, for both + `windowed_seasonal` and the submitted ensemble. On ILI+, `windowed_seasonal` + has little level bias, but shares the NHSN shortfall at the peak (section 1.2). +2. **Why do short and long horizons behave so differently?** An outcome is + only learned from once it has settled, `h + 2` rounds after the forecast, + so the h3 offset is always at least five rounds stale. +3. **What does a leak on the fast term buy?** It stops a stale offset from + persisting into the part of the season where it has the wrong sign. +4. **What does the sqrt transform buy?** Offsets that stay closer to + proportionate between trough and peak. + +All tracker runs use `lr_mult = 0.03`, `lr_window = 20`, `season_policy = +"carry"`, and 2023-24 as the burn-in season. Scores are on counts against the +latest NHSN vintage, over all hub locations (US included) unless stated. "WIS +change %" is positive when calibration improves on the base forecast. Burn-in +rounds are excluded from every score. + +```{r libs} +# Not cached: attaches packages and defines functions every render. +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +source(here::here("scripts/calibration/calibration_ili_backfill.R")) +source(here::here("scripts/calibration/calibration_ws_backfill.R")) +``` + +```{r load, cache=TRUE} +nhsn_arch <- get_nhsn_data_archive("flu") +nhsn_truth <- nhsn_read_truth("flu", nhsn_arch) +hub_fc <- hub_read_forecasts(hub_dir = here::here(params$hub_dir)) +ili <- ili_read_pseudo_hub() +ws <- ws_read_pseudo_hub() + +geo_map <- get_population_data() %>% + distinct(state_id, .keep_all = TRUE) %>% + transmute(geo_value = state_id, abbrev = toupper(state_id), location = state_code, population) +``` + +```{r prep, cache=TRUE, dependson="load"} +LIVE_SEASONS <- c("2024-2025", "2025-2026") + +# windowed_seasonal replay on the hub's submission rounds only, so seasons, +# gaps and round counts match the ensemble's. +ws_fc <- ws %>% + filter(reference_date %in% unique(hub_fc$reference_date)) %>% + select(-as_of_policy) +# Locations both bases cover, for the like-for-like comparison in section 5. +common_loc <- intersect(unique(ws_fc$location), unique(hub_fc$location)) + +ili_rounds <- hub_label_seasons(unique(ili$forecasts$reference_date)) +ili_fc <- ili$forecasts %>% + filter(reference_date %in% ili_rounds$round_date[!ili_rounds$season %in% c("2010-2011", "2020-2021", "2021-2022", "2023-2024")]) + +in_season <- function(d) as.integer(format(d, "%m")) %in% c(10:12, 1:5) +pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) +``` + +```{r runs, cache=TRUE, dependson="prep"} +run <- function(fc, truth, ...) { + args <- modifyList( + list( + burn_in_seasons = "2023-2024", transform = "sqrt", lr_window = 20, season_policy = "carry", + lr_args = list(mult = 0.03, floor = 1e-3), progress = FALSE + ), + list(...) + ) + suppressMessages(do.call(calibrate_hub_forecasts, c(list(fc, truth), args))) +} +operating <- list(slow_init = "burn_in_quantile", lr_slow = list(mult = 0.003), fast_decay = 0.1) + +# The submitted ensemble, all hub locations: the method variants. +ens <- list( + count = run(hub_fc, nhsn_truth, transform = "identity"), + count_leaky = run(hub_fc, nhsn_truth, transform = "identity", fast_decay = 0.1), + sqrt = run(hub_fc, nhsn_truth), + sqrt_leaky = run(hub_fc, nhsn_truth, fast_decay = 0.1), + warm_static = run(hub_fc, nhsn_truth, slow_init = "burn_in_quantile", lr_args = list(mult = 1e-6, floor = 0)), + warm_leaky = run(hub_fc, nhsn_truth, slow_init = "burn_in_quantile", fast_decay = 0.1), + operating = do.call(run, c(list(hub_fc, nhsn_truth), operating)) +) +# Ensemble (A) and windowed_seasonal (B) on their common locations. +ens_common <- hub_fc %>% filter(location %in% common_loc) +ws_common <- ws_fc %>% filter(location %in% common_loc) +ab <- list( + A_single = run(ens_common, nhsn_truth), + A_leaky = run(ens_common, nhsn_truth, fast_decay = 0.1), + A_operating = do.call(run, c(list(ens_common, nhsn_truth), operating)), + B_single = run(ws_common, nhsn_truth), + B_leaky = run(ws_common, nhsn_truth, fast_decay = 0.1), + B_operating = do.call(run, c(list(ws_common, nhsn_truth), operating)) +) + +variant_labels <- c( + count = "count space, single term (the paper)", + count_leaky = "count space, leaky fast (decay 0.1)", + sqrt = "sqrt, single term", + sqrt_leaky = "sqrt, leaky fast (decay 0.1)", + warm_static = "sqrt, warm start only (no tracking)", + warm_leaky = "sqrt, warm start + leaky fast", + operating = "sqrt, warm start + slow 0.003 + leaky fast (operating point)" +) +``` + +```{r score-helpers} +wis_table <- function(cals, labels = names(cals), by = character(0), seasons = NULL) { + purrr::imap(cals, function(cal, nm) { + left_join( + hub_quantile_loss(cal, by = by), + hub_coverage_summary(cal, by = by) %>% select(all_of(c(by, "horizon")), "cal_error_base", "cal_error_cal"), + by = c(by, "horizon") + ) %>% mutate(variant = nm) + }) %>% + bind_rows() %>% + { if (!is.null(seasons)) filter(., season %in% seasons) else . } %>% + mutate(variant = factor(variant, levels = names(cals), labels = labels)) +} +wide <- function(tbl, col, digits = 1, id = "variant") { + tbl %>% + select(all_of(id), horizon, value = all_of(col)) %>% + mutate(value = round(value, digits)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = value, names_prefix = "h") +} +median_rows <- function(fc, truth, base) { + rounds <- hub_label_seasons(unique(fc$reference_date)) %>% select(reference_date = round_date, season) + fc %>% + filter(level == 0.5) %>% + inner_join(truth, by = c("location", "target_end_date")) %>% + inner_join(rounds, by = "reference_date") %>% + mutate(base = base) +} +season_of <- function(d) { + y <- as.integer(format(d, "%Y")) - (as.integer(format(d, "%m")) < 8) + paste0(y, "-", y + 1) +} +fan_rows <- function(fc, loc, seasons, every = 3L) { + rounds <- hub_label_seasons(unique(fc$reference_date)) %>% + filter(season %in% seasons) %>% + group_by(season) %>% + filter((season_round - 1L) %% every == 0L) %>% + ungroup() %>% + select(reference_date = round_date, season) + fc %>% + filter(location == loc, level %in% c(0.1, 0.5, 0.9)) %>% + inner_join(rounds, by = "reference_date") %>% + mutate(level = paste0("q", level * 100)) %>% + select(season, reference_date, target_end_date, level, value) %>% + tidyr::pivot_wider(names_from = level, values_from = value) +} +fan_plot <- function(fans, truth_rows, vintages = NULL, ylab) { + p <- ggplot(fans, aes(target_end_date, group = reference_date)) + if (!is.null(vintages)) { + p <- p + geom_line(aes(time_value, value, group = reference_date), data = vintages, color = "orange3", linewidth = 0.4, alpha = 0.8) + } + p + + geom_line(aes(target_end_date, truth), data = truth_rows, inherit.aes = FALSE, color = "black") + + geom_ribbon(aes(ymin = q10, ymax = q90), fill = "steelblue", alpha = 0.25) + + geom_line(aes(y = q50), color = "steelblue4") + + facet_wrap(~season, scales = "free", ncol = 3) + + labs(x = NULL, y = ylab) +} +``` + +# 1. The base bias + +## 1.1 On NHSN, both bases forecast too low + +The tracker's largest gains come from correcting a persistent bias, so the +first question is whether the base has one. Two bases on NHSN: + +- **the submitted ensemble** (`CMU-TimeSeries`), read from the hub checkout; +- **`windowed_seasonal`**, flu prod's seasonal component, replayed by + `scripts/calibration/calibration_ws_backfill.R` with prod's archive, seed + and substitutions. 2024-25 and 2025-26 are honest as-of replays; 2023-24 + uses finalized data, since no NHSN vintages exist before 2024-11-19. + +```{r bias-data} +med <- bind_rows( + median_rows(ws_fc, nhsn_truth, "windowed_seasonal"), + median_rows(hub_fc, nhsn_truth, "submitted ensemble") +) +bias_by_season <- med %>% + group_by(base, season, horizon) %>% + summarize(above = 100 * mean(truth > value), .groups = "drop") +``` + +```{r bias-plot, fig.height=4} +ggplot(bias_by_season, aes(season, above, color = factor(horizon), group = horizon)) + + geom_hline(yintercept = 50, linetype = "dashed") + + geom_line() + + geom_point() + + facet_wrap(~base) + + scale_color_viridis_d(name = "horizon", end = 0.9) + + labs(x = NULL, y = "% of forecasts with truth above the median", title = "Above the dashed line: forecast too low") +``` + +Pooled over all three seasons: the share of forecasts with truth above the +median, and the summed median as a percentage error on summed truth (negative +means too low; this weights the peak weeks). + +```{r bias-table} +bias_pooled <- med %>% + group_by(base, horizon) %>% + summarize(above = 100 * mean(truth > value), level_err = 100 * (sum(value) / sum(truth) - 1), .groups = "drop") +knitr::kable( + left_join( + wide(bias_pooled, "above", 0, id = "base") %>% rename_with(~ paste(.x, "above %"), starts_with("h")), + wide(bias_pooled, "level_err", 0, id = "base") %>% rename_with(~ paste(.x, "level %"), starts_with("h")), + by = "base" + ) +) +``` + +Where in the season the bias sits: the same two numbers at h0 and h3, split +by how high truth is relative to the location's 90th-percentile in-season +level (trough on the left, peak on the right). + +```{r bias-bins} +loc_scale <- nhsn_truth %>% + filter(target_end_date >= as.Date("2023-10-01"), in_season(target_end_date)) %>% + group_by(location) %>% + summarize(scale = stats::quantile(truth, 0.9, names = FALSE), .groups = "drop") +bins <- med %>% + filter(horizon %in% c(0, 3)) %>% + inner_join(loc_scale, by = "location") %>% + mutate(level_bin = cut(truth / scale, c(0, 0.1, 0.25, 0.5, 0.75, 1, Inf), right = FALSE)) %>% + filter(!is.na(level_bin)) %>% + group_by(base, horizon, level_bin) %>% + summarize(above = round(100 * mean(truth > value)), level = round(100 * (sum(value) / sum(truth) - 1)), .groups = "drop") %>% + mutate(cell = sprintf("%d%% / %+d%%", above, level)) %>% + select(base, horizon, level_bin, cell) %>% + tidyr::pivot_wider(names_from = level_bin, values_from = cell) +knitr::kable(bins, caption = "truth above median % / level error %, by truth ÷ location's in-season 90th percentile") +``` + +- Both bases are low in all three NHSN seasons, and the bias grows with + horizon. At h0 it is a fairly even level shift (roughly 5–20% low across the + range). At h3 it concentrates at the peak: near the top of the season the + median is 60–70% low and truth is above it almost every time. That is the + base failing to anticipate how high the peak goes, not a constant offset. +- `windowed_seasonal` is the less biased of the two at h−1 and the more biased + of the two by h3. The ensemble's h−1 is a different model + (`climate_linear`) and is further off there. + +### A representative state + +The state whose `windowed_seasonal` above-median share (pooled over h0–h3 +and both live seasons) is closest to the pooled value, among states with +population over `r format(params$rep_min_pop, big.mark = ",", scientific = FALSE)`. +Every third round is drawn: the median path from h−1 to h3 with an 80% band, +over finalized truth (black). The orange lines are the NHSN vintage each drawn +round saw (its last eight weeks); where the end of an orange line sits below +the black line, the forecaster was working from data that was later revised +up. + +```{r rep-state} +pooled <- med %>% + filter(base == "windowed_seasonal", horizon >= 0, season %in% LIVE_SEASONS) %>% + summarize(p = mean(truth > value)) %>% + pull(p) +rep_scores <- med %>% + filter(base == "windowed_seasonal", horizon >= 0, season %in% LIVE_SEASONS) %>% + group_by(location) %>% + summarize(score = abs(mean(truth > value) - pooled), .groups = "drop") %>% + inner_join(geo_map, by = "location") %>% + filter(population > params$rep_min_pop) %>% + arrange(score) +rep_loc <- rep_scores$location[1] +rep_abbrev <- rep_scores$abbrev[1] +rep_geo <- rep_scores$geo_value[1] +``` + +```{r nhsn-fans, fig.height=4.5} +fans <- fan_rows(ws_fc, rep_loc, LIVE_SEASONS) +truth_rows <- nhsn_truth %>% + filter(location == rep_loc) %>% + mutate(season = season_of(target_end_date)) %>% + filter(season %in% LIVE_SEASONS, in_season(target_end_date)) +vintages <- purrr::map(unique(fans$reference_date), function(rd) { + as_of <- rd - 3L + if (as_of < min(nhsn_arch$DT$version)) return(NULL) + epix_as_of(nhsn_arch, as_of) %>% + as_tibble() %>% + filter(geo_value == rep_geo, time_value > rd - 7L * 9L) %>% + transmute(reference_date = rd, season = season_of(rd), time_value, value) +}) %>% bind_rows() +fan_plot(fans, truth_rows, vintages, ylab = "weekly flu admissions") + + ggtitle(sprintf("windowed_seasonal on NHSN, %s", rep_abbrev)) +``` + +## 1.2 On ILI+, the same forecaster has little level bias + +The same forecaster replayed on the ILI+ decade +(`scripts/calibration/calibration_ili_backfill.R`: finalized state ILI+, +snapshot cut one week before the forecast date, so hub h0 is one week past the +last observation). The same two numbers as the NHSN table: + +```{r ili-bias} +ili_med <- median_rows(ili_fc, ili$truth, "windowed_seasonal, ILI+ replay") +ili_pooled <- ili_med %>% + group_by(base, horizon) %>% + summarize(above = 100 * mean(truth > value), level_err = 100 * (sum(value) / sum(truth) - 1), .groups = "drop") +knitr::kable( + left_join( + wide(ili_pooled, "above", 0, id = "base") %>% rename_with(~ paste(.x, "above %"), starts_with("h")), + wide(ili_pooled, "level_err", 0, id = "base") %>% rename_with(~ paste(.x, "level %"), starts_with("h")), + by = "base" + ) +) +ili_scale <- ili$truth %>% + group_by(location) %>% + summarize(scale = stats::quantile(truth, 0.9, names = FALSE), .groups = "drop") +ili_bins <- ili_med %>% + filter(horizon %in% c(0, 3)) %>% + inner_join(ili_scale, by = "location") %>% + mutate(level_bin = cut(truth / scale, c(0, 0.1, 0.25, 0.5, 0.75, 1, Inf), right = FALSE)) %>% + filter(!is.na(level_bin)) %>% + group_by(horizon, level_bin) %>% + summarize(above = round(100 * mean(truth > value)), level = round(100 * (sum(value) / sum(truth) - 1)), .groups = "drop") %>% + mutate(cell = sprintf("%d%% / %+d%%", above, level)) %>% + select(horizon, level_bin, cell) %>% + tidyr::pivot_wider(names_from = level_bin, values_from = cell) +knitr::kable(ili_bins, caption = "ILI+: truth above median % / level error %, by truth ÷ location's 90th percentile") +``` + +- On ILI+ the median is about as often above truth as below (43–46% above), + and the summed level is within ±4% at every horizon. The persistent low bias + on NHSN does not show up here. +- The peak shortfall does. At h3, in the top bin, truth is above the median + almost nine times in ten, as on NHSN. At the trough the median is well + above truth. Across the season these roughly cancel on ILI+; on NHSN the peak + shortfall dominates. + +The replay's fans for the representative state: + +```{r ili-fans, fig.height=4.5} +ili_seasons <- c("2016-2017", "2017-2018", "2018-2019") +fans <- fan_rows(ili_fc, rep_loc, ili_seasons) +truth_rows <- ili$truth %>% + filter(location == rep_loc) %>% + mutate(season = season_of(target_end_date)) %>% + filter(season %in% ili_seasons, in_season(target_end_date)) +fan_plot(fans, truth_rows, ylab = "ILI+ (%)") + + ggtitle(sprintf("windowed_seasonal ILI+ replay, %s", rep_abbrev)) +``` + +An earlier version of this replay was wrong at every horizon, and the ILI+ +results in `notes/CALIBRATION.md` dated before 2026-09-23 came from it (the +"bias flips sign on ILI+" reading, the ILI+ burn-in and setup C). The cause +was epipredict's `extend_lags` latency adjustment. It pushes every lag back +by the worst latency of any state, and some ILI+ state series end years +before the forecast date (DC stops in 2015), so on 2018-01-03 every lag was +extended by 1001 days. The replay now leaves such states out of the latency +check. The ILI+ burn-in and setup C have not been re-run on the fixed replay. + +# 2. Staleness: why h−1/h0 and h2/h3 respond differently + +The tracker learns from an outcome only once it has settled (14 days after +the target week), so the outcome of an h-step forecast is revealed `h + 2` +rounds after it was issued, and the step at round `t` is driven by forecasts +issued at `t − h − 2` or earlier. The long horizons have the largest reveal +lag *and* are the hardest to forecast, so they also carry most of the base +error: + +```{r lag-table} +base_wis <- hub_quantile_loss(ens$sqrt) %>% + mutate( + `reveal lag (rounds)` = horizon + 2L, + `base WIS (ensemble)` = round(2 * loss_base, 1), + `relative to h0` = round(loss_base / loss_base[horizon == 0], 2) + ) %>% + select(horizon, `reveal lag (rounds)`, `base WIS (ensemble)`, `relative to h0`) +knitr::kable(base_wis) +``` + +## The offset echoes the past + +For the median of each series, define the *needed* shift as +`sqrt(truth) − sqrt(base median)` (what an oracle would have added, in the +tracker's working units) and the *played* shift as +`sqrt(calibrated median) − sqrt(base median)`. Below: the correlation between +the played shift at round `t` and the needed shift at round `t − k`, averaged +over locations, for the single-term sqrt tracker. + +```{r lag-corr, fig.height=4} +shift_rows <- function(cal) { + cal$forecasts %>% + filter(!is_burn_in, level == 0.5, !is.na(truth), !is.na(value_base)) %>% + mutate(needed = sqrt(truth) - sqrt(value_base), played = sqrt(value_cal) - sqrt(value_base)) %>% + select(location, horizon, round_index, reference_date, season, value_base, value_cal, needed, played) +} +ks <- 0:12 +lag_corr <- shift_rows(ens$sqrt) %>% + group_by(location, horizon) %>% + arrange(round_index, .by_group = TRUE) %>% + reframe(k = ks, r = purrr::map_dbl(ks, function(k) { + n <- length(played) + if (n - k < 10) return(NA_real_) + x <- played[(k + 1):n] + y <- needed[1:(n - k)] + if (sd(x) == 0 || sd(y) == 0) return(NA_real_) + stats::cor(x, y) + })) %>% + group_by(horizon, k) %>% + summarize(r = mean(r, na.rm = TRUE), .groups = "drop") +peaks <- lag_corr %>% group_by(horizon) %>% slice_max(r, n = 1) %>% ungroup() +ggplot(lag_corr, aes(k, r, color = factor(horizon))) + + geom_hline(yintercept = 0, color = "grey50") + + geom_line() + + geom_point(data = peaks, size = 3) + + geom_vline(aes(xintercept = horizon + 2, color = factor(horizon)), data = distinct(lag_corr, horizon), linetype = "dotted") + + scale_color_viridis_d(name = "horizon", end = 0.9) + + scale_x_continuous(breaks = ks) + + labs( + x = "k (rounds): played shift at t vs needed shift at t − k", + y = "mean correlation over locations", + title = "Dotted: reveal lag h + 2. Large dots: best-matching lag." + ) +knitr::kable( + lag_corr %>% + filter(k == 0) %>% + select(horizon, `r at k = 0` = r) %>% + left_join(peaks %>% select(horizon, `best k` = k, `r at best k` = r), by = "horizon") %>% + mutate(`reveal lag` = horizon + 2L, across(starts_with("r "), ~ round(.x, 2))) +) +``` + +- **The current needed shift and the played shift are negatively + correlated** at every horizon from h0 up, and more so at longer horizons: + on average the tracker is pushing the wrong way *right now*. +- **The played shift best matches the needed shift of `h + 4` to `h + 6` + rounds earlier** (h0–h3): the reveal lag plus two to four rounds for the + tracker to build up an offset. At h3 that is nine rounds, about two months. + h−1 has almost no structure: its lag is one round. + +Share of rounds where the calibrated median moved in the right direction +(same sign as the needed shift), among rounds where it moved the median by +more than 1%: + +```{r sign-table} +sign_right <- purrr::imap(ens[c("count", "sqrt", "sqrt_leaky", "warm_static", "operating")], function(cal, nm) { + shift_rows(cal) %>% + filter(abs(value_cal - value_base) > 0.01 * value_base) %>% + group_by(horizon) %>% + summarize(right = 100 * mean(sign(played) == sign(needed)), .groups = "drop") %>% + mutate(variant = variant_labels[[nm]]) +}) %>% bind_rows() +knitr::kable(wide(sign_right, "right", 0), caption = "% of moves in the right direction") +``` + +Because the base is low most of the time, "shift up" is right more often than +not, so every variant is above 50%. The static warm start (a constant +upward shift, no tracking) is right as often as any tracker at h0–h3: what +the trackers add is size and coverage, not timing. + +## One series through a season + +`r rep_abbrev` at h0 and h3 over both live seasons: the needed shift (black) +and the played shift for three variants, in sqrt units. + +```{r series-plot, fig.height=6} +series <- purrr::imap(ens[c("sqrt", "sqrt_leaky", "operating")], function(cal, nm) { + shift_rows(cal) %>% filter(location == rep_loc, horizon %in% c(0, 3)) %>% mutate(variant = variant_labels[[nm]]) +}) %>% bind_rows() +needed <- series %>% distinct(horizon, season, reference_date, needed) +ggplot(series, aes(reference_date)) + + geom_hline(yintercept = 0, color = "grey50") + + geom_line(aes(y = needed), data = needed, color = "black", linewidth = 0.8) + + geom_line(aes(y = played, color = variant)) + + facet_grid(paste0("h", horizon) ~ season, scales = "free") + + scale_color_brewer(palette = "Set1", name = NULL) + + labs(x = NULL, y = "shift of the median (sqrt units)") + + theme(legend.position = "bottom", legend.direction = "vertical") +``` + +The needed shift is dominated by the base missing the peak: large, fast and +changing sign within a few weeks, more so at h3. None of the trackers +follows it; at this learning rate they only build a small, smooth level. That +is by design (a faster tracker overshoots; see the sweep in +`calibration_qt_flu.Rmd`), and it is why the gain is mostly a level +correction plus coverage rather than a fix for the peak miss. + +## Score by horizon + +WIS change % over both live seasons, and calibration error (mean absolute +coverage miss over the 23 levels; base in the first row). + +```{r horizon-table} +tbl <- wis_table(ens, variant_labels[names(ens)]) +knitr::kable(wide(tbl, "pct_improvement"), caption = "WIS change % (positive is better)") +cal_err <- bind_rows( + tbl %>% filter(variant == variant_labels[["sqrt"]]) %>% mutate(variant = "base", cal_error_cal = cal_error_base), + tbl %>% mutate(variant = as.character(variant)) +) +knitr::kable(wide(cal_err, "cal_error_cal", 3), caption = "Calibration error") +``` + +- **h−1 and h0 gain under every tracker.** The lag is one to two rounds, + short enough that recent misses are still informative. At h−1 the + ensemble's base is `climate_linear`, a weaker model, so there is more to + correct. +- **h2 and h3 are where the method choices matter.** A tracker alone is + negative (single term) or roughly neutral (leaky) there; the warm start (a constant correction + learned from the burn-in season, which needs no timely feedback) is what + makes them positive. + +# 3. The leak: letting stale offsets decay + +With `fast_decay = 0.1`, the fast term is multiplied by 0.9 before each +step, so a correction that new outcomes do not refresh fades (half-life about +6.6 rounds). The effect by calendar month of the reference date: + +```{r by-month} +by_month <- function(cal) { + cal$forecasts %>% + filter(!is_burn_in, !is.na(truth), !is.na(value_base)) %>% + mutate(month = format(reference_date, "%b")) %>% + group_by(horizon, month) %>% + summarize( + base = sum(pinball(truth, value_base, level)), + cal = sum(pinball(truth, value_cal, level)), + .groups = "drop" + ) %>% + group_by(horizon) %>% + mutate(pct = 100 * (base - cal) / base, share = 100 * base / sum(base)) %>% + ungroup() +} +month_levels <- month.abb[c(10:12, 1:5)] +month_tbl <- purrr::imap(ens[c("sqrt", "sqrt_leaky", "warm_leaky", "operating")], function(cal, nm) { + by_month(cal) %>% mutate(variant = variant_labels[[nm]]) +}) %>% + bind_rows() %>% + filter(month %in% month_levels) %>% + mutate(month = factor(month, levels = month_levels)) + +ggplot(month_tbl %>% filter(horizon %in% c(0, 3)), aes(month, pct, color = variant, group = variant)) + + geom_hline(yintercept = 0, color = "grey50") + + geom_line() + + geom_point() + + facet_wrap(~ paste0("h", horizon)) + + scale_color_brewer(palette = "Set1", name = NULL) + + coord_cartesian(ylim = c(-60, 30)) + + labs(x = "month of reference date", y = "WIS change % (clipped at −60)") + + theme(legend.position = "bottom", legend.direction = "vertical") +knitr::kable( + month_tbl %>% + filter(horizon == 3) %>% + mutate(pct = round(pct, 1)) %>% + select(variant, month, pct) %>% + arrange(month) %>% + tidyr::pivot_wider(names_from = month, values_from = pct) %>% + bind_rows( + month_tbl %>% + filter(horizon == 3, variant == variant_labels[["sqrt"]]) %>% + transmute(variant = "share of base WIS %", month, pct = round(share, 1)) %>% + arrange(month) %>% + tidyr::pivot_wider(names_from = month, values_from = pct) + ), + caption = "h3: WIS change % by month, and each month's share of the base WIS" +) +``` + +- **The losses are in March–April and October, which carry little WIS.** In + March–April the offset built while chasing the winter peak is still in + place when the base has already come down; in October the offset carried + over from the previous spring has not been unwound yet (the reveal lag + means the first live outcomes arrive weeks into the season). +- **The leak alone cuts those losses by two thirds or more** (h3: March −17% + against −47% for the single-term tracker, October −14% against −48%, April + +9% against −36%), at the cost of coverage: + a correction that must be refreshed every round cannot hold a level through + a five-round reveal lag (calibration error in the table above: the leaky + variant sits between single-term and uncalibrated). +- **The warm start wins in December–January**, the months that carry most of + the base WIS (+4% at h3 against +1–2% without it), and so wins overall at + h1–h3 even though part of the spring loss comes back (March −28 to −33%). The warm start is a constant, + so it is too large once the season comes down; the leak cannot take it + away because it only decays the fast term. + +# 4. The sqrt transform + +In sqrt space the tracker sees `sqrt(truth)` and `sqrt(base quantiles)`, +learns and projects offsets there, and the played quantiles are squared back +(clamped at zero) and scored on counts. The coverage gradient is unchanged, +since a monotone map preserves which side of a quantile the truth falls on; +what changes is the size of a step. A fixed sqrt-space offset `d` on a base +median `b` moves it by `2 d sqrt(b) + d²` counts, so it grows with the square +root of the level instead of staying a fixed count. + +The plot shows the played median shift relative to the base median, split by +where the base median sits within the location's season (base median divided +by that location's 90th-percentile in-season truth). + +```{r relative-shift, fig.height=4} +rel <- purrr::imap(ens[c("count", "sqrt", "count_leaky", "sqrt_leaky")], function(cal, nm) { + cal$forecasts %>% + filter(!is_burn_in, level == 0.5, !is.na(value_base), value_base > 0) %>% + inner_join(loc_scale, by = "location") %>% + mutate( + level_bin = cut(value_base / scale, c(0, 0.05, 0.1, 0.2, 0.4, 0.7, Inf), right = FALSE), + rel_shift = abs(value_cal - value_base) / value_base, + variant = variant_labels[[nm]] + ) +}) %>% bind_rows() +rel_summary <- rel %>% + filter(horizon %in% c(0, 3)) %>% + group_by(variant, horizon, level_bin) %>% + summarize(med = 100 * median(rel_shift), .groups = "drop") +ggplot(rel_summary, aes(level_bin, med, color = variant, group = variant)) + + geom_line() + + geom_point() + + facet_wrap(~ paste0("h", horizon)) + + scale_y_log10() + + scale_color_brewer(palette = "Paired", name = NULL, direction = -1) + + labs( + x = "base median / location's in-season 90th percentile (trough → peak)", + y = "median |shift| / base median (%, log)" + ) + + theme(legend.position = "bottom", legend.direction = "vertical") +knitr::kable( + rel_summary %>% + group_by(variant, horizon) %>% + summarize(`trough ÷ peak` = round(first(med) / last(med), 1), .groups = "drop") %>% + tidyr::pivot_wider(names_from = horizon, values_from = `trough ÷ peak`, names_prefix = "h"), + caption = "Relative shift at the trough bin divided by the relative shift at the peak bin" +) +``` + +```{r transform-table} +tt <- wis_table(ens[c("count", "sqrt", "count_leaky", "sqrt_leaky")], variant_labels[c("count", "sqrt", "count_leaky", "sqrt_leaky")]) +knitr::kable(wide(tt, "pct_improvement"), caption = "WIS change %, both live seasons") +knitr::kable(wide(tt, "cal_error_cal", 3), caption = "Calibration error") +``` + +- **In count space the relative shift spans a much wider range** between + trough and peak than in sqrt space: an offset sized on peak residuals is + far too large for the trough. Sqrt narrows that range but does not remove + it, and it makes the relative shift at the peak somewhat larger. +- **Sqrt is better on WIS at every horizon**, with or without the leak. Its + calibration error is similar (slightly better at h1–h3 for the single-term + tracker, slightly worse with the leak). +- **The transform does not address staleness.** A stale sqrt-space offset is + still wrong-signed, just more modestly sized. The leak and the transform + are complementary: count space with a leak is comparable to sqrt without + one at h1–h3. + +# 5. The same tracker on two bases + +The ensemble (A) and `windowed_seasonal` (B) calibrated the same way, on the +locations both cover. `windowed_seasonal` is less miscalibrated to start with, +so there is less to gain. + +```{r ab-table} +ab_tbl <- wis_table(ab, names(ab), by = "season", seasons = LIVE_SEASONS) %>% + mutate(variant = as.character(variant)) +knitr::kable(wide(ab_tbl, "pct_improvement", id = c("variant", "season")) %>% arrange(season, variant), caption = "WIS change %") +ab_err <- bind_rows( + ab_tbl %>% filter(variant %in% c("A_single", "B_single")) %>% mutate(variant = paste0(substr(variant, 1, 1), "_base"), cal_error_cal = cal_error_base), + ab_tbl +) +knitr::kable(wide(ab_err, "cal_error_cal", 3, id = c("variant", "season")) %>% arrange(season, variant), caption = "Calibration error") +``` + +- The operating point helps both bases in both seasons at h0–h3, with small + gains on `windowed_seasonal` (under 4%). +- On `windowed_seasonal` in 2025-26 the single-term tracker has far better + calibration error than the operating point (0.009–0.014 against + 0.019–0.045) for slightly less WIS gain: the leak is the price of the + spring fix, paid in coverage. +- Whether the warm start transfers to a *different data source* (an ILI+ + burn-in before NHSN) has not been re-run on the fixed ILI+ replay. Within NHSN, the warm start comes from the same + base's burn-in season, and it helps both. + +# Summary + +| question | answer | +|---|---| +| Is there a stable bias to correct? | On NHSN yes: both bases forecast too low in every season, worse at longer horizons and, at h3, concentrated at the peak. On ILI+ the same forecaster is roughly level-unbiased, with the same peak shortfall. | +| Why do h2/h3 behave differently from h−1/h0? | The offset is at least `h + 2` rounds stale. Right now it is negatively correlated with the needed shift, and it best matches the needed shift of `h + 4` to `h + 6` rounds earlier. The long horizons are also the hardest and carry most of the base WIS. | +| What does the leak do? | Cuts the spring and October losses from stale offsets by two thirds or more, at the cost of coverage. | +| What does the warm start do? | Holds the persistent low bias as a constant: gains where the WIS mass is (December–January), which the trackers alone cannot at h2/h3. Its spring losses come back, since the leak does not touch it. | +| What does sqrt do? | Narrows the range of relative offset size between trough and peak, and is better on WIS than count space at every horizon. It does not fix staleness. | diff --git a/reports/writeups/calibration/calibration_qt_covid.Rmd b/reports/writeups/calibration/calibration_qt_covid.Rmd index 82397d1c..a4b83c51 100644 --- a/reports/writeups/calibration/calibration_qt_covid.Rmd +++ b/reports/writeups/calibration/calibration_qt_covid.Rmd @@ -8,6 +8,7 @@ output: toc: true toc_float: true params: + hub_dir: "../covid19-forecast-hub" burn_in_seasons: "" season_policy: "carry" lr_window: 50 @@ -29,11 +30,14 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw()) ``` -```{r load, cache=TRUE, cache.extra=list(params)} +```{r libs} +# Not cached: attaches packages every render. suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` -forecasts <- hub_read_forecasts(hub_dir = HUB_COVID_DIR, target = HUB_COVID_TARGET) -truth <- hub_read_covid_truth() +```{r load, cache=TRUE, cache.extra=list(params)} +forecasts <- hub_read_forecasts(hub_dir = here::here(params$hub_dir), target = HUB_COVID_TARGET) +truth <- nhsn_read_truth("covid") rounds <- hub_label_seasons(unique(forecasts$reference_date)) burn_in <- if (nzchar(params$burn_in_seasons)) { diff --git a/reports/writeups/calibration/calibration_qt_flu.Rmd b/reports/writeups/calibration/calibration_qt_flu.Rmd index ce1e8946..192865d4 100644 --- a/reports/writeups/calibration/calibration_qt_flu.Rmd +++ b/reports/writeups/calibration/calibration_qt_flu.Rmd @@ -8,6 +8,7 @@ output: toc: true toc_float: true params: + hub_dir: "../FluSight-forecast-hub" burn_in_seasons: "2023-2024" season_policy: "carry" lr_window: 50 @@ -29,11 +30,14 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw()) ``` -```{r load, cache=TRUE, cache.extra=list(params)} +```{r libs} +# Not cached: attaches packages every render. suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` -forecasts <- hub_read_forecasts() -truth <- hub_read_truth() +```{r load, cache=TRUE, cache.extra=list(params)} +forecasts <- hub_read_forecasts(hub_dir = here::here(params$hub_dir)) +truth <- nhsn_read_truth("flu") rounds <- hub_label_seasons(unique(forecasts$reference_date)) burn_in <- if (nzchar(params$burn_in_seasons)) { diff --git a/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd b/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd index cbd9b641..59190418 100644 --- a/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd +++ b/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd @@ -9,6 +9,7 @@ output: toc_float: true toc_depth: 2 params: + hub_dir: "../covid19-forecast-hub" burn_in_seasons: "" season_policy: "carry" lr_window: 20 @@ -30,11 +31,14 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw()) ``` -```{r load, cache=TRUE, cache.extra=list(params)} +```{r libs} +# Not cached: attaches packages every render. suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` -forecasts <- hub_read_forecasts(hub_dir = HUB_COVID_DIR, target = HUB_COVID_TARGET) -truth <- hub_read_covid_truth() +```{r load, cache=TRUE, cache.extra=list(params)} +forecasts <- hub_read_forecasts(hub_dir = here::here(params$hub_dir), target = HUB_COVID_TARGET) +truth <- nhsn_read_truth("covid") rounds <- hub_label_seasons(unique(forecasts$reference_date)) burn_in <- if (nzchar(params$burn_in_seasons)) { @@ -99,9 +103,8 @@ Each panel shows the NHSN data as of the Wednesday before the reference date (`reference_date - 3`) alongside the finalized truth. ```{r snapshots, cache=TRUE, dependson="load"} -archive <- nanoparquet::read_parquet( - here::here("cache/calibration/nhsn_archive_covid.parquet") -) %>% +archive <- get_nhsn_data_archive("covid")$DT %>% + mutate(geo_value = ifelse(geo_value == "usa", "us", geo_value)) %>% as_epi_archive(compactify = TRUE) geo_map <- get_population_data() %>% diff --git a/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd b/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd index 00b9cbc6..eb5883c7 100644 --- a/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd +++ b/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd @@ -9,6 +9,7 @@ output: toc_float: true toc_depth: 2 params: + hub_dir: "../FluSight-forecast-hub" burn_in_seasons: "2023-2024" season_policy: "carry" lr_window: 20 @@ -30,11 +31,14 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw()) ``` -```{r load, cache=TRUE, cache.extra=list(params)} +```{r libs} +# Not cached: attaches packages every render. suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` -forecasts <- hub_read_forecasts() -truth <- hub_read_truth() +```{r load, cache=TRUE, cache.extra=list(params)} +forecasts <- hub_read_forecasts(hub_dir = here::here(params$hub_dir)) +truth <- nhsn_read_truth("flu") rounds <- hub_label_seasons(unique(forecasts$reference_date)) burn_in <- if (nzchar(params$burn_in_seasons)) { diff --git a/reports/writeups/calibration/calibration_qt_seasons_covid.Rmd b/reports/writeups/calibration/calibration_qt_seasons_covid.Rmd index 2c941d8c..c0972d60 100644 --- a/reports/writeups/calibration/calibration_qt_seasons_covid.Rmd +++ b/reports/writeups/calibration/calibration_qt_seasons_covid.Rmd @@ -9,6 +9,7 @@ output: toc_float: true toc_depth: 2 params: + hub_dir: "../covid19-forecast-hub" burn_in_seasons: "" season_policy: "carry" lr_window: 20 @@ -38,11 +39,14 @@ effect; burn-in-dependent variants (conformal warm start) are not included. The states are chosen by how much the baseline calibration changed them, pooled over horizons −1…3. -```{r load, cache=TRUE} +```{r libs} +# Not cached: attaches packages every render. suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` -forecasts <- hub_read_forecasts(hub_dir = HUB_COVID_DIR, target = HUB_COVID_TARGET) -truth <- hub_read_covid_truth() +```{r load, cache=TRUE} +forecasts <- hub_read_forecasts(hub_dir = here::here(params$hub_dir), target = HUB_COVID_TARGET) +truth <- nhsn_read_truth("covid") rounds <- hub_label_seasons(unique(forecasts$reference_date)) burn_in <- if (nzchar(params$burn_in_seasons)) { strsplit(params$burn_in_seasons, ",")[[1]] @@ -54,9 +58,8 @@ geo_map <- get_population_data() %>% distinct(state_id, .keep_all = TRUE) %>% transmute(state_id, abbrev = toupper(state_id), location = state_code) -archive <- nanoparquet::read_parquet( - here::here("cache/calibration/nhsn_archive_covid.parquet") -) %>% +archive <- get_nhsn_data_archive("covid")$DT %>% + mutate(geo_value = ifelse(geo_value == "usa", "us", geo_value)) %>% as_epi_archive(compactify = TRUE) live_rounds <- rounds %>% filter(!season %in% burn_in) diff --git a/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd b/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd index 86b30f5e..d637b158 100644 --- a/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd +++ b/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd @@ -9,6 +9,7 @@ output: toc_float: true toc_depth: 2 params: + hub_dir: "../FluSight-forecast-hub" burn_in_seasons: "2023-2024" season_policy: "carry" lr_window: 20 @@ -45,11 +46,14 @@ season with no tracking, the warm start plus a leaky fast tracker, the warm start plus slow and leaky fast terms (the current operating point), and switching off after February 15 as the earlier reference. -```{r load, cache=TRUE} +```{r libs} +# Not cached: attaches packages every render. suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` -forecasts <- hub_read_forecasts() -truth <- hub_read_truth() +```{r load, cache=TRUE} +forecasts <- hub_read_forecasts(hub_dir = here::here(params$hub_dir)) +truth <- nhsn_read_truth("flu") rounds <- hub_label_seasons(unique(forecasts$reference_date)) burn_in <- strsplit(params$burn_in_seasons, ",")[[1]] diff --git a/scripts/calibration/calibration_export_series.R b/scripts/calibration/calibration_export_series.R index 9d2a9df8..b5436c26 100644 --- a/scripts/calibration/calibration_export_series.R +++ b/scripts/calibration/calibration_export_series.R @@ -7,7 +7,7 @@ # the identical problem otherwise. # # Usage: -# Rscript scripts/calibration_export_series.R [outdir] [n_series] +# Rscript scripts/calibration/calibration_export_series.R [outdir] [n_series] suppressPackageStartupMessages(source(here::here("R/load_all.R"))) @@ -18,19 +18,7 @@ n_series <- if (length(args) >= 2) as.integer(args[[2]]) else 4L dir.create(outdir, showWarnings = FALSE, recursive = TRUE) forecasts <- hub_read_forecasts() -truth <- { - arch <- get_nhsn_data_archive("flu") - arch %>% - epix_as_of(arch$versions_end) %>% - mutate(geo_value = ifelse(geo_value == "usa", "us", geo_value)) %>% - left_join( - get_population_data() %>% select("state_id", location = "state_code"), - by = c("geo_value" = "state_id") - ) %>% - select(target_end_date = time_value, location, truth = value) %>% - filter(!is.na(truth)) %>% - arrange(location, target_end_date) -} +truth <- nhsn_read_truth("flu") rounds <- hub_label_seasons(unique(forecasts$reference_date)) round_date <- rounds$round_date diff --git a/scripts/calibration/calibration_harness.R b/scripts/calibration/calibration_harness.R index 4ccdf1a4..e973c091 100644 --- a/scripts/calibration/calibration_harness.R +++ b/scripts/calibration/calibration_harness.R @@ -18,7 +18,7 @@ # ~14% windowed_seasonal, ~0.03% climate_linear. # # Usage, from the repo root: -# source("scripts/calibration_harness.R") +# source("scripts/calibration/calibration_harness.R") # inp <- ch_inputs() # archives, geos dropped # raw <- ch_backfill(inp) # ~6 min cold, instant warm (cached) # ch_verify_against_store(raw) # prove the harness == the DAG diff --git a/scripts/calibration/calibration_ili_backfill.R b/scripts/calibration/calibration_ili_backfill.R index 700c0aea..336a7a01 100644 --- a/scripts/calibration/calibration_ili_backfill.R +++ b/scripts/calibration/calibration_ili_backfill.R @@ -14,7 +14,7 @@ # forecast date, so hub horizon 0 is one week past the last observation. # # Usage, from the repo root (~30 min cold, cached in cache/calibration/): -# distrobox enter rocker -- Rscript scripts/calibration_ili_backfill.R +# distrobox enter rocker -- Rscript scripts/calibration/calibration_ili_backfill.R # Then in R: ili <- ili_read_pseudo_hub() # list(forecasts, truth) suppressPackageStartupMessages(source(here::here("R/load_all.R"))) @@ -72,10 +72,21 @@ ili_schedule <- function(archive) { ili_forecast_one <- function(archive, fd, geo_map) { snapshot <- make_forecast_snapshot(archive, forecast_date = fd, generation_date = fd, as_of_policy = "asof") %>% filter(time_value <= fd - 7L) + # extend_lags uses the worst latency across geos, so one state whose ILI+ + # series ended (DC stops in 2015) would push every lag years back. Leave + # such states out of the latency check; they still train, they just get no forecast. + stale <- snapshot %>% + as_tibble() %>% + group_by(geo_value) %>% + summarize(last = max(time_value)) %>% + filter(last < max(snapshot$time_value)) %>% + pull(geo_value) + params <- ILI_PARAMS + if (length(stale) > 0) params$keys_to_ignore <- list(list("geo_value", stale)) purrr::map(ILI_HORIZONS, function(h) { run_forecaster( snapshot = snapshot, forecaster = scaled_pop_seasonal, aheads = h * 7L, - params = ILI_PARAMS, id = "windowed_seasonal_ili", + params = params, id = "windowed_seasonal_ili", target_date_shift = 3L, sort_quantiles = TRUE ) %>% mutate(horizon = h) diff --git a/scripts/calibration/calibration_ws_backfill.R b/scripts/calibration/calibration_ws_backfill.R index 1593bf6a..a081d9f7 100644 --- a/scripts/calibration/calibration_ws_backfill.R +++ b/scripts/calibration/calibration_ws_backfill.R @@ -2,7 +2,7 @@ # forecaster (scaled_pop_seasonal, seasonal_method = "window", trained on the # nhsn + ILI+ + flusurv prod archive) over the NHSN seasons and write the result # in the FluSight hub schema that calibrate_hub_forecasts() consumes. Together -# with scripts/calibration_ili_backfill.R this gives one forecaster's history +# with scripts/calibration/calibration_ili_backfill.R this gives one forecaster's history # from 2011 to today, so calibration can be studied without the submitted # ensemble's changing composition in the way. # @@ -14,7 +14,7 @@ # (finalized data cut at the forecast date) and is marked as such in the output. # # Usage, from the repo root (~10 min cold, cached in cache/calibration/): -# distrobox enter rocker -- Rscript scripts/calibration_ws_backfill.R +# distrobox enter rocker -- Rscript scripts/calibration/calibration_ws_backfill.R # Then in R: ws <- ws_read_pseudo_hub() # forecasts; truth from get_nhsn_data_archive("flu") suppressPackageStartupMessages(source(here::here("R/load_all.R"))) diff --git a/scripts/calibration/calibration_ws_experiments.R b/scripts/calibration/calibration_ws_experiments.R index cbaef4cc..5268ece9 100644 --- a/scripts/calibration/calibration_ws_experiments.R +++ b/scripts/calibration/calibration_ws_experiments.R @@ -5,32 +5,20 @@ # results are recorded in notes/CALIBRATION.md. # # Inputs (all cached in cache/calibration/): -# scripts/calibration_ili_backfill.R -> windowed_seasonal on ILI+, 2010-2024 -# scripts/calibration_ws_backfill.R -> windowed_seasonal on NHSN, 2023-2026 -# hub_read_forecasts()/hub_read_truth() -> the submitted ensemble and hub truth +# scripts/calibration/calibration_ili_backfill.R -> windowed_seasonal on ILI+, 2010-2024 +# scripts/calibration/calibration_ws_backfill.R -> windowed_seasonal on NHSN, 2023-2026 +# hub_read_forecasts()/nhsn_read_truth() -> the submitted ensemble and NHSN truth # -# Usage: distrobox enter rocker -- Rscript scripts/calibration_ws_experiments.R +# Usage: distrobox enter rocker -- Rscript scripts/calibration/calibration_ws_experiments.R suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -source(here::here("scripts/calibration_ili_backfill.R")) -source(here::here("scripts/calibration_ws_backfill.R")) +source(here::here("scripts/calibration/calibration_ili_backfill.R")) +source(here::here("scripts/calibration/calibration_ws_backfill.R")) ili <- ili_read_pseudo_hub() ws <- ws_read_pseudo_hub() hub_fc <- hub_read_forecasts() -hub_truth <- { - arch <- get_nhsn_data_archive("flu") - arch %>% - epix_as_of(arch$versions_end) %>% - mutate(geo_value = ifelse(geo_value == "usa", "us", geo_value)) %>% - left_join( - get_population_data() %>% select("state_id", location = "state_code"), - by = c("geo_value" = "state_id") - ) %>% - select(target_end_date = time_value, location, truth = value) %>% - filter(!is.na(truth)) %>% - arrange(location, target_end_date) -} +hub_truth <- nhsn_read_truth("flu") LIVE_SEASONS <- c("2024-2025", "2025-2026") ERA_SPLIT <- as.Date("2023-08-01") # ILI+ history before, NHSN after diff --git a/uv.lock b/uv.lock index 6cfa491b..63bbcb8a 100644 --- a/uv.lock +++ b/uv.lock @@ -7,11 +7,15 @@ name = "exploration-tooling" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "numpy" }, { name = "pandas" }, ] [package.metadata] -requires-dist = [{ name = "pandas", specifier = ">=2.3.3" }] +requires-dist = [ + { name = "numpy", specifier = ">=2.0" }, + { name = "pandas", specifier = ">=2.3.3" }, +] [[package]] name = "numpy" From ef4df52e842caf153a99d36e47a88190d228a9f3 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:03:12 -0700 Subject: [PATCH 002/107] docs(calibration): note effective covid ensemble_mix weights --- notes/CALIBRATION.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index ec213b12..7056a835 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -596,6 +596,13 @@ Flu gets the 2023-24 burn-in for free: the hub checkout already contains all pas **Notebooks:** `CMU-TimeSeries-Calibrated` is added to `ongoing_score_report.Rmd` (`our_forecasters`, score table styling, line width). The per-date `forecast_report.Rmd` notebook receives calibrated forecasts for the current round's reference date, converted back to internal schema (state abbr, Wednesday `forecast_date`, count-space quantile values) via the `notebook` target in `prod_shared.R`. +**What the covid submission actually is:** the submitted covid forecast is `ensemble_mix`, whose components are `windowed_seasonal`, `windowed_seasonal_extra_sources` and `revision_aware` (`pipelines/covid_hosp_prod.R`). Its raw weights come from `pipelines/covid_geo_exclusions.csv`, and `ensemble_weighted()` renormalizes them per (geo, ahead). The 2026-09-23 block sets `windowed_seasonal = 0.5`, `windowed_seasonal_extra_sources = 3` and `revision_aware = 1` at aheads −1 and 0 only. The effective mix is therefore: +- aheads 1–3: ~86% `windowed_seasonal_extra_sources`, ~14% `windowed_seasonal`; +- ahead 0: ~67% / ~11% / ~22% `revision_aware`; +- ahead −1: 100% `revision_aware`, because `drop_negative_aheads` strips the two AR components there. + +`windowed_seasonal_extra_sources` excludes mo and wy, so those states fall back to `windowed_seasonal`. Blocks before 2026-09-16 used `windowed_seasonal = 0.05`, which made the split ~98% / ~2%. The `climate_linear` rows in the CSV are not `ensemble_mix` components. So covid forecast-quality work, calibration included, is mostly about `windowed_seasonal_extra_sources`. The CSV is edited by hand each week, so re-derive the weights before relying on these numbers. + **Model metadata:** `CMU-TimeSeries-Calibrated.yml` created in both `../FluSight-forecast-hub/model-metadata/` and `../covid19-forecast-hub/model-metadata/` (`designated_model: false`). **Bug fixed en route:** `nssp_archive` in both `flu_data_targets.R` and `covid_data_targets.R` had a duplicate-key error (pre-existing since August 2026): `compactify = TRUE` can retain multiple rows per `(geo_value, time_value)` when the value changed across issues, and the subsequent `mutate(version = time_value + 7)` flattened all to the same version, producing duplicate keys in `as_epi_archive()`. From 3ecb52f585d35765ee6ef4d30ec06c4ecb24e964 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:25:01 -0700 Subject: [PATCH 003/107] docs(calibration): experiment ledger and vintage-truth caveat --- notes/CALIBRATION.md | 23 ++++++++++ notes/calibration-ledger.md | 89 +++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 notes/calibration-ledger.md diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index 7056a835..9efd9a49 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -45,6 +45,27 @@ projection. Full repro commands live in `notes/calibration-runbook.md`. `windowed_seasonal_extra_sources`, aheads 0:4, `covid_hosp_evaluation` store). WIP. +The index of experiments (what was varied against what, and which results +are stale) is `notes/calibration-ledger.md`. + +## Learning truth: every run so far uses finalized values + +`calibrate_hub_forecasts()` looks up `Y` in one table from `nhsn_read_truth()`, +the latest NHSN vintage. The gradients, the adaptive eta pool and the +`burn_in_quantile` warm start therefore all learn from finalized values. Only +the timing of each reveal is realistic (`settle_days`). This favours short +revision delays, so no number in this file is a realistic backtest yet, and +all of them need to be re-run once the backtest uses vintages. + +A realistic backtest matches prod. Prod re-runs the tracker from scratch each +week on that week's latest vintage, so at round `t` every revealed round's +truth is its value as of `t`. The exact backtest re-runs the tracker once per +round (83 runs per config). A cheaper version uses each round's value as of +the round it is revealed at, which is one run per config and differs from +prod only through revisions that land after the reveal. Vintages exist from +2024-11-19, so October to mid-November 2024 would still fall back to +finalized values. + ## Data FluSight CMU-TimeSeries submissions: 83 rounds (2023-10-14 … 2026-05-30), 53 @@ -612,6 +633,8 @@ Fix: `group_by(geo_value, time_value) %>% slice_max(version, n = 1L, with_ties = Possible next steps, roughly ordered: +0. **Vintage-aware learning truth** (see "Learning truth" above), then + re-run the current experiments on it (E03–E05, E07 in the ledger). 0. **Re-run the ILI+ experiments on the fixed replay**: the ILI+ burn-in and setup C (does a warm start learned on ILI+ transfer to NHSN?). 1. **Review the ranked gallery** — does the WIS cost concentrate at turning diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md new file mode 100644 index 00000000..0274a2e5 --- /dev/null +++ b/notes/calibration-ledger.md @@ -0,0 +1,89 @@ +# Calibration experiment ledger + +Index of every calibration comparison: what was varied, against what, and +whether the result still stands. Findings and numbers live in +`notes/CALIBRATION.md`; this file only tracks the design of each comparison. + +Rule: each experiment names a reference config and varies one axis from it. +When an experiment has to move several axes (E09), it includes bridge rows +that step from the reference one axis at a time. + +## Axes + +| axis | values seen | reference value | +|---|---|---| +| disease | flu, covid | flu | +| base forecaster | submitted ensemble, `windowed_seasonal` | submitted ensemble | +| working scale | count, rate (per 100k), sqrt, log1p, quartic root | sqrt | +| learning truth | finalized (latest vintage), vintage-aware | finalized (**all runs so far**) | +| learning rate type | adaptive+ (`mult`, `floor`, `lr_window`), constant | adaptive+ | +| `lr_mult` | 0.3, 0.1, 0.03, 0.01 | 0.03 | +| `lr_window` | 8, 20, 50, Inf | 20 | +| season policy | carry, reset | carry | +| burn-in | 2023-24 hub, ILI+, none | 2023-24 hub | +| `settle_days` (extra revision delay) | 14 (= 1 week extra) | 14 | +| offset structure | single term, leak (`fast_decay`), warm start, slow term | see REF-op | +| update gating | always, `off_after` date | always | +| eta variants | pooled, per-level, geo-pooled, seasonal window | pooled | + +Reference configs: + +- **REF-paper**: flu, ensemble, count, finalized truth, adaptive+ mult 0.03, + window 20, carry, 2023-24 burn-in, settle 14, single term. +- **REF-op** (current operating point, in prod): REF-paper with sqrt, floor + 1e-3, warm start `burn_in_quantile`, slow mult 0.003, `fast_decay` 0.1. + +## Views + +Standard outputs an experiment can report. The goal is for every live config +to have all of them. + +| id | view | +|---|---| +| V-head | WIS % change and L1 coverage bias by horizon (× season) | +| V-month | WIS % change by month of reference date, with base-WIS share | +| V-state | per-state season panels with as-of vintages (a few states) | +| V-gallery | a few ranked forecasts (see the note below) | +| V-ae | absolute error of the median, change vs base | +| V-curve | coverage bias vs WIS curve over a swept parameter, per location + aggregate | + +V-gallery selection: ranking best/worst n by a single metric filled the old +gallery with small states at the end of the season. There, a mid-season +upward offset that no longer applies inflates the error on a tiny base. A +gallery should show only a few panels and pick them so that one regime +cannot take over. For example, weight the ranking by the forecast's share of +base WIS, and pick within strata (season phase × location size). + +## Experiments + +"Code" is pre-fix or post-fix relative to the 2026-09-17 correctness fixes +(sqrt clamp, issue-round gating). + +| id | question | varies | reference | disease | code | truth | views | where | status | +|---|---|---|---|---|---|---|---|---|---| +| E01 | which eta settings? | season policy × window × mult | REF-paper | flu | pre-fix | final | V-head, V-curve (mult) | `calibration_qt_flu.Rmd` | stale | +| E02 | cutoff and eta variants | `off_after`, per-level, geo-pool, seasonal window (one at a time) | REF-paper | flu | pre-fix | final | V-head, V-month, V-state | `calibration_qt_seasons_flu.Rmd` | stale | +| E03 | which working scale? | count vs sqrt (log1p, quartic pre-fix only) | REF-paper | flu | post-fix | final | V-head, V-month | `calibration_findings_flu.Rmd` | current, needs vintage rerun | +| E04 | leak / two-term offset | single, leak 0.1, slow term | REF-paper at sqrt | flu | post-fix | final | V-head, V-month | `calibration_findings_flu.Rmd` | current, needs vintage rerun | +| E05 | warm start from burn-in | warm start alone, + leak, + slow | REF-paper at sqrt | flu | post-fix | final | V-head, V-month | `calibration_findings_flu.Rmd` | current, needs vintage rerun | +| E06 | ILI+ as burn-in | burn-in source | REF-op | flu | post-fix | final | V-head | notes only | invalid (broken replay) | +| E07 | one forecaster across eras | base forecaster (A ensemble / B `windowed_seasonal`; C invalid) | REF-paper at sqrt | flu | post-fix | final | V-head | `calibration_ws_experiments.R`, findings notebook | current, needs vintage rerun | +| E08 | covid | disease | old REF-paper (count, window 50) | covid | pre-fix | final | V-head, V-state, V-gallery | `calibration_qt_*_covid.Rmd` | stale | +| E09 | constant lr × revision delay | constant lr grid × `settle_days` {7, 14, 21, 28}; bridge rows below | REF-paper | flu | post-fix | vintage | V-curve, V-head | planned: `calibration_lr_delay_flu.Rmd` | planned | + +E09 bridge rows, each one axis from the previous: REF-paper → rate scale → +constant lr (sweep) → `settle_days` (sweep). + +Diagnostics that are analyses rather than configs (all flu, finalized truth): +year-over-year seasonal offset viability, staleness lagged correlation, base +bias on NHSN and ILI+. See `notes/CALIBRATION.md`. + +## Gaps + +- **Learning truth.** Every run learns from finalized NHSN, so no current + number is a realistic backtest. See "Learning truth" in + `notes/CALIBRATION.md`. +- **Covid** has not been run since the fixes or at REF-op. +- **V-ae** exists only as the state-picking criterion in the seasons notebook; + no experiment reports it as a table. +- **V-state and V-gallery** have not been produced for any post-fix config. From efc18512def290ebbb36954d4bf335daa4ec4e57 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:59:42 -0700 Subject: [PATCH 004/107] feat(calibration): vintage-aware learning truth and exact per-round backtest --- R/calibration/calibrate.R | 164 ++++++++++++++++++++++++++++++-- R/calibration/hub_data.R | 34 ++++++- tests/testthat/test-calibrate.R | 49 ++++++++++ 3 files changed, 234 insertions(+), 13 deletions(-) diff --git a/R/calibration/calibrate.R b/R/calibration/calibrate.R index 54435487..e116522b 100644 --- a/R/calibration/calibrate.R +++ b/R/calibration/calibrate.R @@ -64,7 +64,18 @@ hub_series_matrix <- function(series, round_date, n_levels) { #' Online-calibrate a hub model's submitted forecasts, per (location, horizon). #' #' @param forecasts [hub_read_forecasts()] output. -#' @param truth tibble with columns `target_end_date`, `location` (FIPS), `truth` (raw count). +#' @param truth tibble with columns `target_end_date`, `location` (FIPS), `truth` +#' (raw count). Always used for scoring (the `truth` column of the output); +#' also used for learning when `learn_truth` is `NULL`. +#' @param learn_truth what the tracker learns from. `NULL` learns from `truth` +#' (finalized values, not a realistic backtest). A [nhsn_read_vintages()] +#' table (has a `version` column) learns each outcome from its value as of the +#' round it is revealed at, where round `t` sees the data published by +#' `round_date[t] - 3` (the Wednesday submission); as-of dates before the +#' first version use the first version. The warm start then uses the vintage +#' as of the first live round. A table without `version` is used as-is for +#' learning, which is how [calibrate_hub_forecasts_exact()] passes one +#' snapshot per round. See "Learning truth" in `notes/CALIBRATION.md`. #' @param settle_days how long after `target_end_date` the outcome is trusted. #' 14 gives exactly `horizon + 2` rounds of delay on hub coordinates. #' @param burn_in_seasons season labels (as produced by [hub_label_seasons()]) @@ -138,6 +149,7 @@ hub_series_matrix <- function(series, round_date, n_levels) { calibrate_hub_forecasts <- function( forecasts, truth, + learn_truth = NULL, settle_days = 14L, burn_in_seasons = character(0), season_policy = c("carry", "reset", "shrink"), @@ -224,9 +236,17 @@ calibrate_hub_forecasts <- function( season gap policy {.val {season_policy}}." ) - truth_lookup <- stats::setNames( - truth$truth, paste(truth$location, truth$target_end_date) - ) + truth_lookup <- hub_truth_lookup(truth) + vintage_mode <- !is.null(learn_truth) && "version" %in% names(learn_truth) + learn_lookup <- if (is.null(learn_truth)) { + truth_lookup + } else if (!vintage_mode) { + hub_truth_lookup(learn_truth) + } + if (vintage_mode) { + vintages_by_loc <- split(learn_truth, learn_truth$location) + first_version <- min(learn_truth$version) + } grid <- forecasts %>% distinct(.data$location, .data$horizon) %>% @@ -244,8 +264,14 @@ calibrate_hub_forecasts <- function( if (!any(rounds$season %in% burn_in_seasons)) { cli::cli_abort("{.arg slow_init} needs at least one burn-in season.") } + warm_lookup <- if (vintage_mode) { + first_live <- round_date[which(update_from)[1L]] + hub_truth_lookup(hub_vintage_snapshot(learn_truth, hub_round_asof(first_live))) + } else { + learn_lookup + } init_slow_by_h <- hub_burn_in_offsets( - chunks, keys, round_date, m, levels, truth_lookup, + chunks, keys, round_date, m, levels, warm_lookup, burn = rounds$season %in% burn_in_seasons, scales = scales, fwd = fwd ) } @@ -266,13 +292,19 @@ calibrate_hub_forecasts <- function( # truth yet (the live tail of a season), or no base forecast at that round # (nothing was played, so there is no coverage). A round pruned here is not # "revealed late", it is never learned from at all. - learnable <- !is.na(Y) & !apply(is.na(Yhat), 2, any) delay <- qt_delay_from_dates(round_date, target_end_date, settle_days) + Y_learn <- if (vintage_mode) { + hub_vintage_at_reveal(vintages_by_loc[[loc]], target_end_date, round_date, delay, first_version) + } else { + unname(learn_lookup[paste(loc, target_end_date)]) + } + learnable <- !is.na(Y_learn) & !apply(is.na(Yhat), 2, any) delay <- lapply(delay, function(idx) idx[learnable[idx]]) + revealed <- seq_len(n) %in% unlist(delay) s_t <- hub_round_scales(scales, loc, round_date) res <- qt_track( - Y = fwd(Y / s_t), Yhat = fwd(sweep(Yhat, 2L, s_t, "/")), levels = levels, delay = delay, + Y = fwd(Y_learn / s_t), Yhat = fwd(sweep(Yhat, 2L, s_t, "/")), levels = levels, delay = delay, lr = lr, lr_window = lr_window, lr_args = lr_args, projection = projection, nonneg = FALSE, update_from = update_from, hidden_scale = hidden_scale, @@ -298,7 +330,8 @@ calibrate_hub_forecasts <- function( slow = as.vector(res$slow), fast = as.vector(res$fast), lr_level = as.vector(res$lr_levels), - truth = rep(Y, each = m) + truth = rep(Y, each = m), + truth_learned = rep(ifelse(revealed, Y_learn, NA_real_), each = m) ) diagnostics[[i]] <- tibble( location = loc, @@ -326,7 +359,7 @@ calibrate_hub_forecasts <- function( "location", "horizon", "reference_date", "target_end_date", "season", "season_round", "round_index", "level_index", "level", "value_base", "value_cal", "offset", "hidden", "slow", "fast", "lr_level", - "truth", "is_burn_in" + "truth", "truth_learned", "is_burn_in" ) list( @@ -478,6 +511,119 @@ hub_rolling_tradeoff <- function(cal, window = 20L, by = character(0), drop_burn } +#' Calibrate as prod does: re-run the tracker at every round on that round's +#' vintage and keep only the forecast it plays at that round. +#' +#' Prod re-runs [calibrate_hub_forecasts()] from scratch each week on the +#' latest data, so at round `t` every revealed outcome carries its value as of +#' `t`. Passing a vintage table as `learn_truth` to [calibrate_hub_forecasts()] +#' instead fixes each outcome at its value as of the round it was revealed at; +#' this function is the reference that approximation is checked against. It +#' costs one full run per round. +#' +#' @param forecasts,truth as in [calibrate_hub_forecasts()]; `truth` is used for +#' scoring only. +#' @param vintages [nhsn_read_vintages()] output. +#' @param ... passed to [calibrate_hub_forecasts()]. +#' @param workers number of forked processes (rounds are independent runs). +#' @return list with `forecasts` (one round from each run) and `rounds` (from +#' the last run), which the metric functions accept. +#' @export +calibrate_hub_forecasts_exact <- function(forecasts, truth, vintages, ..., workers = 1L) { + rounds <- hub_label_seasons(unique(forecasts$reference_date)) + # A run needs at least one live round, and burn-in rounds are not scored, so + # burn-in rounds are taken from the first live round's run. + burn <- rounds$season %in% (list(...)$burn_in_seasons %||% character(0)) + live <- which(!burn) + run_at <- function(t) { + snapshot <- hub_vintage_snapshot(vintages, hub_round_asof(rounds$round_date[t])) + # Later rounds cannot affect what is played at t, so they are dropped to + # save time; the round axis up to t is unchanged. + cal <- suppressMessages(calibrate_hub_forecasts( + forecasts %>% filter(.data$reference_date <= rounds$round_date[t]), + truth, + learn_truth = snapshot, progress = FALSE, ... + )) + keep <- if (t == live[1L]) c(which(burn), t) else t + list( + forecasts = cal$forecasts %>% filter(.data$round_index %in% keep), + rounds = cal$rounds + ) + } + # Later rounds are the slowest runs, so they are started first. + order <- rev(seq_along(live)) + runs <- parallel::mclapply(live[order], run_at, mc.cores = workers, mc.preschedule = FALSE) + failed <- purrr::map_lgl(runs, inherits, "try-error") + if (any(failed)) { + cli::cli_abort("Exact calibration failed at round{?s} {.val {live[order][failed]}}: {runs[[which(failed)[1]]]}") + } + runs <- runs[order(order)] + list( + forecasts = bind_rows(purrr::map(runs, "forecasts")), + rounds = runs[[length(runs)]]$rounds + ) +} + + +#' The as-of date of a hub round: the Wednesday submission before the Saturday +#' reference date. +#' @keywords internal +hub_round_asof <- function(round_date) round_date - 3L + + +#' Named lookup of `truth` by `"location target_end_date"`. +#' @keywords internal +hub_truth_lookup <- function(truth) { + stats::setNames(truth$truth, paste(truth$location, truth$target_end_date)) +} + + +#' Values as of one date from a [nhsn_read_vintages()] table; dates before the +#' first version use the first version. +#' @keywords internal +hub_vintage_snapshot <- function(vintages, as_of) { + as_of <- max(as_of, min(vintages$version)) + vintages %>% + filter(.data$version <= as_of) %>% + group_by(.data$location, .data$target_end_date) %>% + slice_max(.data$version, n = 1L, with_ties = FALSE) %>% + ungroup() %>% + filter(!is.na(.data$truth)) %>% + select("location", "target_end_date", "truth") +} + + +#' Each round's outcome as of the round that reveals it, for one location. +#' +#' @param vintages one location's rows of a [nhsn_read_vintages()] table. +#' @param delay from [qt_delay_from_dates()], before pruning. +#' @return numeric of length `length(round_date)`; `NA` for rounds that are +#' never revealed or whose value was not yet published at the reveal round. +#' @keywords internal +hub_vintage_at_reveal <- function(vintages, target_date, round_date, delay, first_version) { + n <- length(round_date) + out <- rep(NA_real_, n) + if (is.null(vintages)) { + return(out) + } + reveal_at <- rep(NA_integer_, n) + for (t in seq_len(n)) reveal_at[delay[[t]]] <- t + i <- which(!is.na(reveal_at)) + queries <- tibble( + i = i, + target_end_date = target_date[i], + as_of = pmax(hub_round_asof(round_date[reveal_at[i]]), first_version) + ) + hits <- queries %>% + left_join( + vintages %>% select("target_end_date", "version", "truth"), + by = dplyr::join_by("target_end_date", closest("as_of" >= "version")) + ) + out[hits$i] <- hits$truth + out +} + + #' Conformal warm start for the slow term: per (horizon, level), the #' `level`-quantile of `Y - Yhat[level]` over burn-in rounds, pooled across #' locations, in the tracker's working units. diff --git a/R/calibration/hub_data.R b/R/calibration/hub_data.R index 46f23eec..b06c5188 100644 --- a/R/calibration/hub_data.R +++ b/R/calibration/hub_data.R @@ -147,14 +147,40 @@ nhsn_read_truth <- function(disease = "flu", archive = get_nhsn_data_archive(dis archive %>% epix_as_of(archive$versions_end) %>% as_tibble() %>% + nhsn_to_hub_locations() %>% + select(target_end_date = "time_value", "location", truth = "value") %>% + filter(!is.na(.data$truth)) %>% + arrange(.data$location, .data$target_end_date) +} + + +#' Every NHSN vintage, on hub coordinates, for vintage-aware calibration. +#' +#' One row per (location, week, version) at which the value changed (the +#' archive is compactified), so the value as of a date is the row with the +#' latest `version` at or before it. `NA` rows mean the value was absent in +#' that version. +#' @inheritParams nhsn_read_truth +#' @return tibble with `target_end_date`, `location` (FIPS), `version`, `truth`. +#' @export +nhsn_read_vintages <- function(disease = "flu", archive = get_nhsn_data_archive(disease)) { + archive$DT %>% + as_tibble() %>% + nhsn_to_hub_locations() %>% + select(target_end_date = "time_value", "location", "version", truth = "value") %>% + arrange(.data$location, .data$target_end_date, .data$version) +} + + +#' Map NHSN `geo_value` (state abbreviations, `usa`) to hub FIPS `location`. +#' @keywords internal +nhsn_to_hub_locations <- function(df) { + df %>% mutate(geo_value = ifelse(.data$geo_value == "usa", "us", .data$geo_value)) %>% inner_join( get_population_data() %>% distinct(.data$state_id, .keep_all = TRUE) %>% select("state_id", location = "state_code"), by = c("geo_value" = "state_id") - ) %>% - select(target_end_date = "time_value", "location", truth = "value") %>% - filter(!is.na(.data$truth)) %>% - arrange(.data$location, .data$target_end_date) + ) } diff --git a/tests/testthat/test-calibrate.R b/tests/testthat/test-calibrate.R index 3c9f252c..91f42ec8 100644 --- a/tests/testthat/test-calibrate.R +++ b/tests/testthat/test-calibrate.R @@ -89,3 +89,52 @@ test_that("metrics ignore rounds a location was not forecast at", { expect_false(anyNA(hub_coverage(cal)$coverage_cal)) expect_false(anyNA(hub_rolling_tradeoff(cal, window = 5L)$qloss_cal)) }) + +# Vintages where each week is first reported at half its value on the +# Wednesday after it ends and revised to the final value three weeks later. +make_synthetic_vintages <- function(truth) { + bind_rows( + truth %>% mutate(version = target_end_date + 4L, truth = 0.5 * truth), + truth %>% mutate(version = target_end_date + 25L) + ) +} + +test_that("vintage learning truth is the value as of the reveal round", { + hub <- make_synthetic_hub() + vintages <- make_synthetic_vintages(hub$truth) + learned <- function(settle_days) { + cal <- calibrate_hub_forecasts( + hub$forecasts, hub$truth, learn_truth = vintages, settle_days = settle_days, + burn_in_seasons = "2023-2024", progress = FALSE + ) + cal$forecasts %>% filter(level == 0.5, !is.na(truth_learned)) + } + # settle_days = 7 reveals each week at its first report; 28 after the revision. + # Burn-in weeks at the end of a season are revealed after the off-season gap, + # when they have already been revised, so only live rounds are checked. + first <- learned(7L) %>% filter(!is_burn_in) + expect_gt(nrow(first), 0) + expect_equal(first$truth_learned, 0.5 * first$truth) + settled <- learned(28L) + expect_equal(settled$truth_learned, settled$truth) +}) + +test_that("with no revisions, vintage and exact modes match learning from final truth", { + hub <- make_synthetic_hub() + vintages <- hub$truth %>% mutate(version = target_end_date + 4L) + args <- list( + burn_in_seasons = "2023-2024", transform = "sqrt", lr_window = 20, + lr_args = list(mult = 0.03, floor = 1e-3), slow_init = "burn_in_quantile", + lr_slow = list(mult = 0.003), fast_decay = 0.1 + ) + run <- function(f, ...) { + do.call(f, c(list(hub$forecasts, hub$truth, ...), args))$forecasts %>% + filter(!is_burn_in) %>% + arrange(location, horizon, round_index, level_index) + } + final <- run(calibrate_hub_forecasts, progress = FALSE) + cheap <- run(calibrate_hub_forecasts, learn_truth = vintages, progress = FALSE) + expect_equal(cheap$value_cal, final$value_cal) + exact <- run(calibrate_hub_forecasts_exact, vintages = vintages) + expect_equal(exact$value_cal, final$value_cal) +}) From 9d84a0c8ccb3629ed42df069649ac3b60952679a Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:08:46 -0700 Subject: [PATCH 005/107] feat(calibration): named reference configs and standard views for experiment notebooks --- R/calibration/views.R | 102 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 R/calibration/views.R diff --git a/R/calibration/views.R b/R/calibration/views.R new file mode 100644 index 00000000..6852cb71 --- /dev/null +++ b/R/calibration/views.R @@ -0,0 +1,102 @@ +# Named reference configs and standard views for the calibration experiment +# notebooks. The experiments, their reference configs and the view ids (V-head, +# V-month, ...) are indexed in notes/calibration-ledger.md. + + +#' Arguments to [calibrate_hub_forecasts()] for a named reference config. +#' +#' `"paper"` is REF-paper (count space, single term); `"op"` is REF-op, the +#' operating point in prod. Experiments vary one axis from one of these via +#' `...` in [cal_run()]. +#' @export +cal_ref_args <- function(ref = c("paper", "op")) { + ref <- rlang::arg_match(ref) + paper <- list( + burn_in_seasons = "2023-2024", settle_days = 14L, transform = "identity", + lr = "adaptive+", lr_window = 20, lr_args = list(mult = 0.03), + season_policy = "carry" + ) + switch(ref, + paper = paper, + op = utils::modifyList(paper, list( + transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3), + slow_init = "burn_in_quantile", lr_slow = list(mult = 0.003), fast_decay = 0.1 + )) + ) +} + + +#' Run a reference config with some arguments overridden. +#' +#' @param learn `"final"`, `"vintage"` (value as of the reveal round) or +#' `"exact"` (re-run per round on that round's vintage, as prod does). The +#' last two need `vintages`; `workers` parallelizes `"exact"`. +#' @param ... overrides of [cal_ref_args()]. `lr_args` replaces the reference's +#' `lr_args` whole rather than merging into it. +#' @export +cal_run <- function(forecasts, truth, ref = "paper", ..., learn = c("final", "vintage", "exact"), vintages = NULL, workers = 1L) { + learn <- rlang::arg_match(learn) + args <- cal_ref_args(ref) + overrides <- list(...) + args[names(overrides)] <- overrides + if (learn != "final" && is.null(vintages)) { + cli::cli_abort("{.arg learn} = {.val {learn}} needs {.arg vintages}.") + } + suppressMessages(switch(learn, + final = do.call(calibrate_hub_forecasts, c(list(forecasts, truth, progress = FALSE), args)), + vintage = do.call(calibrate_hub_forecasts, c(list(forecasts, truth, learn_truth = vintages, progress = FALSE), args)), + exact = do.call(calibrate_hub_forecasts_exact, c(list(forecasts, truth, vintages), args, workers = workers)) + )) +} + + +#' V-head: WIS change % and L1 coverage bias by horizon, one row per +#' (variant, `by`, horizon). +#' +#' @param cals named list of [calibrate_hub_forecasts()] outputs. +#' @param from only score rounds with reference date on or after this date. +#' @export +cal_headline <- function(cals, by = character(0), from = NULL) { + purrr::imap(cals, function(cal, nm) { + fc <- cal$forecasts + if (!is.null(from)) fc <- fc %>% filter(.data$reference_date >= from) + left_join( + hub_quantile_loss(fc, by = by), + hub_coverage_summary(fc, by = by) %>% + select(all_of(c(by, "horizon")), "cal_error_base", "cal_error_cal"), + by = c(by, "horizon") + ) %>% mutate(variant = nm, .before = 1) + }) %>% + bind_rows() %>% + mutate(variant = factor(.data$variant, levels = names(cals))) +} + + +#' One column of a [cal_headline()] table, horizons across. +#' @export +cal_wide <- function(tbl, col, digits = 1, id = "variant") { + tbl %>% + select(all_of(id), "horizon", value = all_of(col)) %>% + mutate(value = round(.data$value, digits)) %>% + tidyr::pivot_wider(names_from = "horizon", values_from = "value", names_prefix = "h") +} + + +#' V-month: WIS change % by month of reference date, with each month's share +#' of the base WIS, per horizon. +#' @export +cal_by_month <- function(cal) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + cal$forecasts %>% + filter(!.data$is_burn_in, !is.na(.data$truth), !is.na(.data$value_base)) %>% + mutate(month = factor(format(.data$reference_date, "%b"), levels = month.abb[c(7:12, 1:6)])) %>% + group_by(.data$horizon, .data$month) %>% + summarize( + base = sum(pinball(.data$truth, .data$value_base, .data$level)), + cal = sum(pinball(.data$truth, .data$value_cal, .data$level)), + .groups = "drop" + ) %>% + group_by(.data$horizon) %>% + mutate(pct = 100 * (.data$base - .data$cal) / .data$base, share = 100 * .data$base / sum(.data$base)) %>% + ungroup() +} From 870ad84defdab8b69fdb67c7802a3d938ae407cf Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:16:02 -0700 Subject: [PATCH 006/107] build(calibration): render experiment notebooks and ledger index --- Justfile | 5 ++++ .../calibration_experiments/index.Rmd | 29 +++++++++++++++++++ scripts/calibration/render_experiments.R | 22 ++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 Justfile create mode 100644 reports/writeups/calibration_experiments/index.Rmd create mode 100644 scripts/calibration/render_experiments.R diff --git a/Justfile b/Justfile new file mode 100644 index 00000000..83bd1383 --- /dev/null +++ b/Justfile @@ -0,0 +1,5 @@ +rscript := env_var_or_default("RSCRIPT", "Rscript") + +# Render calibration experiment notebooks (all, or the named ones, e.g. e00_vintage_backtest) and the index. +calibration-experiments *notebooks: + {{rscript}} scripts/calibration/render_experiments.R {{notebooks}} diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd new file mode 100644 index 00000000..2e96f36a --- /dev/null +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -0,0 +1,29 @@ +--- +title: "Calibration experiments" +date: "`r format(Sys.time(), '%d %B %Y')`" +output: + html_document: + toc: true + toc_float: true +--- + +```{r setup, include=FALSE} +knitr::opts_knit$set(root.dir = here::here()) +``` + +# Notebooks + +```{r notebooks, results="asis", echo=FALSE} +src <- list.files(here::here("reports/writeups/calibration_experiments"), pattern = "^e[0-9]+_.*[.]Rmd$", full.names = TRUE) +out_dir <- here::here("reports/calibration_experiments") +for (f in src) { + html <- sub("[.]Rmd$", ".html", basename(f)) + title <- rmarkdown::yaml_front_matter(f)$title + rendered <- file.exists(file.path(out_dir, html)) + stamp <- if (rendered) format(file.mtime(file.path(out_dir, html)), "%Y-%m-%d %H:%M") else "not rendered" + cat(sprintf("- %s (%s)\n", if (rendered) sprintf("[%s](%s)", title, html) else title, stamp)) +} +``` + +```{r ledger, child=here::here("notes/calibration-ledger.md")} +``` diff --git a/scripts/calibration/render_experiments.R b/scripts/calibration/render_experiments.R new file mode 100644 index 00000000..15f79051 --- /dev/null +++ b/scripts/calibration/render_experiments.R @@ -0,0 +1,22 @@ +# Render calibration experiment notebooks into reports/calibration_experiments/ +# and rebuild its index.html (the ledger plus links to every rendered notebook). +# +# Usage: Rscript scripts/calibration/render_experiments.R [e00_vintage_backtest ...] +# With no arguments, renders every notebook; `--index` renders only the index. + +src_dir <- here::here("reports/writeups/calibration_experiments") +out_dir <- here::here("reports/calibration_experiments") +dir.create(out_dir, showWarnings = FALSE, recursive = TRUE) + +args <- commandArgs(trailingOnly = TRUE) +notebooks <- if (identical(args, "--index")) { + character(0) +} else if (length(args) > 0) { + file.path(src_dir, paste0(sub("[.]Rmd$", "", args), ".Rmd")) +} else { + list.files(src_dir, pattern = "^e[0-9]+_.*[.]Rmd$", full.names = TRUE) +} +for (nb in notebooks) { + rmarkdown::render(nb, output_dir = out_dir, envir = new.env(), quiet = TRUE) +} +rmarkdown::render(file.path(src_dir, "index.Rmd"), output_dir = out_dir, envir = new.env(), quiet = TRUE) From 314fa408537dda3b5af654fce7c89baed431be07 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:16:02 -0700 Subject: [PATCH 007/107] docs(calibration): E00 vintage vs finalized learning truth --- notes/CALIBRATION.md | 52 ++++-- notes/calibration-ledger.md | 21 ++- .../e00_vintage_backtest.Rmd | 162 ++++++++++++++++++ 3 files changed, 209 insertions(+), 26 deletions(-) create mode 100644 reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index 9efd9a49..3df5f2bb 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -48,23 +48,37 @@ projection. Full repro commands live in `notes/calibration-runbook.md`. The index of experiments (what was varied against what, and which results are stale) is `notes/calibration-ledger.md`. -## Learning truth: every run so far uses finalized values - -`calibrate_hub_forecasts()` looks up `Y` in one table from `nhsn_read_truth()`, -the latest NHSN vintage. The gradients, the adaptive eta pool and the -`burn_in_quantile` warm start therefore all learn from finalized values. Only -the timing of each reveal is realistic (`settle_days`). This favours short -revision delays, so no number in this file is a realistic backtest yet, and -all of them need to be re-run once the backtest uses vintages. - -A realistic backtest matches prod. Prod re-runs the tracker from scratch each -week on that week's latest vintage, so at round `t` every revealed round's -truth is its value as of `t`. The exact backtest re-runs the tracker once per -round (83 runs per config). A cheaper version uses each round's value as of -the round it is revealed at, which is one run per config and differs from -prod only through revisions that land after the reveal. Vintages exist from -2024-11-19, so October to mid-November 2024 would still fall back to -finalized values. +## Learning truth (2026-09-30) + +Until 2026-09-30, `calibrate_hub_forecasts()` learned only from `truth`, the +latest NHSN vintage: the gradients, the adaptive eta pool and the +`burn_in_quantile` warm start all saw finalized values, and only the timing of +each reveal (`settle_days`) was realistic. Every number in this file below +that date was measured that way. + +`learn_truth` now selects what the tracker learns from (scores stay on +finalized truth). A round with reference date `d` sees data published by +`d - 3`. Modes, as `cal_run(learn = )`: + +- `"final"`: the old behavior. +- `"exact"`: what prod does. Prod re-runs the tracker from scratch each week on + that week's data, so at round `t` every revealed outcome has its value as + of `t`. `calibrate_hub_forecasts_exact()` replays this with one run per + round (about 1.2 min per config with 12 forked workers). +- `"vintage"`: one run in which each outcome keeps its value at the round + that revealed it: a stateful tracker that never revisits a step. + +Findings (E00, `e00_vintage_backtest.Rmd`, flu, both live seasons): + +- Final vs exact is small except at h−1: REF-op at settle 14 is + +8.9/+5.4/+2.6/+1.9/+1.9 final vs +6.8/+5.1/+2.5/+1.8/+1.9 exact, coverage + bias 0.073–0.040 vs 0.080–0.041. REF-paper is within 0.6 points. +- Vintage mode is not a stand-in for exact: at settle 7 it learns once from + the under-reported first report (REF-op h−1 +1.2, coverage bias 0.127). +- Under the exact (prod) design, settle 7 beats settle 14 at every horizon + at both references (REF-op +7.4/+5.3/+2.9/+2.0/+2.2), because next week's + run corrects what an early reveal got wrong. A stateful tracker would + prefer the longer delay. The right revision delay depends on the design. ## Data @@ -633,8 +647,8 @@ Fix: `group_by(geo_value, time_value) %>% slice_max(version, n = 1L, with_ties = Possible next steps, roughly ordered: -0. **Vintage-aware learning truth** (see "Learning truth" above), then - re-run the current experiments on it (E03–E05, E07 in the ledger). +0. **Re-run the current experiments with exact learning truth** (E03–E05, + E07 in the ledger), after E09. 0. **Re-run the ILI+ experiments on the fixed replay**: the ILI+ burn-in and setup C (does a warm start learned on ILI+ transfer to NHSN?). 1. **Review the ranked gallery** — does the WIS cost concentrate at turning diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 0274a2e5..78f55237 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -15,7 +15,7 @@ that step from the reference one axis at a time. | disease | flu, covid | flu | | base forecaster | submitted ensemble, `windowed_seasonal` | submitted ensemble | | working scale | count, rate (per 100k), sqrt, log1p, quartic root | sqrt | -| learning truth | finalized (latest vintage), vintage-aware | finalized (**all runs so far**) | +| learning truth | final (latest vintage), vintage (value at reveal; a stateful tracker), exact (prod's weekly re-run) | exact (E00 on; E01–E08 used final) | | learning rate type | adaptive+ (`mult`, `floor`, `lr_window`), constant | adaptive+ | | `lr_mult` | 0.3, 0.1, 0.03, 0.01 | 0.03 | | `lr_window` | 8, 20, 50, Inf | 20 | @@ -28,11 +28,17 @@ that step from the reference one axis at a time. Reference configs: -- **REF-paper**: flu, ensemble, count, finalized truth, adaptive+ mult 0.03, - window 20, carry, 2023-24 burn-in, settle 14, single term. +- **REF-paper**: flu, ensemble, count, adaptive+ mult 0.03, window 20, + carry, 2023-24 burn-in, settle 14, single term. - **REF-op** (current operating point, in prod): REF-paper with sqrt, floor 1e-3, warm start `burn_in_quantile`, slow mult 0.003, `fast_decay` 0.1. +Both are `cal_ref_args()` in `R/calibration/views.R`; notebooks run them +through `cal_run()`, which also picks the learning truth. Experiment notebooks +live in `reports/writeups/calibration_experiments/` and render to +`reports/calibration_experiments/` (with this ledger as `index.html`) via +`just calibration-experiments [notebook ...]`. + ## Views Standard outputs an experiment can report. The goal is for every live config @@ -61,6 +67,7 @@ base WIS, and pick within strata (season phase × location size). | id | question | varies | reference | disease | code | truth | views | where | status | |---|---|---|---|---|---|---|---|---|---| +| E00 | how much did final truth flatter results; can vintage stand in for exact? | learning truth {final, vintage, exact} × settle {7, 14} | REF-paper, REF-op | flu | post-fix | all three | V-head | `e00_vintage_backtest.Rmd` | done: exact is the default; vintage is not a stand-in | | E01 | which eta settings? | season policy × window × mult | REF-paper | flu | pre-fix | final | V-head, V-curve (mult) | `calibration_qt_flu.Rmd` | stale | | E02 | cutoff and eta variants | `off_after`, per-level, geo-pool, seasonal window (one at a time) | REF-paper | flu | pre-fix | final | V-head, V-month, V-state | `calibration_qt_seasons_flu.Rmd` | stale | | E03 | which working scale? | count vs sqrt (log1p, quartic pre-fix only) | REF-paper | flu | post-fix | final | V-head, V-month | `calibration_findings_flu.Rmd` | current, needs vintage rerun | @@ -69,7 +76,7 @@ base WIS, and pick within strata (season phase × location size). | E06 | ILI+ as burn-in | burn-in source | REF-op | flu | post-fix | final | V-head | notes only | invalid (broken replay) | | E07 | one forecaster across eras | base forecaster (A ensemble / B `windowed_seasonal`; C invalid) | REF-paper at sqrt | flu | post-fix | final | V-head | `calibration_ws_experiments.R`, findings notebook | current, needs vintage rerun | | E08 | covid | disease | old REF-paper (count, window 50) | covid | pre-fix | final | V-head, V-state, V-gallery | `calibration_qt_*_covid.Rmd` | stale | -| E09 | constant lr × revision delay | constant lr grid × `settle_days` {7, 14, 21, 28}; bridge rows below | REF-paper | flu | post-fix | vintage | V-curve, V-head | planned: `calibration_lr_delay_flu.Rmd` | planned | +| E09 | constant lr × revision delay | constant lr grid × `settle_days` {7, 14, 21, 28}; bridge rows below | REF-paper | flu | post-fix | exact (+ vintage, for a stateful tracker) | V-curve, V-head | planned: `e09_lr_delay.Rmd` | planned | E09 bridge rows, each one axis from the previous: REF-paper → rate scale → constant lr (sweep) → `settle_days` (sweep). @@ -80,9 +87,9 @@ bias on NHSN and ILI+. See `notes/CALIBRATION.md`. ## Gaps -- **Learning truth.** Every run learns from finalized NHSN, so no current - number is a realistic backtest. See "Learning truth" in - `notes/CALIBRATION.md`. +- **Learning truth.** E01–E08 learned from finalized NHSN. E00 shows the + error is small outside h−1, but those experiments still need an exact + re-run (E03–E05 first). - **Covid** has not been run since the fixes or at REF-op. - **V-ae** exists only as the state-picking criterion in the seasons notebook; no experiment reports it as a table. diff --git a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd new file mode 100644 index 00000000..2c61b9d1 --- /dev/null +++ b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd @@ -0,0 +1,162 @@ +--- +title: "E00: learning from vintages instead of finalized truth" +date: "`r format(Sys.time(), '%d %B %Y')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true +params: + hub_dir: "../FluSight-forecast-hub" + workers: 12 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +# cache = TRUE chunks are keyed on chunk code, not on functions in R/. After +# changing calibration code, clear cache/calibration/experiments/knitr_e00/. +knitr::opts_chunk$set( + fig.align = "center", fig.width = 10, fig.height = 5, + message = FALSE, warning = FALSE, + cache.path = here::here("cache/calibration/experiments/knitr_e00/") +) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw()) +``` + +Ledger entry **E00** (`notes/calibration-ledger.md`). Every result before this +notebook trained the tracker on finalized NHSN values. Only the *timing* of +each reveal was realistic (`settle_days`). This notebook varies one axis, the +truth the tracker learns from, at both reference configs: + +- **final**: the latest vintage, as in all earlier results. +- **exact**: what prod does. Prod re-runs the tracker from scratch each week + on that week's data, so at round `t` every revealed outcome has its value + as of `t`. We replay that by re-running the tracker once per round. +- **vintage**: one run in which each outcome keeps the value it had at the + round that revealed it. This was meant as a cheap stand-in for exact, but + it is also a deployment design in its own right: a stateful tracker that + steps once per outcome and never revisits it. + +A round with reference date `d` sees data published by `d - 3` (the Wednesday +submission). Vintages start 2024-11-19; earlier as-of dates use that first +vintage. Scores are always against finalized truth, over both live seasons, +all hub locations, with burn-in rounds excluded. "WIS change %" is positive +when calibration improves on the base. + +Questions: (1) how much did learning from finalized values flatter the +earlier results, and (2) is the cheap mode close enough to exact to use in +sweeps? + +# Findings + +Numbers below are from the render of 30 September 2026. + +- **Finalized truth flattered the earlier results only a little.** At REF-op + with `settle_days = 14`, exact vs final WIS change is +6.8 vs +8.9 at h−1, + within 0.3 points at h0–h3, and the h−1 coverage bias is 0.080 vs 0.073. At + REF-paper, exact is within 0.6 points of final at every horizon, slightly + better at h1–h3. Earlier conclusions stand; the h−1 gain was overstated. +- **The cheap mode is not a usable stand-in.** It drifts from exact as the + season goes on (figure below). At `settle_days = 7` it learns once from the + first report, which under-reports and is never corrected: REF-op h−1 falls + to +1.2 (exact +7.4) and coverage bias rises to 0.127 (exact 0.082). Every + later experiment uses exact. +- **Under prod's re-run design, a shorter settle is better.** Exact with + `settle_days = 7` beats 14 at every horizon, at both references (REF-op + +7.4/+5.3/+2.9/+2.0/+2.2 vs +6.8/+5.1/+2.5/+1.8/+1.9), with the same + coverage bias. Learning early from a first report costs little when next + week's run replaces it with the revised value. A stateful tracker (the + vintage mode) shows the opposite. So the value of an extra revision delay + depends on the deployment design, and E09 should report both. + +```{r libs, cache=FALSE} +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` + +```{r load, cache=TRUE} +archive <- get_nhsn_data_archive("flu") +truth <- nhsn_read_truth("flu", archive) +vintages <- nhsn_read_vintages("flu", archive) +hub_fc <- hub_read_forecasts(hub_dir = here::here(params$hub_dir)) +``` + +```{r runs, cache=TRUE, dependson="load"} +grid <- tidyr::expand_grid(ref = c("paper", "op"), settle_days = c(7L, 14L), learn = c("final", "vintage", "exact")) +cals <- purrr::pmap(grid, function(ref, settle_days, learn) { + cal_run(hub_fc, truth, ref, settle_days = settle_days, learn = learn, vintages = vintages, workers = params$workers) +}) +names(cals) <- with(grid, paste(ref, settle_days, learn, sep = "/")) +``` + +```{r headline} +LIVE <- c("2024-2025", "2025-2026") +head_tbl <- cal_headline(cals) %>% + tidyr::separate(variant, c("ref", "settle_days", "learn"), sep = "/", remove = FALSE) %>% + mutate(learn = factor(learn, levels = c("final", "vintage", "exact"))) +first_version <- min(vintages$version) +n_clamped <- sum(unique(hub_fc$reference_date) - 3L < first_version & + hub_label_seasons(unique(hub_fc$reference_date))$season %in% LIVE) +``` + +`r n_clamped` live rounds are issued before the first vintage (2024-11-19); only +burn-in rounds use that vintage in place of their own. + +# Headline + +WIS change % by horizon: + +```{r wis-table} +knitr::kable(cal_wide(head_tbl, "pct_improvement", id = c("ref", "settle_days", "learn")) %>% arrange(ref, settle_days, learn)) +``` + +L1 coverage bias (mean over levels of |coverage − level|); the `base` row is +the uncalibrated forecast: + +```{r cal-table} +knitr::kable( + bind_rows( + cal_wide(head_tbl, "cal_error_cal", 3, id = c("ref", "settle_days", "learn")), + cal_wide(head_tbl %>% filter(learn == "final", ref == "paper", settle_days == "14") %>% mutate(ref = "base", settle_days = "", learn = ""), "cal_error_base", 3, id = c("ref", "settle_days", "learn")) + ) %>% arrange(ref, settle_days, learn) +) +``` + +The same by season: + +```{r season-table} +knitr::kable( + cal_wide( + cal_headline(cals, by = "season") %>% filter(season %in% LIVE) %>% + tidyr::separate(variant, c("ref", "settle_days", "learn"), sep = "/"), + "pct_improvement", id = c("ref", "settle_days", "learn", "season") + ) %>% arrange(ref, settle_days, season, learn) +) +``` + +# How far the cheap mode drifts from exact + +Median over locations of |cheap − exact| in the played median, relative to +the base median, by month: + +```{r drift, fig.height=6} +drift <- purrr::map(unique(paste(grid$ref, grid$settle_days, sep = "/")), function(cfg) { + pick <- function(mode) { + cals[[paste(cfg, mode, sep = "/")]]$forecasts %>% + filter(level == 0.5, !is_burn_in, !is.na(value_base)) %>% + select(location, horizon, reference_date, value_base, value_cal) + } + inner_join(pick("vintage"), pick("exact"), by = c("location", "horizon", "reference_date", "value_base"), suffix = c("_cheap", "_exact")) %>% + mutate(config = cfg) +}) %>% + bind_rows() %>% + mutate(month = lubridate::floor_date(reference_date, "month")) %>% + group_by(config, horizon, month) %>% + summarize(rel = median(abs(value_cal_cheap - value_cal_exact) / pmax(value_base, 1)), .groups = "drop") +ggplot(drift, aes(month, 100 * rel, colour = factor(horizon), group = horizon)) + + geom_line() + + geom_point(size = 0.8) + + facet_wrap(~config) + + labs(x = NULL, y = "median |cheap − exact| / base median (%)", colour = "horizon") +``` From ac1d6c7a47f7abb56109ce1f44c2682442c3fb8a Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:22:24 -0700 Subject: [PATCH 008/107] docs(calibration): E09 notebook, constant lr x revision delay on the rate scale --- .../calibration_experiments/e09_lr_delay.Rmd | 286 ++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 reports/writeups/calibration_experiments/e09_lr_delay.Rmd diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd new file mode 100644 index 00000000..8baca6ca --- /dev/null +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -0,0 +1,286 @@ +--- +title: "E09: constant learning rate × revision delay (rate scale)" +date: "`r format(Sys.time(), '%d %B %Y')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true +params: + hub_dir: "../FluSight-forecast-hub" + workers: 14 + wis_max: 50 + smoke: false +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set( + fig.align = "center", fig.width = 11, fig.height = 11, + message = FALSE, warning = FALSE, + cache.path = here::here("cache/calibration/experiments/knitr_e09/") +) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Ledger entry **E09** (`notes/calibration-ledger.md`). The question, from a +collaborator: for the plain MultiQT tracker applied on the **rate scale** +(admissions per 100k) with a **constant** learning rate, how do coverage and +WIS trade off across learning rates, and how does that change with the +**extra delay** before an outcome is learned from? + +**Extra delay.** The first NHSN report of the week ending on Saturday `X` is +published on Wednesday `X + 4`, in time for the round with reference date +`X + 7`. Learning from it then is `settle_days = 7` (`h + 1` rounds), zero +extra delay. Each further week adds 7 days: our reference `settle_days = 14` +is one extra week. The sweep is 0–3 extra weeks. + +**Learning truth.** The tracker learns from NHSN vintages, not finalized +values (E00). Two deployment designs are shown, since E00 found they respond +to the delay in opposite directions: + +- **re-run** (`exact`, what prod does): the tracker is re-run from scratch each + week on that week's data, so a value learned early is replaced by its + revision the next week. +- **stateful**: the tracker steps once per outcome, on the value available at + the round it is revealed, and never revisits it. + +Everything else is REF-paper (`cal_ref_args("paper")`): single term, carry +across seasons, 2023-24 burn-in (no steps; the constant rate needs no +warm-up). Scores are on counts against finalized truth, over the two live +seasons, all 53 hub locations. "WIS inflation %" is `100 × (WIS_cal / WIS_base − 1)`, +so negative is better than the base. "L1 coverage bias" is the mean over the +23 levels of |empirical coverage − level|. + +```{r libs, cache=FALSE} +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +LRS <- 10^seq(-3, 0.5, by = 0.25) +SETTLE <- c(7L, 14L, 21L, 28L) +# smoke = TRUE checks the notebook end to end in a few minutes: a small grid, +# and the re-run design computed with the fast stateful mode. +if (params$smoke) { + LRS <- LRS[c(1, 9, 13)] + SETTLE <- c(7L, 14L) +} +LEARN_LABELS <- c(exact = "re-run (prod)", vintage = "stateful") +``` + +```{r load, cache=TRUE} +archive <- get_nhsn_data_archive("flu") +truth <- nhsn_read_truth("flu", archive) +vintages <- nhsn_read_vintages("flu", archive) +hub_fc <- hub_read_forecasts(hub_dir = here::here(params$hub_dir)) +rate_scales <- get_population_data() %>% + distinct(state_code, .keep_all = TRUE) %>% + filter(state_code %in% unique(hub_fc$location)) %>% + transmute(location = state_code, from = as.Date("2000-01-01"), scale = population / 1e5) +stopifnot(setequal(rate_scales$location, unique(hub_fc$location))) +``` + +```{r runs, cache=FALSE} +# Each run is cached to its own file, so a change to the grid only computes the +# new runs. Clear cache/calibration/experiments/e09/ after changing R/calibration/. +run_dir <- here::here("cache/calibration/experiments", if (params$smoke) "e09_smoke" else "e09") +dir.create(run_dir, showWarnings = FALSE, recursive = TRUE) +cached_run <- function(key, ..., learn) { + path <- file.path(run_dir, paste0(key, ".rds")) + if (file.exists(path)) return(readRDS(path)) + if (params$smoke) learn <- "vintage" + cal <- cal_run(hub_fc, truth, ..., learn = learn, vintages = vintages, workers = params$workers) + # Only what the metrics need, to keep the cache small. + cal <- list(forecasts = cal$forecasts %>% + select(location, horizon, reference_date, season, level, value_base, value_cal, truth, is_burn_in)) + saveRDS(cal, path) + cal +} +rate <- list(transform = "identity", scales = rate_scales) + +sweep_grid <- tidyr::expand_grid(learn = c("exact", "vintage"), settle_days = SETTLE, lr = LRS) +sweep <- purrr::pmap(sweep_grid, function(learn, settle_days, lr) { + key <- sprintf("sweep_%s_s%d_lr%.4f", learn, settle_days, lr) + do.call(cached_run, c(list(key, "paper", settle_days = settle_days, lr = lr, learn = learn), rate)) +}) + +# Reference points and the bridge from REF-paper, per (learn, settle_days). +ref_grid <- tidyr::expand_grid(learn = c("exact", "vintage"), settle_days = SETTLE) +refs <- purrr::pmap(ref_grid, function(learn, settle_days) { + s <- sprintf("%s_s%d", learn, settle_days) + list( + `REF-paper (count, adaptive)` = cached_run(paste0("paper_", s), "paper", settle_days = settle_days, learn = learn), + `rate, adaptive` = do.call(cached_run, c(list(paste0("rate_adaptive_", s), "paper", + settle_days = settle_days, learn = learn, lr_args = list(mult = 0.03, floor = 1e-3)), rate)), + `REF-op` = cached_run(paste0("op_", s), "op", settle_days = settle_days, learn = learn) + ) +}) +``` + +```{r metrics, cache=FALSE} +pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) +# Per (location, horizon) and pooled over locations: WIS inflation and L1 +# coverage bias, for the calibrated forecast and the base. +metrics <- function(cal) { + fc <- cal$forecasts %>% filter(!is_burn_in, !is.na(truth), !is.na(value_base)) + one <- function(df, by) { + df %>% + group_by(across(all_of(c(by, "horizon", "level")))) %>% + summarize( + cov_base = mean(truth <= value_base), cov_cal = mean(truth <= value_cal), + wis_base = sum(pinball(truth, value_base, level)), wis_cal = sum(pinball(truth, value_cal, level)), + .groups = "drop" + ) %>% + group_by(across(all_of(c(by, "horizon")))) %>% + summarize( + bias_base = mean(abs(cov_base - level)), bias_cal = mean(abs(cov_cal - level)), + wis_infl = 100 * (sum(wis_cal) / sum(wis_base) - 1), + .groups = "drop" + ) + } + bind_rows(one(fc, "location"), one(fc, character(0)) %>% mutate(location = "pooled")) +} +sweep_m <- purrr::map2(sweep, seq_along(sweep), function(cal, i) metrics(cal) %>% bind_cols(sweep_grid[i, ])) %>% + bind_rows() +ref_m <- purrr::map2(refs, seq_along(refs), function(r, i) { + purrr::imap(r, function(cal, nm) metrics(cal) %>% mutate(ref = nm)) %>% bind_rows() %>% bind_cols(ref_grid[i, ]) +}) %>% + bind_rows() %>% + filter(location == "pooled") +label_panels <- function(df) { + df %>% mutate( + extra = factor(sprintf("+%d wk extra delay", (settle_days - 7L) %/% 7L)), + horizon = factor(paste0("h", horizon), levels = paste0("h", -1:3)) + ) +} +sweep_m <- label_panels(sweep_m) +ref_m <- label_panels(ref_m) +``` + +# Bridge from REF-paper + +One axis at a time at `settle_days = 14` (the reference), re-run design: first +the rate scale with the adaptive learning rate (its `floor` moves from 0.1 +count to 0.001 rate units, since the floor is a scale-dependent constant), +then the constant learning rate with the lowest pooled WIS at each horizon. +REF-op is for orientation only; it differs in several axes. + +```{r bridge} +best_const <- sweep_m %>% + filter(location == "pooled", learn == "exact", settle_days == 14L) %>% + group_by(horizon) %>% + slice_min(wis_infl, n = 1, with_ties = FALSE) %>% + ungroup() %>% + mutate(ref = "rate, constant lr (best WIS per horizon)") +bridge <- bind_rows( + ref_m %>% filter(learn == "exact", settle_days == 14L), + best_const +) %>% + mutate(ref = factor(ref, levels = c("REF-paper (count, adaptive)", "rate, adaptive", "rate, constant lr (best WIS per horizon)", "REF-op"))) +base_row <- bridge %>% filter(ref == "REF-paper (count, adaptive)") %>% mutate(ref = "base", wis_infl = 0, bias_cal = bias_base) +tbl <- bind_rows(base_row, bridge) %>% + transmute(config = ref, horizon, cell = sprintf("%+.1f%% / %.3f", wis_infl, bias_cal)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = cell) +knitr::kable(tbl, caption = "WIS inflation % / L1 coverage bias, pooled over locations") +knitr::kable( + best_const %>% select(horizon, lr) %>% mutate(lr = signif(lr, 2)) %>% tidyr::pivot_wider(names_from = horizon, values_from = lr), + caption = "Constant learning rate with the lowest pooled WIS (rate units per step)" +) +``` + +# Coverage bias vs WIS, per location + +Each thin grey line is one location traced across learning rates; the bold +line is the pooled result, labelled with the learning rate at each decade. Learning rates whose pooled WIS inflation exceeds +`r params$wis_max`% in a panel are dropped from that panel; the axes are +clipped to the bulk of the per-location points, so a few location lines run +off the edge. The open circle is the base forecast (no calibration) and the +triangles are the reference configs at the same delay. + +Per-location coverage comes from about 50 forecasts per horizon, so a +location's L1 bias has a noise floor of a few points even for a perfectly +calibrated forecaster. Read the shape of the pooled line; the grey lines show +spread. + +```{r curve-fn} +ACCENT <- "#1f5fa8" +curve_plot <- function(which_learn) { + d <- sweep_m %>% filter(learn == which_learn) + keep <- d %>% + filter(location == "pooled", wis_infl <= params$wis_max) %>% + select(horizon, extra, lr) + d <- d %>% semi_join(keep, by = c("horizon", "extra", "lr")) %>% arrange(lr) + pooled <- d %>% filter(location == "pooled") + locs <- d %>% filter(location != "pooled") + base <- pooled %>% distinct(horizon, extra, bias_base) + marks <- ref_m %>% filter(learn == which_learn, ref %in% c("REF-paper (count, adaptive)", "REF-op")) + lab <- pooled %>% filter(abs(log10(lr) - round(log10(lr))) < 1e-9) + xlim <- c(0, max(stats::quantile(locs$bias_cal, 0.97), base$bias_base) * 1.05) + ylim <- range(stats::quantile(locs$wis_infl, c(0.03, 0.97)), pooled$wis_infl, marks$wis_infl, 0) + ggplot() + + geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + + geom_path(data = locs, aes(bias_cal, wis_infl, group = location), colour = "grey45", alpha = 0.25, linewidth = 0.3) + + geom_path(data = pooled, aes(bias_cal, wis_infl), colour = ACCENT, linewidth = 1.1) + + geom_point(data = pooled, aes(bias_cal, wis_infl), colour = ACCENT, size = 1.2) + + geom_text(data = lab, aes(bias_cal, wis_infl, label = format(lr, scientific = FALSE, drop0trailing = TRUE)), + colour = "grey15", size = 2.6, vjust = -0.9) + + geom_point(data = base, aes(bias_base, 0), shape = 21, fill = "white", colour = "black", size = 2.2) + + geom_point(data = marks, aes(bias_cal, wis_infl, shape = ref), colour = "black", size = 2) + + scale_shape_manual(values = c(`REF-paper (count, adaptive)` = 2, `REF-op` = 17), name = NULL) + + facet_grid(horizon ~ extra) + + coord_cartesian(xlim = xlim, ylim = ylim) + + labs(x = "L1 coverage bias", y = "WIS inflation % (negative = better than base)") + + theme(legend.position = "bottom") +} +``` + +## Re-run design (prod) + +```{r curve-exact} +curve_plot("exact") +``` + +## Stateful design + +```{r curve-vintage} +curve_plot("vintage") +``` + +# Pooled curves by delay + +The bold lines from above, one colour per extra delay (darker is longer), so +the delays can be compared directly. Solid is the re-run design, dashed the +stateful one. + +```{r pooled-overlay, fig.height=6} +DELAY_COLS <- setNames(RColorBrewer::brewer.pal(9, "Blues")[c(4, 6, 8, 9)], levels(sweep_m$extra)) +pooled_all <- sweep_m %>% + filter(location == "pooled") %>% + semi_join(sweep_m %>% filter(location == "pooled", wis_infl <= params$wis_max) %>% select(horizon, extra, lr, learn), + by = c("horizon", "extra", "lr", "learn")) %>% + arrange(lr) %>% + mutate(design = factor(LEARN_LABELS[learn], levels = LEARN_LABELS)) +ggplot(pooled_all, aes(bias_cal, wis_infl, colour = extra, linetype = design, group = interaction(extra, design))) + + geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + + geom_path(linewidth = 0.8) + + geom_point(data = pooled_all %>% distinct(horizon, bias_base), aes(bias_base, 0), inherit.aes = FALSE, + shape = 21, fill = "white", colour = "black", size = 2) + + scale_colour_manual(values = DELAY_COLS, name = NULL) + + scale_linetype_manual(values = c(`re-run (prod)` = "solid", stateful = "22"), name = NULL) + + facet_wrap(~horizon, nrow = 1) + + labs(x = "L1 coverage bias", y = "WIS inflation %") + + theme(legend.position = "bottom", legend.box = "vertical") +``` + +# Table + +Pooled WIS inflation % / L1 coverage bias for every learning rate, re-run +design, by horizon (columns) and extra delay (row blocks). + +```{r table} +sweep_m %>% + filter(location == "pooled", learn == "exact") %>% + transmute(extra, lr = signif(lr, 2), horizon, cell = sprintf("%+.1f / %.3f", wis_infl, bias_cal)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = cell) %>% + arrange(extra, lr) %>% + knitr::kable() +``` From c7de2a23afc6a3cea199f87e97822dda7270ecb0 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:28:04 -0700 Subject: [PATCH 009/107] feat(calibration): cached runs, absolute-error, state-panel and gallery views --- R/calibration/views.R | 203 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) diff --git a/R/calibration/views.R b/R/calibration/views.R index 6852cb71..19e59da9 100644 --- a/R/calibration/views.R +++ b/R/calibration/views.R @@ -100,3 +100,206 @@ cal_by_month <- function(cal) { mutate(pct = 100 * (.data$base - .data$cal) / .data$base, share = 100 * .data$base / sum(.data$base)) %>% ungroup() } + + +#' [cal_run()], cached on disk under a hash of its inputs. +#' +#' Keeps only the forecast columns the views use. Clear `cache_dir` after +#' changing code in `R/calibration/`: the key covers the arguments and data, +#' not the code. +#' @export +cal_run_cached <- function(forecasts, truth, ref = "paper", ..., learn = "exact", vintages = NULL, workers = 1L, + cache_dir = here::here("cache/calibration/experiments/runs")) { + key <- rlang::hash(list(forecasts, truth, ref, list(...), learn, if (learn != "final") vintages)) + path <- file.path(cache_dir, paste0(key, ".rds")) + if (file.exists(path)) { + return(readRDS(path)) + } + cal <- cal_run(forecasts, truth, ref, ..., learn = learn, vintages = vintages, workers = workers) + cal <- list(forecasts = cal$forecasts %>% select( + "location", "horizon", "reference_date", "target_end_date", "season", "level", + "value_base", "value_cal", "truth", "is_burn_in" + )) + dir.create(cache_dir, showWarnings = FALSE, recursive = TRUE) + saveRDS(cal, path) + cal +} + + +#' V-ae: summed absolute error of the median, change % vs base (positive is +#' better), per (variant, `by`, horizon). +#' @export +cal_ae <- function(cals, by = character(0)) { + purrr::imap(cals, function(cal, nm) { + cal$forecasts %>% + filter(!.data$is_burn_in, .data$level == 0.5, !is.na(.data$truth), !is.na(.data$value_base)) %>% + group_by(across(all_of(c(by, "horizon")))) %>% + summarize( + ae_base = sum(abs(.data$truth - .data$value_base)), + ae_cal = sum(abs(.data$truth - .data$value_cal)), + .groups = "drop" + ) %>% + mutate(variant = nm, .before = 1, ae_pct = 100 * (.data$ae_base - .data$ae_cal) / .data$ae_base) + }) %>% + bind_rows() %>% + mutate(variant = factor(.data$variant, levels = names(cals))) +} + + +#' The fixed V-state locations, so state panels are comparable across +#' experiments: national, then states from large to very small. +#' @export +CAL_STATES <- c(US = "US", CA = "06", GA = "13", MN = "27", VT = "50") + +CAL_FAN_COLS <- c(base = "#0072B2", calibrated = "#D55E00") + + +#' Each round's NHSN snapshot for one location, as the forecaster saw it +#' (published by `reference_date - 3`), limited to `weeks` before that. +#' @keywords internal +cal_round_snapshots <- function(vintages, loc, ref_dates, weeks = 12L) { + v <- vintages %>% filter(.data$location == loc) + purrr::map(ref_dates, function(rd) { + hub_vintage_snapshot(v, hub_round_asof(rd)) %>% + filter(.data$target_end_date >= rd - 7L * weeks) %>% + mutate(reference_date = rd) + }) %>% bind_rows() +} + + +#' V-state: one location over one season. Every other round's fan (h0–h3, +#' base 80% band and median, calibrated 10/50/90%), finalized truth in grey and, +#' in black, the snapshot each plotted round saw, over the fortnight before it. +#' @export +cal_state_panel <- function(cal, loc, season, truth, vintages, title = NULL) { + refs <- cal$forecasts %>% + filter(.data$season == !!season, .data$location == loc, !is.na(.data$value_base)) %>% + distinct(.data$reference_date) %>% + arrange(.data$reference_date) %>% + pull() + refs <- refs[seq(1L, length(refs), by = 2L)] + fan <- cal$forecasts %>% + filter(.data$location == loc, .data$reference_date %in% refs, .data$horizon %in% 0:3, .data$level %in% c(0.1, 0.5, 0.9)) %>% + mutate(lvl = c(`0.1` = "lo", `0.5` = "med", `0.9` = "hi")[as.character(.data$level)]) %>% + select("reference_date", "target_end_date", "lvl", base = "value_base", calibrated = "value_cal") %>% + tidyr::pivot_longer(c("base", "calibrated"), names_to = "which") %>% + tidyr::pivot_wider(names_from = "lvl", values_from = "value") + fin <- truth %>% + filter(.data$location == loc, .data$target_end_date >= min(refs) - 21L, .data$target_end_date <= max(refs) + 28L) + snap <- cal_round_snapshots(vintages, loc, refs, weeks = 2L) + ggplot2::ggplot() + + ggplot2::geom_line(data = fin, ggplot2::aes(.data$target_end_date, .data$truth), colour = "grey65") + + ggplot2::geom_ribbon( + data = filter(fan, .data$which == "base"), + ggplot2::aes(.data$target_end_date, ymin = .data$lo, ymax = .data$hi, fill = .data$which, group = .data$reference_date), + alpha = 0.2 + ) + + ggplot2::geom_line( + data = filter(fan, .data$which == "calibrated") %>% tidyr::pivot_longer(c("lo", "hi"), names_to = "edge", values_to = "q"), + ggplot2::aes(.data$target_end_date, .data$q, colour = .data$which, group = interaction(.data$reference_date, .data$edge)), + linewidth = 0.3 + ) + + ggplot2::geom_line( + data = fan, + ggplot2::aes(.data$target_end_date, .data$med, colour = .data$which, group = interaction(.data$reference_date, .data$which)) + ) + + ggplot2::geom_line(data = snap, ggplot2::aes(.data$target_end_date, .data$truth, group = .data$reference_date), colour = "black") + + ggplot2::scale_fill_manual(values = CAL_FAN_COLS, aesthetics = c("fill", "colour")) + + ggplot2::labs(x = NULL, y = "admissions", colour = NULL, fill = NULL, subtitle = title %||% paste(names(CAL_STATES)[CAL_STATES == loc], season)) +} + + +#' V-gallery selection: a few forecasts, the best and the worst per season +#' phase and location size, each from a different location. +#' +#' Ranking by a single metric fills a gallery with small states late in the +#' season, where a stale offset on a tiny base inflates relative error, and +#' ranking by WIS per 100k favours small places too (their rates are noisier). +#' Here each forecast is scored by its WIS change as a share of its own +#' location-season's base WIS, so a late-season forecast on a tiny base only +#' ranks high if it moved a real part of that location's season error. Small +#' places still vary more, so slots are split by size (population above or +#' below the median) as well as by phase (before, around, after the location's +#' peak in finalized truth). +#' +#' @param population tibble of `location`, `population`. +#' @param peak_weeks half-width of the "peak" phase. +#' @return one row per picked forecast: `location`, `reference_date`, `phase`, +#' `size`, `kind` (gain/loss), `share` (% of the location-season's base WIS), +#' `wis_pct`. +#' @export +cal_gallery_pick <- function(cal, population, peak_weeks = 3L) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + fc <- cal$forecasts %>% filter(!.data$is_burn_in, !is.na(.data$truth), !is.na(.data$value_base)) + median_pop <- stats::median(population$population[population$location %in% fc$location]) + peaks <- fc %>% + distinct(.data$location, .data$season, .data$target_end_date, .data$truth) %>% + group_by(.data$location, .data$season) %>% + slice_max(.data$truth, n = 1L, with_ties = FALSE) %>% + ungroup() %>% + select("location", "season", peak = "target_end_date") + scored <- fc %>% + group_by(.data$location, .data$season, .data$reference_date) %>% + summarize( + wis_base = sum(pinball(.data$truth, .data$value_base, .data$level)), + wis_cal = sum(pinball(.data$truth, .data$value_cal, .data$level)), + .groups = "drop" + ) %>% + group_by(.data$location, .data$season) %>% + mutate(share = 100 * (.data$wis_base - .data$wis_cal) / sum(.data$wis_base)) %>% + ungroup() %>% + inner_join(peaks, by = c("location", "season")) %>% + inner_join(population, by = "location") %>% + mutate( + size = if_else(.data$population >= median_pop, "large", "small"), + weeks_from_peak = as.numeric(.data$reference_date - .data$peak) / 7, + phase = factor( + dplyr::case_when( + .data$weeks_from_peak < -peak_weeks ~ "before peak", + .data$weeks_from_peak > peak_weeks ~ "after peak", + TRUE ~ "around peak" + ), + levels = c("before peak", "around peak", "after peak") + ), + wis_pct = 100 * (.data$wis_base - .data$wis_cal) / .data$wis_base + ) + # Greedy, one slot at a time, skipping locations already picked. + slots <- tidyr::expand_grid(phase = levels(scored$phase), size = c("large", "small"), kind = c("gain", "loss")) + used <- character(0) + picks <- purrr::map(seq_len(nrow(slots)), function(i) { + cand <- scored %>% filter(.data$phase == slots$phase[i], .data$size == slots$size[i], !(.data$location %in% used)) + cand <- if (slots$kind[i] == "gain") arrange(cand, desc(.data$share)) else arrange(cand, .data$share) + pick <- cand[1, ] %>% mutate(kind = slots$kind[i]) + used <<- c(used, pick$location) + pick + }) + bind_rows(picks) %>% + select("location", "reference_date", "phase", "size", "kind", "share", "wis_pct") +} + + +#' V-gallery panel: one forecast (all horizons), base and calibrated 50% and +#' 90% bands, the snapshot the forecaster saw (black) and finalized truth (grey). +#' @export +cal_gallery_panel <- function(cal, loc, ref_date, truth, vintages, subtitle = NULL) { + levels_map <- c("0.05" = "lo90", "0.25" = "lo50", "0.5" = "med", "0.75" = "hi50", "0.95" = "hi90") + fin <- truth %>% filter(.data$location == loc, .data$target_end_date >= ref_date - 84L, .data$target_end_date <= ref_date + 28L) + snap <- cal_round_snapshots(vintages, loc, ref_date) + fan <- cal$forecasts %>% + filter(.data$location == loc, .data$reference_date == ref_date, .data$level %in% as.numeric(names(levels_map))) %>% + mutate(lvl = levels_map[as.character(.data$level)]) %>% + select("target_end_date", "lvl", base = "value_base", calibrated = "value_cal") %>% + tidyr::pivot_longer(c("base", "calibrated"), names_to = "which") %>% + tidyr::pivot_wider(names_from = "lvl", values_from = "value") + ggplot2::ggplot() + + ggplot2::geom_line(data = fin, ggplot2::aes(.data$target_end_date, .data$truth), colour = "grey65") + + ggplot2::geom_point(data = fin, ggplot2::aes(.data$target_end_date, .data$truth), colour = "grey65", size = 0.7) + + ggplot2::geom_ribbon(data = fan, ggplot2::aes(.data$target_end_date, ymin = .data$lo90, ymax = .data$hi90, fill = .data$which), alpha = 0.16) + + ggplot2::geom_ribbon(data = fan, ggplot2::aes(.data$target_end_date, ymin = .data$lo50, ymax = .data$hi50, fill = .data$which), alpha = 0.3) + + ggplot2::geom_line(data = fan, ggplot2::aes(.data$target_end_date, .data$med, colour = .data$which)) + + ggplot2::geom_line(data = snap, ggplot2::aes(.data$target_end_date, .data$truth), colour = "black") + + ggplot2::geom_point(data = snap, ggplot2::aes(.data$target_end_date, .data$truth), colour = "black", size = 0.7) + + ggplot2::geom_vline(xintercept = hub_round_asof(ref_date), linetype = "dashed", colour = "grey40") + + ggplot2::scale_fill_manual(values = CAL_FAN_COLS, aesthetics = c("fill", "colour")) + + ggplot2::labs(x = NULL, y = "admissions", colour = NULL, fill = NULL, subtitle = subtitle) +} From 9991ba0e6c5924c4f93cb66e9e3f75d4d418741b Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:28:05 -0700 Subject: [PATCH 010/107] docs(calibration): E03-E05 notebooks on exact learning truth with standard views --- .../calibration_experiments/_load.Rmd | 16 +++ .../_standard_views.Rmd | 105 ++++++++++++++++++ .../calibration_experiments/e03_scale.Rmd | 51 +++++++++ .../e04_offset_structure.Rmd | 51 +++++++++ .../e05_warm_start.Rmd | 56 ++++++++++ 5 files changed, 279 insertions(+) create mode 100644 reports/writeups/calibration_experiments/_load.Rmd create mode 100644 reports/writeups/calibration_experiments/_standard_views.Rmd create mode 100644 reports/writeups/calibration_experiments/e03_scale.Rmd create mode 100644 reports/writeups/calibration_experiments/e04_offset_structure.Rmd create mode 100644 reports/writeups/calibration_experiments/e05_warm_start.Rmd diff --git a/reports/writeups/calibration_experiments/_load.Rmd b/reports/writeups/calibration_experiments/_load.Rmd new file mode 100644 index 00000000..3387f754 --- /dev/null +++ b/reports/writeups/calibration_experiments/_load.Rmd @@ -0,0 +1,16 @@ +```{r load-data, cache=FALSE} +# Shared by the experiment notebooks: flu hub submissions, finalized truth for +# scoring, NHSN vintages for learning, and population for rates. +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +archive <- get_nhsn_data_archive("flu") +truth <- nhsn_read_truth("flu", archive) +vintages <- nhsn_read_vintages("flu", archive) +hub_fc <- suppressMessages(hub_read_forecasts(hub_dir = here::here(params$hub_dir))) +population <- get_population_data() %>% + distinct(state_code, .keep_all = TRUE) %>% + filter(state_code %in% unique(hub_fc$location)) %>% + select(location = state_code, population) +rate_scales <- population %>% transmute(location, from = as.Date("2000-01-01"), scale = population / 1e5) +LIVE <- c("2024-2025", "2025-2026") +run <- function(...) cal_run_cached(hub_fc, truth, ..., vintages = vintages, workers = params$workers) +``` diff --git a/reports/writeups/calibration_experiments/_standard_views.Rmd b/reports/writeups/calibration_experiments/_standard_views.Rmd new file mode 100644 index 00000000..197abb94 --- /dev/null +++ b/reports/writeups/calibration_experiments/_standard_views.Rmd @@ -0,0 +1,105 @@ +# Standard views + +Expects `cals` (a named list of runs) and `focus` (the name of the run the +state panels and gallery show). Every score is over both live seasons, all +hub locations, against finalized truth; burn-in rounds are excluded. + +## V-head: WIS and coverage + +WIS change % (positive is better than the base): + +```{r v-head} +head_all <- cal_headline(cals) +knitr::kable(cal_wide(head_all, "pct_improvement")) +``` + +L1 coverage bias (mean over levels of |coverage − level|), with the base first: + +```{r v-head-cov} +knitr::kable(bind_rows( + cal_wide(head_all %>% filter(variant == names(cals)[1]) %>% mutate(variant = "base"), "cal_error_base", 3), + cal_wide(head_all %>% mutate(variant = as.character(variant)), "cal_error_cal", 3) +)) +``` + +WIS change % by season: + +```{r v-head-season} +knitr::kable( + cal_wide(cal_headline(cals, by = "season") %>% filter(season %in% LIVE), "pct_improvement", id = c("variant", "season")) %>% + arrange(season, variant) +) +``` + +## V-ae: absolute error of the median + +Summed |truth − median|, change % vs base (positive is better): + +```{r v-ae} +knitr::kable(cal_wide(cal_ae(cals), "ae_pct")) +``` + +## V-month: where in the season + +WIS change % by month of the reference date (clipped at −60), h0 and h3, +and each month's share of the base WIS at h3: + +```{r v-month, fig.height=4.5} +month_tbl <- purrr::imap(cals, function(cal, nm) cal_by_month(cal) %>% mutate(variant = nm)) %>% + bind_rows() %>% + filter(month %in% month.abb[c(10:12, 1:5)]) %>% + mutate(variant = factor(variant, levels = names(cals)), month = droplevels(month)) +ggplot(month_tbl %>% filter(horizon %in% c(0, 3)), aes(month, pct, colour = variant, group = variant)) + + geom_hline(yintercept = 0, colour = "grey50") + + geom_line() + + geom_point() + + facet_wrap(~ paste0("h", horizon)) + + scale_colour_brewer(palette = "Dark2", name = NULL) + + coord_cartesian(ylim = c(-60, 30)) + + labs(x = "month of reference date", y = "WIS change %") + + theme(legend.position = "bottom", legend.direction = "vertical") +knitr::kable( + month_tbl %>% + filter(horizon == 3) %>% + transmute(variant, month, pct = round(pct, 1)) %>% + tidyr::pivot_wider(names_from = month, values_from = pct) %>% + bind_rows(month_tbl %>% filter(horizon == 3, variant == names(cals)[1]) %>% + transmute(variant = "share of base WIS %", month, pct = round(share, 1)) %>% + tidyr::pivot_wider(names_from = month, values_from = pct)) +) +``` + +## V-state: fixed states, `r focus` {.tabset} + +The same states in every experiment notebook (`CAL_STATES`). Every other +round's fan at h0–h3: blue ribbon and line are the base 80% band and median, +orange lines the calibrated 10/50/90% quantiles, grey the finalized truth, +black the NHSN snapshot each plotted round saw. + +```{r v-state, results="asis", fig.height=4} +for (nm in names(CAL_STATES)) { + cat("\n\n### ", nm, "\n\n", sep = "") + for (s in LIVE) print(cal_state_panel(cals[[focus]], CAL_STATES[[nm]], s, truth, vintages, title = paste(nm, s))) +} +``` + +## V-gallery: `r focus` {.tabset} + +For each season phase and location size (population above or below the +median), the forecast with the largest WIS gain and the one with the largest +loss, each from a different location. Forecasts are ranked by their +WIS change as a share of their own location-season's base WIS (see +`cal_gallery_pick()`), so small states late in the season, where the base is +tiny, do not crowd the gallery. Ribbons are the 50% +and 90% bands, the dashed line is the forecast date. + +```{r v-gallery, results="asis", fig.height=3.5} +picks <- cal_gallery_pick(cals[[focus]], population) +abbrev <- setNames(toupper(get_population_data()$state_id), get_population_data()$state_code) +for (i in seq_len(nrow(picks))) { + p <- picks[i, ] + if (i == 1L || p$phase != picks$phase[i - 1L]) cat("\n\n### ", as.character(p$phase), "\n\n", sep = "") + sub <- sprintf("%s (%s), %s, %s: WIS %+.1f%% (%+.1f%% of its season's base WIS)", abbrev[[p$location]], p$size, p$reference_date, p$kind, p$wis_pct, p$share) + print(cal_gallery_panel(cals[[focus]], p$location, p$reference_date, truth, vintages, subtitle = sub)) +} +``` diff --git a/reports/writeups/calibration_experiments/e03_scale.Rmd b/reports/writeups/calibration_experiments/e03_scale.Rmd new file mode 100644 index 00000000..6e70bada --- /dev/null +++ b/reports/writeups/calibration_experiments/e03_scale.Rmd @@ -0,0 +1,51 @@ +--- +title: "E03: working scale (count, sqrt, rate)" +date: "`r format(Sys.time(), '%d %B %Y')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + hub_dir: "../FluSight-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} +``` + +Ledger entry **E03** (`notes/calibration-ledger.md`). One axis: the scale +the tracker works on. Everything else is REF-paper (single term, adaptive +learning rate with `mult` 0.03 and window 20, carry, 2023-24 burn-in, +`settle_days` 14), learning from vintages the way prod does (re-run each +week; E00). + +- **count**: REF-paper itself; additive offsets in admissions. +- **sqrt**: offsets on the square root of admissions, mapped back. +- **rate**: offsets in admissions per 100k, so one location's step size is + comparable to another's. + +The adaptive learning rate has a floor (0.1 by default) in the tracker's +working units, so the sqrt and rate variants use a floor of 0.001; that floor +change goes with the scale. + +```{r runs} +cals <- list( + count = run("paper"), + sqrt = run("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)), + rate = run("paper", scales = rate_scales, lr_args = list(mult = 0.03, floor = 1e-3)) +) +focus <- "sqrt" +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd new file mode 100644 index 00000000..62d58a88 --- /dev/null +++ b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd @@ -0,0 +1,51 @@ +--- +title: "E04: offset structure (leak and slow term)" +date: "`r format(Sys.time(), '%d %B %Y')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + hub_dir: "../FluSight-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} +``` + +Ledger entry **E04** (`notes/calibration-ledger.md`). One axis: the +structure of the offset. Everything else is REF-paper on the sqrt scale (the +E03 sqrt variant), learning from vintages the way prod does (E00). + +- **single**: the paper's tracker, one offset per level. +- **leak 0.1**: the offset shrinks by 10% each round (`fast_decay = 0.1`), so + a correction that new outcomes do not refresh fades (half-life about 6.6 + rounds). +- **slow 0.003**: a second, persistent offset term with a small learning rate + (`lr_slow = list(mult = 0.003)`) added to the paper's one. +- **slow 0.003 + leak 0.1**: both. This is REF-op without its warm start + (E05). + +```{r runs} +sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) +cals <- list( + single = do.call(run, sqrt_args), + `leak 0.1` = do.call(run, c(sqrt_args, fast_decay = 0.1)), + `slow 0.003` = do.call(run, c(sqrt_args, list(lr_slow = list(mult = 0.003)))), + `slow 0.003 + leak 0.1` = do.call(run, c(sqrt_args, list(lr_slow = list(mult = 0.003)), fast_decay = 0.1)) +) +focus <- "leak 0.1" +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e05_warm_start.Rmd b/reports/writeups/calibration_experiments/e05_warm_start.Rmd new file mode 100644 index 00000000..5a2f97c8 --- /dev/null +++ b/reports/writeups/calibration_experiments/e05_warm_start.Rmd @@ -0,0 +1,56 @@ +--- +title: "E05: warm start from the burn-in season" +date: "`r format(Sys.time(), '%d %B %Y')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + hub_dir: "../FluSight-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} +``` + +Ledger entry **E05** (`notes/calibration-ledger.md`). One axis: whether the +persistent offset starts from the burn-in season's conformal offset +(`slow_init = "burn_in_quantile"`: per horizon and level, the level-quantile +of the burn-in residuals pooled over locations, on the sqrt scale) or from +zero. Everything else is REF-paper on the sqrt scale, learning from vintages +the way prod does (E00). Each warm-started run is paired with its cold twin +from E04: + +- **warm only**: the warm start, then no tracking (learning rate ≈ 0). +- **single** / **warm + single**: the paper's tracker. +- **leak 0.1** / **warm + leak 0.1**. +- **slow + leak** / **warm + slow + leak**: the second is REF-op, the + operating point in prod. + +```{r runs} +sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) +warm <- list(slow_init = "burn_in_quantile") +cals <- list( + `warm only` = do.call(run, c(sqrt_args[1:2], warm, list(lr_args = list(mult = 1e-6, floor = 0)))), + single = do.call(run, sqrt_args), + `warm + single` = do.call(run, c(sqrt_args, warm)), + `leak 0.1` = do.call(run, c(sqrt_args, fast_decay = 0.1)), + `warm + leak 0.1` = do.call(run, c(sqrt_args, warm, fast_decay = 0.1)), + `slow + leak` = do.call(run, c(sqrt_args, list(lr_slow = list(mult = 0.003)), fast_decay = 0.1)), + `warm + slow + leak (REF-op)` = run("op") +) +focus <- "warm + slow + leak (REF-op)" +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} +``` From bb449ed1816f13798f4a76f8cdf740cfd83968b2 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:44:22 -0700 Subject: [PATCH 011/107] docs(calibration): E09 findings in the notebook --- .../calibration_experiments/e09_lr_delay.Rmd | 72 ++++++++++++++++--- 1 file changed, 61 insertions(+), 11 deletions(-) diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd index 8baca6ca..2b1aec95 100644 --- a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -9,7 +9,7 @@ output: params: hub_dir: "../FluSight-forecast-hub" workers: 14 - wis_max: 50 + wis_max: 20 smoke: false editor_options: chunk_output_type: console @@ -54,6 +54,42 @@ seasons, all 53 hub locations. "WIS inflation %" is `100 × (WIS_cal / WIS_base so negative is better than the base. "L1 coverage bias" is the mean over the 23 levels of |empirical coverage − level|. +# Findings + +Numbers are from the render of 30 September 2026, pooled over locations. +"Knee" below means a constant learning rate of 0.056 admissions per 100k per +step. + +- **The trade-off is L-shaped.** Raising the constant learning rate cuts the + coverage bias from about 0.10 to 0.03 at almost no WIS cost, up to a knee + near 0.05–0.1 (h1–h3; up to 0.3 at h−1). Past the knee WIS climbs steeply + and coverage stops improving. The per-location lines spread widely: past + 0.3 many locations inflate WIS by 20% or more. +- **On the rate scale, the knee beats the paper's adaptive tracker.** At the + reference delay (+1 week, re-run design), the knee gives WIS inflation + −3.9/−2.3/−1.0/−0.6/−0.1% and coverage bias 0.043/0.031/0.026/0.026/0.026 + at h−1…h3. REF-paper gives −4.2/−2.4/+0.5/+2.9/+5.2% and + 0.047/0.030/0.032/0.034/0.032, so the knee is better on both at h1–h3 and + within 0.3 points at h−1/h0. +- **The rate scale only matters with a constant learning rate.** "rate, + adaptive" matches REF-paper exactly. The adaptive rate is a quantile of the + location's own residuals, so dividing a location by its population rescales + the steps and the offsets together. +- **REF-op makes a different trade.** Its WIS gain (−6.8/−5.1/−2.5/−1.8/−1.9%) + is larger than any constant rate reaches at h0–h3, but at about twice the + coverage bias (0.080/0.054/0.045/0.046/0.041). The leak and the warm start + buy WIS; the plain tracker buys coverage. +- **Extra delay, re-run design (prod): shorter is better, but it matters only + at h−1/h0.** At a learning rate of 0.1, h−1 WIS inflation is −6.2/−4.6/−2.8% + at +0/+1/+2 weeks. At h1–h3 the curves nearly coincide, and coverage bias + barely moves with delay. A value learned early is replaced by its revision + in the next week's run, so waiting only delays the signal. +- **Extra delay, stateful design: one week is needed.** At +0 weeks the + tracker learns once from the first report, which under-reports, so the + offsets are pushed down: at the knee, h−1 coverage bias is 0.143, worse than + the uncalibrated 0.118. +1 week recovers most of it (0.067); +2 and +3 + weeks change little at h1–h3 and lose WIS at h−1. + ```{r libs, cache=FALSE} suppressPackageStartupMessages(source(here::here("R/load_all.R"))) LRS <- 10^seq(-3, 0.5, by = 0.25) @@ -271,16 +307,30 @@ ggplot(pooled_all, aes(bias_cal, wis_infl, colour = extra, linetype = design, gr theme(legend.position = "bottom", legend.box = "vertical") ``` -# Table +# Tables + +Pooled WIS inflation % / L1 coverage bias for every learning rate, by horizon +(columns) and extra delay (row blocks). + +```{r table-fn} +sweep_table <- function(which_learn) { + sweep_m %>% + filter(location == "pooled", learn == which_learn) %>% + transmute(extra, lr = signif(lr, 2), horizon, cell = sprintf("%+.1f / %.3f", wis_infl, bias_cal)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = cell) %>% + arrange(extra, lr) %>% + knitr::kable() +} +``` + +## Re-run design (prod) + +```{r table-exact} +sweep_table("exact") +``` -Pooled WIS inflation % / L1 coverage bias for every learning rate, re-run -design, by horizon (columns) and extra delay (row blocks). +## Stateful design -```{r table} -sweep_m %>% - filter(location == "pooled", learn == "exact") %>% - transmute(extra, lr = signif(lr, 2), horizon, cell = sprintf("%+.1f / %.3f", wis_infl, bias_cal)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = cell) %>% - arrange(extra, lr) %>% - knitr::kable() +```{r table-vintage} +sweep_table("vintage") ``` From dbe7016e301584264b5eac859ea9d31a5c602dd8 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:58:22 -0700 Subject: [PATCH 012/107] docs(calibration): E03-E05 findings; E09 and E03-E05 summaries in notes and ledger --- notes/CALIBRATION.md | 42 ++++++++++++++++++- notes/calibration-ledger.md | 27 ++++++------ .../calibration_experiments/e03_scale.Rmd | 19 +++++++++ .../e04_offset_structure.Rmd | 21 ++++++++++ .../e05_warm_start.Rmd | 25 +++++++++++ 5 files changed, 119 insertions(+), 15 deletions(-) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index 3df5f2bb..a2d4f77a 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -80,6 +80,25 @@ Findings (E00, `e00_vintage_backtest.Rmd`, flu, both live seasons): run corrects what an early reveal got wrong. A stateful tracker would prefer the longer delay. The right revision delay depends on the design. +## Constant learning rate × revision delay (E09, 2026-09-30) + +`e09_lr_delay.Rmd`: the plain tracker (REF-paper) on the rate scale (per +100k) with a constant learning rate, swept over 0.001–3.2 × extra delay 0–3 +weeks, exact and stateful learning truth. Pooled over locations: + +- The coverage/WIS trade-off is L-shaped, with a knee at a learning rate of + about 0.05–0.1 per 100k (up to 0.3 at h−1). At 0.056 and +1 week (re-run): + WIS −3.9/−2.3/−1.0/−0.6/−0.1% (negative is better), coverage bias + 0.043/0.031/0.026/0.026/0.026. That beats REF-paper on both at h1–h3. +- With the adaptive rate, rate scale and count scale give identical results + (the adaptive rate rescales with the location), so the rate scale matters + only with a constant rate. +- REF-op gains more WIS than any constant rate at h0–h3, at about twice the + coverage bias. +- Re-run design (prod): shorter delay is better, mostly at h−1/h0. Stateful + design: zero extra delay makes h−1 coverage worse than base (learning from + the under-reported first report); one extra week fixes it. + ## Data FluSight CMU-TimeSeries submissions: 83 rounds (2023-10-14 … 2026-05-30), 53 @@ -472,7 +491,28 @@ shortfall at h3 as NHSN. So the "bias flips sign with the data source" reading does not hold. On ILI+ the base is about level on average, not biased high. -### Current numbers (2026-09-23) +### Current numbers (2026-09-23; superseded by the exact re-runs below) + +Re-run 2026-09-30 with exact learning truth (the prod re-run design) in +`e03_scale.Rmd`, `e04_offset_structure.Rmd`, `e05_warm_start.Rmd`. WIS change +% (positive is better) and coverage bias, same setup: + +| variant | h−1 | h0 | h1 | h2 | h3 | cal err h−1…h3 | +|---|---|---|---|---|---|---| +| count space, single term | +4.2 | +2.4 | −0.5 | −2.9 | −5.2 | 0.047/0.030/0.032/0.034/0.032 | +| sqrt, single term | +6.4 | +4.1 | +0.8 | −0.6 | −1.3 | 0.054/0.032/0.026/0.028/0.026 | +| sqrt, leak 0.1 | +6.2 | +3.9 | +1.8 | +1.0 | +0.4 | 0.096/0.082/0.075/0.080/0.082 | +| warm start only | +0.4 | +1.1 | +0.9 | +1.1 | +2.0 | 0.105/0.075/0.062/0.065/0.062 | +| warm start + single term | +6.4 | +4.7 | +1.3 | +0.3 | +0.4 | 0.050/0.021/0.013/0.016/0.014 | +| warm start + leak 0.1 | +6.3 | +4.7 | +2.5 | +1.9 | +2.1 | 0.087/0.060/0.051/0.052/0.047 | +| operating point | +6.8 | +5.1 | +2.5 | +1.8 | +1.9 | 0.080/0.054/0.045/0.046/0.041 | + +The ordering of the variants is unchanged from the finalized-truth table. +h−1 WIS of the tracking sqrt variants drops by 1.5–2 points; count space and +the warm start alone are unchanged, and h0–h3 move by at most 0.5. New: warm start + single +term (not run before) is the best-calibrated variant, and with REF-op it +brackets the WIS/coverage trade-off. The finalized-truth numbers follow for +the record. From `reports/writeups/calibration/calibration_findings_flu.Rmd`: post-fix, truth from `nhsn_read_truth()`, all 53 hub locations, the submitted ensemble, diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 78f55237..eacdebbc 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -16,12 +16,12 @@ that step from the reference one axis at a time. | base forecaster | submitted ensemble, `windowed_seasonal` | submitted ensemble | | working scale | count, rate (per 100k), sqrt, log1p, quartic root | sqrt | | learning truth | final (latest vintage), vintage (value at reveal; a stateful tracker), exact (prod's weekly re-run) | exact (E00 on; E01–E08 used final) | -| learning rate type | adaptive+ (`mult`, `floor`, `lr_window`), constant | adaptive+ | +| learning rate type | adaptive+ (`mult`, `floor`, `lr_window`), constant (0.001–3.2 per 100k, E09) | adaptive+ | | `lr_mult` | 0.3, 0.1, 0.03, 0.01 | 0.03 | | `lr_window` | 8, 20, 50, Inf | 20 | | season policy | carry, reset | carry | | burn-in | 2023-24 hub, ILI+, none | 2023-24 hub | -| `settle_days` (extra revision delay) | 14 (= 1 week extra) | 14 | +| `settle_days` (extra revision delay) | 7, 14, 21, 28 (= 0–3 weeks extra) | 14 | | offset structure | single term, leak (`fast_decay`), warm start, slow term | see REF-op | | update gating | always, `off_after` date | always | | eta variants | pooled, per-level, geo-pooled, seasonal window | pooled | @@ -70,13 +70,15 @@ base WIS, and pick within strata (season phase × location size). | E00 | how much did final truth flatter results; can vintage stand in for exact? | learning truth {final, vintage, exact} × settle {7, 14} | REF-paper, REF-op | flu | post-fix | all three | V-head | `e00_vintage_backtest.Rmd` | done: exact is the default; vintage is not a stand-in | | E01 | which eta settings? | season policy × window × mult | REF-paper | flu | pre-fix | final | V-head, V-curve (mult) | `calibration_qt_flu.Rmd` | stale | | E02 | cutoff and eta variants | `off_after`, per-level, geo-pool, seasonal window (one at a time) | REF-paper | flu | pre-fix | final | V-head, V-month, V-state | `calibration_qt_seasons_flu.Rmd` | stale | -| E03 | which working scale? | count vs sqrt (log1p, quartic pre-fix only) | REF-paper | flu | post-fix | final | V-head, V-month | `calibration_findings_flu.Rmd` | current, needs vintage rerun | -| E04 | leak / two-term offset | single, leak 0.1, slow term | REF-paper at sqrt | flu | post-fix | final | V-head, V-month | `calibration_findings_flu.Rmd` | current, needs vintage rerun | -| E05 | warm start from burn-in | warm start alone, + leak, + slow | REF-paper at sqrt | flu | post-fix | final | V-head, V-month | `calibration_findings_flu.Rmd` | current, needs vintage rerun | +| E03 | which working scale? | count vs sqrt vs rate (log1p, quartic pre-fix only) | REF-paper | flu | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e03_scale.Rmd` | done | +| E04 | leak / two-term offset | single, leak 0.1, slow term, slow + leak | REF-paper at sqrt | flu | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e04_offset_structure.Rmd` | done | +| E05 | warm start from burn-in | warm vs cold, paired, for single / leak / slow + leak; warm alone | REF-paper at sqrt | flu | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e05_warm_start.Rmd` | done | | E06 | ILI+ as burn-in | burn-in source | REF-op | flu | post-fix | final | V-head | notes only | invalid (broken replay) | | E07 | one forecaster across eras | base forecaster (A ensemble / B `windowed_seasonal`; C invalid) | REF-paper at sqrt | flu | post-fix | final | V-head | `calibration_ws_experiments.R`, findings notebook | current, needs vintage rerun | -| E08 | covid | disease | old REF-paper (count, window 50) | covid | pre-fix | final | V-head, V-state, V-gallery | `calibration_qt_*_covid.Rmd` | stale | -| E09 | constant lr × revision delay | constant lr grid × `settle_days` {7, 14, 21, 28}; bridge rows below | REF-paper | flu | post-fix | exact (+ vintage, for a stateful tracker) | V-curve, V-head | planned: `e09_lr_delay.Rmd` | planned | +| E08 | covid | disease | old REF-paper (count, window 50) | covid | pre-fix | final | V-head, V-state, V-gallery | `calibration_qt_*_covid.Rmd` | stale, replaced by E10 | +| E09 | constant lr × revision delay | constant lr grid × `settle_days` {7, 14, 21, 28}; bridge rows below | REF-paper | flu | post-fix | exact (+ vintage, for a stateful tracker) | V-curve, V-head | `e09_lr_delay.Rmd` | done | + +| E10 | covid at the references | disease (REF-paper, REF-op without warm start as in covid prod) | REF-paper, REF-op | covid | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e10_covid.Rmd` | planned | E09 bridge rows, each one axis from the previous: REF-paper → rate scale → constant lr (sweep) → `settle_days` (sweep). @@ -87,10 +89,7 @@ bias on NHSN and ILI+. See `notes/CALIBRATION.md`. ## Gaps -- **Learning truth.** E01–E08 learned from finalized NHSN. E00 shows the - error is small outside h−1, but those experiments still need an exact - re-run (E03–E05 first). -- **Covid** has not been run since the fixes or at REF-op. -- **V-ae** exists only as the state-picking criterion in the seasons notebook; - no experiment reports it as a table. -- **V-state and V-gallery** have not been produced for any post-fix config. +- **Learning truth.** E01, E02 and E06–E08 learned from finalized NHSN (E00 + shows the error is small outside h−1). E03–E05 have been re-run exactly. +- **Covid** has not been run since the fixes (E10 planned). +- **E07** (ensemble vs `windowed_seasonal`) has not been re-run exactly. diff --git a/reports/writeups/calibration_experiments/e03_scale.Rmd b/reports/writeups/calibration_experiments/e03_scale.Rmd index 6e70bada..d975db3e 100644 --- a/reports/writeups/calibration_experiments/e03_scale.Rmd +++ b/reports/writeups/calibration_experiments/e03_scale.Rmd @@ -38,6 +38,25 @@ The adaptive learning rate has a floor (0.1 by default) in the tracker's working units, so the sqrt and rate variants use a floor of 0.001; that floor change goes with the scale. +# Findings + +Numbers are from the render of 30 September 2026, pooled over both live seasons and all locations; WIS and absolute-error changes are % vs base, positive is better. + +- **sqrt beats count at every horizon.** WIS +6.4/+4.1/+0.8/−0.6/−1.3 vs + +4.2/+2.4/−0.5/−2.9/−5.2 at h−1…h3, with similar coverage bias + (0.054–0.026 vs 0.047–0.032). The median improves too: absolute error + +12.0/+8.6/+2.6/+2.4/+1.7 vs +6.3/+5.7/+1.9/−2.1/−4.8. +- **The gain is in the spring.** At h3, count loses 164% of April's and 92% of + May's base WIS; sqrt loses 33% and gains 20%. An offset learned at the peak + is sized in admissions, so in count space it is far too large for the tail. + Both lose 43–44% in October (0.2% of the base WIS). +- **rate equals count with the adaptive learning rate.** The adaptive rate is + a quantile of the location's own residuals, so dividing a location by its + population rescales steps and offsets together. The rate scale matters only + with a constant learning rate (E09). +- **2025-26 is harder for both.** sqrt loses 2.5–3.3% at h1–h3 there (count + 5.6–6.9%), against gains in 2024-25. + ```{r runs} cals <- list( count = run("paper"), diff --git a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd index 62d58a88..5a677d85 100644 --- a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd +++ b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd @@ -36,6 +36,27 @@ E03 sqrt variant), learning from vintages the way prod does (E00). - **slow 0.003 + leak 0.1**: both. This is REF-op without its warm start (E05). +# Findings + +Numbers are from the render of 30 September 2026, pooled over both live seasons and all locations; WIS and absolute-error changes are % vs base, positive is better. + +- **The leak buys WIS at h1–h3 with coverage.** `leak 0.1` moves WIS from + +0.8/−0.6/−1.3 to +1.8/+1.0/+0.4 at h1–h3, but coverage bias nearly + triples (0.026–0.028 → 0.075–0.082), close to the base's 0.095–0.103. + A correction that fades between refreshes cannot hold coverage through a + 3–5 round reveal lag. +- **The leak is what repairs the tail and the second season.** At h3 it cuts + the October loss from 44% to 11% and March from 46% to 16%, and turns April + from −33% to +9%. In 2025-26, h1–h3 go from −2.5/−3.3/−2.7 to + +1.7/+1.1/+0.8. +- **A slow term alone does nothing.** `slow 0.003` is within 0.3 points of + `single` everywhere, and `slow 0.003 + leak 0.1` is within 0.5 points of + `leak 0.1` (slightly better coverage, 0.068–0.089). At this learning rate + the slow term never accumulates enough to matter without a warm start + (E05). +- The median follows WIS: the leak improves absolute error at h1–h3 + (+3.7/+3.1/+3.3 vs +2.6/+2.4/+1.7) and costs about a point at h−1/h0. + ```{r runs} sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) cals <- list( diff --git a/reports/writeups/calibration_experiments/e05_warm_start.Rmd b/reports/writeups/calibration_experiments/e05_warm_start.Rmd index 5a2f97c8..d1c29391 100644 --- a/reports/writeups/calibration_experiments/e05_warm_start.Rmd +++ b/reports/writeups/calibration_experiments/e05_warm_start.Rmd @@ -37,6 +37,31 @@ from E04: - **slow + leak** / **warm + slow + leak**: the second is REF-op, the operating point in prod. +# Findings + +Numbers are from the render of 30 September 2026, pooled over both live seasons and all locations; WIS and absolute-error changes are % vs base, positive is better. + +- **The warm start helps every tracker, and never hurts.** Each warm run is + at least as good as its cold twin on WIS at every horizon, and better on + coverage: `single` +6.4/+4.1/+0.8/−0.6/−1.3 → `warm + single` + +6.4/+4.7/+1.3/+0.3/+0.4, coverage bias 0.054–0.026 → 0.050–0.013. +- **`warm + single` is the best-calibrated config so far.** Its coverage bias + is 0.013–0.021 at h0–h3, a fifth of the base's, with WIS positive at every + horizon. REF-op (`warm + slow + leak`) has the best WIS + (+6.8/+5.1/+2.5/+1.8/+1.9) at about three times the coverage bias + (0.080–0.041). These two mark the ends of the WIS/coverage trade-off. +- **The warm start's gain is in the first live season.** In 2024-25 it adds + 1.5–3 points at h2/h3 to every tracker; in 2025-26 warm and cold runs are + within 0.5 points, since by then the tracker has learned the level itself. + `warm + single` still loses 2.5–3.4% at h1–h3 in 2025-26; only the leaky + variants gain there. +- **The warm start alone** (no tracking) gains 0.4–2.0% WIS and cuts + coverage bias by a third: the persistent low bias is real and cheap to + correct. +- **Where REF-op loses** is unchanged from the finalized-truth runs: at h3, + October (−28%) and March (−32%), 5% of the base WIS between them; + December–January (75% of it) gain about 4%. + ```{r runs} sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) warm <- list(slow_init = "burn_in_quantile") From d8f8c8f92ca83a11ae03f45327c29e9e2cf17ed4 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:58:31 -0700 Subject: [PATCH 013/107] docs(calibration): E10 notebook, covid at the reference configs --- .../calibration_experiments/_load.Rmd | 15 +++-- .../calibration_experiments/e10_covid.Rmd | 61 +++++++++++++++++++ 2 files changed, 70 insertions(+), 6 deletions(-) create mode 100644 reports/writeups/calibration_experiments/e10_covid.Rmd diff --git a/reports/writeups/calibration_experiments/_load.Rmd b/reports/writeups/calibration_experiments/_load.Rmd index 3387f754..36f668c3 100644 --- a/reports/writeups/calibration_experiments/_load.Rmd +++ b/reports/writeups/calibration_experiments/_load.Rmd @@ -1,11 +1,14 @@ ```{r load-data, cache=FALSE} -# Shared by the experiment notebooks: flu hub submissions, finalized truth for -# scoring, NHSN vintages for learning, and population for rates. +# Shared by the experiment notebooks: hub submissions, finalized truth for +# scoring, NHSN vintages for learning, and population for rates. The disease +# comes from `params$disease` (default flu). suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -archive <- get_nhsn_data_archive("flu") -truth <- nhsn_read_truth("flu", archive) -vintages <- nhsn_read_vintages("flu", archive) -hub_fc <- suppressMessages(hub_read_forecasts(hub_dir = here::here(params$hub_dir))) +disease <- params$disease %||% "flu" +archive <- get_nhsn_data_archive(disease) +truth <- nhsn_read_truth(disease, archive) +vintages <- nhsn_read_vintages(disease, archive) +hub_target <- c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET)[[disease]] +hub_fc <- suppressMessages(hub_read_forecasts(hub_dir = here::here(params$hub_dir), target = hub_target)) population <- get_population_data() %>% distinct(state_code, .keep_all = TRUE) %>% filter(state_code %in% unique(hub_fc$location)) %>% diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd new file mode 100644 index 00000000..2ea5718c --- /dev/null +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -0,0 +1,61 @@ +--- +title: "E10: covid at the reference configs" +date: "`r format(Sys.time(), '%d %B %Y')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + disease: "covid" + hub_dir: "../covid19-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} +``` + +Ledger entry **E10** (`notes/calibration-ledger.md`). One axis: the +disease. Both reference configs run on the covid hub's `CMU-TimeSeries` +submissions, learning from NHSN covid vintages the way prod does (E00), and +scored against finalized covid NHSN. + +Covid submissions start 2024-11-23, so there is no burn-in season. As in the +covid prod pipeline, both configs run with `burn_in_seasons = character(0)`, +and REF-op runs without its warm start (`slow_init = NULL`), leaving sqrt + +slow term 0.003 + leak 0.1. The tracker therefore starts from zero offsets at +the first round. + +Covid is submitted year-round, so the hub round axis has no off-season gap +and `hub_label_seasons()` puts every round in one season. For the per-season +views below, seasons are relabelled as August–July years. + +```{r runs} +no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) +cals <- list( + `REF-paper` = do.call(run, c(list("paper"), no_burn_in)), + `REF-op (no warm start)` = do.call(run, c(list("op"), no_burn_in)) +) +season_of <- function(d) { + y <- as.integer(format(d, "%Y")) - (as.integer(format(d, "%m")) < 8) + paste0(y, "-", y + 1) +} +cals <- purrr::map(cals, function(cal) { + cal$forecasts <- cal$forecasts %>% mutate(season = season_of(reference_date)) + cal +}) +LIVE <- c("2024-2025", "2025-2026") +focus <- "REF-op (no warm start)" +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} +``` From 86b24d786a0621d9d1ac4886018404ba33b3b834 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:05:00 -0700 Subject: [PATCH 014/107] fix(calibration): month view shows every month with forecasts --- reports/writeups/calibration_experiments/_standard_views.Rmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reports/writeups/calibration_experiments/_standard_views.Rmd b/reports/writeups/calibration_experiments/_standard_views.Rmd index 197abb94..4bee16b0 100644 --- a/reports/writeups/calibration_experiments/_standard_views.Rmd +++ b/reports/writeups/calibration_experiments/_standard_views.Rmd @@ -42,12 +42,12 @@ knitr::kable(cal_wide(cal_ae(cals), "ae_pct")) ## V-month: where in the season WIS change % by month of the reference date (clipped at −60), h0 and h3, -and each month's share of the base WIS at h3: +and each month's share of the base WIS at h3. Months run July to June; only +months with forecasts are shown. ```{r v-month, fig.height=4.5} month_tbl <- purrr::imap(cals, function(cal, nm) cal_by_month(cal) %>% mutate(variant = nm)) %>% bind_rows() %>% - filter(month %in% month.abb[c(10:12, 1:5)]) %>% mutate(variant = factor(variant, levels = names(cals)), month = droplevels(month)) ggplot(month_tbl %>% filter(horizon %in% c(0, 3)), aes(month, pct, colour = variant, group = variant)) + geom_hline(yintercept = 0, colour = "grey50") + From 367a73440e045544ceeb8eccc28d631fd73dff78 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:05:00 -0700 Subject: [PATCH 015/107] docs(calibration): E10 covid findings --- notes/CALIBRATION.md | 9 +++++++ notes/calibration-ledger.md | 3 +-- .../calibration_experiments/e10_covid.Rmd | 26 +++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index a2d4f77a..4ee55aad 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -99,6 +99,15 @@ weeks, exact and stateful learning truth. Pooled over locations: design: zero extra delay makes h−1 coverage worse than base (learning from the under-reported first report); one extra week fixes it. +## Covid at the references (E10, 2026-09-30) + +`e10_covid.Rmd`, exact learning truth, REF-op without the warm start (as in +covid prod). The covid base is already calibrated at h1–h3 (coverage bias +0.043/0.034/0.027), so calibration there is neutral (REF-op −0.1/−0.2/−0.8% +WIS) or harmful (REF-paper −5.6/−6.8/−9.4%). h−1, which is 100% +`revision_aware`, gains +9.5% WIS. REF-op's losses come from 2025-26 and from +September–October. Candidate change: calibrate covid at h−1 (maybe h0) only. + ## Data FluSight CMU-TimeSeries submissions: 83 rounds (2023-10-14 … 2026-05-30), 53 diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index eacdebbc..67d4e279 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -78,7 +78,7 @@ base WIS, and pick within strata (season phase × location size). | E08 | covid | disease | old REF-paper (count, window 50) | covid | pre-fix | final | V-head, V-state, V-gallery | `calibration_qt_*_covid.Rmd` | stale, replaced by E10 | | E09 | constant lr × revision delay | constant lr grid × `settle_days` {7, 14, 21, 28}; bridge rows below | REF-paper | flu | post-fix | exact (+ vintage, for a stateful tracker) | V-curve, V-head | `e09_lr_delay.Rmd` | done | -| E10 | covid at the references | disease (REF-paper, REF-op without warm start as in covid prod) | REF-paper, REF-op | covid | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e10_covid.Rmd` | planned | +| E10 | covid at the references | disease (REF-paper, REF-op without warm start as in covid prod) | REF-paper, REF-op | covid | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e10_covid.Rmd` | done | E09 bridge rows, each one axis from the previous: REF-paper → rate scale → constant lr (sweep) → `settle_days` (sweep). @@ -91,5 +91,4 @@ bias on NHSN and ILI+. See `notes/CALIBRATION.md`. - **Learning truth.** E01, E02 and E06–E08 learned from finalized NHSN (E00 shows the error is small outside h−1). E03–E05 have been re-run exactly. -- **Covid** has not been run since the fixes (E10 planned). - **E07** (ensemble vs `windowed_seasonal`) has not been re-run exactly. diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd index 2ea5718c..a83a689c 100644 --- a/reports/writeups/calibration_experiments/e10_covid.Rmd +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -39,6 +39,32 @@ Covid is submitted year-round, so the hub round axis has no off-season gap and `hub_label_seasons()` puts every round in one season. For the per-season views below, seasons are relabelled as August–July years. +# Findings + +Numbers are from the render of 30 September 2026, pooled over all rounds +(2024-11-23 to 2026-09-19) and locations; WIS changes are % vs base, positive +is better. + +- **The covid base is already calibrated at h1–h3,** so there is little to + fix there. Its coverage bias is 0.094/0.063/0.043/0.034/0.027 at h−1…h3, + against about 0.1 at every horizon for flu. +- **h−1 is a clear win.** WIS +9.5% (REF-op) and +9.2% (REF-paper), absolute + error of the median +18% and +16%. At h−1 the submission is 100% + `revision_aware` (`notes/CALIBRATION.md`), and h−1 is where the base is + least calibrated. +- **At h1–h3, REF-op is neutral and REF-paper is harmful.** REF-op: WIS + −0.1/−0.2/−0.8%, coverage bias slightly worse than base at h2/h3 + (0.037/0.035 vs 0.034/0.027). REF-paper: WIS −5.6/−6.8/−9.4%, coverage bias + worse than base from h2 on (0.043/0.048). An offset learned where the base + is calibrated is noise. +- **The losses are in 2025-26 and around September.** REF-op gains + +1.8/+3.0/+2.4% at h1–h3 in 2024-25 and loses 3.0–4.1% in 2025-26. By month + at h3, REF-op loses 16% in September and 12% in October (19% of the base + WIS between them), around the late-summer wave. +- **Implication for prod:** for covid, calibrating h−1 (and perhaps h0) and + passing h1–h3 through would keep the gain and drop the losses. The covid prod + pipeline calibrates every horizon today. + ```{r runs} no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) cals <- list( From 3e1e1ebf3061eb8daecf7f287f186ef93b0a5d7f Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:09:52 -0700 Subject: [PATCH 016/107] docs(calibration): notebooks and index readable without the repo; Pacific timestamps --- notes/calibration-ledger.md | 3 +- .../_standard_views.Rmd | 20 ++-- .../e00_vintage_backtest.Rmd | 25 ++--- .../calibration_experiments/e03_scale.Rmd | 12 +-- .../e04_offset_structure.Rmd | 9 +- .../e05_warm_start.Rmd | 10 +- .../calibration_experiments/e09_lr_delay.Rmd | 94 +++++++++++++------ .../calibration_experiments/e10_covid.Rmd | 20 ++-- .../calibration_experiments/index.Rmd | 58 +++++++++++- scripts/calibration/render_experiments.R | 2 +- 10 files changed, 168 insertions(+), 85 deletions(-) diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 67d4e279..f88ea316 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -36,7 +36,7 @@ Reference configs: Both are `cal_ref_args()` in `R/calibration/views.R`; notebooks run them through `cal_run()`, which also picks the learning truth. Experiment notebooks live in `reports/writeups/calibration_experiments/` and render to -`reports/calibration_experiments/` (with this ledger as `index.html`) via +`reports/calibration_experiments/` (with a reader-facing `index.html`) via `just calibration-experiments [notebook ...]`. ## Views @@ -77,7 +77,6 @@ base WIS, and pick within strata (season phase × location size). | E07 | one forecaster across eras | base forecaster (A ensemble / B `windowed_seasonal`; C invalid) | REF-paper at sqrt | flu | post-fix | final | V-head | `calibration_ws_experiments.R`, findings notebook | current, needs vintage rerun | | E08 | covid | disease | old REF-paper (count, window 50) | covid | pre-fix | final | V-head, V-state, V-gallery | `calibration_qt_*_covid.Rmd` | stale, replaced by E10 | | E09 | constant lr × revision delay | constant lr grid × `settle_days` {7, 14, 21, 28}; bridge rows below | REF-paper | flu | post-fix | exact (+ vintage, for a stateful tracker) | V-curve, V-head | `e09_lr_delay.Rmd` | done | - | E10 | covid at the references | disease (REF-paper, REF-op without warm start as in covid prod) | REF-paper, REF-op | covid | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e10_covid.Rmd` | done | E09 bridge rows, each one axis from the previous: REF-paper → rate scale → diff --git a/reports/writeups/calibration_experiments/_standard_views.Rmd b/reports/writeups/calibration_experiments/_standard_views.Rmd index 4bee16b0..97107a8c 100644 --- a/reports/writeups/calibration_experiments/_standard_views.Rmd +++ b/reports/writeups/calibration_experiments/_standard_views.Rmd @@ -1,10 +1,11 @@ # Standard views -Expects `cals` (a named list of runs) and `focus` (the name of the run the -state panels and gallery show). Every score is over both live seasons, all + + +Every score is over both live seasons, all hub locations, against finalized truth; burn-in rounds are excluded. -## V-head: WIS and coverage +## WIS and coverage WIS change % (positive is better than the base): @@ -31,7 +32,7 @@ knitr::kable( ) ``` -## V-ae: absolute error of the median +## Absolute error of the median Summed |truth − median|, change % vs base (positive is better): @@ -39,7 +40,7 @@ Summed |truth − median|, change % vs base (positive is better): knitr::kable(cal_wide(cal_ae(cals), "ae_pct")) ``` -## V-month: where in the season +## Where in the season WIS change % by month of the reference date (clipped at −60), h0 and h3, and each month's share of the base WIS at h3. Months run July to June; only @@ -69,9 +70,9 @@ knitr::kable( ) ``` -## V-state: fixed states, `r focus` {.tabset} +## Fixed states, `r focus` {.tabset} -The same states in every experiment notebook (`CAL_STATES`). Every other +The same states (US, CA, GA, MN, VT) in every experiment notebook. Every other round's fan at h0–h3: blue ribbon and line are the base 80% band and median, orange lines the calibrated 10/50/90% quantiles, grey the finalized truth, black the NHSN snapshot each plotted round saw. @@ -83,13 +84,12 @@ for (nm in names(CAL_STATES)) { } ``` -## V-gallery: `r focus` {.tabset} +## Gallery: `r focus` {.tabset} For each season phase and location size (population above or below the median), the forecast with the largest WIS gain and the one with the largest loss, each from a different location. Forecasts are ranked by their -WIS change as a share of their own location-season's base WIS (see -`cal_gallery_pick()`), so small states late in the season, where the base is +WIS change as a share of their own location-season's base WIS, so small states late in the season, where the base is tiny, do not crowd the gallery. Ribbons are the 50% and 90% bands, the dashed line is the forecast date. diff --git a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd index 2c61b9d1..44b97f27 100644 --- a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd +++ b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd @@ -1,6 +1,6 @@ --- title: "E00: learning from vintages instead of finalized truth" -date: "`r format(Sys.time(), '%d %B %Y')`" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" output: html_document: code_folding: hide @@ -25,15 +25,16 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw()) ``` -Ledger entry **E00** (`notes/calibration-ledger.md`). Every result before this -notebook trained the tracker on finalized NHSN values. Only the *timing* of -each reveal was realistic (`settle_days`). This notebook varies one axis, the -truth the tracker learns from, at both reference configs: +Every result before this notebook trained the tracker on finalized NHSN +values. Only the *timing* of each reveal was realistic (when the tracker +learns from an outcome). This notebook varies one thing, the data the tracker +learns from, at both reference configs (REF-paper and REF-op; see the index): - **final**: the latest vintage, as in all earlier results. -- **exact**: what prod does. Prod re-runs the tracker from scratch each week - on that week's data, so at round `t` every revealed outcome has its value - as of `t`. We replay that by re-running the tracker once per round. +- **exact**: what prod does. Prod re-runs the tracker from the first round + each week on that week's data, so at round `t` every revealed outcome has + its value as published by `t`. We replay that by re-running the tracker + once per round. - **vintage**: one run in which each outcome keeps the value it had at the round that revealed it. This was meant as a cheap stand-in for exact, but it is also a deployment design in its own right: a stateful tracker that @@ -54,17 +55,17 @@ sweeps? Numbers below are from the render of 30 September 2026. - **Finalized truth flattered the earlier results only a little.** At REF-op - with `settle_days = 14`, exact vs final WIS change is +6.8 vs +8.9 at h−1, + with one week of extra delay, exact vs final WIS change is +6.8 vs +8.9 at h−1, within 0.3 points at h0–h3, and the h−1 coverage bias is 0.080 vs 0.073. At REF-paper, exact is within 0.6 points of final at every horizon, slightly better at h1–h3. Earlier conclusions stand; the h−1 gain was overstated. - **The cheap mode is not a usable stand-in.** It drifts from exact as the - season goes on (figure below). At `settle_days = 7` it learns once from the + season goes on (figure below). With no extra delay it learns once from the first report, which under-reports and is never corrected: REF-op h−1 falls to +1.2 (exact +7.4) and coverage bias rises to 0.127 (exact 0.082). Every later experiment uses exact. -- **Under prod's re-run design, a shorter settle is better.** Exact with - `settle_days = 7` beats 14 at every horizon, at both references (REF-op +- **Under prod's re-run design, a shorter delay is better.** Exact with no + extra delay beats one week of extra delay at every horizon, at both references (REF-op +7.4/+5.3/+2.9/+2.0/+2.2 vs +6.8/+5.1/+2.5/+1.8/+1.9), with the same coverage bias. Learning early from a first report costs little when next week's run replaces it with the revised value. A stateful tracker (the diff --git a/reports/writeups/calibration_experiments/e03_scale.Rmd b/reports/writeups/calibration_experiments/e03_scale.Rmd index d975db3e..6bd97c03 100644 --- a/reports/writeups/calibration_experiments/e03_scale.Rmd +++ b/reports/writeups/calibration_experiments/e03_scale.Rmd @@ -1,6 +1,6 @@ --- title: "E03: working scale (count, sqrt, rate)" -date: "`r format(Sys.time(), '%d %B %Y')`" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" output: html_document: code_folding: hide @@ -23,11 +23,11 @@ ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplo ```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} ``` -Ledger entry **E03** (`notes/calibration-ledger.md`). One axis: the scale -the tracker works on. Everything else is REF-paper (single term, adaptive -learning rate with `mult` 0.03 and window 20, carry, 2023-24 burn-in, -`settle_days` 14), learning from vintages the way prod does (re-run each -week; E00). +One axis: the scale the tracker works on. Everything else is REF-paper (one +offset per level, adaptive learning rate of 0.03 × the 90th percentile of the +last 20 rounds' absolute errors, offsets carried across seasons, 2023-24 +burn-in, one week of extra delay), learning from vintages the way prod does +(re-run each week; E00). - **count**: REF-paper itself; additive offsets in admissions. - **sqrt**: offsets on the square root of admissions, mapped back. diff --git a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd index 5a677d85..88c7e8d4 100644 --- a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd +++ b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd @@ -1,6 +1,6 @@ --- title: "E04: offset structure (leak and slow term)" -date: "`r format(Sys.time(), '%d %B %Y')`" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" output: html_document: code_folding: hide @@ -23,16 +23,15 @@ ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplo ```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} ``` -Ledger entry **E04** (`notes/calibration-ledger.md`). One axis: the -structure of the offset. Everything else is REF-paper on the sqrt scale (the +One axis: the structure of the offset. Everything else is REF-paper on the sqrt scale (the E03 sqrt variant), learning from vintages the way prod does (E00). - **single**: the paper's tracker, one offset per level. -- **leak 0.1**: the offset shrinks by 10% each round (`fast_decay = 0.1`), so +- **leak 0.1**: the offset shrinks by 10% each round, so a correction that new outcomes do not refresh fades (half-life about 6.6 rounds). - **slow 0.003**: a second, persistent offset term with a small learning rate - (`lr_slow = list(mult = 0.003)`) added to the paper's one. + (0.003 × the error percentile, against 0.03 for the main term) added to the paper's one. - **slow 0.003 + leak 0.1**: both. This is REF-op without its warm start (E05). diff --git a/reports/writeups/calibration_experiments/e05_warm_start.Rmd b/reports/writeups/calibration_experiments/e05_warm_start.Rmd index d1c29391..bc9453e5 100644 --- a/reports/writeups/calibration_experiments/e05_warm_start.Rmd +++ b/reports/writeups/calibration_experiments/e05_warm_start.Rmd @@ -1,6 +1,6 @@ --- title: "E05: warm start from the burn-in season" -date: "`r format(Sys.time(), '%d %B %Y')`" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" output: html_document: code_folding: hide @@ -23,11 +23,9 @@ ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplo ```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} ``` -Ledger entry **E05** (`notes/calibration-ledger.md`). One axis: whether the -persistent offset starts from the burn-in season's conformal offset -(`slow_init = "burn_in_quantile"`: per horizon and level, the level-quantile -of the burn-in residuals pooled over locations, on the sqrt scale) or from -zero. Everything else is REF-paper on the sqrt scale, learning from vintages +One axis: whether the persistent offset starts from the burn-in season's +conformal offset (per horizon and level, the level-quantile of the burn-in +season's errors pooled over locations, on the sqrt scale) or from zero. Everything else is REF-paper on the sqrt scale, learning from vintages the way prod does (E00). Each warm-started run is paired with its cold twin from E04: diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd index 2b1aec95..31197241 100644 --- a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -1,6 +1,6 @@ --- title: "E09: constant learning rate × revision delay (rate scale)" -date: "`r format(Sys.time(), '%d %B %Y')`" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" output: html_document: code_folding: hide @@ -25,34 +25,72 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` -Ledger entry **E09** (`notes/calibration-ledger.md`). The question, from a -collaborator: for the plain MultiQT tracker applied on the **rate scale** +The question: for the plain quantile tracker applied on the **rate scale** (admissions per 100k) with a **constant** learning rate, how do coverage and WIS trade off across learning rates, and how does that change with the -**extra delay** before an outcome is learned from? +**extra delay** before the tracker learns from an outcome? + +**The tracker.** For each location, horizon and quantile level, the tracker +keeps an additive offset that is added to the submitted quantile. When an +outcome is learned, each offset moves by `lr × (1{y > q} − (1 − τ))`: up by +`lr × τ` if the outcome landed above the played quantile `q`, down by +`lr × (1 − τ)` if below. `lr` is the learning rate (the step size). + +- **Constant learning rate** (the sweep in this notebook): `lr` is a fixed + number, the same for every location and round, and does not depend on the + forecast errors. On the rate scale it is in admissions per 100k, so one + step moves an offset by at most `lr` per 100k. The curves below trace this + one number from 0.001 to about 3.2. +- **Adaptive learning rate** (the paper's default, used by both reference + configs): `lr` is recomputed every round as 0.03 × the 90th percentile of + the base forecast's absolute errors over the location's last 20 observed + rounds. The paper uses 0.1 and 50 rounds; an earlier sweep found 0.03 gives + similar coverage at a much smaller WIS cost, and that the window length + (8, 20, 50 or all history) barely matters. The window has no role with a + constant learning rate, so it is not varied here; the reference marks use + 20. **Extra delay.** The first NHSN report of the week ending on Saturday `X` is published on Wednesday `X + 4`, in time for the round with reference date -`X + 7`. Learning from it then is `settle_days = 7` (`h + 1` rounds), zero -extra delay. Each further week adds 7 days: our reference `settle_days = 14` -is one extra week. The sweep is 0–3 extra weeks. - -**Learning truth.** The tracker learns from NHSN vintages, not finalized -values (E00). Two deployment designs are shown, since E00 found they respond -to the delay in opposite directions: - -- **re-run** (`exact`, what prod does): the tracker is re-run from scratch each - week on that week's data, so a value learned early is replaced by its - revision the next week. -- **stateful**: the tracker steps once per outcome, on the value available at - the round it is revealed, and never revisits it. - -Everything else is REF-paper (`cal_ref_args("paper")`): single term, carry -across seasons, 2023-24 burn-in (no steps; the constant rate needs no -warm-up). Scores are on counts against finalized truth, over the two live -seasons, all 53 hub locations. "WIS inflation %" is `100 × (WIS_cal / WIS_base − 1)`, -so negative is better than the base. "L1 coverage bias" is the mean over the -23 levels of |empirical coverage − level|. +`X + 7`. Learning from it at that round is zero extra delay. Each further +week of waiting is one week of extra delay, giving later revisions time to +come in. The reference configs wait one extra week; the sweep covers 0–3. + +**What the tracker learns from.** At each round the tracker learns from the +NHSN data as published by then (a "vintage"), not from finalized values, +because finalized values are not available in real time (see E00). There are +two ways to deploy that, and E00 found they respond to the delay in opposite +directions, so both are shown: + +- **re-run** (what prod does): each week the tracker is run again from the + first round over the whole history, on that week's vintage. An outcome + first learned from its early report is learned from its revised value in + every later week's run. +- **stateful**: the tracker runs once, stepping forward a round at a time, + and learns each outcome once, from the value published at the round it + becomes available. It never sees later revisions of that outcome. + +This is a separate choice from what happens to the offsets between seasons. +In both designs the offsets carry over the summer off-season rather than +resetting to zero. + +**Other settings.** Everything else follows the REF-paper reference config: +one offset per level, carried across seasons, and a 2023-24 burn-in season +whose outcomes take no steps (the adaptive rate uses it to warm up; the +constant rate needs no warm-up). Scores are on counts against finalized +truth, over the two live seasons (2024-25 and 2025-26), all 53 hub +locations. "WIS inflation %" is `100 × (WIS_cal / WIS_base − 1)`, so negative +is better than the base. "L1 coverage bias" is the mean over the 23 quantile +levels of |empirical coverage − level|. + +**Reference configs** (marked on the plots): + +- **REF-paper**: the paper's tracker on counts, adaptive learning rate as + above. +- **REF-op**: the configuration running in prod. REF-paper on the + square-root scale, plus a 10% per-round shrink of the offset toward zero, a + second slow-moving offset, and a starting offset taken from the burn-in + season's errors (E03–E05). # Findings @@ -194,11 +232,11 @@ ref_m <- label_panels(ref_m) # Bridge from REF-paper -One axis at a time at `settle_days = 14` (the reference), re-run design: first -the rate scale with the adaptive learning rate (its `floor` moves from 0.1 -count to 0.001 rate units, since the floor is a scale-dependent constant), +One change at a time at one week of extra delay, re-run design: first the +rate scale with the adaptive learning rate (its minimum step drops from 0.1 +admissions to 0.001 per 100k, since that minimum is in the working units), then the constant learning rate with the lowest pooled WIS at each horizon. -REF-op is for orientation only; it differs in several axes. +REF-op is for orientation only; it differs in several settings. ```{r bridge} best_const <- sweep_m %>% diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd index a83a689c..a9a8833e 100644 --- a/reports/writeups/calibration_experiments/e10_covid.Rmd +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -1,6 +1,6 @@ --- title: "E10: covid at the reference configs" -date: "`r format(Sys.time(), '%d %B %Y')`" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" output: html_document: code_folding: hide @@ -24,20 +24,18 @@ ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplo ```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} ``` -Ledger entry **E10** (`notes/calibration-ledger.md`). One axis: the -disease. Both reference configs run on the covid hub's `CMU-TimeSeries` +One axis: the disease. Both reference configs run on the covid hub's `CMU-TimeSeries` submissions, learning from NHSN covid vintages the way prod does (E00), and scored against finalized covid NHSN. Covid submissions start 2024-11-23, so there is no burn-in season. As in the -covid prod pipeline, both configs run with `burn_in_seasons = character(0)`, -and REF-op runs without its warm start (`slow_init = NULL`), leaving sqrt + -slow term 0.003 + leak 0.1. The tracker therefore starts from zero offsets at +covid prod pipeline, both configs run with no burn-in season, +and REF-op runs without its warm start, leaving sqrt + slow term 0.003 + +leak 0.1. The tracker therefore starts from zero offsets at the first round. -Covid is submitted year-round, so the hub round axis has no off-season gap -and `hub_label_seasons()` puts every round in one season. For the per-season -views below, seasons are relabelled as August–July years. +Covid is submitted year-round, so there is no off-season gap between +seasons. For the per-season views below, seasons are August–July years. # Findings @@ -49,8 +47,8 @@ is better. fix there. Its coverage bias is 0.094/0.063/0.043/0.034/0.027 at h−1…h3, against about 0.1 at every horizon for flu. - **h−1 is a clear win.** WIS +9.5% (REF-op) and +9.2% (REF-paper), absolute - error of the median +18% and +16%. At h−1 the submission is 100% - `revision_aware` (`notes/CALIBRATION.md`), and h−1 is where the base is + error of the median +18% and +16%. At h−1 the submission comes entirely + from one revision-aware component model, and h−1 is where the base is least calibrated. - **At h1–h3, REF-op is neutral and REF-paper is harmful.** REF-op: WIS −0.1/−0.2/−0.8%, coverage bias slightly worse than base at h2/h3 diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index 2e96f36a..41f4a60c 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -1,6 +1,6 @@ --- title: "Calibration experiments" -date: "`r format(Sys.time(), '%d %B %Y')`" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" output: html_document: toc: true @@ -11,6 +11,13 @@ output: knitr::opts_knit$set(root.dir = here::here()) ``` +These notebooks test a post-hoc calibration step for our FluSight and covid +hub submissions. The step adjusts each submitted quantile forecast by an +additive offset that an online quantile tracker (MultiQT; Ding, Gibbs & +Tibshirani, 2025) learns from past coverage misses. Each notebook changes one +setting of that tracker and compares the result with the uncalibrated +submission. + # Notebooks ```{r notebooks, results="asis", echo=FALSE} @@ -20,10 +27,53 @@ for (f in src) { html <- sub("[.]Rmd$", ".html", basename(f)) title <- rmarkdown::yaml_front_matter(f)$title rendered <- file.exists(file.path(out_dir, html)) - stamp <- if (rendered) format(file.mtime(file.path(out_dir, html)), "%Y-%m-%d %H:%M") else "not rendered" + stamp <- if (rendered) format(file.mtime(file.path(out_dir, html)), "%Y-%m-%d %H:%M %Z", tz = "America/Los_Angeles") else "not rendered" cat(sprintf("- %s (%s)\n", if (rendered) sprintf("[%s](%s)", title, html) else title, stamp)) } ``` -```{r ledger, child=here::here("notes/calibration-ledger.md")} -``` +| | question | short answer | +|---|---|---| +| E00 | Should the tracker learn from finalized data or from data as published at the time? | As published, re-run each week as prod does. Finalized data overstated the earlier gains only a little (about 2 points at h−1). | +| E03 | Which scale should the offsets live on: counts, square root, or rate per 100k? | Square root, at every horizon, mostly by avoiding spring losses. Rate equals counts with the adaptive learning rate. | +| E04 | Should the offset shrink between updates, or have a second slow component? | Shrinking buys WIS at h1–h3 but nearly triples coverage error. A slow component alone does nothing. | +| E05 | Should the offset start from the burn-in season's errors instead of zero? | Yes; it never hurts. With the plain tracker it gives the best-calibrated config so far. | +| E09 | With a constant learning rate on the rate scale, how do coverage and WIS trade off, and how much revision delay is best? | An L-shaped trade-off whose knee beats the paper's adaptive rate at h1–h3. Under prod's weekly re-run, less delay is better. | +| E10 | Do the flu results carry over to covid? | Only at h−1 (+9.5% WIS). The covid base is already calibrated at h1–h3, where calibrating is neutral or harmful. | + +# How to read the results + +- **Horizons** h−1 to h3 are weeks relative to the forecast's reference date. + h−1 is the latest week NHSN has reported, still open to revision; h0 and + later have not been reported yet. +- **WIS change %** compares the calibrated forecast's weighted interval score + with the uncalibrated submission's, pooled over locations. The sign + convention is stated in each notebook. +- **L1 coverage bias** is the mean over the 23 quantile levels of + |empirical coverage − level|. 0 is perfect; the uncalibrated flu + submission sits near 0.1. +- **Scoring.** Every score is against finalized NHSN data, over the live + seasons (2024-25 and 2025-26 for flu), all hub locations. A burn-in season + (2023-24 for flu) is used only to warm up the tracker and is not scored. +- **Extra delay.** The tracker can learn from a week's outcome as soon as + NHSN first reports it (zero extra delay) or wait for revisions. The + reference configs wait one extra week. +- **Re-run vs stateful.** Prod re-runs the tracker from the first round each + week on that week's data, so early reports are replaced by their revisions. + A stateful tracker instead learns each outcome once, from the first value + it sees. All notebooks use the re-run design; E09 also shows the stateful + one. + +## Reference configs + +Each experiment starts from one of two configurations and changes one +setting. + +- **REF-paper**: the paper's tracker on counts. One offset per quantile level, + updated with an adaptive learning rate (0.03 × the 90th percentile of the + last 20 rounds' absolute errors), carried across seasons, with one week of + extra delay. +- **REF-op**: the configuration running in prod. REF-paper on the + square-root scale, with the offset shrinking 10% per round, a second + slow-moving offset, and a starting offset taken from the burn-in season's + errors. diff --git a/scripts/calibration/render_experiments.R b/scripts/calibration/render_experiments.R index 15f79051..bf07134e 100644 --- a/scripts/calibration/render_experiments.R +++ b/scripts/calibration/render_experiments.R @@ -1,5 +1,5 @@ # Render calibration experiment notebooks into reports/calibration_experiments/ -# and rebuild its index.html (the ledger plus links to every rendered notebook). +# and rebuild its index.html (an overview plus links to every rendered notebook). # # Usage: Rscript scripts/calibration/render_experiments.R [e00_vintage_backtest ...] # With no arguments, renders every notebook; `--index` renders only the index. From 38199191212daeeb165f7b2f962675ad3c443a12 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:37:21 -0700 Subject: [PATCH 017/107] =?UTF-8?q?fix(calibration):=20state=20panels=20st?= =?UTF-8?q?art=20at=20h=E2=88=921=20so=20each=20snapshot=20leads=20into=20?= =?UTF-8?q?its=20own=20fan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- R/calibration/views.R | 15 +++++++++++---- .../calibration_experiments/_standard_views.Rmd | 17 ++++++++++++++--- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/R/calibration/views.R b/R/calibration/views.R index 19e59da9..530c0cf0 100644 --- a/R/calibration/views.R +++ b/R/calibration/views.R @@ -167,9 +167,11 @@ cal_round_snapshots <- function(vintages, loc, ref_dates, weeks = 12L) { } -#' V-state: one location over one season. Every other round's fan (h0–h3, +#' V-state: one location over one season. Every other round's fan (h−1–h3, #' base 80% band and median, calibrated 10/50/90%), finalized truth in grey and, -#' in black, the snapshot each plotted round saw, over the fortnight before it. +#' in black, the last two reported weeks of the snapshot each plotted round saw, +#' with a dot on the latest. That is normally the fan's h−1 week, so each black +#' tail leads into its own fan; after a late release it ends a week earlier. #' @export cal_state_panel <- function(cal, loc, season, truth, vintages, title = NULL) { refs <- cal$forecasts %>% @@ -179,14 +181,18 @@ cal_state_panel <- function(cal, loc, season, truth, vintages, title = NULL) { pull() refs <- refs[seq(1L, length(refs), by = 2L)] fan <- cal$forecasts %>% - filter(.data$location == loc, .data$reference_date %in% refs, .data$horizon %in% 0:3, .data$level %in% c(0.1, 0.5, 0.9)) %>% + filter(.data$location == loc, .data$reference_date %in% refs, .data$horizon %in% -1:3, .data$level %in% c(0.1, 0.5, 0.9)) %>% mutate(lvl = c(`0.1` = "lo", `0.5` = "med", `0.9` = "hi")[as.character(.data$level)]) %>% select("reference_date", "target_end_date", "lvl", base = "value_base", calibrated = "value_cal") %>% tidyr::pivot_longer(c("base", "calibrated"), names_to = "which") %>% tidyr::pivot_wider(names_from = "lvl", values_from = "value") fin <- truth %>% filter(.data$location == loc, .data$target_end_date >= min(refs) - 21L, .data$target_end_date <= max(refs) + 28L) - snap <- cal_round_snapshots(vintages, loc, refs, weeks = 2L) + snap <- cal_round_snapshots(vintages, loc, refs, weeks = 4L) %>% + group_by(.data$reference_date) %>% + slice_max(.data$target_end_date, n = 2L) %>% + ungroup() + snap_end <- snap %>% group_by(.data$reference_date) %>% slice_max(.data$target_end_date, n = 1L) %>% ungroup() ggplot2::ggplot() + ggplot2::geom_line(data = fin, ggplot2::aes(.data$target_end_date, .data$truth), colour = "grey65") + ggplot2::geom_ribbon( @@ -204,6 +210,7 @@ cal_state_panel <- function(cal, loc, season, truth, vintages, title = NULL) { ggplot2::aes(.data$target_end_date, .data$med, colour = .data$which, group = interaction(.data$reference_date, .data$which)) ) + ggplot2::geom_line(data = snap, ggplot2::aes(.data$target_end_date, .data$truth, group = .data$reference_date), colour = "black") + + ggplot2::geom_point(data = snap_end, ggplot2::aes(.data$target_end_date, .data$truth), colour = "black", size = 1) + ggplot2::scale_fill_manual(values = CAL_FAN_COLS, aesthetics = c("fill", "colour")) + ggplot2::labs(x = NULL, y = "admissions", colour = NULL, fill = NULL, subtitle = title %||% paste(names(CAL_STATES)[CAL_STATES == loc], season)) } diff --git a/reports/writeups/calibration_experiments/_standard_views.Rmd b/reports/writeups/calibration_experiments/_standard_views.Rmd index 97107a8c..2803bdb7 100644 --- a/reports/writeups/calibration_experiments/_standard_views.Rmd +++ b/reports/writeups/calibration_experiments/_standard_views.Rmd @@ -73,9 +73,20 @@ knitr::kable( ## Fixed states, `r focus` {.tabset} The same states (US, CA, GA, MN, VT) in every experiment notebook. Every other -round's fan at h0–h3: blue ribbon and line are the base 80% band and median, -orange lines the calibrated 10/50/90% quantiles, grey the finalized truth, -black the NHSN snapshot each plotted round saw. +round's fan at h−1–h3: blue ribbon and line are the base 80% band and median, +orange lines the calibrated 10/50/90% quantiles, grey the finalized truth. + +The black line is the last two reported weeks of NHSN data as published when +that round was forecast. Its dot is normally the round's h−1 week, where its +fan starts. +Plotted rounds are two weeks apart, so each black line also runs across the +first weeks of the previous round's fan. Black sits below grey where first +reports were later revised up. A black line with no fan in front of it, or a +gap between fans, is a round with no submission (for flu, 2025-10-25 to +2025-11-15 and 2026-04-18). Around Christmas and New Year 2024 the Wednesday +releases slipped to Thursday and Friday, so the 2024-12-28 and 2025-01-04 +rounds saw one week less data than usual; their dot sits a week before the +fan starts. ```{r v-state, results="asis", fig.height=4} for (nm in names(CAL_STATES)) { From 413bba584d0283f278a60b04882dbf2f35bdf3a7 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:03:27 -0700 Subject: [PATCH 018/107] =?UTF-8?q?fix(calibration):=20drop=20spoiled=20fl?= =?UTF-8?q?u=20submissions=20(zero=20h=E2=88=921=20on=202025-11-22,=20halv?= =?UTF-8?q?ed=20US)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- R/calibration/hub_data.R | 25 ++++++++- notes/calibration-ledger.md | 5 ++ notes/spoiled-submissions.md | 51 +++++++++++++++++++ .../calibration_experiments/index.Rmd | 5 ++ 4 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 notes/spoiled-submissions.md diff --git a/R/calibration/hub_data.R b/R/calibration/hub_data.R index b06c5188..3777aa5b 100644 --- a/R/calibration/hub_data.R +++ b/R/calibration/hub_data.R @@ -29,6 +29,16 @@ HUB_QUANTILE_LEVELS <- c( HUB_FLU_TARGET <- "wk inc flu hosp" HUB_COVID_TARGET <- "wk inc covid hosp" +# Submitted forecasts broken by a pipeline bug rather than by the model; see +# notes/spoiled-submissions.md. `NA` location or horizon matches all of them. +HUB_SPOILED_SUBMISSIONS <- tibble::tribble( + ~model, ~target, ~reference_date, ~location, ~horizon, + "CMU-TimeSeries", HUB_FLU_TARGET, "2025-11-22", NA, -1L, + "CMU-TimeSeries", HUB_FLU_TARGET, "2024-12-14", "US", NA, + "CMU-TimeSeries", HUB_FLU_TARGET, "2024-12-21", "US", NA, + "CMU-TimeSeries", HUB_FLU_TARGET, "2025-01-04", "US", NA +) %>% dplyr::mutate(reference_date = as.Date(.data$reference_date)) + # Parent directory containing the flu hub checkout. Override with DELPHI_HUB_PARENT_DIR # or HUB_FLU_DIR to point at a different layout. @@ -74,6 +84,7 @@ hub_level_index <- function(x, levels = HUB_QUANTILE_LEVELS, tol = 1e-8) { #' `wk flu hosp rate change` pmf rows, so both `target` and #' `output_type == "quantile"` are filtered. #' @param locations optionally restrict to these hub location codes. +#' @param drop_spoiled drop the rows listed in `HUB_SPOILED_SUBMISSIONS`. #' @return long tibble: `reference_date`, `horizon`, `target_end_date`, #' `location`, `level_index`, `level`, `value`. #' @export @@ -81,7 +92,8 @@ hub_read_forecasts <- function( hub_dir = HUB_FLU_DIR, model = "CMU-TimeSeries", target = HUB_FLU_TARGET, - locations = NULL + locations = NULL, + drop_spoiled = TRUE ) { model_dir <- file.path(path.expand(hub_dir), "model-output", model) files <- list.files(model_dir, pattern = "\\.csv$", full.names = TRUE) @@ -122,6 +134,17 @@ hub_read_forecasts <- function( if (!is.null(locations)) { out <- out %>% filter(.data$location %in% locations) } + if (drop_spoiled) { + spoiled <- HUB_SPOILED_SUBMISSIONS %>% filter(.data$model == !!model, .data$target == !!target) + for (i in seq_len(nrow(spoiled))) { + sp <- spoiled[i, ] + out <- out %>% filter(!( + .data$reference_date == sp$reference_date & + (is.na(sp$location) | .data$location == sp$location) & + (is.na(sp$horizon) | .data$horizon == sp$horizon) + )) + } + } # The hub's own invariant: target_end_date is determined by the other two. bad <- out %>% filter(.data$target_end_date != .data$reference_date + 7L * .data$horizon) diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index f88ea316..f353aac1 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -88,6 +88,11 @@ bias on NHSN and ILI+. See `notes/CALIBRATION.md`. ## Gaps +- **Spoiled submissions.** From 2026-09-30, hub reads drop the flu + submissions listed in `notes/spoiled-submissions.md` (2025-11-22 h−1, three + halved US rounds). E00, E03–E05 and E09 were re-run without them; every + older result, and the numbers in `notes/CALIBRATION.md` before that date, + include them. - **Learning truth.** E01, E02 and E06–E08 learned from finalized NHSN (E00 shows the error is small outside h−1). E03–E05 have been re-run exactly. - **E07** (ensemble vs `windowed_seasonal`) has not been re-run exactly. diff --git a/notes/spoiled-submissions.md b/notes/spoiled-submissions.md new file mode 100644 index 00000000..049d1bf6 --- /dev/null +++ b/notes/spoiled-submissions.md @@ -0,0 +1,51 @@ +# Spoiled hub submissions + +Forecasts we submitted to the hubs that were broken by a pipeline bug, not by +the model. They stay in the hub forever, so anything that reads our hub +submissions (the calibration experiments, scoring) should drop them. +`hub_read_forecasts()` does this by default from `HUB_SPOILED_SUBMISSIONS` in +`R/calibration/hub_data.R`; add new cases there and here. + +Covid submissions were checked for both patterns below and are clean. + +## Flu, 2025-11-22 round: h−1 is zero everywhere + +Every quantile at horizon −1 is exactly 0, in all 53 locations. h0–h3 are +normal. + +Cause: the first forecast after the autumn 2025 data gap (forecast date +2025-11-19). That week the AR models were kept out of negative aheads, so h−1 +came only from the `climate_linear` ensemble. The weights block added that day +in `pipelines/flu_geo_exclusions.csv` (commit `492bade`) dated the `linear`, +`climate_base` and other climate rows `2024-11-19` instead of `2025-11-19`. +The weights parser matches dates exactly, so those forecasters had no weight, +and the `climate_linear` ensemble came out as 0. The AR models were let into +h−1 from 2025-11-26 (`24f366d`), and the typo was fixed on 2026-09-30. +Before that fix, replays of 2025-11-19 in the prod targets store reproduced +the zero `ensemble_clim_lin`. + +## Flu, US, rounds 2024-12-14, 2024-12-21 and 2025-01-04: halved + +The US forecast is about half the sum of the state forecasts at every horizon +(ratio 0.45–0.54). In every other live round the US median is within a few +percent of the state sum at h−1 and h0. The cause is not confirmed. Commits +from early 2025 fixed several `us` / `usa` / `US` mixups, and averaging the +US forecast with a zero duplicate would halve it exactly. + +## How to find more + +`scripts/one_offs/sweep_submission_deviations.R [flu|covid]` joins every +submitted forecast with the NHSN value known when it was made and the +finalized value, writes `cache/submission_sweep_{disease}.csv`, and prints +two flags at h−1 and h0: + +- **suspect**: the median is off by more than 2x from both the value known + then and the finalized value. This is the useful one for pipeline bugs. +- **missed**: the finalized value is outside the 95% interval and the median + is off by more than 2x. This also catches data problems (bad first + reports, holiday delays, the autumn 2025 gap). + +A round where many locations are flagged together usually means a pipeline +problem; scattered single locations are usually model error or bad data. The +halved-US check (US median vs the sum of state medians) is not in the script; +it is a one-line duckdb query over the hub CSVs. diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index 41f4a60c..8d19a995 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -55,6 +55,11 @@ for (f in src) { - **Scoring.** Every score is against finalized NHSN data, over the live seasons (2024-25 and 2025-26 for flu), all hub locations. A burn-in season (2023-24 for flu) is used only to warm up the tracker and is not scored. +- **Excluded submissions.** A few flu submissions were broken by pipeline + bugs and are left out, both from what the tracker learns and from the + scores: h−1 of the 2025-11-22 round (every quantile was 0) and the US + forecasts of the 2024-12-14, 2024-12-21 and 2025-01-04 rounds (about half + the sum of the states). - **Extra delay.** The tracker can learn from a week's outcome as soon as NHSN first reports it (zero extra delay) or wait for revisions. The reference configs wait one extra week. From 57ba70e1fa25db762b7dfcfaed66f11a6c1fb48d Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:23:31 -0700 Subject: [PATCH 019/107] feat(one_offs): sweep hub submissions for forecasts far from the data --- .../one_offs/sweep_submission_deviations.R | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 scripts/one_offs/sweep_submission_deviations.R diff --git a/scripts/one_offs/sweep_submission_deviations.R b/scripts/one_offs/sweep_submission_deviations.R new file mode 100644 index 00000000..834464cc --- /dev/null +++ b/scripts/one_offs/sweep_submission_deviations.R @@ -0,0 +1,76 @@ +#!/usr/bin/env Rscript +# Sweep our hub submissions for forecasts far from the data, to catch pipeline +# mistakes. See notes/spoiled-submissions.md for what it has found. +# +# For every round, location and horizon it joins the submitted 2.5/50/97.5% +# quantiles with the NHSN value as published when the round was forecast +# (`asof_value`; `latest_value` is the latest week then reported) and the +# finalized value. It writes the joined table to +# cache/submission_sweep_{disease}.csv and prints two flags at h−1 and h0: +# +# - suspect: the median is off by more than 2x from BOTH the value known at +# the time and the finalized value, which a model rarely does on its own. +# - missed: the finalized value is outside the 95% interval and the median is +# off from it by more than 2x. This also catches data problems (bad first +# reports, data gaps), so read it alongside `asof_value`. +# +# Usage: Rscript scripts/one_offs/sweep_submission_deviations.R [flu|covid ...] + +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) + +diseases <- commandArgs(trailingOnly = TRUE) +if (length(diseases) == 0) diseases <- c("flu", "covid") +hub_dirs <- c(flu = "../FluSight-forecast-hub", covid = "../covid19-forecast-hub") +targets <- c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET) + +sweep_one <- function(disease) { + archive <- get_nhsn_data_archive(disease) + vintages <- nhsn_read_vintages(disease, archive) + final <- nhsn_read_truth(disease, archive) %>% rename(final = truth) + fc <- suppressMessages(hub_read_forecasts(here::here(hub_dirs[[disease]]), target = targets[[disease]], drop_spoiled = FALSE)) %>% + filter(level %in% c(0.025, 0.5, 0.975), reference_date >= min(vintages$version) + 3) %>% + mutate(q = c(`0.025` = "lo", `0.5` = "med", `0.975` = "hi")[as.character(level)]) %>% + select(reference_date, horizon, target_end_date, location, q, value) %>% + tidyr::pivot_wider(names_from = q, values_from = value) + asof <- purrr::map(sort(unique(fc$reference_date)), function(rd) { + hub_vintage_snapshot(vintages, hub_round_asof(rd)) %>% + filter(target_end_date >= rd - 28) %>% + group_by(location) %>% + mutate(latest_week = max(target_end_date), latest_value = truth[target_end_date == max(target_end_date)]) %>% + ungroup() %>% + mutate(reference_date = rd) + }) %>% bind_rows() + fc %>% + left_join(select(asof, reference_date, location, target_end_date, asof_value = truth), + by = c("reference_date", "location", "target_end_date")) %>% + left_join(distinct(asof, reference_date, location, latest_week, latest_value), by = c("reference_date", "location")) %>% + left_join(final, by = c("location", "target_end_date")) %>% + mutate( + disease = disease, + known = coalesce(asof_value, latest_value), + lr_known = log(pmax(med, 0.5) / known), + lr_final = log(pmax(med, 0.5) / final) + ) +} + +show_flag <- function(df, title) { + cat("\n", title, "\n", sep = "") + df %>% + arrange(desc(abs(lr_final))) %>% + group_by(disease, reference_date, horizon) %>% + summarize( + n = n(), + `worst (median / known then / final)` = paste(head(sprintf("%s %.0f/%.0f/%.0f", location, med, known, final), 4), collapse = ", "), + .groups = "drop" + ) %>% + arrange(disease, reference_date, horizon) %>% + print(n = Inf, width = Inf) +} + +for (disease in diseases) { + res <- sweep_one(disease) + readr::write_csv(res, here::here("cache", paste0("submission_sweep_", disease, ".csv"))) + near <- res %>% filter(horizon %in% c(-1, 0), final >= 20, known >= 20) + show_flag(near %>% filter(abs(lr_known) > log(2), abs(lr_final) > log(2)), paste0("== ", disease, ": suspect")) + show_flag(near %>% filter(final < lo | final > hi, abs(lr_final) > log(2)), paste0("== ", disease, ": missed")) +} From db68877f854cae4505e9413763d3c0baaac35b36 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:37:37 -0700 Subject: [PATCH 020/107] =?UTF-8?q?docs(calibration):=20causes=20of=20thre?= =?UTF-8?q?e=20spoiled-submission=20candidates;=20covid=20h=E2=88=921=20wa?= =?UTF-8?q?s=20climate=5Flinear=20before=202026-09-23?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notes/CALIBRATION.md | 7 +- notes/spoiled-submissions.md | 66 ++++++++++++++++++- .../calibration_experiments/e10_covid.Rmd | 7 +- 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index 4ee55aad..25c64d3f 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -104,8 +104,9 @@ weeks, exact and stateful learning truth. Pooled over locations: `e10_covid.Rmd`, exact learning truth, REF-op without the warm start (as in covid prod). The covid base is already calibrated at h1–h3 (coverage bias 0.043/0.034/0.027), so calibration there is neutral (REF-op −0.1/−0.2/−0.8% -WIS) or harmful (REF-paper −5.6/−6.8/−9.4%). h−1, which is 100% -`revision_aware`, gains +9.5% WIS. REF-op's losses come from 2025-26 and from +WIS) or harmful (REF-paper −5.6/−6.8/−9.4%). h−1, which over +E10's rounds is 100% the `climate_linear` ensemble (`revision_aware` took +over h−1 only on 2026-09-23), gains +9.5% WIS. REF-op's losses come from 2025-26 and from September–October. Candidate change: calibrate covid at h−1 (maybe h0) only. ## Data @@ -685,6 +686,8 @@ The per-date `forecast_report.Rmd` notebook receives calibrated forecasts for th - ahead 0: ~67% / ~11% / ~22% `revision_aware`; - ahead −1: 100% `revision_aware`, because `drop_negative_aheads` strips the two AR components there. +Before 2026-09-23 (all of E10's rounds), the mix had no `revision_aware`: ahead −1 was 100% the `climate_linear` ensemble, since the AR components were filtered out of negative aheads. + `windowed_seasonal_extra_sources` excludes mo and wy, so those states fall back to `windowed_seasonal`. Blocks before 2026-09-16 used `windowed_seasonal = 0.05`, which made the split ~98% / ~2%. The `climate_linear` rows in the CSV are not `ensemble_mix` components. So covid forecast-quality work, calibration included, is mostly about `windowed_seasonal_extra_sources`. The CSV is edited by hand each week, so re-derive the weights before relying on these numbers. **Model metadata:** `CMU-TimeSeries-Calibrated.yml` created in both `../FluSight-forecast-hub/model-metadata/` and `../covid19-forecast-hub/model-metadata/` (`designated_model: false`). diff --git a/notes/spoiled-submissions.md b/notes/spoiled-submissions.md index 049d1bf6..38d9763f 100644 --- a/notes/spoiled-submissions.md +++ b/notes/spoiled-submissions.md @@ -6,7 +6,9 @@ submissions (the calibration experiments, scoring) should drop them. `hub_read_forecasts()` does this by default from `HUB_SPOILED_SUBMISSIONS` in `R/calibration/hub_data.R`; add new cases there and here. -Covid submissions were checked for both patterns below and are clean. +Covid submissions were checked for the two flu patterns (all-zero quantiles, +halved US) and are clean. The candidates found by the sweep below are +recorded with their causes but are not yet excluded. ## Flu, 2025-11-22 round: h−1 is zero everywhere @@ -32,6 +34,61 @@ percent of the state sum at h−1 and h0. The cause is not confirmed. Commits from early 2025 fixed several `us` / `usa` / `US` mixups, and averaging the US forecast with a zero duplicate would halve it exactly. +## Covid, 2024-11-23 round: about 2.4x too high everywhere (candidate) + +At h−1 and h0 the median is about 2.2–2.6x both the value known at the time +and the finalized value, in 41 of 49 locations; h1–h3 are off by a similar +factor. + +Cause: NHSN's 2024-11-20 release inflated week 2024-11-16 in many states +(covid: PA 6124, MI 3732, US 17593; corrected on 2024-11-21 to 1005, 600 and +7437). Flu had the same bad release (PA 729 → 111). The first run for this +round (2024-11-20) used that data; the AR models are pooled across states, +so a jump in a few large states raised every state's forecast. On 2024-11-21 +we re-ran on the corrected data and re-submitted flu (FluSight hub commit +`c160da00`, "updated 11-23 forecast for new data"), but the covid file was +never re-submitted, so the hub holds the 2024-11-20 version. A prod report +rendered on 2024-11-21 (`reports/2024-11-21_covid_prod_on_2024-11-21.html`) +shows the forecasts we would have sent (GA h−1 median 64 against the +submitted 339). A May 2025 replay of the store also gives forecasts about +half the submitted ones. + +## Covid, 2026-04-25 round: h−1 near zero in many states (candidate) + +The h−1 median is far below both the value known at the time and the +finalized value in about half the locations (WI 2 vs 27 known, VA 3 vs 28, +MA 9 vs 44, IN 9 vs 38), and far below the same round's own h0 (WI 10, VA 25). +h0–h3 look normal. The data were fine: the 2026-04-22 release already had +these values for week 2026-04-18, and later revisions barely changed them. + +Cause, not confirmed: at the time, covid h−1 came only from the +`climate_linear` ensemble (the AR models were filtered out of negative +aheads), and at short aheads that ensemble is mostly the linear baseline, +which extrapolates the last few reported weeks. The near-zeros are what a +linear trend through 2026-04-04 → 2026-04-11 gives (WI 23 → 14, VA 51 → 26) +if week 2026-04-18 is missing, so the run most likely did not have that +week. That day the cast API moved from v2 to v5; the fix (`6d91648`, "keep +up with cast-api updates") was committed at 16:11 PDT, nine minutes after +the submission (covid hub `a6b94bb`, 16:02), so the submission probably ran +on a stale archive fetch. There is no covid prod store for that date to +check. + +## Flu, 2026-04-04 round: h−1 about 1.8x too high (candidate, design issue) + +The h−1 median is about 1.6–1.8x both the value known then and the finalized +value in 22 of 35 locations (OH 204 vs 60, CO 90 vs 34, MN 92 vs 36). h0 is +fine. The prod store replay reproduces the submission exactly, so this is +what the pipeline does, not a one-off failure. + +Cause: week 2026-03-28 fell abruptly (OH 257 → 60, US about 5900 → 3300; the +finalized data agree). At h−1 the AR models are run through +`filter_minus_one_ahead()`, which drops the week being nowcast, so they +extrapolate from 2026-03-21 and never see the drop, even though it was +already reported. The same mechanism puts 2025-26 h−1 above the reported +value by 10–30% in other weeks of the decline. This is a design choice in +how h−1 is built, not a bug in one round, so excluding only this round would +leave the milder cases in. + ## How to find more `scripts/one_offs/sweep_submission_deviations.R [flu|covid]` joins every @@ -49,3 +106,10 @@ A round where many locations are flagged together usually means a pipeline problem; scattered single locations are usually model error or bad data. The halved-US check (US median vs the sum of state medians) is not in the script; it is a one-line duckdb query over the hub CSVs. + +## To do + +- Run the same sweep on `windowed_seasonal` (the main ensemble component, + used as a proxy for the ensemble in the calibration harness) to check it + does not hit similar pathologies or data errors. Needs a variant of the + script that reads the harness forecasts instead of the hub files. diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd index a9a8833e..90179e13 100644 --- a/reports/writeups/calibration_experiments/e10_covid.Rmd +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -47,9 +47,10 @@ is better. fix there. Its coverage bias is 0.094/0.063/0.043/0.034/0.027 at h−1…h3, against about 0.1 at every horizon for flu. - **h−1 is a clear win.** WIS +9.5% (REF-op) and +9.2% (REF-paper), absolute - error of the median +18% and +16%. At h−1 the submission comes entirely - from one revision-aware component model, and h−1 is where the base is - least calibrated. + error of the median +18% and +16%. Over these rounds the submitted h−1 + came entirely from a climatology-plus-linear-trend ensemble (the + autoregressive models were left out of negative horizons), and h−1 is + where the base is least calibrated. - **At h1–h3, REF-op is neutral and REF-paper is harmful.** REF-op: WIS −0.1/−0.2/−0.8%, coverage bias slightly worse than base at h2/h3 (0.037/0.035 vs 0.034/0.027). REF-paper: WIS −5.6/−6.8/−9.4%, coverage bias From b4d6354efd7a68a305888861aa2b0711e2c296b5 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:40:36 -0700 Subject: [PATCH 021/107] =?UTF-8?q?fix(calibration):=20exclude=20covid=202?= =?UTF-8?q?024-11-23=20round=20and=202026-04-25=20h=E2=88=921?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- R/calibration/hub_data.R | 4 +++- notes/calibration-ledger.md | 10 +++++----- notes/spoiled-submissions.md | 15 ++++++++------- .../writeups/calibration_experiments/index.Rmd | 13 ++++++++----- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/R/calibration/hub_data.R b/R/calibration/hub_data.R index 3777aa5b..632c2746 100644 --- a/R/calibration/hub_data.R +++ b/R/calibration/hub_data.R @@ -36,7 +36,9 @@ HUB_SPOILED_SUBMISSIONS <- tibble::tribble( "CMU-TimeSeries", HUB_FLU_TARGET, "2025-11-22", NA, -1L, "CMU-TimeSeries", HUB_FLU_TARGET, "2024-12-14", "US", NA, "CMU-TimeSeries", HUB_FLU_TARGET, "2024-12-21", "US", NA, - "CMU-TimeSeries", HUB_FLU_TARGET, "2025-01-04", "US", NA + "CMU-TimeSeries", HUB_FLU_TARGET, "2025-01-04", "US", NA, + "CMU-TimeSeries", HUB_COVID_TARGET, "2024-11-23", NA, NA, + "CMU-TimeSeries", HUB_COVID_TARGET, "2026-04-25", NA, -1L ) %>% dplyr::mutate(reference_date = as.Date(.data$reference_date)) diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index f353aac1..2fc39370 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -88,11 +88,11 @@ bias on NHSN and ILI+. See `notes/CALIBRATION.md`. ## Gaps -- **Spoiled submissions.** From 2026-09-30, hub reads drop the flu - submissions listed in `notes/spoiled-submissions.md` (2025-11-22 h−1, three - halved US rounds). E00, E03–E05 and E09 were re-run without them; every - older result, and the numbers in `notes/CALIBRATION.md` before that date, - include them. +- **Spoiled submissions.** From 2026-09-30, hub reads drop the submissions + listed in `notes/spoiled-submissions.md` (flu: 2025-11-22 h−1, three halved + US rounds; covid: the 2024-11-23 round, 2026-04-25 h−1). E00, E03–E05, E09 + and E10 are re-run without them; every older result, and the numbers in + `notes/CALIBRATION.md` before that date, include them. - **Learning truth.** E01, E02 and E06–E08 learned from finalized NHSN (E00 shows the error is small outside h−1). E03–E05 have been re-run exactly. - **E07** (ensemble vs `windowed_seasonal`) has not been re-run exactly. diff --git a/notes/spoiled-submissions.md b/notes/spoiled-submissions.md index 38d9763f..1f37de56 100644 --- a/notes/spoiled-submissions.md +++ b/notes/spoiled-submissions.md @@ -7,8 +7,8 @@ submissions (the calibration experiments, scoring) should drop them. `R/calibration/hub_data.R`; add new cases there and here. Covid submissions were checked for the two flu patterns (all-zero quantiles, -halved US) and are clean. The candidates found by the sweep below are -recorded with their causes but are not yet excluded. +halved US) and are clean. Of the cases the sweep found, the two covid ones +are excluded; the flu 2026-04-04 one is kept on purpose (see its section). ## Flu, 2025-11-22 round: h−1 is zero everywhere @@ -34,7 +34,7 @@ percent of the state sum at h−1 and h0. The cause is not confirmed. Commits from early 2025 fixed several `us` / `usa` / `US` mixups, and averaging the US forecast with a zero duplicate would halve it exactly. -## Covid, 2024-11-23 round: about 2.4x too high everywhere (candidate) +## Covid, 2024-11-23 round: about 2.4x too high everywhere (excluded, all horizons) At h−1 and h0 the median is about 2.2–2.6x both the value known at the time and the finalized value, in 41 of 49 locations; h1–h3 are off by a similar @@ -53,7 +53,7 @@ shows the forecasts we would have sent (GA h−1 median 64 against the submitted 339). A May 2025 replay of the store also gives forecasts about half the submitted ones. -## Covid, 2026-04-25 round: h−1 near zero in many states (candidate) +## Covid, 2026-04-25 round: h−1 near zero in many states (excluded, h−1) The h−1 median is far below both the value known at the time and the finalized value in about half the locations (WI 2 vs 27 known, VA 3 vs 28, @@ -73,7 +73,7 @@ the submission (covid hub `a6b94bb`, 16:02), so the submission probably ran on a stale archive fetch. There is no covid prod store for that date to check. -## Flu, 2026-04-04 round: h−1 about 1.8x too high (candidate, design issue) +## Flu, 2026-04-04 round: h−1 about 1.8x too high (kept) The h−1 median is about 1.6–1.8x both the value known then and the finalized value in 22 of 35 locations (OH 204 vs 60, CO 90 vs 34, MN 92 vs 36). h0 is @@ -86,8 +86,9 @@ finalized data agree). At h−1 the AR models are run through extrapolate from 2026-03-21 and never see the drop, even though it was already reported. The same mechanism puts 2025-26 h−1 above the reported value by 10–30% in other weeks of the decline. This is a design choice in -how h−1 is built, not a bug in one round, so excluding only this round would -leave the milder cases in. +how h−1 is built, not a bug in one round, so it is kept: it is a systematic +h−1 bias, which is what the calibrator should learn to correct. The fix +belongs in how prod builds h−1 (let the AR models see the reported week). ## How to find more diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index 8d19a995..d3d46709 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -55,11 +55,14 @@ for (f in src) { - **Scoring.** Every score is against finalized NHSN data, over the live seasons (2024-25 and 2025-26 for flu), all hub locations. A burn-in season (2023-24 for flu) is used only to warm up the tracker and is not scored. -- **Excluded submissions.** A few flu submissions were broken by pipeline - bugs and are left out, both from what the tracker learns and from the - scores: h−1 of the 2025-11-22 round (every quantile was 0) and the US - forecasts of the 2024-12-14, 2024-12-21 and 2025-01-04 rounds (about half - the sum of the states). +- **Excluded submissions.** A few submissions were broken by pipeline bugs + or bad data and are left out, both from what the tracker learns and from + the scores. Flu: h−1 of the 2025-11-22 round (every quantile was 0) and + the US forecasts of the 2024-12-14, 2024-12-21 and 2025-01-04 rounds + (about half the sum of the states). Covid: the whole 2024-11-23 round + (made from a bad NHSN release that was corrected the next day) and h−1 of + the 2026-04-25 round (near zero in many states, most likely from a stale + data fetch). - **Extra delay.** The tracker can learn from a week's outcome as soon as NHSN first reports it (zero extra delay) or wait for revisions. The reference configs wait one extra week. From b8e1d019e61aa263f5b57ef03153db9c5c78ebb9 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:44:12 -0700 Subject: [PATCH 022/107] doc: record TODOs --- notes/CALIBRATION2.md | 53 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 notes/CALIBRATION2.md diff --git a/notes/CALIBRATION2.md b/notes/CALIBRATION2.md new file mode 100644 index 00000000..5ca89c06 --- /dev/null +++ b/notes/CALIBRATION2.md @@ -0,0 +1,53 @@ +Open threads and TODOs + +In progress +1. Full re-render: E00, E03–E05, E09, E10 and the index, on the final exclusion list. +2. Refresh the findings numbers afterwards. Every notebook's "Findings" section, notes/CALIBRATION.md and the ledger quote numbers from before the exclusions. Expect shifts mostly at h−1, in flu 2024-25 (the US is a large share of pooled count WIS) and in E10. +3. Re-check E10's conclusions once that's done. Its h−1 gain was partly measured on rounds now excluded. Its h−1 base was also a climatology-plus-trend nowcast, not the revision-aware one now in prod, so "calibrate covid at h−1 only" needs re-reading in that light. + +Decided but not done +4. Sweep windowed_seasonal, the main ensemble component used as a proxy in the calibration harness. It needs a variant of the sweep script that reads the harness forecasts. Recorded in notes/spoiled-submissions.md. +5. Fix how prod builds h−1 (flu, and covid before 2026-09-23). The AR models drop the already-reported week at h−1 (filter_minus_one_ahead). That's the 2026-04-04 overshoot and the 10–30% h−1 bias during 2025-26 declines. This is a prod change, not a calibration one. + +Open questions from earlier work +6. Halved US cause (2024-12-14, 2024-12-21, 2025-01-04): unconfirmed. The us/usa mixup is my best guess. +7. Covid 2026-04-25 cause (a stale fetch during the cast API v2→v5 change): likely but unconfirmed; there's no store to replay. +8. Experiments not yet re-run on vintages: E07 (ensemble vs windowed_seasonal), and E01/E02, still marked stale. +9. Covid per-horizon calibration (h−1, maybe h0, only): untested. +10. Replay drift: the July 2026 flu replay doesn't reproduce the 2025-11-22 submission, and the May 2025 covid replay doesn't reproduce 2024-11-23. That's expected, since code and data changed, but don't read replays as "what was submitted". + +Housekeeping +11. Old run caches: E09 and the knitr caches were moved aside (cache/calibration/experiments/*.pre-spoiled). I can delete them once the new renders check out. +12. just calibration-experiments doesn't work on the host, because Rscript isn't installed there. It needs RSCRIPT="distrobox enter rocker -- Rscript". Should I make that the Justfile default? +13. renv warns on every run: the library has renv 1.1.6 but the lockfile wants 1.2.3, and some lockfile packages aren't installed. It's harmless so far. +14. No bookmark yet: the branch is about a dozen jj commits on top of ds/calibrate2. Tell me when you want one moved for a PR. + +--- + +The faster snapshot gives identical output and is about 25x faster: 0.15–0.3 s instead of 3.6–4.9 s. + +Why E09 is slow + +- Each "exact" config is really about 55 tracker runs. Prod re-runs the tracker from the first round each week, so replaying it means one full run per live round, each over a longer history. Total work grows with the square of the number of rounds. +- E09 has about 72 exact configs. That's 60 sweep runs plus 12 reference runs, around 4,000 full tracker runs, at roughly 65 s per config on 14 cores. +- The stateful runs are cheap, since each config is a single run. + +Profile of one full run (13.5 s at the last round), plus the snapshot step: + +┌──────────────────────────────────────────────────────────┬─────────────────────────────────────┬─────────────────────────────────────────────────────────────────┬──────────────────────────────────────────┐ +│ cost │ share │ fix │ exact? │ +├──────────────────────────────────────────────────────────┼─────────────────────────────────────┼─────────────────────────────────────────────────────────────────┼──────────────────────────────────────────┤ +│ Rebuilding "data as of round t" with a grouped slice_max │ ~4–5 s per round, on top of the run │ Sort-based lookup (tested above) │ Yes, identical output │ +├──────────────────────────────────────────────────────────┼─────────────────────────────────────┼─────────────────────────────────────────────────────────────────┼──────────────────────────────────────────┤ +│ rlang::arg_match called on every tracker step │ ~15% of the run │ Validate once, outside the loop │ Yes │ +├──────────────────────────────────────────────────────────┼─────────────────────────────────────┼─────────────────────────────────────────────────────────────────┼──────────────────────────────────────────┤ +│ Isotonic projection (isoreg) on every step │ ~8% │ Skip it when quantiles are already ordered, which is most steps │ Yes, PAVA leaves ordered input unchanged │ +├──────────────────────────────────────────────────────────┼─────────────────────────────────────┼─────────────────────────────────────────────────────────────────┼──────────────────────────────────────────┤ +│ tibble/glue building inside the per-series loop │ ~15% │ Plain vectors in the hot path │ Yes │ +└──────────────────────────────────────────────────────────┴─────────────────────────────────────┴─────────────────────────────────────────────────────────────────┴──────────────────────────────────────────┘ + +Quick wins. Together those should make exact runs about 2–2.5x faster, so E09 drops from about 1.5 h to about 40 min. They're all exact, and I'd check that by comparing results bit-for-bit against the cached runs. About 30 minutes of work. + +Bigger, algorithmic win. Two consecutive weekly re-runs see identical data for everything older than the last few weeks, because revisions settle within a few weeks. So the run for round t could start from a checkpoint of round t−1's run at the first round where their data differ, instead of from the beginning. That turns the quadratic cost into roughly linear: probably another 5–10x, bringing E09 to a few minutes. It needs the tracker to save and resume its state (offsets, learning-rate window, slow term). That's a moderate refactor to qt_track(), and it can be tested for exact equality against the current full re-run. Half a day or so, with tests. + +About the running render. It started with the old code and has roughly 1–1.5 hours left. Restarting it after the quick wins would save little once you count the implementation time. I'd let it finish and do the quick wins next, so later sweeps benefit. Then decide on the checkpointing based on how many more exact sweeps you expect. Want me to start on the quick wins now? Editing the code won't affect the running render, which loaded everything at start. From 7f07d3190006d90ee069800f1d1d72fcc560b140 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:22:16 -0700 Subject: [PATCH 023/107] docs(calibration): findings refreshed after excluding spoiled submissions --- notes/CALIBRATION.md | 64 +++++++++++-------- notes/CALIBRATION2.md | 5 -- .../e00_vintage_backtest.Rmd | 23 +++---- .../calibration_experiments/e03_scale.Rmd | 25 +++++--- .../e04_offset_structure.Rmd | 21 +++--- .../e05_warm_start.Rmd | 30 +++++---- .../calibration_experiments/e09_lr_delay.Rmd | 32 +++++----- .../calibration_experiments/e10_covid.Rmd | 35 +++++----- .../calibration_experiments/index.Rmd | 4 +- 9 files changed, 129 insertions(+), 110 deletions(-) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index 25c64d3f..ec074d26 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -68,16 +68,17 @@ finalized truth). A round with reference date `d` sees data published by - `"vintage"`: one run in which each outcome keeps its value at the round that revealed it: a stateful tracker that never revisits a step. -Findings (E00, `e00_vintage_backtest.Rmd`, flu, both live seasons): +Findings (E00, `e00_vintage_backtest.Rmd`, flu, both live seasons, spoiled +submissions excluded): - Final vs exact is small except at h−1: REF-op at settle 14 is - +8.9/+5.4/+2.6/+1.9/+1.9 final vs +6.8/+5.1/+2.5/+1.8/+1.9 exact, coverage - bias 0.073–0.040 vs 0.080–0.041. REF-paper is within 0.6 points. + +7.6/+5.4/+2.4/+1.8/+2.2 final vs +5.4/+4.9/+2.4/+1.7/+2.2 exact, coverage + bias 0.067–0.040 vs 0.074–0.041. REF-paper is within 0.6 points. - Vintage mode is not a stand-in for exact: at settle 7 it learns once from - the under-reported first report (REF-op h−1 +1.2, coverage bias 0.127). -- Under the exact (prod) design, settle 7 beats settle 14 at every horizon - at both references (REF-op +7.4/+5.3/+2.9/+2.0/+2.2), because next week's - run corrects what an early reveal got wrong. A stateful tracker would + the under-reported first report (REF-op h−1 0.0, coverage bias 0.121). +- Under the exact (prod) design, settle 7 beats or ties settle 14 at every + horizon at both references (REF-op +6.1/+5.1/+2.6/+1.8/+2.2), because next + week's run corrects what an early reveal got wrong. A stateful tracker would prefer the longer delay. The right revision delay depends on the design. ## Constant learning rate × revision delay (E09, 2026-09-30) @@ -87,9 +88,11 @@ Findings (E00, `e00_vintage_backtest.Rmd`, flu, both live seasons): weeks, exact and stateful learning truth. Pooled over locations: - The coverage/WIS trade-off is L-shaped, with a knee at a learning rate of - about 0.05–0.1 per 100k (up to 0.3 at h−1). At 0.056 and +1 week (re-run): - WIS −3.9/−2.3/−1.0/−0.6/−0.1% (negative is better), coverage bias - 0.043/0.031/0.026/0.026/0.026. That beats REF-paper on both at h1–h3. + about 0.05–0.1 per 100k. At 0.056 and +1 week (re-run): WIS + −3.2/−2.5/−1.1/−0.5/−0.2% (negative is better), coverage bias + 0.040/0.031/0.026/0.027/0.027. That matches or beats REF-paper + (−3.2/−1.9/+2.0/+2.7/+5.3%, 0.044/0.030/0.032/0.034/0.032) at every + horizon. - With the adaptive rate, rate scale and count scale give identical results (the adaptive rate rescales with the location), so the rate scale matters only with a constant rate. @@ -102,11 +105,12 @@ weeks, exact and stateful learning truth. Pooled over locations: ## Covid at the references (E10, 2026-09-30) `e10_covid.Rmd`, exact learning truth, REF-op without the warm start (as in -covid prod). The covid base is already calibrated at h1–h3 (coverage bias -0.043/0.034/0.027), so calibration there is neutral (REF-op −0.1/−0.2/−0.8% -WIS) or harmful (REF-paper −5.6/−6.8/−9.4%). h−1, which over +covid prod), rounds 2024-11-30 to 2026-09-19 (the spoiled 2024-11-23 round +excluded). The covid base is already calibrated at h1–h3 (coverage bias +0.046/0.036/0.028), so calibration there is neutral to slightly harmful +(REF-op 0.0/−0.6/−1.3% WIS) or harmful (REF-paper −4.5/−7.3/−10.3%). h−1, which over E10's rounds is 100% the `climate_linear` ensemble (`revision_aware` took -over h−1 only on 2026-09-23), gains +9.5% WIS. REF-op's losses come from 2025-26 and from +over h−1 only on 2026-09-23), gains +10.2% WIS (REF-paper +11.4%). REF-op's losses come from 2025-26 and from September–October. Candidate change: calibrate covid at h−1 (maybe h0) only. ## Data @@ -166,7 +170,9 @@ the central integration design issue. +8.9/+5.4/+2.6/+1.9/+1.9 at h−1…3 over both live seasons, calibration error 0.040–0.073 against 0.098–0.118 uncalibrated. Current numbers for every variant are in `reports/writeups/calibration/calibration_findings_flu.Rmd` -(section "Current numbers" below). +(section "Current numbers" below). With exact learning truth and spoiled +submissions excluded (2026-09-30): +5.4/+4.9/+2.4/+1.7/+2.2, coverage bias +0.041–0.074 against 0.095–0.111 uncalibrated. **Numbers dated before 2026-09-17 predate the two correctness fixes** ("Correctness fixes" below) and used the hub target-data CSV as truth. The @@ -503,25 +509,27 @@ biased high. ### Current numbers (2026-09-23; superseded by the exact re-runs below) -Re-run 2026-09-30 with exact learning truth (the prod re-run design) in +Re-run 2026-09-30 with exact learning truth (the prod re-run design) and +spoiled submissions excluded (`notes/spoiled-submissions.md`) in `e03_scale.Rmd`, `e04_offset_structure.Rmd`, `e05_warm_start.Rmd`. WIS change -% (positive is better) and coverage bias, same setup: +% (positive is better) and coverage bias, same setup (base +0.111/0.105/0.095/0.098/0.102): | variant | h−1 | h0 | h1 | h2 | h3 | cal err h−1…h3 | |---|---|---|---|---|---|---| -| count space, single term | +4.2 | +2.4 | −0.5 | −2.9 | −5.2 | 0.047/0.030/0.032/0.034/0.032 | -| sqrt, single term | +6.4 | +4.1 | +0.8 | −0.6 | −1.3 | 0.054/0.032/0.026/0.028/0.026 | -| sqrt, leak 0.1 | +6.2 | +3.9 | +1.8 | +1.0 | +0.4 | 0.096/0.082/0.075/0.080/0.082 | -| warm start only | +0.4 | +1.1 | +0.9 | +1.1 | +2.0 | 0.105/0.075/0.062/0.065/0.062 | -| warm start + single term | +6.4 | +4.7 | +1.3 | +0.3 | +0.4 | 0.050/0.021/0.013/0.016/0.014 | -| warm start + leak 0.1 | +6.3 | +4.7 | +2.5 | +1.9 | +2.1 | 0.087/0.060/0.051/0.052/0.047 | -| operating point | +6.8 | +5.1 | +2.5 | +1.8 | +1.9 | 0.080/0.054/0.045/0.046/0.041 | +| count space, single term | +3.2 | +1.9 | −2.0 | −2.7 | −5.3 | 0.044/0.030/0.032/0.034/0.032 | +| sqrt, single term | +5.1 | +4.0 | +0.7 | −1.3 | −1.4 | 0.049/0.032/0.026/0.028/0.026 | +| sqrt, leak 0.1 | +4.8 | +3.7 | +1.6 | +0.8 | +0.6 | 0.090/0.082/0.075/0.079/0.081 | +| warm start only | +0.5 | +1.3 | +0.9 | +1.2 | +2.1 | 0.098/0.075/0.062/0.065/0.062 | +| warm start + single term | +5.1 | +4.7 | +1.3 | −0.2 | +0.3 | 0.046/0.020/0.013/0.016/0.014 | +| warm start + leak 0.1 | +5.0 | +4.5 | +2.3 | +1.8 | +2.4 | 0.080/0.060/0.050/0.052/0.047 | +| operating point | +5.4 | +4.9 | +2.4 | +1.7 | +2.2 | 0.074/0.054/0.045/0.046/0.041 | The ordering of the variants is unchanged from the finalized-truth table. -h−1 WIS of the tracking sqrt variants drops by 1.5–2 points; count space and -the warm start alone are unchanged, and h0–h3 move by at most 0.5. New: warm start + single -term (not run before) is the best-calibrated variant, and with REF-op it -brackets the WIS/coverage trade-off. The finalized-truth numbers follow for +Warm start + single term (not run before) is the best-calibrated variant, and +with REF-op it brackets the WIS/coverage trade-off. At h3, sqrt single term +loses more than count in October (−35% vs −27%) and March (−42% vs −26%); +the leak brings those to −12% and −14%. The finalized-truth numbers follow for the record. From `reports/writeups/calibration/calibration_findings_flu.Rmd`: post-fix, diff --git a/notes/CALIBRATION2.md b/notes/CALIBRATION2.md index 5ca89c06..dae33539 100644 --- a/notes/CALIBRATION2.md +++ b/notes/CALIBRATION2.md @@ -1,10 +1,5 @@ Open threads and TODOs -In progress -1. Full re-render: E00, E03–E05, E09, E10 and the index, on the final exclusion list. -2. Refresh the findings numbers afterwards. Every notebook's "Findings" section, notes/CALIBRATION.md and the ledger quote numbers from before the exclusions. Expect shifts mostly at h−1, in flu 2024-25 (the US is a large share of pooled count WIS) and in E10. -3. Re-check E10's conclusions once that's done. Its h−1 gain was partly measured on rounds now excluded. Its h−1 base was also a climatology-plus-trend nowcast, not the revision-aware one now in prod, so "calibrate covid at h−1 only" needs re-reading in that light. - Decided but not done 4. Sweep windowed_seasonal, the main ensemble component used as a proxy in the calibration harness. It needs a variant of the sweep script that reads the harness forecasts. Recorded in notes/spoiled-submissions.md. 5. Fix how prod builds h−1 (flu, and covid before 2026-09-23). The AR models drop the already-reported week at h−1 (filter_minus_one_ahead). That's the 2026-04-04 overshoot and the 10–30% h−1 bias during 2025-26 declines. This is a prod change, not a calibration one. diff --git a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd index 44b97f27..d9351c5a 100644 --- a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd +++ b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd @@ -55,22 +55,23 @@ sweeps? Numbers below are from the render of 30 September 2026. - **Finalized truth flattered the earlier results only a little.** At REF-op - with one week of extra delay, exact vs final WIS change is +6.8 vs +8.9 at h−1, - within 0.3 points at h0–h3, and the h−1 coverage bias is 0.080 vs 0.073. At - REF-paper, exact is within 0.6 points of final at every horizon, slightly - better at h1–h3. Earlier conclusions stand; the h−1 gain was overstated. + with one week of extra delay, exact vs final WIS change is +5.4 vs +7.6 at + h−1, within 0.5 points at h0–h3, and the h−1 coverage bias is 0.074 vs + 0.067. At REF-paper, exact is within 0.6 points of final at every horizon: + slightly better at h2/h3, slightly worse at h1. Earlier conclusions stand; + the h−1 gain was overstated by about 2 points. - **The cheap mode is not a usable stand-in.** It drifts from exact as the season goes on (figure below). With no extra delay it learns once from the first report, which under-reports and is never corrected: REF-op h−1 falls - to +1.2 (exact +7.4) and coverage bias rises to 0.127 (exact 0.082). Every + to 0.0 (exact +6.1) and coverage bias rises to 0.121 (exact 0.076). Every later experiment uses exact. - **Under prod's re-run design, a shorter delay is better.** Exact with no - extra delay beats one week of extra delay at every horizon, at both references (REF-op - +7.4/+5.3/+2.9/+2.0/+2.2 vs +6.8/+5.1/+2.5/+1.8/+1.9), with the same - coverage bias. Learning early from a first report costs little when next - week's run replaces it with the revised value. A stateful tracker (the - vintage mode) shows the opposite. So the value of an extra revision delay - depends on the deployment design, and E09 should report both. + extra delay beats or ties one week of extra delay at every horizon, at both + references (REF-op +6.1/+5.1/+2.6/+1.8/+2.2 vs +5.4/+4.9/+2.4/+1.7/+2.2), + with nearly the same coverage bias. Learning early from a first report costs + little when next week's run replaces it with the revised value. A stateful + tracker (the vintage mode) shows the opposite. So the value of an extra + revision delay depends on the deployment design, and E09 reports both. ```{r libs, cache=FALSE} suppressPackageStartupMessages(source(here::here("R/load_all.R"))) diff --git a/reports/writeups/calibration_experiments/e03_scale.Rmd b/reports/writeups/calibration_experiments/e03_scale.Rmd index 6bd97c03..7d254686 100644 --- a/reports/writeups/calibration_experiments/e03_scale.Rmd +++ b/reports/writeups/calibration_experiments/e03_scale.Rmd @@ -40,22 +40,27 @@ change goes with the scale. # Findings -Numbers are from the render of 30 September 2026, pooled over both live seasons and all locations; WIS and absolute-error changes are % vs base, positive is better. +Numbers are from the render of 30 September 2026, pooled over both live +seasons and all locations; WIS and absolute-error changes are % vs base, +positive is better. -- **sqrt beats count at every horizon.** WIS +6.4/+4.1/+0.8/−0.6/−1.3 vs - +4.2/+2.4/−0.5/−2.9/−5.2 at h−1…h3, with similar coverage bias - (0.054–0.026 vs 0.047–0.032). The median improves too: absolute error - +12.0/+8.6/+2.6/+2.4/+1.7 vs +6.3/+5.7/+1.9/−2.1/−4.8. -- **The gain is in the spring.** At h3, count loses 164% of April's and 92% of - May's base WIS; sqrt loses 33% and gains 20%. An offset learned at the peak +- **sqrt beats count at every horizon.** WIS +5.1/+4.0/+0.7/−1.3/−1.4 vs + +3.2/+1.9/−2.0/−2.7/−5.3 at h−1…h3, with similar coverage bias + (0.049–0.026 vs 0.044–0.032). The median improves too: absolute error + +9.5/+7.3/+2.4/−0.6/+1.2 vs +5.0/+4.7/+0.2/−1.3/−4.9. +- **The gain is in the spring.** At h3, count loses 122% of April's and 63% of + May's base WIS; sqrt loses 20% and gains 18%. An offset learned at the peak is sized in admissions, so in count space it is far too large for the tail. - Both lose 43–44% in October (0.2% of the base WIS). +- **sqrt is worse at the start of the season and in March.** At h3 it loses + 35% in October and 42% in March, against 27% and 26% for count. These + months carry little of the base WIS (0.3% and 5.4%), so the spring gain + dominates. - **rate equals count with the adaptive learning rate.** The adaptive rate is a quantile of the location's own residuals, so dividing a location by its population rescales steps and offsets together. The rate scale matters only with a constant learning rate (E09). -- **2025-26 is harder for both.** sqrt loses 2.5–3.3% at h1–h3 there (count - 5.6–6.9%), against gains in 2024-25. +- **2025-26 is harder for both.** sqrt loses 1.8–4.6% at h1–h3 there (count + 6.9–7.9%), against gains in 2024-25. ```{r runs} cals <- list( diff --git a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd index 88c7e8d4..fcc01567 100644 --- a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd +++ b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd @@ -37,24 +37,27 @@ E03 sqrt variant), learning from vintages the way prod does (E00). # Findings -Numbers are from the render of 30 September 2026, pooled over both live seasons and all locations; WIS and absolute-error changes are % vs base, positive is better. +Numbers are from the render of 30 September 2026, pooled over both live +seasons and all locations; WIS and absolute-error changes are % vs base, +positive is better. - **The leak buys WIS at h1–h3 with coverage.** `leak 0.1` moves WIS from - +0.8/−0.6/−1.3 to +1.8/+1.0/+0.4 at h1–h3, but coverage bias nearly - triples (0.026–0.028 → 0.075–0.082), close to the base's 0.095–0.103. + +0.7/−1.3/−1.4 to +1.6/+0.8/+0.6 at h1–h3, but coverage bias nearly + triples (0.026–0.028 → 0.075–0.081), close to the base's 0.095–0.102. A correction that fades between refreshes cannot hold coverage through a 3–5 round reveal lag. - **The leak is what repairs the tail and the second season.** At h3 it cuts - the October loss from 44% to 11% and March from 46% to 16%, and turns April - from −33% to +9%. In 2025-26, h1–h3 go from −2.5/−3.3/−2.7 to - +1.7/+1.1/+0.8. -- **A slow term alone does nothing.** `slow 0.003` is within 0.3 points of + the October loss from 35% to 12% and March from 42% to 14%, and turns April + from −20% to +10%. In 2025-26, h1–h3 go from −1.8/−4.6/−3.5 to + +1.5/+1.0/+0.9. +- **A slow term alone does nothing.** `slow 0.003` is within 0.2 points of `single` everywhere, and `slow 0.003 + leak 0.1` is within 0.5 points of - `leak 0.1` (slightly better coverage, 0.068–0.089). At this learning rate + `leak 0.1` (slightly better coverage, 0.068–0.084). At this learning rate the slow term never accumulates enough to matter without a warm start (E05). - The median follows WIS: the leak improves absolute error at h1–h3 - (+3.7/+3.1/+3.3 vs +2.6/+2.4/+1.7) and costs about a point at h−1/h0. + (+3.4/+2.8/+3.4 vs +2.4/−0.6/+1.2) and costs 1.1 points at h−1 and 0.3 at + h0. ```{r runs} sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) diff --git a/reports/writeups/calibration_experiments/e05_warm_start.Rmd b/reports/writeups/calibration_experiments/e05_warm_start.Rmd index bc9453e5..bc271f58 100644 --- a/reports/writeups/calibration_experiments/e05_warm_start.Rmd +++ b/reports/writeups/calibration_experiments/e05_warm_start.Rmd @@ -37,28 +37,30 @@ from E04: # Findings -Numbers are from the render of 30 September 2026, pooled over both live seasons and all locations; WIS and absolute-error changes are % vs base, positive is better. +Numbers are from the render of 30 September 2026, pooled over both live +seasons and all locations; WIS and absolute-error changes are % vs base, +positive is better. - **The warm start helps every tracker, and never hurts.** Each warm run is at least as good as its cold twin on WIS at every horizon, and better on - coverage: `single` +6.4/+4.1/+0.8/−0.6/−1.3 → `warm + single` - +6.4/+4.7/+1.3/+0.3/+0.4, coverage bias 0.054–0.026 → 0.050–0.013. + coverage: `single` +5.1/+4.0/+0.7/−1.3/−1.4 → `warm + single` + +5.1/+4.7/+1.3/−0.2/+0.3, coverage bias 0.049–0.026 → 0.046–0.014. - **`warm + single` is the best-calibrated config so far.** Its coverage bias - is 0.013–0.021 at h0–h3, a fifth of the base's, with WIS positive at every - horizon. REF-op (`warm + slow + leak`) has the best WIS - (+6.8/+5.1/+2.5/+1.8/+1.9) at about three times the coverage bias - (0.080–0.041). These two mark the ends of the WIS/coverage trade-off. + is 0.013–0.020 at h0–h3, a fifth of the base's, with WIS within 0.2 points + of the base or better at every horizon. REF-op (`warm + slow + leak`) has + the best WIS (+5.4/+4.9/+2.4/+1.7/+2.2) at about three times the coverage + bias (0.074–0.041). These two mark the ends of the WIS/coverage trade-off. - **The warm start's gain is in the first live season.** In 2024-25 it adds - 1.5–3 points at h2/h3 to every tracker; in 2025-26 warm and cold runs are + 1.7–3.2 points at h2/h3 to every tracker; in 2025-26 warm and cold runs are within 0.5 points, since by then the tracker has learned the level itself. - `warm + single` still loses 2.5–3.4% at h1–h3 in 2025-26; only the leaky + `warm + single` still loses 1.8–4.3% at h1–h3 in 2025-26; only the leaky variants gain there. -- **The warm start alone** (no tracking) gains 0.4–2.0% WIS and cuts - coverage bias by a third: the persistent low bias is real and cheap to - correct. +- **The warm start alone** (no tracking) gains 0.5–2.1% WIS and cuts + coverage bias by about a third at h0–h3 (12% at h−1): the persistent low + bias is real and cheap to correct. - **Where REF-op loses** is unchanged from the finalized-truth runs: at h3, - October (−28%) and March (−32%), 5% of the base WIS between them; - December–January (75% of it) gain about 4%. + October (−30%) and March (−29%), 6% of the base WIS between them; + December–January (71% of it) gain about 5%. ```{r runs} sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd index 31197241..2a2592b5 100644 --- a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -100,33 +100,35 @@ step. - **The trade-off is L-shaped.** Raising the constant learning rate cuts the coverage bias from about 0.10 to 0.03 at almost no WIS cost, up to a knee - near 0.05–0.1 (h1–h3; up to 0.3 at h−1). Past the knee WIS climbs steeply - and coverage stops improving. The per-location lines spread widely: past - 0.3 many locations inflate WIS by 20% or more. -- **On the rate scale, the knee beats the paper's adaptive tracker.** At the - reference delay (+1 week, re-run design), the knee gives WIS inflation - −3.9/−2.3/−1.0/−0.6/−0.1% and coverage bias 0.043/0.031/0.026/0.026/0.026 - at h−1…h3. REF-paper gives −4.2/−2.4/+0.5/+2.9/+5.2% and - 0.047/0.030/0.032/0.034/0.032, so the knee is better on both at h1–h3 and - within 0.3 points at h−1/h0. + near 0.05–0.1. Past the knee WIS climbs steeply and coverage barely + improves. At the reference delay, h−1 has its best WIS at 0.1 (at 0.32 + with no extra delay). The per-location lines spread widely: past 0.3 many + locations inflate WIS by 20% or more. +- **On the rate scale, the knee matches or beats the paper's adaptive + tracker at every horizon.** At the reference delay (+1 week, re-run + design), the knee gives WIS inflation −3.2/−2.5/−1.1/−0.5/−0.2% and coverage + bias 0.040/0.031/0.026/0.027/0.027 at h−1…h3. REF-paper gives + −3.2/−1.9/+2.0/+2.7/+5.3% and 0.044/0.030/0.032/0.034/0.032: the same WIS + at h−1, worse WIS from h0 on, and worse coverage at h−1 and h1–h3. - **The rate scale only matters with a constant learning rate.** "rate, adaptive" matches REF-paper exactly. The adaptive rate is a quantile of the location's own residuals, so dividing a location by its population rescales the steps and the offsets together. -- **REF-op makes a different trade.** Its WIS gain (−6.8/−5.1/−2.5/−1.8/−1.9%) +- **REF-op makes a different trade.** Its WIS gain (−5.4/−4.9/−2.4/−1.7/−2.2%) is larger than any constant rate reaches at h0–h3, but at about twice the - coverage bias (0.080/0.054/0.045/0.046/0.041). The leak and the warm start + coverage bias (0.074/0.054/0.045/0.046/0.041). The leak and the warm start buy WIS; the plain tracker buys coverage. - **Extra delay, re-run design (prod): shorter is better, but it matters only - at h−1/h0.** At a learning rate of 0.1, h−1 WIS inflation is −6.2/−4.6/−2.8% + at h−1/h0.** At a learning rate of 0.1, h−1 WIS inflation is −5.3/−3.6/−1.0% at +0/+1/+2 weeks. At h1–h3 the curves nearly coincide, and coverage bias barely moves with delay. A value learned early is replaced by its revision in the next week's run, so waiting only delays the signal. - **Extra delay, stateful design: one week is needed.** At +0 weeks the tracker learns once from the first report, which under-reports, so the - offsets are pushed down: at the knee, h−1 coverage bias is 0.143, worse than - the uncalibrated 0.118. +1 week recovers most of it (0.067); +2 and +3 - weeks change little at h1–h3 and lose WIS at h−1. + offsets are pushed down: at the knee, h−1 coverage bias is 0.142, worse than + the uncalibrated 0.111, and h−1 WIS is 0.9% worse than the base. +1 week + recovers most of it (0.066); +2 and +3 weeks change little at h1–h3 and + lose WIS at h−1. ```{r libs, cache=FALSE} suppressPackageStartupMessages(source(here::here("R/load_all.R"))) diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd index 90179e13..1025b7da 100644 --- a/reports/writeups/calibration_experiments/e10_covid.Rmd +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -40,26 +40,29 @@ seasons. For the per-season views below, seasons are August–July years. # Findings Numbers are from the render of 30 September 2026, pooled over all rounds -(2024-11-23 to 2026-09-19) and locations; WIS changes are % vs base, positive -is better. +(2024-11-30 to 2026-09-19) and locations; WIS changes are % vs base, positive +is better. The first covid round (2024-11-23) is excluded: it was made from a +bad NHSN release that was corrected the next day. - **The covid base is already calibrated at h1–h3,** so there is little to - fix there. Its coverage bias is 0.094/0.063/0.043/0.034/0.027 at h−1…h3, + fix there. Its coverage bias is 0.098/0.067/0.046/0.036/0.028 at h−1…h3, against about 0.1 at every horizon for flu. -- **h−1 is a clear win.** WIS +9.5% (REF-op) and +9.2% (REF-paper), absolute - error of the median +18% and +16%. Over these rounds the submitted h−1 - came entirely from a climatology-plus-linear-trend ensemble (the - autoregressive models were left out of negative horizons), and h−1 is - where the base is least calibrated. -- **At h1–h3, REF-op is neutral and REF-paper is harmful.** REF-op: WIS - −0.1/−0.2/−0.8%, coverage bias slightly worse than base at h2/h3 - (0.037/0.035 vs 0.034/0.027). REF-paper: WIS −5.6/−6.8/−9.4%, coverage bias - worse than base from h2 on (0.043/0.048). An offset learned where the base - is calibrated is noise. +- **h−1 is a clear win.** WIS +10.2% (REF-op) and +11.4% (REF-paper), + absolute error of the median +20% for both. REF-paper's h−1 coverage bias + is 0.012, the lowest of any config in these notebooks. Over these rounds + the submitted h−1 came entirely from a climatology-plus-linear-trend + ensemble (the autoregressive models were left out of negative horizons), + and h−1 is where the base is least calibrated. +- **At h1–h3, REF-op is neutral to slightly harmful and REF-paper is + harmful.** REF-op: WIS 0.0/−0.6/−1.3%, coverage bias slightly worse than + base at h2/h3 (0.039/0.037 vs 0.036/0.028). REF-paper: WIS + −4.5/−7.3/−10.3%, coverage bias worse than base from h2 on (0.044/0.049). + An offset learned where the base is calibrated is noise. - **The losses are in 2025-26 and around September.** REF-op gains - +1.8/+3.0/+2.4% at h1–h3 in 2024-25 and loses 3.0–4.1% in 2025-26. By month - at h3, REF-op loses 16% in September and 12% in October (19% of the base - WIS between them), around the late-summer wave. + +2.4/+2.8/+1.9% at h1–h3 in 2024-25 and loses 3.0–4.3% in 2025-26. By month + at h3, REF-op loses 16% in September and 12% in October (20% of the base + WIS between them), around the late-summer wave. REF-paper also loses 42–49% + in April and May. - **Implication for prod:** for covid, calibrating h−1 (and perhaps h0) and passing h1–h3 through would keep the gain and drop the losses. The covid prod pipeline calibrates every horizon today. diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index d3d46709..56672181 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -38,8 +38,8 @@ for (f in src) { | E03 | Which scale should the offsets live on: counts, square root, or rate per 100k? | Square root, at every horizon, mostly by avoiding spring losses. Rate equals counts with the adaptive learning rate. | | E04 | Should the offset shrink between updates, or have a second slow component? | Shrinking buys WIS at h1–h3 but nearly triples coverage error. A slow component alone does nothing. | | E05 | Should the offset start from the burn-in season's errors instead of zero? | Yes; it never hurts. With the plain tracker it gives the best-calibrated config so far. | -| E09 | With a constant learning rate on the rate scale, how do coverage and WIS trade off, and how much revision delay is best? | An L-shaped trade-off whose knee beats the paper's adaptive rate at h1–h3. Under prod's weekly re-run, less delay is better. | -| E10 | Do the flu results carry over to covid? | Only at h−1 (+9.5% WIS). The covid base is already calibrated at h1–h3, where calibrating is neutral or harmful. | +| E09 | With a constant learning rate on the rate scale, how do coverage and WIS trade off, and how much revision delay is best? | An L-shaped trade-off whose knee matches or beats the paper's adaptive rate at every horizon. Under prod's weekly re-run, less delay is better. | +| E10 | Do the flu results carry over to covid? | Only at h−1 (+10% WIS). The covid base is already calibrated at h1–h3, where calibrating is neutral or harmful. | # How to read the results From 22a96eb2c17c4c048cd8c0db85fe2577b9a64d44 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 30 Sep 2026 23:02:48 -0700 Subject: [PATCH 024/107] =?UTF-8?q?docs(calibration):=20h=E2=88=921=20spli?= =?UTF-8?q?t=20out=20of=20calibration;=20revision=5Faware=20vs=20baseline?= =?UTF-8?q?=20is=20its=20own=20track?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notes/CALIBRATION2.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/notes/CALIBRATION2.md b/notes/CALIBRATION2.md index dae33539..5ac0d0ec 100644 --- a/notes/CALIBRATION2.md +++ b/notes/CALIBRATION2.md @@ -2,13 +2,14 @@ Open threads and TODOs Decided but not done 4. Sweep windowed_seasonal, the main ensemble component used as a proxy in the calibration harness. It needs a variant of the sweep script that reads the harness forecasts. Recorded in notes/spoiled-submissions.md. -5. Fix how prod builds h−1 (flu, and covid before 2026-09-23). The AR models drop the already-reported week at h−1 (filter_minus_one_ahead). That's the 2026-04-04 overshoot and the 10–30% h−1 bias during 2025-26 declines. This is a prod change, not a calibration one. +5. h−1 is its own track, separate from calibration. Since 2026-09-23 prod no longer uses the AR models at h−1: `ensemble_mix` drops them at negative aheads and h−1 is about 99.9% `revision_aware`, which predicts the finalized value from the week's first report (checked on the 2026-04-04 round). Next: score `revision_aware` against the `revision_ratio_nowcast` baseline at h−1 through explore (explore's `g_aheads` has no −7 yet). On 2026-04-04 the baseline gave OH 64 / CO 34 / MN 34 (finalized ≈60/34/36) vs `revision_aware` 84/41/43; its lag-7 term drags toward the previous week in sharp declines. + Calibration focuses on h0 and above. Harness h−1 history predates 2026-09-23 (AR with the reported week dropped for flu, `climate_linear` for covid), so h−1 offsets learned from it don't apply to the current h−1. Revisit h−1 calibration only after the h−1 component is settled. Open questions from earlier work 6. Halved US cause (2024-12-14, 2024-12-21, 2025-01-04): unconfirmed. The us/usa mixup is my best guess. 7. Covid 2026-04-25 cause (a stale fetch during the cast API v2→v5 change): likely but unconfirmed; there's no store to replay. 8. Experiments not yet re-run on vintages: E07 (ensemble vs windowed_seasonal), and E01/E02, still marked stale. -9. Covid per-horizon calibration (h−1, maybe h0, only): untested. +9. Covid per-horizon calibration (h0 only; h−1 deferred, see 5): untested. 10. Replay drift: the July 2026 flu replay doesn't reproduce the 2025-11-22 submission, and the May 2025 covid replay doesn't reproduce 2024-11-23. That's expected, since code and data changed, but don't read replays as "what was submitted". Housekeeping From e9f6fe611937e1c2535bba89ceb10c5679fbc07d Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:18:50 -0700 Subject: [PATCH 025/107] docs(calibration): reorganize open threads by priority --- notes/CALIBRATION2.md | 65 +++++++++++++------------------------------ 1 file changed, 20 insertions(+), 45 deletions(-) diff --git a/notes/CALIBRATION2.md b/notes/CALIBRATION2.md index 5ac0d0ec..51316d36 100644 --- a/notes/CALIBRATION2.md +++ b/notes/CALIBRATION2.md @@ -1,49 +1,24 @@ Open threads and TODOs -Decided but not done -4. Sweep windowed_seasonal, the main ensemble component used as a proxy in the calibration harness. It needs a variant of the sweep script that reads the harness forecasts. Recorded in notes/spoiled-submissions.md. -5. h−1 is its own track, separate from calibration. Since 2026-09-23 prod no longer uses the AR models at h−1: `ensemble_mix` drops them at negative aheads and h−1 is about 99.9% `revision_aware`, which predicts the finalized value from the week's first report (checked on the 2026-04-04 round). Next: score `revision_aware` against the `revision_ratio_nowcast` baseline at h−1 through explore (explore's `g_aheads` has no −7 yet). On 2026-04-04 the baseline gave OH 64 / CO 34 / MN 34 (finalized ≈60/34/36) vs `revision_aware` 84/41/43; its lag-7 term drags toward the previous week in sharp declines. - Calibration focuses on h0 and above. Harness h−1 history predates 2026-09-23 (AR with the reported week dropped for flu, `climate_linear` for covid), so h−1 offsets learned from it don't apply to the current h−1. Revisit h−1 calibration only after the h−1 component is settled. - -Open questions from earlier work -6. Halved US cause (2024-12-14, 2024-12-21, 2025-01-04): unconfirmed. The us/usa mixup is my best guess. -7. Covid 2026-04-25 cause (a stale fetch during the cast API v2→v5 change): likely but unconfirmed; there's no store to replay. -8. Experiments not yet re-run on vintages: E07 (ensemble vs windowed_seasonal), and E01/E02, still marked stale. -9. Covid per-horizon calibration (h0 only; h−1 deferred, see 5): untested. -10. Replay drift: the July 2026 flu replay doesn't reproduce the 2025-11-22 submission, and the May 2025 covid replay doesn't reproduce 2024-11-23. That's expected, since code and data changed, but don't read replays as "what was submitted". +Main priority (calibration) +1. Run all E00–E10 notebooks. +2. Pick a constant learning rate from E09: the bottom-left of the elbow. Add that setting to a notebook that compares it with the adaptive learning rate. Use the "rerun" method rather than "stateful". + +Secondary (tech debt and other methods) +3. Reduced-set explore run: the most promising families, not the full sweep. +4. Revision-method backtests: `revision_aware` against the `revision_ratio` baseline at h−1 (explore h−1 families plus `nowcast_notebook`, both diseases), then choose prod's h−1 component. Covid NSSP families need re-running, since covid explore NSSP now has real vintages. + +Low priority +5. Calibrate h0 and above only. Revisit h−1 calibration after 4 settles; the harness h−1 history predates the 2026-09-23 switch to `revision_aware`. +6. Sweep windowed_seasonal, the harness proxy, for spoiled submissions; needs a variant of the sweep script that reads the harness forecasts (`notes/spoiled-submissions.md`). +7. Re-run E07, E01 and E02 on vintages (still marked stale). +8. Covid per-horizon calibration (h0 only): untested. +9. Speed up exact tracker runs, if more exact sweeps are coming. An exact config replays prod's weekly re-run, about 55 full tracker runs, so cost grows with the square of the number of rounds (E09: about 72 configs, about 1.5 h on 14 cores). + - Quick wins, all exact, about 2–2.5x: sort-based "data as of round t" lookup instead of a grouped slice_max (about 4–5 s per round); validate `arg_match` once outside the loop (about 15%); skip isoreg when quantiles are already ordered (about 8%); plain vectors instead of tibble/glue in the per-series loop (about 15%). Check bit-for-bit against cached runs. + - Checkpointing, another 5–10x: start round t's run from round t−1's tracker state at the first round whose data differ. Needs `qt_track()` to save and resume offsets, the learning-rate window and the slow term; test for exact equality with the full re-run. Housekeeping -11. Old run caches: E09 and the knitr caches were moved aside (cache/calibration/experiments/*.pre-spoiled). I can delete them once the new renders check out. -12. just calibration-experiments doesn't work on the host, because Rscript isn't installed there. It needs RSCRIPT="distrobox enter rocker -- Rscript". Should I make that the Justfile default? -13. renv warns on every run: the library has renv 1.1.6 but the lockfile wants 1.2.3, and some lockfile packages aren't installed. It's harmless so far. -14. No bookmark yet: the branch is about a dozen jj commits on top of ds/calibrate2. Tell me when you want one moved for a PR. - ---- - -The faster snapshot gives identical output and is about 25x faster: 0.15–0.3 s instead of 3.6–4.9 s. - -Why E09 is slow - -- Each "exact" config is really about 55 tracker runs. Prod re-runs the tracker from the first round each week, so replaying it means one full run per live round, each over a longer history. Total work grows with the square of the number of rounds. -- E09 has about 72 exact configs. That's 60 sweep runs plus 12 reference runs, around 4,000 full tracker runs, at roughly 65 s per config on 14 cores. -- The stateful runs are cheap, since each config is a single run. - -Profile of one full run (13.5 s at the last round), plus the snapshot step: - -┌──────────────────────────────────────────────────────────┬─────────────────────────────────────┬─────────────────────────────────────────────────────────────────┬──────────────────────────────────────────┐ -│ cost │ share │ fix │ exact? │ -├──────────────────────────────────────────────────────────┼─────────────────────────────────────┼─────────────────────────────────────────────────────────────────┼──────────────────────────────────────────┤ -│ Rebuilding "data as of round t" with a grouped slice_max │ ~4–5 s per round, on top of the run │ Sort-based lookup (tested above) │ Yes, identical output │ -├──────────────────────────────────────────────────────────┼─────────────────────────────────────┼─────────────────────────────────────────────────────────────────┼──────────────────────────────────────────┤ -│ rlang::arg_match called on every tracker step │ ~15% of the run │ Validate once, outside the loop │ Yes │ -├──────────────────────────────────────────────────────────┼─────────────────────────────────────┼─────────────────────────────────────────────────────────────────┼──────────────────────────────────────────┤ -│ Isotonic projection (isoreg) on every step │ ~8% │ Skip it when quantiles are already ordered, which is most steps │ Yes, PAVA leaves ordered input unchanged │ -├──────────────────────────────────────────────────────────┼─────────────────────────────────────┼─────────────────────────────────────────────────────────────────┼──────────────────────────────────────────┤ -│ tibble/glue building inside the per-series loop │ ~15% │ Plain vectors in the hot path │ Yes │ -└──────────────────────────────────────────────────────────┴─────────────────────────────────────┴─────────────────────────────────────────────────────────────────┴──────────────────────────────────────────┘ - -Quick wins. Together those should make exact runs about 2–2.5x faster, so E09 drops from about 1.5 h to about 40 min. They're all exact, and I'd check that by comparing results bit-for-bit against the cached runs. About 30 minutes of work. - -Bigger, algorithmic win. Two consecutive weekly re-runs see identical data for everything older than the last few weeks, because revisions settle within a few weeks. So the run for round t could start from a checkpoint of round t−1's run at the first round where their data differ, instead of from the beginning. That turns the quadratic cost into roughly linear: probably another 5–10x, bringing E09 to a few minutes. It needs the tracker to save and resume its state (offsets, learning-rate window, slow term). That's a moderate refactor to qt_track(), and it can be tested for exact equality against the current full re-run. Half a day or so, with tests. - -About the running render. It started with the old code and has roughly 1–1.5 hours left. Restarting it after the quick wins would save little once you count the implementation time. I'd let it finish and do the quick wins next, so later sweeps benefit. Then decide on the checkpointing based on how many more exact sweeps you expect. Want me to start on the quick wins now? Editing the code won't affect the running render, which loaded everything at start. +10. Delete the moved-aside caches (`cache/calibration/experiments/*.pre-spoiled`) once the new renders check out. +11. `just calibration-experiments` needs `RSCRIPT="distrobox enter rocker -- Rscript"` on the host; make it the Justfile default? +12. renv warns on every run (library renv 1.1.6, lockfile 1.2.3; some lockfile packages not installed). Harmless so far. +13. `ds/calibrate2` mixes calibration, h−1, explore and prod-health work; consider splitting it into separate PRs. From 6f3bd28773351733de82a5aea6a43e0fcd5c7daf Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:25:53 -0700 Subject: [PATCH 026/107] docs: move unexplained spoiled-submission causes to the spoiled-submissions note --- notes/spoiled-submissions.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/notes/spoiled-submissions.md b/notes/spoiled-submissions.md index 1f37de56..f76ea563 100644 --- a/notes/spoiled-submissions.md +++ b/notes/spoiled-submissions.md @@ -108,6 +108,15 @@ problem; scattered single locations are usually model error or bad data. The halved-US check (US median vs the sum of state medians) is not in the script; it is a one-line duckdb query over the hub CSVs. +## Open threads + +- Halved US (flu 2024-12-14, 2024-12-21, 2025-01-04): cause unconfirmed; a + `us`/`usa` mixup is the best guess. +- Covid 2026-04-25: a stale fetch during the cast API v2→v5 change is likely + but unconfirmed; there is no store to replay. +- Covid 2024-11-23: the May 2025 replay doesn't reproduce the submission, for + reasons not yet explained. Don't read replays as "what was submitted". + ## To do - Run the same sweep on `windowed_seasonal` (the main ensemble component, From f2392699a629713c089b61cbde2fb2abbea61c41 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:26:54 -0700 Subject: [PATCH 027/107] docs: run just recipes inside the rocker distrobox --- Justfile | 1 + notes/CALIBRATION2.md | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Justfile b/Justfile index 83bd1383..8e0a0b93 100644 --- a/Justfile +++ b/Justfile @@ -1,3 +1,4 @@ +# R isn't installed on the host; run recipes inside the distrobox: `distrobox enter rocker -- just `. rscript := env_var_or_default("RSCRIPT", "Rscript") # Render calibration experiment notebooks (all, or the named ones, e.g. e00_vintage_backtest) and the index. diff --git a/notes/CALIBRATION2.md b/notes/CALIBRATION2.md index 51316d36..b0a1eb71 100644 --- a/notes/CALIBRATION2.md +++ b/notes/CALIBRATION2.md @@ -19,6 +19,4 @@ Low priority Housekeeping 10. Delete the moved-aside caches (`cache/calibration/experiments/*.pre-spoiled`) once the new renders check out. -11. `just calibration-experiments` needs `RSCRIPT="distrobox enter rocker -- Rscript"` on the host; make it the Justfile default? -12. renv warns on every run (library renv 1.1.6, lockfile 1.2.3; some lockfile packages not installed). Harmless so far. -13. `ds/calibrate2` mixes calibration, h−1, explore and prod-health work; consider splitting it into separate PRs. +11. renv warns on every run (library renv 1.1.6, lockfile 1.2.3; some lockfile packages not installed). Harmless so far. From bbdf6478cb266380dc3f0a6681717bd28ea0b0af Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:29:34 -0700 Subject: [PATCH 028/107] docs(calibration): E11 notebook comparing a constant rate-scale learning rate with adaptive ones --- .../e11_constant_lr.Rmd | 85 +++++++++++++++++++ .../calibration_experiments/index.Rmd | 1 + 2 files changed, 86 insertions(+) create mode 100644 reports/writeups/calibration_experiments/e11_constant_lr.Rmd diff --git a/reports/writeups/calibration_experiments/e11_constant_lr.Rmd b/reports/writeups/calibration_experiments/e11_constant_lr.Rmd new file mode 100644 index 00000000..5e2954c8 --- /dev/null +++ b/reports/writeups/calibration_experiments/e11_constant_lr.Rmd @@ -0,0 +1,85 @@ +--- +title: "E11: constant learning rate vs adaptive" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + hub_dir: "../FluSight-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} +``` + +One axis: the learning rate. The constant rate is the corner of E09's +trade-off curve, re-run design, one week of extra delay: 0.1 admissions per +100k per step on the rate scale. Below that rate, coverage bias falls with +little change in WIS. Above it, coverage stops improving and WIS gets worse +at every horizon. It is compared with the adaptive learning rate (0.03 × the +90th percentile of the last 20 rounds' absolute errors) in the configs from +E03 and E05. Everything else is REF-paper, learning from vintages the way +prod does (re-run each week; E00). + +- **REF-paper**: counts, adaptive. +- **sqrt, adaptive**: the E03 winner. +- **warm + sqrt, adaptive**: the E05 `warm + single`, the best-calibrated + adaptive config. +- **rate, constant 0.1**: the E09 corner. +- **REF-op**: prod, for orientation; it differs in several settings. + +# Findings + +Numbers are from the render of 1 October 2026, pooled over both live +seasons and all locations; WIS and absolute-error changes are % vs base, +positive is better. The constant run matches E09's table at 0.1, +1 week, +re-run. + +- **Against the cold adaptive trackers, the constant rate trades WIS at + h−1/h0 for better coverage everywhere.** WIS +3.6/+2.4/+0.7/+0.3/−0.5 at + h−1…h3, against +5.1/+4.0/+0.7/−1.3/−1.4 for `sqrt, adaptive`; coverage + bias 0.032/0.022/0.018/0.017/0.018 against 0.049/0.032/0.026/0.028/0.026. + It beats REF-paper on both at every horizon. +- **The warm-started adaptive tracker still beats it at h0–h1.** + `warm + sqrt, adaptive` has better WIS at h−1–h1 (+5.1/+4.7/+1.3) and + better coverage at h0–h3 (0.013–0.020). The constant rate keeps an edge only + in h−1 coverage (0.032 vs 0.046) and is within 0.8 points on WIS at h2/h3. +- **It is the most stable tracker across seasons and months.** In 2025-26 it + loses 1.8–2.0% at h1–h3, against up to 4.6% for the adaptive sqrt trackers + (REF-op is the only config that gains there). By month at h3 it has the + smallest March loss of the plain trackers (16% vs 42–51% for sqrt), but is + nearly neutral in December–January (about +1%, 71% of the base WIS), where + the warm start and REF-op gain 4–6%. Its h−1 is the weakest in 2025-26 + (−1.6%). +- **The median gains less near the present.** Absolute error improves + +6.3/+4.9% at h−1/h0, against about +10/+8% for the sqrt trackers; it is + better than them at h2 (+1.8%). +- **Untested:** a warm start on the constant-rate tracker. The warm start + is the largest single gain for the adaptive tracker (E05), and the constant + rate keeps coverage without a leak. + +```{r runs} +sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) +cals <- list( + `REF-paper` = run("paper"), + `sqrt, adaptive` = do.call(run, sqrt_args), + `warm + sqrt, adaptive` = do.call(run, c(sqrt_args, slow_init = "burn_in_quantile")), + `rate, constant 0.1` = run("paper", scales = rate_scales, lr = 0.1), + `REF-op` = run("op") +) +focus <- "rate, constant 0.1" +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index 56672181..cfa05023 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -40,6 +40,7 @@ for (f in src) { | E05 | Should the offset start from the burn-in season's errors instead of zero? | Yes; it never hurts. With the plain tracker it gives the best-calibrated config so far. | | E09 | With a constant learning rate on the rate scale, how do coverage and WIS trade off, and how much revision delay is best? | An L-shaped trade-off whose knee matches or beats the paper's adaptive rate at every horizon. Under prod's weekly re-run, less delay is better. | | E10 | Do the flu results carry over to covid? | Only at h−1 (+10% WIS). The covid base is already calibrated at h1–h3, where calibrating is neutral or harmful. | +| E11 | How does E09's best constant learning rate compare with the adaptive trackers? | Better coverage than the cold adaptive trackers at less WIS gain near the present, and the most stable across seasons. The warm-started adaptive tracker still wins at h0–h1. | # How to read the results From d82cbe2737a86913019506b34ac8d230cc59f791 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:35:13 -0700 Subject: [PATCH 029/107] =?UTF-8?q?docs(calibration):=20close=20the=20E00?= =?UTF-8?q?=E2=80=93E10=20render=20and=20constant-rate=20threads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notes/CALIBRATION2.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/notes/CALIBRATION2.md b/notes/CALIBRATION2.md index b0a1eb71..61990602 100644 --- a/notes/CALIBRATION2.md +++ b/notes/CALIBRATION2.md @@ -1,22 +1,20 @@ Open threads and TODOs Main priority (calibration) -1. Run all E00–E10 notebooks. -2. Pick a constant learning rate from E09: the bottom-left of the elbow. Add that setting to a notebook that compares it with the adaptive learning rate. Use the "rerun" method rather than "stateful". +1. Warm start on the constant-rate tracker. E11 (constant 0.1 per 100k, rate scale, +1 wk, re-run) beats the cold adaptive trackers on coverage and is the most stable across seasons, but `warm + sqrt, adaptive` still wins at h0–h1. Try E11's constant rate with `slow_init = "burn_in_quantile"`. Secondary (tech debt and other methods) -3. Reduced-set explore run: the most promising families, not the full sweep. -4. Revision-method backtests: `revision_aware` against the `revision_ratio` baseline at h−1 (explore h−1 families plus `nowcast_notebook`, both diseases), then choose prod's h−1 component. Covid NSSP families need re-running, since covid explore NSSP now has real vintages. +2. Reduced-set explore run: the most promising families, not the full sweep. +3. Revision-method backtests: `revision_aware` against the `revision_ratio` baseline at h−1 (explore h−1 families plus `nowcast_notebook`, both diseases), then choose prod's h−1 component. Covid NSSP families need re-running, since covid explore NSSP now has real vintages. Low priority -5. Calibrate h0 and above only. Revisit h−1 calibration after 4 settles; the harness h−1 history predates the 2026-09-23 switch to `revision_aware`. -6. Sweep windowed_seasonal, the harness proxy, for spoiled submissions; needs a variant of the sweep script that reads the harness forecasts (`notes/spoiled-submissions.md`). -7. Re-run E07, E01 and E02 on vintages (still marked stale). -8. Covid per-horizon calibration (h0 only): untested. -9. Speed up exact tracker runs, if more exact sweeps are coming. An exact config replays prod's weekly re-run, about 55 full tracker runs, so cost grows with the square of the number of rounds (E09: about 72 configs, about 1.5 h on 14 cores). +4. Calibrate h0 and above only. Revisit h−1 calibration after 3 settles; the harness h−1 history predates the 2026-09-23 switch to `revision_aware`. +5. Sweep windowed_seasonal, the harness proxy, for spoiled submissions; needs a variant of the sweep script that reads the harness forecasts (`notes/spoiled-submissions.md`). +6. Re-run E07, E01 and E02 on vintages (still marked stale). +7. Covid per-horizon calibration (h0 only): untested. +8. Speed up exact tracker runs, if more exact sweeps are coming. An exact config replays prod's weekly re-run, about 55 full tracker runs, so cost grows with the square of the number of rounds (E09: about 72 configs, about 1.5 h on 14 cores). - Quick wins, all exact, about 2–2.5x: sort-based "data as of round t" lookup instead of a grouped slice_max (about 4–5 s per round); validate `arg_match` once outside the loop (about 15%); skip isoreg when quantiles are already ordered (about 8%); plain vectors instead of tibble/glue in the per-series loop (about 15%). Check bit-for-bit against cached runs. - Checkpointing, another 5–10x: start round t's run from round t−1's tracker state at the first round whose data differ. Needs `qt_track()` to save and resume offsets, the learning-rate window and the slow term; test for exact equality with the full re-run. Housekeeping -10. Delete the moved-aside caches (`cache/calibration/experiments/*.pre-spoiled`) once the new renders check out. -11. renv warns on every run (library renv 1.1.6, lockfile 1.2.3; some lockfile packages not installed). Harmless so far. +9. renv warns on every run (library renv 1.1.6, lockfile 1.2.3; some lockfile packages not installed). Harmless so far. From e5398554a392e165eb49495c572c6728b84d607d Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 01:00:17 -0700 Subject: [PATCH 030/107] docs: add notes/ROADMAP.md and fold CALIBRATION2.md into CALIBRATION.md --- AGENTS.md | 5 ++- notes/CALIBRATION.md | 73 +++++++++++++++++++++++++------------ notes/CALIBRATION2.md | 20 ---------- notes/ROADMAP.md | 81 +++++++++++++++++++++++++++++++++++++++++ notes/refactor-ideas.md | 3 +- 5 files changed, 135 insertions(+), 47 deletions(-) delete mode 100644 notes/CALIBRATION2.md create mode 100644 notes/ROADMAP.md diff --git a/AGENTS.md b/AGENTS.md index ecc90e84..c12c4b25 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,8 +111,9 @@ directly on the shared stack whenever it is picked up. History lives in `notes/`: the annotated commit log (`notes/2026-07-22-ds-refactor2-annotated-commit-log.md`) is the source of -truth for what was done and how it was verified; `notes/refactor-ideas.md` -holds all open threads and future refactor designs; the remaining dated +truth for what was done and how it was verified; `notes/ROADMAP.md` lists +open tasks and tech debt, `notes/refactor-ideas.md` holds the refactor +designs behind them; the remaining dated files are experiment records. CLAUDE.md describes only the current state. ## Refactoring practice diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index ec074d26..b64fa249 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -703,33 +703,58 @@ Before 2026-09-23 (all of E10's rounds), the mix had no `revision_aware`: ahead **Bug fixed en route:** `nssp_archive` in both `flu_data_targets.R` and `covid_data_targets.R` had a duplicate-key error (pre-existing since August 2026): `compactify = TRUE` can retain multiple rows per `(geo_value, time_value)` when the value changed across issues, and the subsequent `mutate(version = time_value + 7)` flattened all to the same version, producing duplicate keys in `as_epi_archive()`. Fix: `group_by(geo_value, time_value) %>% slice_max(version, n = 1L, with_ties = FALSE) %>% ungroup()` before the version clobber. -# Roadmap - -Possible next steps, roughly ordered: - -0. **Re-run the current experiments with exact learning truth** (E03–E05, - E07 in the ledger), after E09. -0. **Re-run the ILI+ experiments on the fixed replay**: the ILI+ burn-in and +# Open threads + +Calibration-specific open items, roughly in priority order. Repo-wide items +(evaluation testbed, NSSP-target backtesting, revision-aware backtests) are +in `notes/ROADMAP.md`. + +1. **Warm start on the constant-rate tracker.** E11 (constant 0.1 per 100k, + rate scale, +1 week, re-run) beats the cold adaptive trackers on coverage + and is the most stable across seasons, but `warm + sqrt, adaptive` still + wins at h0–h1. Try E11's constant rate with + `slow_init = "burn_in_quantile"`. +2. **Calibrate h0 and above only.** Revisit h−1 calibration once prod's h−1 + component is chosen (`notes/ROADMAP.md`, revision-method backtests); the + harness h−1 history predates the 2026-09-23 switch to `revision_aware`. +3. **Covid per-horizon calibration** (calibrate h−1 and perhaps h0, pass + h1–h3 through; E10): untested. +4. **Re-run E07, E01 and E02 on vintages** (still marked stale in the + ledger). E03–E05 already use the re-run design. +5. **Re-run the ILI+ experiments on the fixed replay**: the ILI+ burn-in and setup C (does a warm start learned on ILI+ transfer to NHSN?). -1. **Review the ranked gallery** — does the WIS cost concentrate at turning - points (the `h + 2` staleness prediction)? Are the biggest offsets fixing - real miscalibration or chasing data-revision artifacts? -2. **Send the collaborator email** (sweep findings; h −1 free win; carry vs - reset a wash). -3. **`windowed_seasonal_extra_sources` retrospective** via - `scripts/calibration/calibration_harness.R` (covid, evaluation store) — does +6. **Sweep `windowed_seasonal`, the harness proxy, for spoiled + submissions.** Needs a variant of the sweep script that reads the harness + forecasts (`notes/spoiled-submissions.md`). +7. **`windowed_seasonal_extra_sources` retrospective** via + `scripts/calibration/calibration_harness.R` (covid, evaluation store): does calibration help our best component forecaster, not just the submitted ensemble? -4. **Integration design** for prod: where the tracker lives (per-forecaster - vs post-ensemble), how state persists week to week, and how to handle the - warm-up (burn-in on a past season? carry state across seasons? a - burn-in-free learning-rate schedule?). -5. **Method extensions**, in the order argued under "Directions this points +8. **Review the ranked gallery.** Does the WIS cost concentrate at turning + points (the `h + 2` staleness prediction)? Are the biggest offsets fixing + real miscalibration or chasing data-revision artifacts? +9. **Method extensions**, in the order argued under "Directions this points to": slow relative level term + fast tracker; cross-horizon proxy gradients for staleness; data-driven phase gating; scale proxy. Pooling offsets across locations and per-level learning rates are lower priority. -6. **Agentic triage** of gallery panels against a fixed rubric (data revision - visible? base missed the turn? calibration helped/hurt?) if manual - skimming of the ranked gallery proves insufficient. -7. **Dynamic gallery app** (Shiny/Posit or a served page with dynamic data - fetch) if the static top-N format becomes limiting. +10. **Speed up exact tracker runs**, if more exact sweeps are coming. An + exact config replays prod's weekly re-run, about 55 full tracker runs, + so cost grows with the square of the number of rounds (E09: about 72 + configs, about 1.5 h on 14 cores). + - Quick wins, all exact, about 2–2.5x: sort-based "data as of round t" + lookup instead of a grouped slice_max (about 4–5 s per round); + validate `arg_match` once outside the loop (about 15%); skip isoreg + when quantiles are already ordered (about 8%); plain vectors instead + of tibble/glue in the per-series loop (about 15%). Check bit-for-bit + against cached runs. + - Checkpointing, another 5–10x: start round t's run from round t−1's + tracker state at the first round whose data differ. Needs + `qt_track()` to save and resume offsets, the learning-rate window and + the slow term; test for exact equality with the full re-run. +11. **Send the collaborator email** (sweep findings; h−1 free win; carry vs + reset a wash). +12. **Agentic triage** of gallery panels against a fixed rubric (data + revision visible? base missed the turn? calibration helped/hurt?) if + manual skimming of the ranked gallery proves insufficient. +13. **Dynamic gallery app** (Shiny/Posit or a served page with dynamic data + fetch) if the static top-N format becomes limiting. diff --git a/notes/CALIBRATION2.md b/notes/CALIBRATION2.md deleted file mode 100644 index 61990602..00000000 --- a/notes/CALIBRATION2.md +++ /dev/null @@ -1,20 +0,0 @@ -Open threads and TODOs - -Main priority (calibration) -1. Warm start on the constant-rate tracker. E11 (constant 0.1 per 100k, rate scale, +1 wk, re-run) beats the cold adaptive trackers on coverage and is the most stable across seasons, but `warm + sqrt, adaptive` still wins at h0–h1. Try E11's constant rate with `slow_init = "burn_in_quantile"`. - -Secondary (tech debt and other methods) -2. Reduced-set explore run: the most promising families, not the full sweep. -3. Revision-method backtests: `revision_aware` against the `revision_ratio` baseline at h−1 (explore h−1 families plus `nowcast_notebook`, both diseases), then choose prod's h−1 component. Covid NSSP families need re-running, since covid explore NSSP now has real vintages. - -Low priority -4. Calibrate h0 and above only. Revisit h−1 calibration after 3 settles; the harness h−1 history predates the 2026-09-23 switch to `revision_aware`. -5. Sweep windowed_seasonal, the harness proxy, for spoiled submissions; needs a variant of the sweep script that reads the harness forecasts (`notes/spoiled-submissions.md`). -6. Re-run E07, E01 and E02 on vintages (still marked stale). -7. Covid per-horizon calibration (h0 only): untested. -8. Speed up exact tracker runs, if more exact sweeps are coming. An exact config replays prod's weekly re-run, about 55 full tracker runs, so cost grows with the square of the number of rounds (E09: about 72 configs, about 1.5 h on 14 cores). - - Quick wins, all exact, about 2–2.5x: sort-based "data as of round t" lookup instead of a grouped slice_max (about 4–5 s per round); validate `arg_match` once outside the loop (about 15%); skip isoreg when quantiles are already ordered (about 8%); plain vectors instead of tibble/glue in the per-series loop (about 15%). Check bit-for-bit against cached runs. - - Checkpointing, another 5–10x: start round t's run from round t−1's tracker state at the first round whose data differ. Needs `qt_track()` to save and resume offsets, the learning-rate window and the slow term; test for exact equality with the full re-run. - -Housekeeping -9. renv warns on every run (library renv 1.1.6, lockfile 1.2.3; some lockfile packages not installed). Harmless so far. diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md new file mode 100644 index 00000000..a7cd0395 --- /dev/null +++ b/notes/ROADMAP.md @@ -0,0 +1,81 @@ +# Roadmap + +Repo-wide tasks, TODOs and tech debt, roughly in priority order within each +section. Calibration-specific threads live under "Open threads" in +`notes/CALIBRATION.md`; refactor designs (E2 ensemble sweep, snapshot +validator, simplification inventory) live in `notes/refactor-ideas.md`. + +## Forecast evaluation + +1. **Choose a testbed for comparing calibration with forecaster choice.** + Calibrating needs full quantile forecasts over many rounds, not just + scores. Candidates: + - the calibration harness on hub submissions (what we actually + submitted, already in use); + - the evaluation project (`flu_hosp_evaluation` / `covid_hosp_evaluation`), + which replays the current prod components and ensemble over past dates. + It also replays the hand-edited weights and geo exclusions of each past + week, which biases it toward reproducing what was done; + - the explore stores, which already hold per-forecaster quantile forecasts + for every family on S3 (NHSN target only); + - a new "explore-lite" project with a handful of families. + Undecided (2026-10-01). +2. **NSSP-target backtesting is second-class and needs dedicated + attention.** + - Explore forecasts NHSN only: every family sets `outcome = "hhs"`. NSSP + as a target exists only for the prod components, in the prod and + evaluation pipelines and the `*_nssp_backtesting_*` reports, so no + NSSP-target sweep has ever been run. + - Flu NSSP scoring looks broken. In `flu_nssp_backtesting_2024_2025_on_2026-05-06` + and `2026-07-01_flu_nssp_scoring_*`, every model, the hub ensemble + included, has 90% coverage of 0 and WIS about equal to absolute error. + Most likely the truth and the forecasts are on different scales + (proportion vs percent). Fix this before reading any flu NSSP result. + - Covid NSSP scoring looks sane (May 2026 backtest, 2024-25): our best + component is `windowed_seasonal_latest` (mean WIS 0.11), + `CMU-TimeSeries` 0.14, the CovidHub ensemble 0.06. + +## Revision-aware forecasting and h−1 + +3. **Revision-method backtests.** `revision_aware` against the + `revision_ratio` baseline at h−1 (explore h−1 families plus + `nowcast_notebook`, both diseases), then choose prod's h−1 component. + Calibrating h−1 waits on this (`notes/CALIBRATION.md`). +4. **Flu explore has no current scores for the revision families.** In the + flu explore store on S3 (`joined_scores_2024_2026`, 2026-09-19), none of + `revision_aware`, `revision_aware_augmented`, `revision_aware_nssp`, + `revision_aware_no_season`, `revision_aware_beds_no_season`, + `revision_aware_beds_seasonal` or `revision_ratio` has scores under its + current id. The 32 orphaned ids in that store score 20–60× flatline in + 2024-25, consistent with the pre-fix US population-scaling bug + (`notes/revision_aware_forecaster_devlog.md`). The two `beds` grids are + 24 forecasters each and have never been scored for flu. +5. **Re-run covid explore families that take NSSP as an input.** Covid + explore NSSP now has real vintages (`up_to_date_nssp_state_archive`); the + covid store on S3 (2026-09-28) predates that. `scaled_pop_exogenous` + scores 0.55 relative WIS against flatline there, better than the + CovidHub ensemble (0.62), which suggests leakage from finalized NSSP. + `revision_aware_nssp` needs the same re-run. + +## Explore grid + +6. **Trim `climate_linear`.** It is 73 of 182 flu forecasters and 49 of 97 + covid forecasters. In the S3 stores (WIS relative to flatline, h0–h4, + mean of 2024-25 and 2025-26): + - `drop_non_seasons` gives identical scores either way, for both + diseases. Either it is a no-op on these dates or it is not wired + through; check before dropping it. + - `nonlin_method` moves the best score by less than 0.001. + - `model_used` is the axis that matters, and the winner differs by + disease: `climate` for flu (best 0.865; `climate_linear` 0.963, + `climatological_forecaster` 1.04), `climate_linear` for covid (best + 0.854; `climate` 1.99, `climatological_forecaster` 5.3). + - `quantiles_by_geo = TRUE` helps flu, hurts covid. + Fixing `drop_non_seasons` and `nonlin_method` alone cuts the family to + about a quarter (flu ~19, covid ~13). + +## Tech debt + +7. renv warns on every run (library renv 1.1.6, lockfile 1.2.3; some + lockfile packages not installed). Harmless so far. +8. Refactor and cleanup threads: `notes/refactor-ideas.md`. diff --git a/notes/refactor-ideas.md b/notes/refactor-ideas.md index 8893aff7..cd2adbfa 100644 --- a/notes/refactor-ideas.md +++ b/notes/refactor-ideas.md @@ -1,6 +1,7 @@ # Open threads and future refactor ideas -The single home for everything not yet implemented. Consolidated 2026-07-22 +Refactor designs and cleanup inventories. The task list itself is +`notes/ROADMAP.md`. Consolidated 2026-07-22 from CLAUDE.md's "Open threads" and the three 2026-07-19 design/cleanup scouts (`e2-explore-ensemble-design`, `validate-snapshot-design`, `clarity-simplification-scout` — originals in git history); items the From 1f8b7d3c9e6a988217e248c0f1c10b4771605c6a Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 01:46:54 -0700 Subject: [PATCH 031/107] docs: plan the evaluation-project work incrementally; add an incremental-first principle to AGENTS.md --- AGENTS.md | 13 ++++++++ notes/ROADMAP.md | 77 ++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c12c4b25..cc9e9baf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,6 +132,19 @@ whole grid a dependency of every branch); `tar_target`'s default `tidy_eval = FALSE` to defer); `rlang::syms` on the trainer column is load-bearing. +## Working incrementally + +Most work here is a chain of small decisions about data, edge cases and +forecasters, and full runs (explore sweeps, evaluation replays, exact +calibration sweeps) take hours, so a wrong early choice is expensive to find. +Build up from the cheapest check that can answer the current question: a +standalone script on one geo and a few dates, then one forecaster on a few +dates (`EVALUATION_N_DATES`, a filtered grid), then one forecaster on all +dates, and only last the full run. Look at each stage's output (plots, a +diff against a known value) before scaling up, and write each data or +edge-case decision into the relevant note when it is made, so it can be +revisited. + ## Conventions - Testing focuses on utility functions; forecaster quality is assessed by inspecting results/reports, not unit tests. diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index a7cd0395..5b079658 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -7,19 +7,70 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. ## Forecast evaluation -1. **Choose a testbed for comparing calibration with forecaster choice.** - Calibrating needs full quantile forecasts over many rounds, not just - scores. Candidates: - - the calibration harness on hub submissions (what we actually - submitted, already in use); - - the evaluation project (`flu_hosp_evaluation` / `covid_hosp_evaluation`), - which replays the current prod components and ensemble over past dates. - It also replays the hand-edited weights and geo exclusions of each past - week, which biases it toward reproducing what was done; - - the explore stores, which already hold per-forecaster quantile forecasts - for every family on S3 (NHSN target only); - - a new "explore-lite" project with a handful of families. - Undecided (2026-10-01). +1. **Use the evaluation project as the calibration testbed** (current plan, + 2026-10-01). Calibrating needs full quantile forecasts over many rounds, + not just scores. The evaluation project (`flu_hosp_evaluation` / + `covid_hosp_evaluation`) replays the current prod components and + ensemble weekly since 2024-11-20, which is what calibration would see in + prod from now on. The hub-submission harness stays the record of what was + actually submitted; the explore stores (per-forecaster quantile forecasts, + NHSN only, on S3) can answer "does calibrating a candidate beat picking a + different one" without a new project. Three gaps, below. Work them in + small steps (AGENTS.md, "Working incrementally"); the S3 evaluation + stores are empty, so even the baseline needs a run. + + **Critical path for evaluating calibration:** + + a. *Clean replay, one component.* Start with `windowed_seasonal` alone + (the harness proxy). A single component has no ensemble weights, so + the only hand edit to switch off is the data substitutions + (`make_forecast_snapshot(substitutions = NULL)`). Steps: a way to run + one forecaster (e.g. an env var that filters the grid), 2–3 dates, + diff against the edited run (changes should appear only at the + substituted geo-dates), then all dates, timed. Feed it to the + calibration harness. + b. *2023-24 as a burn-in season.* The best configs so far (E05 + `warm + single`, REF-op) warm-start from a burn-in season, and the + replay has none. Delphi's `hhs` source + (`confirmed_admissions_{influenza,covid}_1d`) has real issue history + for 2023-24 (CA 2023-12-01: issues 12-06, 12-08, 12-20, 12-22), until + HHS reporting ended 2024-04-30. Steps, each checked before the next: + - standalone script: fetch, daily to weekly, check the finalized + values against NHSN's finalized 2023-24 values; decide the week + ending and how daily issues map to weekly versions; + - cut the ILI+/flusurv training extras at each forecast date. They run + into mid-2024, so as-is they leak finalized 2023-24 data (the + archive's `stopifnot` would fail, correctly); + - check how far back NSSP vintages go. `windowed_seasonal_extra_sources` + needs NSSP as of each 2023-24 date; if there are none, decide whether + it sits out 2023-24 or runs on finalized NSSP with a caveat; + - `windowed_seasonal` on 3 dates in 2023-24, fan plots against the + 2023-24 hub submissions; then all 2023-24 dates. + c. *The ensemble with fixed weights.* The geo-exclusions CSVs carry the + ensemble weights as well as exclusions, so "no hand edits" for + `ensemble_mix` means one fixed weight block for every date. Decide + which block (the default one at the top of the file, or the current + one), check the resolved weights on one date, then replay. Only + needed once a single component's calibration results look worth + extending to what we submit. + + **Later (not needed to evaluate calibration):** + + d. *Hand edits on vs off as its own question.* A separate evaluation + project (like the `_regr` ones) replaying with the edits, to measure + whether the weekly substitutions and weight changes help. The only + past check found is the 2025 revision report + (`revision_summary_report_2025.Rmd`), which scored substitutions by + whether they moved values toward the final value. Calibration needs + only the clean replay. + e. *ILINet as a target*, to check our methods on a dataset we have not + tuned on. `fluview` has real issue history (CA wILI for week 2023-50 + is revised weekly through May 2024) back to about 2010. Follows the + `nssp_target_archive` + `primary_source` pattern; the ILI+ training + rows must be dropped in this mode, and wILI is a percentage, so it + shares the scale question with NSSP-target scoring (item 2). Prototype + on one forecaster and a few seasons; `calibration_ili_backfill.R` + (ILI+, no vintages) is a reference, not the base to extend. 2. **NSSP-target backtesting is second-class and needs dedicated attention.** - Explore forecasts NHSN only: every family sets `outcome = "hhs"`. NSSP From e47542773f39bc15e50b5e8aa3a547072156ff4e Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 01:52:49 -0700 Subject: [PATCH 032/107] docs(roadmap): record ILI+ extras leak and NSSP vintage depth findings for 2023-24 burn-in --- notes/ROADMAP.md | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index 5b079658..cd87a31a 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -38,12 +38,26 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. - standalone script: fetch, daily to weekly, check the finalized values against NHSN's finalized 2023-24 values; decide the week ending and how daily issues map to weekly versions; - - cut the ILI+/flusurv training extras at each forecast date. They run - into mid-2024, so as-is they leak finalized 2023-24 data (the - archive's `stopifnot` would fail, correctly); - - check how far back NSSP vintages go. `windowed_seasonal_extra_sources` - needs NSSP as of each 2023-24 date; if there are none, decide whether - it sits out 2023-24 or runs on finalized NSSP with a caveat; + - ILI+/flusurv training extras (checked 2026-10-01). They are folded + into `nhsn_prod_archive` (`pipelines/flu_hosp_prod.R`) with + `version = time_value`, and ILI+ runs to 2024-07-24. A 2023-12-06 + snapshot would train on finalized ILI+ up to the forecast week + itself. The snapshot's version-faithfulness abort can't see this + (no row is newer than its version); only the archive-build + `stopifnot` catches it. Decision: version each extras row at its + season's end (`group_by(source, season)`, + `version = max(time_value)`) and replace that `stopifnot` with a + check on versions. Dates from 2024-11-21 on see every extras row + either way, so the golden diff should be empty. Covid has no + extras; + - NSSP vintages (checked 2026-10-01): none before 2024-04-18 from any + source (v5 API, epidatr `nssp`, the S3 Socrata snapshots, the hub + mirror). Decision: `windowed_seasonal_extra_sources` sits out + 2023-24. Gotcha: an NSSP snapshot before the first vintage + silently returns 0 rows, so a 2023-24 replay needs an explicit skip + or abort for every component that reads NSSP. NHSN versions start + 2024-11-19, so the HHS archive must supply every 2023-24 target + row; - `windowed_seasonal` on 3 dates in 2023-24, fan plots against the 2023-24 hub submissions; then all 2023-24 dates. c. *The ensemble with fixed weights.* The geo-exclusions CSVs carry the From da649c97119faa9f818ed7d50cbaf2f6186ddaa8 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 01:58:20 -0700 Subject: [PATCH 033/107] feat(one_offs): build a weekly versioned 2023-24 archive from hhs daily signals --- notes/ROADMAP.md | 22 ++- scripts/one_offs/hhs_2023_24_archive.R | 184 +++++++++++++++++++++++++ 2 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 scripts/one_offs/hhs_2023_24_archive.R diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index cd87a31a..da48a8d9 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -35,9 +35,25 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. (`confirmed_admissions_{influenza,covid}_1d`) has real issue history for 2023-24 (CA 2023-12-01: issues 12-06, 12-08, 12-20, 12-22), until HHS reporting ended 2024-04-30. Steps, each checked before the next: - - standalone script: fetch, daily to weekly, check the finalized - values against NHSN's finalized 2023-24 values; decide the week - ending and how daily issues map to weekly versions; + - HHS weekly archive (done 2026-10-01, + `scripts/one_offs/hhs_2023_24_archive.R`). Decisions: NHSN's week + ending Saturday S is the sum of hhs daily `time_value` S−7..S−1 + (exact match in 71% of flu geo-weeks; other alignments 17–24%), + labelled `time_value = S − 3` (Wednesday). Every daily issue is a + version; a week's value as of v sums its 7 days at their latest + issue ≤ v, and a week appears only once all 7 days exist (first + report lag is a median of 4 days, as with NHSN). US is hhs + `nation`, which equals the sum of states plus territories, as NHSN + `us` does. The `hhs` source matches the cached healthdata.gov + snapshots exactly, so the gap below is between sources, not a + fetch problem. **Open: level.** Finalized HHS runs below finalized + NHSN (in-season sum ratio flu 0.968, covid 0.941, falling to ~0.91 + by April; tx/in/ks/ar/pr 15–20% low). The gap is already there in + NHSN's first 2023-24 version, so it isn't an NHSN revision. Choose: + use as-is with a caveat, rescale per geo by the finalized ratio + (uses finalized data), or drop the worst geos. Also decide how to + stitch: NHSN's archive holds 2023-24 from version 2024-11-19, so + dates after that see NHSN history, a level jump from HHS; - ILI+/flusurv training extras (checked 2026-10-01). They are folded into `nhsn_prod_archive` (`pipelines/flu_hosp_prod.R`) with `version = time_value`, and ILI+ runs to 2024-07-24. A 2023-12-06 diff --git a/scripts/one_offs/hhs_2023_24_archive.R b/scripts/one_offs/hhs_2023_24_archive.R new file mode 100644 index 00000000..e6aafc75 --- /dev/null +++ b/scripts/one_offs/hhs_2023_24_archive.R @@ -0,0 +1,184 @@ +# Build a weekly, versioned 2023-24 hospital-admissions archive from Delphi's +# daily `hhs` source, for use as a burn-in season ahead of the NHSN-era replay, +# and validate its finalized values against NHSN's finalized 2023-24 values. +# +# Usage: Rscript scripts/one_offs/hhs_2023_24_archive.R [out_dir] +# Writes hhs_2023_24_weekly_archive.rds (epi_archive DT, both diseases), +# hhs_2023_24_weekly_archive.parquet and validation CSVs into out_dir. +# +# Decisions (see notes/ROADMAP.md item 1b): +# - Week: the hhs `time_value` is the admission day (healthdata.gov `date` - 1). +# NHSN's week ending Saturday S equals the sum over admission days S-7..S-1 +# (Sat..Fri, i.e. collection dates Sun..Sat); other alignments match far worse. +# Output time_value is the canonical Wednesday label, S - 3. +# - Versions: every daily issue is a version. A week's value as of v is the sum of +# its 7 daily values as of v; a week is only emitted once all 7 days exist. +# - US: the hhs `nation` signal, which equals the sum of all states incl. as/pr/vi. + +suppressPackageStartupMessages(source(here::here("R", "load_all.R"))) + +args <- commandArgs(trailingOnly = TRUE) +out_dir <- if (length(args) >= 1) args[[1]] else file.path(tempdir(), "hhs_2023_24") +dir.create(out_dir, recursive = TRUE, showWarnings = FALSE) +raw_cache <- file.path(out_dir, "hhs_daily_raw.rds") + +signals <- c(flu = "confirmed_admissions_influenza_1d", covid = "confirmed_admissions_covid_1d") +time_range <- epidatr::epirange(20230701, 20240531) + +# ---- Fetch daily data with all issues ---- +if (file.exists(raw_cache)) { + daily <- readRDS(raw_cache) +} else { + daily <- tidyr::expand_grid(disease = names(signals), geo_type = c("state", "nation")) %>% + purrr::pmap(\(disease, geo_type) { + epidatr::pub_covidcast( + "hhs", signals[[disease]], geo_type, "day", "*", + time_values = time_range, issues = "*" + ) %>% + transmute(disease = disease, geo_value, time_value, version = issue, value) + }) %>% + bind_rows() + saveRDS(daily, raw_cache) +} +stopifnot(!anyNA(daily$value)) + +# Saturday week-ending for an hhs admission day: the Saturday after it. +week_ending_saturday <- function(admission_day) { + d <- admission_day + 1 + d + (6 - lubridate::wday(d, week_start = 7) + 1) %% 7 +} +stopifnot(week_ending_saturday(as.Date("2023-11-25")) == as.Date("2023-12-02")) # Sat -> next Sat +stopifnot(week_ending_saturday(as.Date("2023-12-01")) == as.Date("2023-12-02")) # Fri -> next day + +daily <- daily %>% mutate(week_end = week_ending_saturday(time_value)) + +# ---- Daily issues -> weekly versions ---- +versions <- sort(unique(daily$version)) +weekly_long <- purrr::map(versions, \(v) { + daily %>% + filter(version <= v) %>% + group_by(disease, geo_value, time_value) %>% + slice_max(version, n = 1, with_ties = FALSE) %>% + group_by(disease, geo_value, week_end) %>% + summarize(value = sum(value), n_days = n(), .groups = "drop") %>% + filter(n_days == 7L) %>% + mutate(version = v) +}) %>% + bind_rows() + +# Weeks that were incomplete at some version, for the record. +incomplete_seen <- purrr::map(versions, \(v) { + daily %>% + filter(version <= v) %>% + distinct(disease, geo_value, week_end, time_value) %>% + count(disease, geo_value, week_end) %>% + filter(n < 7L) %>% + mutate(version = v) +}) %>% + bind_rows() + +# Canonical shape: Saturday season info, then the Wednesday label. +archive_dt <- weekly_long %>% + rename(time_value = week_end) %>% + select(disease, geo_value, time_value, version, value) %>% + add_season_info() %>% + mutate(time_value = time_value - 3L, source = "hhs") + +archives <- purrr::map(c(flu = "flu", covid = "covid"), \(d) { + archive_dt %>% + filter(disease == d) %>% + select(-disease) %>% + as_epi_archive(other_keys = "source", compactify = TRUE) +}) +saveRDS(purrr::map(archives, \(a) a$DT), file.path(out_dir, "hhs_2023_24_weekly_archive.rds")) +nanoparquet::write_parquet( + purrr::imap(archives, \(a, d) as_tibble(a$DT) %>% mutate(disease = d)) %>% bind_rows(), + file.path(out_dir, "hhs_2023_24_weekly_archive.parquet") +) + +# ---- Validation against NHSN ---- +nhsn_final <- purrr::map(c("flu", "covid"), \(d) { + get_nhsn_data_archive(d) %>% + epix_as_of_current() %>% + as_tibble() %>% + transmute(disease = d, geo_value = ifelse(geo_value == "usa", "us", geo_value), time_value = time_value - 3L, nhsn = value) +}) %>% + bind_rows() + +hhs_final <- purrr::imap(archives, \(a, d) epix_as_of_current(a) %>% as_tibble() %>% mutate(disease = d)) %>% + bind_rows() %>% + select(disease, geo_value, time_value, hhs = value) + +cmp <- inner_join(hhs_final, nhsn_final, by = c("disease", "geo_value", "time_value")) %>% + mutate(diff = hhs - nhsn, ratio = hhs / nhsn) +in_season <- \(x) filter(x, time_value >= as.Date("2023-10-04"), time_value <= as.Date("2024-04-24")) + +overall <- cmp %>% + in_season() %>% + group_by(disease) %>% + summarize( + geo_weeks = n(), + exact = mean(diff == 0), + sum_ratio = sum(hhs) / sum(nhsn), + median_ratio = median(ratio, na.rm = TRUE), + median_abs_diff = median(abs(diff)), + .groups = "drop" + ) +per_geo <- cmp %>% + in_season() %>% + group_by(disease, geo_value) %>% + summarize( + weeks = n(), + exact = mean(diff == 0), + sum_ratio = sum(hhs) / sum(nhsn), + median_ratio = median(ratio, na.rm = TRUE), + max_abs_diff = max(abs(diff)), + nhsn_total = sum(nhsn), + .groups = "drop" + ) +missing_geos <- anti_join( + nhsn_final %>% in_season() %>% distinct(disease, geo_value), + hhs_final %>% distinct(disease, geo_value), + by = c("disease", "geo_value") +) + +# ---- Revisions ---- +rev <- purrr::imap(archives, \(a, d) as_tibble(a$DT) %>% mutate(disease = d)) %>% + bind_rows() %>% + in_season() %>% + group_by(disease, geo_value, time_value) %>% + arrange(version, .by_group = TRUE) %>% + summarize( + n_versions = n(), + first_version = first(version), + first_value = first(value), + final_value = last(value), + .groups = "drop" + ) %>% + mutate( + lag_days = as.integer(first_version - (time_value + 3L)), + rel_revision = (final_value - first_value) / pmax(final_value, 1) + ) +revision_summary <- rev %>% + group_by(disease, us = geo_value == "us") %>% + summarize( + geo_weeks = n(), + median_versions = median(n_versions), + max_versions = max(n_versions), + median_lag_days = median(lag_days), + median_abs_rel_rev = median(abs(rel_revision)), + p90_abs_rel_rev = quantile(abs(rel_revision), 0.9), + .groups = "drop" + ) + +readr::write_csv(cmp, file.path(out_dir, "validation_weekly.csv")) +readr::write_csv(per_geo, file.path(out_dir, "validation_per_geo.csv")) +readr::write_csv(rev, file.path(out_dir, "revisions.csv")) + +print(knitr::kable(overall, digits = 3)) +print(knitr::kable(per_geo %>% arrange(disease, abs(log(sum_ratio)) * -1) %>% group_by(disease) %>% slice_head(n = 8), digits = 3)) +print(knitr::kable(revision_summary, digits = 3)) +cat("NHSN geos with no hhs data:\n") +print(missing_geos) +cat("Week-versions withheld as incomplete:", nrow(distinct(incomplete_seen, disease, geo_value, week_end, version)), "\n") +cat("Output:", out_dir, "\n") From 328181f5d759e29e67fc96ff38e6b1ddb6498576 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:12:46 -0700 Subject: [PATCH 034/107] =?UTF-8?q?docs(roadmap):=20record=20evaluation=20?= =?UTF-8?q?scope-control=20results=20(1a=20steps=20a1=E2=80=93a2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notes/ROADMAP.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index da48a8d9..b09dc0f3 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -29,6 +29,21 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. diff against the edited run (changes should appear only at the substituted geo-dates), then all dates, timed. Feed it to the calibration harness. + - a1–a2 done 2026-10-01: `EVALUATION_FORECASTERS`, `EVALUATION_DATES`, + `EVALUATION_SUBSTITUTIONS=false` (AGENTS.md "Key env vars"); prod + manifests unchanged. Filtering out any ensemble component skips all + ensemble, submission, report and calibration targets. On vs off, + the snapshot differs exactly at the CSV cells (flu 2025-01-08, + 2025-02-12, 2026-01-28; covid 2025-02-19; control date + bit-identical). The forecasts move a lot at the substituted geos + and slightly (median ~0.1%, max ~6%) at every other geo on the + same date, because `windowed_seasonal` fits one model pooled + across geos. So the expected diff is "only on substituted dates", + not "only at substituted geos". Single-forecaster cost: ~40 + CPU-s per date plus ~4 min fixed overhead per run. + - Side finding: the h=−1 forecast never changes when a substitution + edits only the latest week. Worth checking that h=−1 is meant to + ignore that week. b. *2023-24 as a burn-in season.* The best configs so far (E05 `warm + single`, REF-op) warm-start from a burn-in season, and the replay has none. Delphi's `hhs` source From 4beb2145c578b140b33fa8348af5276577a1bbda Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:37:20 -0700 Subject: [PATCH 035/107] docs(roadmap): record the clean windowed_seasonal replay (1a step a3) --- notes/ROADMAP.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index b09dc0f3..1dd80c5f 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -41,6 +41,14 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. across geos. So the expected diff is "only on substituted dates", not "only at substituted geos". Single-forecaster cost: ~40 CPU-s per date plus ~4 min fixed overhead per run. + - a3 done 2026-10-01: `EVALUATION_FORECASTERS=windowed_seasonal + EVALUATION_SUBSTITUTIONS=false make eval-{flu,covid}` into the + local stores (not pushed). Flu 11.5 min, covid 9 min, 0 errors, 98 + dates (2024-11-20 to 2026-09-30), all 53 geos, no NAs. Forecasts + are in `forecast_nhsn_full`. Gaps: `local_scores_nhsn` never + scores `us` (52 of 53 geos, likely a geo-name mismatch with + `nhsn_latest_data`); NSSP inputs drop `mo` on 81 dates, `wy` on 28 + and `nh` on 5, for both diseases. - Side finding: the h=−1 forecast never changes when a substitution edits only the latest week. Worth checking that h=−1 is meant to ignore that week. From 1bc0a8adad681090217bf4bb2756f064a7a646f0 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:41:39 -0700 Subject: [PATCH 036/107] feat(calibration): harness configs for windowed_seasonal (flu, covid) on the clean evaluation replay, with store reader and hub adapters --- scripts/calibration/calibration_harness.R | 149 +++++++++++++++++++--- 1 file changed, 133 insertions(+), 16 deletions(-) diff --git a/scripts/calibration/calibration_harness.R b/scripts/calibration/calibration_harness.R index e973c091..8fdb0c05 100644 --- a/scripts/calibration/calibration_harness.R +++ b/scripts/calibration/calibration_harness.R @@ -17,6 +17,10 @@ # submitted ensemble) renormalizes to ~86% windowed_seasonal_extra_sources, # ~14% windowed_seasonal, ~0.03% climate_linear. # +# Other targets: ch_use("flu_windowed_seasonal") or ch_use("covid_windowed_seasonal") +# switch every CH_* setting to that forecaster's clean evaluation replay (see +# CH_CONFIGS); ch_read_store() then reads its forecasts instead of backfilling. +# # Usage, from the repo root: # source("scripts/calibration/calibration_harness.R") # inp <- ch_inputs() # archives, geos dropped @@ -72,6 +76,86 @@ CH_SPEC <- list( join_extra_data = TRUE, sort_quantiles = TRUE ) +# What the store's forecasts were built with, for ch_verify_against_store(): the +# substitutions csv (NULL for a replay run with EVALUATION_SUBSTITUTIONS=false) +# and the grid row's output-level `excluded_geos`. +CH_STORE_SUBSTITUTIONS <- here::here("pipelines/covid_data_substitutions.csv") +CH_EXCLUDED_GEOS <- c("mo", "wy") +# Holiday/outage weeks where the evaluation replay ran after the nominal date, +# from the schedule in pipelines/_hosp_prod.R. +CH_SCHEDULE_DELAYS <- c( + "2024-12-25" = "2024-12-26", + "2025-01-01" = "2025-01-02", + "2025-12-24" = "2025-12-29" +) + +# Alternative targets for ch_use(). Each entry overrides the CH_* globals above +# (names are the globals without the prefix); spec values are copied from the +# grid row in pipelines/_hosp_prod.R, split as make_forecaster_grid() +# splits it. The replays they point at were run with substitutions off, and +# windowed_seasonal excludes no geos, so nothing is dropped. +CH_CONFIGS <- list( + flu_windowed_seasonal = list( + STORE = "flu_hosp_evaluation", + DISEASE = "flu", + TARGET = "wk inc flu hosp", + FORECASTER_ID = "windowed_seasonal", + AHEADS = -1:3, + DROP_GEOS = character(0), + EXCLUDED_GEOS = NULL, + STORE_SUBSTITUTIONS = NULL, + SCHEDULE_DELAYS = c("2024-11-20" = "2024-11-21", CH_SCHEDULE_DELAYS), + SPEC = list( + forecaster = "scaled_pop_seasonal", + params = list( + outcome = "value", + trainer = epipredict::quantile_reg(), + seasonal_method = "window", + pop_scaling = FALSE, + lags = c(0, 7), + keys_to_ignore = list(list(c("source"), c("flusurv", "ILI+"))) + ), + ahead_multiplier = 7L, + target_date_shift = 3L, + join_extra_data = FALSE, + sort_quantiles = TRUE + ) + ), + covid_windowed_seasonal = list( + STORE = "covid_hosp_evaluation", + DISEASE = "covid", + TARGET = "wk inc covid hosp", + FORECASTER_ID = "windowed_seasonal", + AHEADS = -1:3, + DROP_GEOS = character(0), + EXCLUDED_GEOS = NULL, + STORE_SUBSTITUTIONS = NULL, + SCHEDULE_DELAYS = CH_SCHEDULE_DELAYS, + SPEC = list( + forecaster = "scaled_pop_seasonal", + params = list( + outcome = "value", + trainer = epipredict::quantile_reg(), + seasonal_method = "none", + drop_non_seasons = TRUE, + pop_scaling = FALSE, + lags = c(0, 7) + ), + ahead_multiplier = 7L, + target_date_shift = 3L, + join_extra_data = FALSE, + sort_quantiles = TRUE + ) + ) +) + +#' Point every CH_* setting at one entry of CH_CONFIGS. +ch_use <- function(name) { + config <- CH_CONFIGS[[rlang::arg_match(name, names(CH_CONFIGS))]] + for (nm in names(config)) assign(paste0("CH_", nm), config[[nm]], envir = globalenv()) + cli::cli_alert_info("Harness target: {.field {CH_FORECASTER_ID}} ({CH_DISEASE}), store {.path {CH_STORE}}") + invisible(config) +} # ================================ INPUTS ================================== @@ -138,13 +222,7 @@ ch_truth_asof <- function(truth_archive, version) { #' than as the two parallel seq.Date() vectors prod uses. That is exactly #' equivalent for the dates prod covers but can't silently misalign the two #' columns if the window is changed. -ch_schedule <- function(through, from = as.Date("2024-11-20")) { - # Holiday/outage weeks where the forecast actually ran after its nominal date. - delays <- c( - "2024-12-25" = "2024-12-26", - "2025-01-01" = "2025-01-02", - "2025-12-24" = "2025-12-29" - ) +ch_schedule <- function(through, from = as.Date("2024-11-20"), delays = CH_SCHEDULE_DELAYS) { through <- as.Date(through) # Back up to the last Wednesday so the weekly sequence stays on schedule. through <- through - ((as.integer(format(through, "%u")) - 3) %% 7) @@ -227,7 +305,7 @@ ch_backfill <- function( params = CH_SPEC$params, id = CH_FORECASTER_ID, target_date_shift = CH_SPEC$target_date_shift, - join_extra_data = CH_SPEC$join_extra_data, + join_extra_data = CH_SPEC$join_extra_data %||% FALSE, extra_data = extra, excluded_geos = excluded_geos, sort_quantiles = CH_SPEC$sort_quantiles @@ -272,6 +350,42 @@ ch_backfill <- function( } +#' Read this forecaster's forecasts straight from the store's `forecast_nhsn_full`. +#' +#' Same shape as ch_backfill() output. Only equivalent to a backfill when the +#' store was built the way the backfill runs (CH_STORE_SUBSTITUTIONS NULL, no +#' DROP_GEOS), which is the case for the CH_CONFIGS replays; +#' ch_verify_against_store() checks the plumbing on a few dates. +ch_read_store <- function(store = CH_STORE, aheads = CH_AHEADS) { + fc <- targets::tar_read(forecast_nhsn_full, store = store) %>% + filter(forecaster == CH_FORECASTER_ID) %>% + mutate(ahead = as.integer(target_end_date - forecast_date - CH_SPEC$target_date_shift) %/% + CH_SPEC$ahead_multiplier) %>% + filter(ahead %in% aheads) + if (nrow(fc) == 0) cli::cli_abort("No {.field {CH_FORECASTER_ID}} forecasts in {.path {store}}.") + fc %>% + select(geo_value, forecast_date, target_end_date, quantile, value, ahead) %>% + left_join(ch_schedule(through = max(fc$forecast_date)), by = "forecast_date") %>% + arrange(forecast_date, ahead, geo_value, quantile) +} + +#' Harness forecasts in hub schema, for calibrate_hub_forecasts() and cal_run(). +ch_to_hub <- function(forecasts, disease = CH_DISEASE) { + out <- internal_to_hub_forecasts(forecasts, disease) + if (anyNA(out$location)) cli::cli_abort("Some geos have no hub location code.") + out +} + +#' Finalized truth and every vintage of the store's NHSN archive, on hub +#' coordinates: the `truth` and `vintages` arguments of cal_run(). +ch_hub_truth <- function(inputs, disease = CH_DISEASE) { + list( + truth = nhsn_read_truth(disease, inputs$truth), + vintages = nhsn_read_vintages(disease, inputs$truth) + ) +} + + # ============================== CALIBRATION =============================== #' Placeholder online calibrator: an additive per-(geo, ahead, quantile) offset @@ -422,8 +536,10 @@ ch_summarize <- function(scores) { #' Forecast dates for which the store has a cached prod forecast target. ch_cached_prod_dates <- function(store = CH_STORE) { + # The date must follow the id directly, so `windowed_seasonal` does not also + # match `windowed_seasonal_extra_sources`. prefix <- glue::glue("forecast_nhsn_{CH_FORECASTER_ID}_") - list.files(file.path(store, "objects"), pattern = glue::glue("^{prefix}")) %>% + list.files(file.path(store, "objects"), pattern = glue::glue("^{prefix}\\d{{4}}\\.")) %>% stringr::str_extract("\\d{4}\\.\\d{2}\\.\\d{2}") %>% unique() %>% stats::na.omit() %>% @@ -445,7 +561,9 @@ ch_cached_prod_dates <- function(store = CH_STORE) { #' configuration (all geos in the archive, prod's substitutions, output-level geo #' exclusion) over a couple of the cached dates and requires exact equality #' there. What it proves is that the plumbing is identical; the configured -#' deviations are then the only difference that remains. +#' deviations are then the only difference that remains. The store's own +#' substitutions and output-level exclusions come from CH_STORE_SUBSTITUTIONS +#' and CH_EXCLUDED_GEOS. ch_verify_against_store <- function(store = CH_STORE, n_dates = 2, tolerance = 0) { dates <- ch_cached_prod_dates(store) if (length(dates) == 0) { @@ -455,14 +573,13 @@ ch_verify_against_store <- function(store = CH_STORE, n_dates = 2, tolerance = 0 cli::cli_alert_info("Verifying against prod on {.val {format(dates)}}") faithful <- ch_inputs(store = store, drop_geos = character(0)) - # Prod's aheads are -1:3; overlap on the non-negative ones is enough, and the - # harness does not model ahead -1 (see CH_AHEADS). + # Prod's aheads are -1:3; compare on the ones the harness also models. mine <- ch_backfill( faithful, - aheads = intersect(CH_AHEADS, 0:3), + aheads = intersect(CH_AHEADS, -1:3), schedule = ch_schedule(through = max(dates)) %>% filter(forecast_date %in% dates), - substitutions = here::here("pipelines/covid_data_substitutions.csv"), - excluded_geos = CH_DROP_GEOS, + substitutions = CH_STORE_SUBSTITUTIONS, + excluded_geos = CH_EXCLUDED_GEOS, cache_dir = file.path(CH_CACHE_DIR, "verify") ) @@ -484,7 +601,7 @@ ch_verify_against_store <- function(store = CH_STORE, n_dates = 2, tolerance = 0 cli::cli_abort( "Harness diverges from the prod DAG: max abs diff {.val {worst}} over {.val {nrow(joined)}} overlapping rows. CH_SPEC is out of sync with - scripts/covid_hosp_prod.R, or the shared runner changed." + pipelines/{CH_DISEASE}_hosp_prod.R, or the shared runner changed." ) } cli::cli_alert_success( From f4c315cdd9cda50e62ac6808fab7ad77f611ff82 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:54:03 -0700 Subject: [PATCH 037/107] feat(calibration): calibrate the clean windowed_seasonal replay (flu, covid) with no-burn-in configs --- scripts/calibration/calibration_ws_replay.R | 128 ++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 scripts/calibration/calibration_ws_replay.R diff --git a/scripts/calibration/calibration_ws_replay.R b/scripts/calibration/calibration_ws_replay.R new file mode 100644 index 00000000..d4723962 --- /dev/null +++ b/scripts/calibration/calibration_ws_replay.R @@ -0,0 +1,128 @@ +# Calibrate the clean evaluation replay of `windowed_seasonal` (no data +# substitutions, ROADMAP 1a) for flu and covid, and print base vs calibrated WIS +# and coverage per season and horizon as markdown tables. +# +# Forecasts come straight from the evaluation stores through the harness +# (ch_use() + ch_read_store()); truth and vintages from the same store's NHSN +# archive. Rounds are the hub's CMU-TimeSeries submission rounds, so the round +# axis matches the hub-based experiments in notes/CALIBRATION.md. No config +# here uses a burn-in season (none exists before 2024-11-20 in the replay). +# +# Usage: distrobox enter rocker -- Rscript scripts/calibration/calibration_ws_replay.R [workers] + +source(here::here("scripts/calibration/calibration_harness.R")) + +args <- commandArgs(trailingOnly = TRUE) +WORKERS <- if (length(args) >= 1) as.integer(args[[1]]) else 12L +HUB_DIRS <- c(flu = "../FluSight-forecast-hub", covid = "../covid19-forecast-hub") +LIVE <- c("2024-2025", "2025-2026") + +# Seasons are August-July years, as in E10 (covid has no off-season gap). +season_of <- function(d) { + y <- as.integer(format(d, "%Y")) - (as.integer(format(d, "%m")) < 8) + paste0(y, "-", y + 1) +} + +# E11's constant rate (0.1 admissions per 100k per step, rate scale) and the +# covid prod config (REF-op without its warm start), both with no burn-in. +ws_configs <- function(rate_scales) { + no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) + list( + `E11 constant 0.1` = c(list(ref = "paper", scales = rate_scales, lr = 0.1), no_burn_in), + `REF-op cold` = c(list(ref = "op"), no_burn_in) + ) +} + +ws_run_disease <- function(disease) { + ch_use(paste0(disease, "_windowed_seasonal")) + inp <- ch_inputs() + tv <- ch_hub_truth(inp) + hub_rounds <- suppressMessages(hub_read_forecasts( + hub_dir = here::here(HUB_DIRS[[disease]]), + target = c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET)[[disease]] + )) %>% distinct(reference_date) %>% pull() + fc <- ch_to_hub(ch_read_store(), disease) %>% filter(reference_date %in% hub_rounds) + cli::cli_alert_info("{disease}: {n_distinct(fc$reference_date)} rounds, {n_distinct(fc$location)} locations") + rate_scales <- get_population_data() %>% + distinct(state_code, .keep_all = TRUE) %>% + filter(state_code %in% unique(fc$location)) %>% + transmute(location = state_code, from = as.Date("2000-01-01"), scale = population / 1e5) + cals <- purrr::map(ws_configs(rate_scales), function(cfg) { + t0 <- Sys.time() + cal <- do.call(cal_run_cached, c(list(fc, tv$truth), cfg, list(learn = "exact", vintages = tv$vintages, workers = WORKERS))) + cli::cli_alert_success("run took {format(round(Sys.time() - t0, 1))}") + cal$forecasts %>% mutate(season = season_of(reference_date)) + }) + list(cals = cals, truth = tv$truth) +} + +#' Base vs calibrated WIS (2 x mean pinball), 50%/90% interval coverage and L1 +#' coverage bias, per (season, horizon); season "both" pools the live seasons. +ws_scores <- function(fc) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + fc <- fc %>% filter(!is.na(truth), !is.na(value_base), season %in% LIVE) + fc <- bind_rows(fc, fc %>% mutate(season = "both")) + long <- fc %>% + tidyr::pivot_longer(c(value_base, value_cal), names_to = "which", values_to = "q") %>% + mutate(which = sub("value_", "", which)) + wis <- long %>% + group_by(season, horizon, which) %>% + summarize(wis = 2 * mean(pinball(truth, q, level)), .groups = "drop") + cov_int <- long %>% + filter(level %in% c(0.05, 0.25, 0.75, 0.95)) %>% + select(season, horizon, which, location, reference_date, truth, level, q) %>% + tidyr::pivot_wider(names_from = level, values_from = q, names_prefix = "q") %>% + group_by(season, horizon, which) %>% + summarize( + cov50 = mean(truth >= q0.25 & truth <= q0.75), + cov90 = mean(truth >= q0.05 & truth <= q0.95), + .groups = "drop" + ) + bias <- long %>% + group_by(season, horizon, which, level) %>% + summarize(gap = mean(truth <= q) - first(level), .groups = "drop") %>% + group_by(season, horizon, which) %>% + summarize(cal_err = mean(abs(gap)), .groups = "drop") + above <- fc %>% + filter(level == 0.5) %>% + group_by(season, horizon) %>% + summarize(above_median = mean(truth > value_base), .groups = "drop") + wis %>% + left_join(cov_int, by = c("season", "horizon", "which")) %>% + left_join(bias, by = c("season", "horizon", "which")) %>% + left_join(above, by = c("season", "horizon")) +} + +ws_report <- function(disease, res) { + scores <- purrr::imap(res$cals, function(fc, nm) ws_scores(fc) %>% mutate(config = nm)) %>% bind_rows() + base <- scores %>% filter(which == "base", config == names(res$cals)[1]) %>% mutate(config = "base") + tbl <- bind_rows(base, scores %>% filter(which == "cal")) %>% + group_by(season, horizon) %>% + mutate(wis_pct = 100 * (wis[config == "base"] - wis) / wis[config == "base"]) %>% + ungroup() %>% + mutate(config = factor(config, levels = c("base", names(res$cals))), season = factor(season, levels = c(LIVE, "both"))) + readr::write_csv(tbl, file.path(CH_CACHE_DIR, glue::glue("ws_replay_scores_{disease}.csv"))) + wide <- function(col, digits) { + tbl %>% + transmute(season, config, horizon, v = round(.data[[col]], digits)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = v, names_prefix = "h") %>% + arrange(season, config) + } + cat("\n\n## ", disease, "\n", sep = "") + for (spec in list( + c("wis", "WIS (2 x mean pinball)", 1), c("wis_pct", "WIS change % vs base (positive is better)", 1), + c("cov50", "50% interval coverage", 3), c("cov90", "90% interval coverage", 3), + c("cal_err", "L1 coverage bias", 3) + )) { + cat("\n### ", spec[2], "\n\n", sep = "") + print(knitr::kable(wide(spec[1], as.integer(spec[3])))) + } + cat("\n### Fraction of truth above the base median\n\n") + print(knitr::kable(tbl %>% filter(config == "base") %>% transmute(season, horizon, p = round(above_median, 2)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = p, names_prefix = "h"))) + invisible(tbl) +} + +if (sys.nframe() == 0L) { + for (disease in c("flu", "covid")) ws_report(disease, ws_run_disease(disease)) +} From a312c9315565e8f55acec71ed609ae7427dbe36e Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:55:08 -0700 Subject: [PATCH 038/107] docs(calibration): record E12, cold calibration of the clean windowed_seasonal replay (1a step a4) --- notes/CALIBRATION.md | 28 ++++++++++++++++++++++++++++ notes/ROADMAP.md | 9 +++++++++ 2 files changed, 37 insertions(+) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index b64fa249..d403e619 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -113,6 +113,34 @@ E10's rounds is 100% the `climate_linear` ensemble (`revision_aware` took over h−1 only on 2026-09-23), gains +10.2% WIS (REF-paper +11.4%). REF-op's losses come from 2025-26 and from September–October. Candidate change: calibrate covid at h−1 (maybe h0) only. +## Clean `windowed_seasonal` replay, no burn-in (E12, 2026-10-01) + +`scripts/calibration/calibration_ws_replay.R` with `ch_use("{flu,covid}_windowed_seasonal")`, +forecasts read from the evaluation store's clean replay (substitutions off, all +53 geos, aheads −1..3; harness backfill matches it exactly), exact learning +truth, hub rounds only (flu 55, covid 89). This changes several things at +once compared with E07-B (no burn-in, clean replay, store truth, all geos), +so it is not a one-axis comparison. Pooled WIS change, h−1..h3: + +| disease | config | h−1 | h0 | h1 | h2 | h3 | +|---|---|---|---|---|---|---| +| flu | E11 constant rate | −0.3 | +1.1 | −0.4 | −0.9 | −1.2 | +| flu | REF-op cold | +1.0 | +2.9 | +1.1 | +0.5 | 0.0 | +| covid | E11 constant rate | +1.4 | +0.5 | −4.9 | −6.9 | −5.1 | +| covid | REF-op cold | +2.5 | +2.8 | −1.3 | −2.6 | −2.3 | + +- Flu base bias matches E07-B's `windowed_seasonal` (share of truth above + the median to within 0.01), so the substitutions barely move the + aggregate bias. +- Flu E11 cuts coverage bias 4–5× (L1 ~0.02) at a ~1% WIS cost, but + under-covers the 50% interval (0.45–0.48 at h0–h3). Check that before + adopting it cold. +- Covid: gains at h−1/h0, losses at h1–h3, worst in 2025-26, the same + season pattern as E10. Supports calibrating covid at h−1/h0 only. +- Per-season tables: `cache/calibration/ws_replay_scores_{flu,covid}.csv`. + Rerun with `burn_in_seasons = "2023-2024"` once the HHS burn-in exists + (ROADMAP 1b). + ## Data FluSight CMU-TimeSeries submissions: 83 rounds (2023-10-14 … 2026-05-30), 53 diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index 1dd80c5f..b5a140ae 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -49,6 +49,15 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. scores `us` (52 of 53 geos, likely a geo-name mismatch with `nhsn_latest_data`); NSSP inputs drop `mo` on 81 dates, `wy` on 28 and `nh` on 5, for both diseases. + - a4 done 2026-10-01: `ch_use("{flu,covid}_windowed_seasonal")` in + `scripts/calibration/calibration_harness.R` reads forecasts from + the clean store (decision: read the store, backfill only for spec + changes; the two paths match exactly on 5 dates per disease). + Results are E12 in `notes/CALIBRATION.md`. The two diseases' + replays differ on 2024-11-20: flu generates on 11-21, covid on + 11-20 (NHSN's bad release; the hub-round filter drops that round). + Open: which aheads to calibrate for `windowed_seasonal` (h−1 is + its own here, which no submission uses). - Side finding: the h=−1 forecast never changes when a substitution edits only the latest week. Worth checking that h=−1 is meant to ignore that week. From 347872bd7cf0356e66e516edc816f4c753303abd Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 09:14:55 -0700 Subject: [PATCH 039/107] =?UTF-8?q?docs(calibration):=20calibrate=20window?= =?UTF-8?q?ed=5Fseasonal=20at=20h0=E2=80=93h3=20only;=20note=20the=20h?= =?UTF-8?q?=E2=88=921=20substitution=20gap=20as=20low=20priority?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notes/CALIBRATION.md | 21 +++++++++++---------- notes/ROADMAP.md | 15 +++++++++------ scripts/calibration/calibration_harness.R | 5 +++-- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index d403e619..ddb7cfbd 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -117,17 +117,18 @@ September–October. Candidate change: calibrate covid at h−1 (maybe h0) only. `scripts/calibration/calibration_ws_replay.R` with `ch_use("{flu,covid}_windowed_seasonal")`, forecasts read from the evaluation store's clean replay (substitutions off, all -53 geos, aheads −1..3; harness backfill matches it exactly), exact learning +53 geos; harness backfill matches it exactly), h0–h3 (h−1 is left to the +revision-aware methods), exact learning truth, hub rounds only (flu 55, covid 89). This changes several things at once compared with E07-B (no burn-in, clean replay, store truth, all geos), -so it is not a one-axis comparison. Pooled WIS change, h−1..h3: +so it is not a one-axis comparison. Pooled WIS change %, h0..h3: -| disease | config | h−1 | h0 | h1 | h2 | h3 | -|---|---|---|---|---|---|---| -| flu | E11 constant rate | −0.3 | +1.1 | −0.4 | −0.9 | −1.2 | -| flu | REF-op cold | +1.0 | +2.9 | +1.1 | +0.5 | 0.0 | -| covid | E11 constant rate | +1.4 | +0.5 | −4.9 | −6.9 | −5.1 | -| covid | REF-op cold | +2.5 | +2.8 | −1.3 | −2.6 | −2.3 | +| disease | config | h0 | h1 | h2 | h3 | +|---|---|---|---|---|---| +| flu | E11 constant rate | +1.1 | −0.4 | −0.9 | −1.2 | +| flu | REF-op cold | +2.9 | +1.1 | +0.5 | 0.0 | +| covid | E11 constant rate | +0.5 | −4.9 | −6.9 | −5.1 | +| covid | REF-op cold | +2.8 | −1.3 | −2.6 | −2.3 | - Flu base bias matches E07-B's `windowed_seasonal` (share of truth above the median to within 0.01), so the substitutions barely move the @@ -135,8 +136,8 @@ so it is not a one-axis comparison. Pooled WIS change, h−1..h3: - Flu E11 cuts coverage bias 4–5× (L1 ~0.02) at a ~1% WIS cost, but under-covers the 50% interval (0.45–0.48 at h0–h3). Check that before adopting it cold. -- Covid: gains at h−1/h0, losses at h1–h3, worst in 2025-26, the same - season pattern as E10. Supports calibrating covid at h−1/h0 only. +- Covid: gains at h0, losses at h1–h3, worst in 2025-26, the same + season pattern as E10. Supports calibrating covid at h0 only. - Per-season tables: `cache/calibration/ws_replay_scores_{flu,covid}.csv`. Rerun with `burn_in_seasons = "2023-2024"` once the HHS burn-in exists (ROADMAP 1b). diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index b5a140ae..a43ee16e 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -56,11 +56,9 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. Results are E12 in `notes/CALIBRATION.md`. The two diseases' replays differ on 2024-11-20: flu generates on 11-21, covid on 11-20 (NHSN's bad release; the hub-round filter drops that round). - Open: which aheads to calibrate for `windowed_seasonal` (h−1 is - its own here, which no submission uses). - - Side finding: the h=−1 forecast never changes when a substitution - edits only the latest week. Worth checking that h=−1 is meant to - ignore that week. + Decision (2026-10-01): calibrate h0 and up only. h−1 belongs to + the revision-aware methods (item 3) and gets integrated with them + later. b. *2023-24 as a burn-in season.* The best configs so far (E05 `warm + single`, REF-op) warm-start from a burn-in season, and the replay has none. Delphi's `hhs` source @@ -191,4 +189,9 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. 7. renv warns on every run (library renv 1.1.6, lockfile 1.2.3; some lockfile packages not installed). Harmless so far. -8. Refactor and cleanup threads: `notes/refactor-ideas.md`. +8. Low priority: `windowed_seasonal`'s h−1 forecast ignores the latest + reported week, so a data substitution to that week never reaches h−1. + Most substitutions are to that week (flu 20 of 25 rows, covid 10 of + 10). Submitted h−1 comes from other components, so nothing shipped is + affected; revisit with the revision-aware h−1 work (item 3). +9. Refactor and cleanup threads: `notes/refactor-ideas.md`. diff --git a/scripts/calibration/calibration_harness.R b/scripts/calibration/calibration_harness.R index 8fdb0c05..858cb5d7 100644 --- a/scripts/calibration/calibration_harness.R +++ b/scripts/calibration/calibration_harness.R @@ -94,13 +94,14 @@ CH_SCHEDULE_DELAYS <- c( # grid row in pipelines/_hosp_prod.R, split as make_forecaster_grid() # splits it. The replays they point at were run with substitutions off, and # windowed_seasonal excludes no geos, so nothing is dropped. +# windowed_seasonal is calibrated at h0 and up; h−1 is left to the revision-aware methods. CH_CONFIGS <- list( flu_windowed_seasonal = list( STORE = "flu_hosp_evaluation", DISEASE = "flu", TARGET = "wk inc flu hosp", FORECASTER_ID = "windowed_seasonal", - AHEADS = -1:3, + AHEADS = 0:3, DROP_GEOS = character(0), EXCLUDED_GEOS = NULL, STORE_SUBSTITUTIONS = NULL, @@ -126,7 +127,7 @@ CH_CONFIGS <- list( DISEASE = "covid", TARGET = "wk inc covid hosp", FORECASTER_ID = "windowed_seasonal", - AHEADS = -1:3, + AHEADS = 0:3, DROP_GEOS = character(0), EXCLUDED_GEOS = NULL, STORE_SUBSTITUTIONS = NULL, From 0c1e53aae0f37cf5db3bfabc26f56a564392877e Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 09:26:48 -0700 Subject: [PATCH 040/107] docs(roadmap): record the US-handling audit and a single-policy design --- notes/ROADMAP.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index a43ee16e..9581e481 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -146,6 +146,41 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. component is `windowed_seasonal_latest` (mean WIS 0.11), `CMU-TimeSeries` 0.14, the CovidHub ensemble 0.06. +2b. **US is handled several different ways; make it one declared policy** + (audited 2026-10-01). + - `score_forecasts()` drops location `US` on purpose + (`R/targets/score_targets.R:84`), so every local, external and + calibrated NHSN score omits US. Behind that, its location → geo join + goes through `get_population_data()`, which has both `us` and an + `usa` alias for code `US`, so dropping only the filter double-counts + US. The calibrated-forecast joins in `prod_shared.R` and both prod + pipelines have the same latent duplicate. A patch (one-row-per-location + `hub_location_crosswalk()`, used in all those joins) is in + `_local/patches/us_scoring.patch`, checked on 2 dates per disease. It + roughly doubles cross-geo mean WIS (US WIS ≈ sum of states), so ship it + with a `geo_level` column and state-only means in the reports. + - Submissions are not affected: US is present in all covid files and + all flu files since 2024 (US ÷ sum of states median ratio 1.00 flu, + 1.03 covid). But flu 2025-11-22 to 12-06 had US at 0.58–0.71 of the + state sum at h2–h3, and nothing checks that. + - `climate_geo_agged` pools US into the state climatology unscaled, so + CMU-climate_baseline's US is about state-sized relative to the pool + (submitted US 0.5–0.9 of the state sum). + - Flu explore: 4 `scaled_pop_data_augmented` forecasters + (`filter_agg_level = ""`) forecast and score US; the other 129 drop it. + The comparison notebook ranks them by mean WIS, so these 4 look worst. + Covid explore drops US everywhere. + - Proposed design: one canonical `"us"` from archive build on, with one + crosswalk for every geo ↔ location join (then delete the `usa` alias + and the redundant downstream renames). A spec column `us_method` + (`direct` default, `sum_of_states`, `none`) in + `FORECASTER_SPEC_DEFAULTS` and the ensemble specs, applied once in + `run_forecaster()`, replacing the data-dependent `filter_agg_level` + drop. Contracts: `validate_forecast_output()` requires `us` unless + `us_method = "none"`; prod health warns when the US median is outside + about 0.8–1.25 × the sum of state medians; `score_forecasts()` never + drops a location silently and has unique keys. + ## Revision-aware forecasting and h−1 3. **Revision-method backtests.** `revision_aware` against the From e65c268f62285c749a258c28c28713e81f612830 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:55:25 -0700 Subject: [PATCH 041/107] feat(hhs-archive): optional start date and version floor for the HHS weekly archive --- scripts/one_offs/hhs_2023_24_archive.R | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/one_offs/hhs_2023_24_archive.R b/scripts/one_offs/hhs_2023_24_archive.R index e6aafc75..6eed4c65 100644 --- a/scripts/one_offs/hhs_2023_24_archive.R +++ b/scripts/one_offs/hhs_2023_24_archive.R @@ -2,7 +2,11 @@ # daily `hhs` source, for use as a burn-in season ahead of the NHSN-era replay, # and validate its finalized values against NHSN's finalized 2023-24 values. # -# Usage: Rscript scripts/one_offs/hhs_2023_24_archive.R [out_dir] +# Usage: Rscript scripts/one_offs/hhs_2023_24_archive.R [out_dir] [start] [versions_from] +# `start` (default 20230701) is the first admission day fetched; the 2023-24 +# burn-in uses 20200801 so a 2023-24 snapshot sees earlier seasons' history. +# Issues before `versions_from` (default 2023-07-01) collapse into one version +# at that date, which keeps the per-version loop small. # Writes hhs_2023_24_weekly_archive.rds (epi_archive DT, both diseases), # hhs_2023_24_weekly_archive.parquet and validation CSVs into out_dir. # @@ -23,7 +27,9 @@ dir.create(out_dir, recursive = TRUE, showWarnings = FALSE) raw_cache <- file.path(out_dir, "hhs_daily_raw.rds") signals <- c(flu = "confirmed_admissions_influenza_1d", covid = "confirmed_admissions_covid_1d") -time_range <- epidatr::epirange(20230701, 20240531) +start <- if (length(args) >= 2) args[[2]] else "20230701" +versions_from <- as.Date(if (length(args) >= 3) args[[3]] else "2023-07-01") +time_range <- epidatr::epirange(as.integer(start), 20240531) # ---- Fetch daily data with all issues ---- if (file.exists(raw_cache)) { @@ -40,7 +46,9 @@ if (file.exists(raw_cache)) { bind_rows() saveRDS(daily, raw_cache) } -stopifnot(!anyNA(daily$value)) +# Flu has missing days before mid-2021 (reporting was optional); a week with one +# becomes NA, as in NHSN. From 2023-07 on there must be none. +stopifnot(!anyNA(daily$value[daily$time_value >= as.Date("2023-07-01")])) # Saturday week-ending for an hhs admission day: the Saturday after it. week_ending_saturday <- function(admission_day) { @@ -53,7 +61,7 @@ stopifnot(week_ending_saturday(as.Date("2023-12-01")) == as.Date("2023-12-02")) daily <- daily %>% mutate(week_end = week_ending_saturday(time_value)) # ---- Daily issues -> weekly versions ---- -versions <- sort(unique(daily$version)) +versions <- sort(unique(pmax(daily$version, versions_from))) weekly_long <- purrr::map(versions, \(v) { daily %>% filter(version <= v) %>% From ecf3fb04bd30232a7a58087b0472afb45dd726f8 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:58:51 -0700 Subject: [PATCH 042/107] feat(calibration): stitch the HHS archive before NHSN for a 2023-24 burn-in in the harness --- notes/ROADMAP.md | 11 ++++++++++ scripts/calibration/calibration_harness.R | 26 +++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index 9581e481..f146cb85 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -220,6 +220,17 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. Fixing `drop_non_seasons` and `nonlin_method` alone cuts the family to about a quarter (flu ~19, covid ~13). +## Parked (2026-10-01, not on the calibration path; don't pick up until item 1 is settled) + +- Covid `windowed_seasonal_extra_sources` calibration (CALIBRATION open + thread 7). It is ~86% of covid `ensemble_mix` but has no 2023-24 NSSP, so + no burn-in. +- CMU-climate_baseline US forecast is too low (item 2b). +- `us_method` spec column and removing the `usa` alias (item 2b). +- Calibration WIS summaries pool raw counts over all locations, so US is + about half of every "WIS change %" in CALIBRATION.md. b4 reports states + only as well; re-reading the older ledger waits. + ## Tech debt 7. renv warns on every run (library renv 1.1.6, lockfile 1.2.3; some diff --git a/scripts/calibration/calibration_harness.R b/scripts/calibration/calibration_harness.R index 858cb5d7..2d8bdfd8 100644 --- a/scripts/calibration/calibration_harness.R +++ b/scripts/calibration/calibration_harness.R @@ -204,6 +204,32 @@ ch_inputs <- function(store = CH_STORE, drop_geos = CH_DROP_GEOS) { inputs } +#' ch_inputs() with the HHS weekly archive stitched in before NHSN's first version. +#' +#' HHS rows (from scripts/one_offs/hhs_2023_24_archive.R, run from 2020-08) are +#' relabelled `source = "nhsn"` and kept only at versions before NHSN's first, +#' so a 2023-24 snapshot sees HHS alone and later snapshots see only NHSN. +#' HHS levels are used as-is (ROADMAP 1b). +ch_inputs_with_burn_in <- function( + hhs_path = file.path(CH_CACHE_DIR, "hhs_weekly_archive_2020_2024.rds"), + inputs = ch_inputs() +) { + train <- as_tibble(inputs$nhsn_train$DT) + nhsn <- train %>% filter(source == "nhsn") + first_version <- min(nhsn$version) + hhs <- as_tibble(readRDS(hhs_path)[[CH_DISEASE]]) %>% + filter(version < first_version, geo_value %in% nhsn$geo_value) %>% + mutate(source = "nhsn") %>% + select(all_of(names(train))) + # Every HHS key must be overwritten at NHSN's first version, or HHS would leak past it. + stray <- anti_join(hhs, nhsn %>% filter(version == first_version), by = c("geo_value", "time_value")) + if (nrow(stray) > 0) cli::cli_abort("{nrow(stray)} HHS row{?s} have no NHSN row at {first_version}.") + inputs$nhsn_train <- bind_rows(train, hhs) %>% + as_epi_archive(other_keys = "source", compactify = TRUE) + cli::cli_alert_success("Stitched {nrow(hhs)} HHS rows before NHSN version {.val {format(first_version)}}") + inputs +} + #' Truth as it was published at `version`, on the forecasts' target_end_date key. #' #' This is the honest input for an online update at time `version`: it contains From 9816030514786633cbbc4f4f889606f78f6236f2 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:09:40 -0700 Subject: [PATCH 043/107] feat(calibration): burn-in configs and states/US score splits in the windowed_seasonal replay --- scripts/calibration/calibration_ws_replay.R | 90 +++++++++++++++------ 1 file changed, 67 insertions(+), 23 deletions(-) diff --git a/scripts/calibration/calibration_ws_replay.R b/scripts/calibration/calibration_ws_replay.R index d4723962..0d6e909c 100644 --- a/scripts/calibration/calibration_ws_replay.R +++ b/scripts/calibration/calibration_ws_replay.R @@ -5,15 +5,22 @@ # Forecasts come straight from the evaluation stores through the harness # (ch_use() + ch_read_store()); truth and vintages from the same store's NHSN # archive. Rounds are the hub's CMU-TimeSeries submission rounds, so the round -# axis matches the hub-based experiments in notes/CALIBRATION.md. No config -# here uses a burn-in season (none exists before 2024-11-20 in the replay). +# axis matches the hub-based experiments in notes/CALIBRATION.md. The E12 +# configs use no burn-in season. # -# Usage: distrobox enter rocker -- Rscript scripts/calibration/calibration_ws_replay.R [workers] +# With `burn_in`, warm-started configs also run with 2023-24 as a burn-in +# season: those rounds are backfilled weekly from 2023-10-11 to 2024-04-24 on +# the HHS-stitched archive (ch_inputs_with_burn_in(), ROADMAP 1b). Scores are +# reported over all locations, states only and US only. +# +# Usage: distrobox enter rocker -- Rscript scripts/calibration/calibration_ws_replay.R [workers] [burn_in] source(here::here("scripts/calibration/calibration_harness.R")) args <- commandArgs(trailingOnly = TRUE) WORKERS <- if (length(args) >= 1) as.integer(args[[1]]) else 12L +BURN_IN <- length(args) >= 2 && args[[2]] == "burn_in" +BURN_IN_DATES <- c(from = "2023-10-11", through = "2024-04-24") HUB_DIRS <- c(flu = "../FluSight-forecast-hub", covid = "../covid19-forecast-hub") LIVE <- c("2024-2025", "2025-2026") @@ -24,13 +31,32 @@ season_of <- function(d) { } # E11's constant rate (0.1 admissions per 100k per step, rate scale) and the -# covid prod config (REF-op without its warm start), both with no burn-in. -ws_configs <- function(rate_scales) { +# covid prod config (REF-op without its warm start), both with no burn-in. With +# `burn_in`, also E05's sqrt single term cold and warm, REF-op with its warm +# start, and E11's constant rate warm-started (CALIBRATION open thread 1). +ws_configs <- function(rate_scales, burn_in = BURN_IN) { no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) - list( + warm <- list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile") + sqrt_single <- list(ref = "paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) + cold <- list( `E11 constant 0.1` = c(list(ref = "paper", scales = rate_scales, lr = 0.1), no_burn_in), `REF-op cold` = c(list(ref = "op"), no_burn_in) ) + if (!burn_in) { + return(cold) + } + c(cold, list( + `E05 single cold` = c(sqrt_single, no_burn_in), + `E05 warm + single` = c(sqrt_single, warm), + `REF-op warm` = c(list(ref = "op"), warm), + `E11 constant 0.1 warm` = c(list(ref = "paper", scales = rate_scales, lr = 0.1), warm) + )) +} + +# 2023-24 burn-in rounds, backfilled on the HHS-stitched archive (cached). +ws_burn_in_forecasts <- function(disease) { + sched <- ch_schedule(through = BURN_IN_DATES[["through"]], from = as.Date(BURN_IN_DATES[["from"]])) + ch_to_hub(ch_backfill(ch_inputs_with_burn_in(), schedule = sched), disease) } ws_run_disease <- function(disease) { @@ -42,6 +68,8 @@ ws_run_disease <- function(disease) { target = c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET)[[disease]] )) %>% distinct(reference_date) %>% pull() fc <- ch_to_hub(ch_read_store(), disease) %>% filter(reference_date %in% hub_rounds) + # Every backfilled 2023-24 round is kept, not only hub rounds (covid has none). + fc_burn_in <- if (BURN_IN) bind_rows(ws_burn_in_forecasts(disease), fc) cli::cli_alert_info("{disease}: {n_distinct(fc$reference_date)} rounds, {n_distinct(fc$location)} locations") rate_scales <- get_population_data() %>% distinct(state_code, .keep_all = TRUE) %>% @@ -49,7 +77,8 @@ ws_run_disease <- function(disease) { transmute(location = state_code, from = as.Date("2000-01-01"), scale = population / 1e5) cals <- purrr::map(ws_configs(rate_scales), function(cfg) { t0 <- Sys.time() - cal <- do.call(cal_run_cached, c(list(fc, tv$truth), cfg, list(learn = "exact", vintages = tv$vintages, workers = WORKERS))) + fc_cfg <- if (length(cfg$burn_in_seasons) > 0) fc_burn_in else fc + cal <- do.call(cal_run_cached, c(list(fc_cfg, tv$truth), cfg, list(learn = "exact", vintages = tv$vintages, workers = WORKERS))) cli::cli_alert_success("run took {format(round(Sys.time() - t0, 1))}") cal$forecasts %>% mutate(season = season_of(reference_date)) }) @@ -93,33 +122,48 @@ ws_scores <- function(fc) { left_join(above, by = c("season", "horizon")) } +# Location subsets scored separately: raw-count pooling makes US about half of +# the all-locations WIS. +WS_GEOS <- list(all = \(loc) rep(TRUE, length(loc)), states = \(loc) loc != "US", us = \(loc) loc == "US") + ws_report <- function(disease, res) { - scores <- purrr::imap(res$cals, function(fc, nm) ws_scores(fc) %>% mutate(config = nm)) %>% bind_rows() + scores <- purrr::imap(WS_GEOS, function(keep, geos) { + purrr::imap(res$cals, function(fc, nm) ws_scores(fc %>% filter(keep(location))) %>% mutate(config = nm)) %>% + bind_rows() %>% + mutate(geos = geos) + }) %>% bind_rows() base <- scores %>% filter(which == "base", config == names(res$cals)[1]) %>% mutate(config = "base") tbl <- bind_rows(base, scores %>% filter(which == "cal")) %>% - group_by(season, horizon) %>% + group_by(geos, season, horizon) %>% mutate(wis_pct = 100 * (wis[config == "base"] - wis) / wis[config == "base"]) %>% ungroup() %>% - mutate(config = factor(config, levels = c("base", names(res$cals))), season = factor(season, levels = c(LIVE, "both"))) - readr::write_csv(tbl, file.path(CH_CACHE_DIR, glue::glue("ws_replay_scores_{disease}.csv"))) - wide <- function(col, digits) { + mutate( + config = factor(config, levels = c("base", names(res$cals))), season = factor(season, levels = c(LIVE, "both")), + geos = factor(geos, levels = names(WS_GEOS)) + ) + suffix <- if (BURN_IN) "_burn_in" else "" + readr::write_csv(tbl, file.path(CH_CACHE_DIR, glue::glue("ws_replay_scores{suffix}_{disease}.csv"))) + wide <- function(col, digits, g) { tbl %>% + filter(geos == g) %>% transmute(season, config, horizon, v = round(.data[[col]], digits)) %>% tidyr::pivot_wider(names_from = horizon, values_from = v, names_prefix = "h") %>% arrange(season, config) } - cat("\n\n## ", disease, "\n", sep = "") - for (spec in list( - c("wis", "WIS (2 x mean pinball)", 1), c("wis_pct", "WIS change % vs base (positive is better)", 1), - c("cov50", "50% interval coverage", 3), c("cov90", "90% interval coverage", 3), - c("cal_err", "L1 coverage bias", 3) - )) { - cat("\n### ", spec[2], "\n\n", sep = "") - print(knitr::kable(wide(spec[1], as.integer(spec[3])))) + for (g in names(WS_GEOS)) { + cat("\n\n## ", disease, ", ", g, " locations\n", sep = "") + for (spec in list( + c("wis", "WIS (2 x mean pinball)", 1), c("wis_pct", "WIS change % vs base (positive is better)", 1), + c("cov50", "50% interval coverage", 3), c("cov90", "90% interval coverage", 3), + c("cal_err", "L1 coverage bias", 3) + )) { + cat("\n### ", spec[2], "\n\n", sep = "") + print(knitr::kable(wide(spec[1], as.integer(spec[3]), g))) + } + cat("\n### Fraction of truth above the base median\n\n") + print(knitr::kable(tbl %>% filter(config == "base", geos == g) %>% transmute(season, horizon, p = round(above_median, 2)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = p, names_prefix = "h"))) } - cat("\n### Fraction of truth above the base median\n\n") - print(knitr::kable(tbl %>% filter(config == "base") %>% transmute(season, horizon, p = round(above_median, 2)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = p, names_prefix = "h"))) invisible(tbl) } From af0b95d4f49b2f3544946d81ddbe1feee95f5d0d Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:46:48 -0700 Subject: [PATCH 044/107] docs(calibration): record E13 (2023-24 HHS burn-in) and park the evaluation-testbed item --- notes/CALIBRATION.md | 24 ++++++++++++++++++++++++ notes/ROADMAP.md | 14 +++++++++----- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index ddb7cfbd..8be2da25 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -142,6 +142,30 @@ so it is not a one-axis comparison. Pooled WIS change %, h0..h3: Rerun with `burn_in_seasons = "2023-2024"` once the HHS burn-in exists (ROADMAP 1b). +## Clean replay with a 2023-24 HHS burn-in (E13, 2026-10-01) + +E12's setup plus 2023-24 as a burn-in (HHS data at real versions, back to +2020, stitched before NHSN in `ch_inputs_with_burn_in()`; run +`calibration_ws_replay.R 12 burn_in`). WIS change %, h0/h1/h2/h3, both +seasons pooled, states only (US excluded): + +| disease | config | cold | warm | +|---|---|---|---| +| flu | REF-op | 2.0/0.6/0.1/−0.2 | 2.8/1.1/0.9/1.1 | +| flu | E05 single | 1.9/−0.2/−1.1/−1.7 | 2.7/0.4/−0.1/−0.1 | +| flu | E11 constant 0.1 | 0.7/−0.2/−0.7/−1.0 | 1.0/0.0/−0.7/−1.1 | +| covid | REF-op | 1.3/−1.1/−1.7/−1.6 | −0.1/−1.8/−0.4/2.8 | +| covid | E11 constant 0.1 | −0.9/−4.3/−4.8/−3.4 | −2.9/−9.2/−13.2/−17.9 | + +- Flu: the warm start adds 0.5–1.6 points for REF-op and E05, but the + warm configs under-cover the 50% interval (0.40–0.44). +- Covid: 2023-24 over-predicted while the live seasons under-predict at + h0, so the warm start is wrong-signed there. E11 warm is badly harmful. +- Open thread 1 answered: a warm start doesn't help E11's constant rate + (flu) and hurts it (covid). +- US is ~45% of all-locations WIS, but states-only and US-only changes + point the same way. Tables: `cache/calibration/ws_replay_scores_burn_in_*.csv`. + ## Data FluSight CMU-TimeSeries submissions: 83 rounds (2023-10-14 … 2026-05-30), 53 diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index f146cb85..c4df841f 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -7,8 +7,10 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. ## Forecast evaluation -1. **Use the evaluation project as the calibration testbed** (current plan, - 2026-10-01). Calibrating needs full quantile forecasts over many rounds, +1. **Use the evaluation project as the calibration testbed** (parked + 2026-10-01 as validation for after the operating point is chosen; the + operating-point work runs on the hub harness, `notes/CALIBRATION.md`). + Done so far: a1–a4, b1–b4 (E12, E13). Calibrating needs full quantile forecasts over many rounds, not just scores. The evaluation project (`flu_hosp_evaluation` / `covid_hosp_evaluation`) replays the current prod components and ensemble weekly since 2024-11-20, which is what calibration would see in @@ -104,8 +106,10 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. or abort for every component that reads NSSP. NHSN versions start 2024-11-19, so the HHS archive must supply every 2023-24 target row; - - `windowed_seasonal` on 3 dates in 2023-24, fan plots against the - 2023-24 hub submissions; then all 2023-24 dates. + - Done 2026-10-01 in the harness, not the pipeline: HHS back to + 2020-08 (issues before 2023-07 collapse to one version), stitched + before NHSN's first version; a 2024-12-04 control matches the + store exactly. Results: E13 in `notes/CALIBRATION.md`. c. *The ensemble with fixed weights.* The geo-exclusions CSVs carry the ensemble weights as well as exclusions, so "no hand edits" for `ensemble_mix` means one fixed weight block for every date. Decide @@ -220,7 +224,7 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. Fixing `drop_non_seasons` and `nonlin_method` alone cuts the family to about a quarter (flu ~19, covid ~13). -## Parked (2026-10-01, not on the calibration path; don't pick up until item 1 is settled) +## Parked (2026-10-01, not on the calibration path) - Covid `windowed_seasonal_extra_sources` calibration (CALIBRATION open thread 7). It is ~86% of covid `ensemble_mix` but has no 2023-24 NSSP, so From a80dc6f8b4f2a5d04fdf9069ca035b47dce2c1a6 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:00:17 -0700 Subject: [PATCH 045/107] refactor(calibration): factor ws_inputs() out of the windowed_seasonal replay so notebooks can source it --- scripts/calibration/calibration_ws_replay.R | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/scripts/calibration/calibration_ws_replay.R b/scripts/calibration/calibration_ws_replay.R index 0d6e909c..3aea39a7 100644 --- a/scripts/calibration/calibration_ws_replay.R +++ b/scripts/calibration/calibration_ws_replay.R @@ -17,7 +17,8 @@ source(here::here("scripts/calibration/calibration_harness.R")) -args <- commandArgs(trailingOnly = TRUE) +# Command-line arguments apply only when run as a script, not when sourced by a notebook. +args <- if (sys.nframe() == 0L) commandArgs(trailingOnly = TRUE) else character(0) WORKERS <- if (length(args) >= 1) as.integer(args[[1]]) else 12L BURN_IN <- length(args) >= 2 && args[[2]] == "burn_in" BURN_IN_DATES <- c(from = "2023-10-11", through = "2024-04-24") @@ -59,7 +60,9 @@ ws_burn_in_forecasts <- function(disease) { ch_to_hub(ch_backfill(ch_inputs_with_burn_in(), schedule = sched), disease) } -ws_run_disease <- function(disease) { +#' The replay's hub-round forecasts, truth and vintages, and per-100k rate +#' scales for one disease. Also used by the E14/E15 notebooks. +ws_inputs <- function(disease) { ch_use(paste0(disease, "_windowed_seasonal")) inp <- ch_inputs() tv <- ch_hub_truth(inp) @@ -68,14 +71,21 @@ ws_run_disease <- function(disease) { target = c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET)[[disease]] )) %>% distinct(reference_date) %>% pull() fc <- ch_to_hub(ch_read_store(), disease) %>% filter(reference_date %in% hub_rounds) - # Every backfilled 2023-24 round is kept, not only hub rounds (covid has none). - fc_burn_in <- if (BURN_IN) bind_rows(ws_burn_in_forecasts(disease), fc) cli::cli_alert_info("{disease}: {n_distinct(fc$reference_date)} rounds, {n_distinct(fc$location)} locations") rate_scales <- get_population_data() %>% distinct(state_code, .keep_all = TRUE) %>% filter(state_code %in% unique(fc$location)) %>% transmute(location = state_code, from = as.Date("2000-01-01"), scale = population / 1e5) - cals <- purrr::map(ws_configs(rate_scales), function(cfg) { + list(fc = fc, truth = tv$truth, vintages = tv$vintages, rate_scales = rate_scales) +} + +ws_run_disease <- function(disease) { + wi <- ws_inputs(disease) + fc <- wi$fc + tv <- wi[c("truth", "vintages")] + # Every backfilled 2023-24 round is kept, not only hub rounds (covid has none). + fc_burn_in <- if (BURN_IN) bind_rows(ws_burn_in_forecasts(disease), fc) + cals <- purrr::map(ws_configs(wi$rate_scales), function(cfg) { t0 <- Sys.time() fc_cfg <- if (length(cfg$burn_in_seasons) > 0) fc_burn_in else fc cal <- do.call(cal_run_cached, c(list(fc_cfg, tv$truth), cfg, list(learn = "exact", vintages = tv$vintages, workers = WORKERS))) From 56baed1f1b49f1937fdcf5a15049324dd9eb482b Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:00:23 -0700 Subject: [PATCH 046/107] feat(calibration): E14 notebook, constant learning rate sweep split by season on the clean replay --- .../e14_lr_by_season.Rmd | 244 ++++++++++++++++++ .../calibration_experiments/index.Rmd | 1 + 2 files changed, 245 insertions(+) create mode 100644 reports/writeups/calibration_experiments/e14_lr_by_season.Rmd diff --git a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd new file mode 100644 index 00000000..28991639 --- /dev/null +++ b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd @@ -0,0 +1,244 @@ +--- +title: "E14: constant learning rate, season by season" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +E09 traced how coverage and WIS trade off as the constant learning rate of +the plain tracker changes, pooled over both live seasons. The question +here, from the collaborators: split those curves by season. **Is there a +range of learning rates that is good in both 2024-25 and 2025-26?** + +**Setup.** The clean `windowed_seasonal` replay for flu (E12: substitutions +off, all 53 locations, the 55 hub rounds, h0–h3; 27 rounds in 2024-25 and +28 in 2025-26), learning from data as published each week and re-run from +scratch each week as prod does, one week of extra revision delay (E09's +best re-run setting), no burn-in season. One axis: the constant learning +rate of the plain tracker on the rate scale (admissions per 100k per step), +12 values from 0.0032 to 1.8, a quarter decade apart. This is E11's +config with its learning rate swept; 0.1 is E11 itself. + +Reference points, run on the same data: + +- **REF-op cold**: prod's tracker without its warm start. +- **sqrt, adaptive**: the plain tracker on the square-root scale with the + adaptive learning rate (E03's winner; E13's "E05 single cold"). + +"WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive +is better** (the opposite sign of E09's "inflation"). "L1 coverage bias" is +the mean over the 23 levels of |empirical coverage − level|. Each season is +scored on its own rounds; the tracker still runs over both seasons, so +2025-26 starts from the offsets 2024-25 left behind. Results lead with states +only (US excluded), since US alone is about 45% of the all-locations WIS. + +# Findings + +Numbers are from the render of 1 October 2026, states only, WIS change % +(positive is better) / L1 coverage bias. All-locations numbers point the +same way. + +- **Yes: 0.018–0.032 per 100k is good in both seasons.** At those rates WIS + improves at every horizon in both seasons (2024-25 +0.2 to +1.0, 2025-26 + +0.8 to +1.9) and coverage bias falls in both. At 0.056 WIS is still + non-negative everywhere (2024-25 h3 −0.0). From 0.1 up WIS gets worse at + h1–h3 in both seasons, and past about 0.18 it falls fast. +- **The seasons disagree about coverage, not WIS.** In 2025-26 coverage + bias reaches its floor (about 0.01–0.015) by 0.032. In 2024-25 it keeps + falling until about 0.3, and 0.032 leaves it at 0.10–0.12 (base + 0.13–0.17). The extra coverage 2024-25 gets from a larger rate costs WIS + in both seasons. Part of the gap is the cold start: with no burn-in, the + tracker starts at zero in November 2024, so 2024-25 is also the season it + is still learning in. +- **The knee is lower than E09's.** E09 (submitted ensemble, 2023-24 + burn-in) put the knee at 0.05–0.1. Here the WIS-best rate is 0.018–0.056 + at every horizon and season, and E11's 0.1 is already past it: −0.5/−0.9 + at h2/h3 in 2024-25 and −1.0/−1.2 in 2025-26. +- **The references.** REF-op cold has the most WIS gain at h0 (+2.2/+1.8) + but keeps 2024-25's coverage bias near the base (0.106–0.124). The adaptive + sqrt tracker reaches a 2025-26 coverage bias of 0.008–0.016 with +2.2 at + h0, but loses WIS at h2/h3 in 2024-25 (−1.4/−2.6). Neither is good in both + seasons at every horizon; a constant rate of 0.018–0.032 is, at a smaller + gain. +- **The 50% interval.** At 0.018–0.032 the 50% interval covers 0.47–0.51 + at h0/h2, close to nominal; 0.1 and above under-cover it in 2024-25 + (0.39–0.45 at h2). + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +LRS <- signif(10^seq(-2.5, 0.25, by = 0.25), 2) +all_cfgs <- ws_configs(wi$rate_scales, burn_in = TRUE) +sweep_cfgs <- purrr::map(LRS, function(lr) utils::modifyList(all_cfgs[["E11 constant 0.1"]], list(lr = lr))) +ref_cfgs <- list(`REF-op cold` = all_cfgs[["REF-op cold"]], `sqrt, adaptive` = all_cfgs[["E05 single cold"]]) +# Cached under cache/calibration/experiments/runs; lr 0.1 and the references are E12/E13's runs. +run <- function(cfg) { + do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +} +sweep <- purrr::map(sweep_cfgs, run) +refs <- purrr::map(ref_cfgs, run) +``` + +```{r metrics} +GEOS <- c(states = "states only (US excluded)", all = "all locations") +SEASONS <- c(LIVE, "both") +# ws_scores() per location subset; wis_pct is the change vs base, positive is better. +score <- function(cal) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) + purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% + bind_rows() %>% + select(geos, season, horizon, which, wis, cal_err, cov50, cov90) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err, cov50, cov90)) %>% + mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) +} +label <- function(df) { + df %>% mutate( + geos = factor(geos, levels = names(GEOS), labels = GEOS), + season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +} +sweep_m <- purrr::map2(sweep, LRS, function(cal, lr) score(cal) %>% mutate(lr = lr)) %>% bind_rows() %>% label() +ref_m <- purrr::imap(refs, function(cal, nm) score(cal) %>% mutate(ref = nm)) %>% bind_rows() %>% label() +SEASON_COLS <- c(`2024-2025` = "#0072B2", `2025-2026` = "#D55E00", `both seasons` = "grey45") +``` + +# Coverage bias vs WIS, by season + +One line per season, tracing the learning rate from 0.0032 (near the +base, the open circle) to 1.8. Labels mark every half decade. The grey +dashed line pools both seasons, as E09 did. Triangles are REF-op cold, +squares the adaptive sqrt tracker, in the season's colour. The y axis is +clipped at −10%. + +```{r curve-fn} +LABELLED <- c(0.01, 0.032, 0.1, 0.32, 1) +curve_plot <- function(g) { + d <- sweep_m %>% filter(geos == g) %>% arrange(lr) + r <- ref_m %>% filter(geos == g) + base <- d %>% distinct(h, season, cal_err_base) + ggplot(d, aes(cal_err_cal, wis_pct, colour = season)) + + geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + + geom_path(aes(linetype = season == "both seasons"), linewidth = 0.8) + + geom_point(size = 1) + + geom_text(data = d %>% filter(lr %in% LABELLED, season != "both seasons"), + aes(label = format(lr, drop0trailing = TRUE)), size = 2.6, vjust = -0.8, show.legend = FALSE) + + geom_point(data = base, aes(cal_err_base, 0), shape = 21, fill = "white", size = 2.2) + + geom_point(data = r %>% filter(season != "both seasons"), aes(shape = ref), size = 2.4) + + scale_colour_manual(values = SEASON_COLS, name = NULL) + + scale_linetype_manual(values = c(`FALSE` = "solid", `TRUE` = "22"), guide = "none") + + scale_shape_manual(values = c(`REF-op cold` = 17, `sqrt, adaptive` = 15), name = NULL) + + facet_wrap(~h, nrow = 1) + + coord_cartesian(ylim = c(max(-10, min(d$wis_pct)), max(c(d$wis_pct, r$wis_pct)) + 0.5)) + + labs(x = "L1 coverage bias", y = "WIS change % (positive is better)", title = g) + + theme(legend.position = "bottom") +} +``` + +## States only + +```{r curve-states} +curve_plot(GEOS[["states"]]) +``` + +## All locations + +```{r curve-all} +curve_plot(GEOS[["all"]]) +``` + +# Each metric against the learning rate + +The same runs with the learning rate on the x axis, so the seasons can be +compared at the same rate. Top: WIS change % (positive is better). Bottom: +L1 coverage bias, with each season's base as a dotted line. States only. + +```{r vs-lr, fig.height=6.5} +long <- sweep_m %>% + filter(geos == GEOS[["states"]], season != "both seasons") %>% + transmute(h, season, lr, `WIS change %` = wis_pct, `L1 coverage bias` = cal_err_cal) %>% + tidyr::pivot_longer(c(`WIS change %`, `L1 coverage bias`), names_to = "metric") %>% + mutate(metric = factor(metric, levels = c("WIS change %", "L1 coverage bias"))) +base_lines <- sweep_m %>% + filter(geos == GEOS[["states"]], season != "both seasons") %>% + distinct(h, season, value = cal_err_base) %>% + mutate(metric = factor("L1 coverage bias", levels = levels(long$metric))) +zero <- tibble(metric = factor("WIS change %", levels = levels(long$metric)), value = 0) +ggplot(long, aes(lr, value, colour = season)) + + geom_hline(data = zero, aes(yintercept = value), colour = "grey60", linewidth = 0.3) + + geom_hline(data = base_lines, aes(yintercept = value, colour = season), linetype = "dotted") + + geom_line() + + geom_point(size = 1) + + scale_x_log10(breaks = c(0.003, 0.01, 0.03, 0.1, 0.3, 1), labels = c("0.003", "0.01", "0.03", "0.1", "0.3", "1")) + + scale_colour_manual(values = SEASON_COLS, name = NULL) + + facet_grid(metric ~ h, scales = "free_y") + + labs(x = "constant learning rate (per 100k per step)", y = NULL) + + theme(legend.position = "bottom") +``` + +# Tables + +WIS change % / L1 coverage bias, per learning rate (rows) and season × +horizon (columns). The base row shows the base's coverage bias. The +references follow the sweep. + +```{r tables} +season_table <- function(g) { + rows <- bind_rows( + sweep_m %>% filter(lr == LRS[1]) %>% mutate(config = "base", wis_pct = 0, cal_err_cal = cal_err_base), + sweep_m %>% mutate(config = format(lr, drop0trailing = TRUE)), + ref_m %>% mutate(config = ref) + ) %>% + filter(geos == g) %>% + mutate(config = factor(config, levels = unique(config))) + rows %>% + transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(config) +} +``` + +## States only + +```{r table-states} +knitr::kable(season_table(GEOS[["states"]])) +``` + +## All locations + +```{r table-all} +knitr::kable(season_table(GEOS[["all"]])) +``` + +## 50% and 90% interval coverage, states only + +E12 found the constant rate at 0.1 under-covers the 50% interval. Coverage +of the 50% / 90% intervals, per season, at h0 and h2. + +```{r table-cov} +bind_rows( + sweep_m %>% filter(lr == LRS[1]) %>% mutate(config = "base", cov50_cal = cov50_base, cov90_cal = cov90_base), + sweep_m %>% mutate(config = format(lr, drop0trailing = TRUE)), + ref_m %>% mutate(config = ref) +) %>% + filter(geos == GEOS[["states"]], h %in% c("h0", "h2"), season != "both seasons") %>% + mutate(config = factor(config, levels = unique(config))) %>% + transmute(config, col = paste(season, h), cell = sprintf("%.2f / %.2f", cov50_cal, cov90_cal)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(config) %>% + knitr::kable() +``` diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index cfa05023..14733847 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -41,6 +41,7 @@ for (f in src) { | E09 | With a constant learning rate on the rate scale, how do coverage and WIS trade off, and how much revision delay is best? | An L-shaped trade-off whose knee matches or beats the paper's adaptive rate at every horizon. Under prod's weekly re-run, less delay is better. | | E10 | Do the flu results carry over to covid? | Only at h−1 (+10% WIS). The covid base is already calibrated at h1–h3, where calibrating is neutral or harmful. | | E11 | How does E09's best constant learning rate compare with the adaptive trackers? | Better coverage than the cold adaptive trackers at less WIS gain near the present, and the most stable across seasons. The warm-started adaptive tracker still wins at h0–h1. | +| E14 | On the clean `windowed_seasonal` replay, is there a constant learning rate that is good in both seasons? | Yes, 0.018–0.032 per 100k: better WIS at every horizon in both seasons and lower coverage bias. 2024-25 needs a larger rate for good coverage, which costs WIS. | # How to read the results From 18f932f607d094fd2d1fd8df2587b072d2f654ec Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:06:24 -0700 Subject: [PATCH 047/107] feat(calibration): E15 notebook, where the WIS gains come from (per level, offset size, median vs width) --- .../e15_wis_sources.Rmd | 328 ++++++++++++++++++ .../calibration_experiments/index.Rmd | 1 + 2 files changed, 329 insertions(+) create mode 100644 reports/writeups/calibration_experiments/e15_wis_sources.Rmd diff --git a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd new file mode 100644 index 00000000..b4c3fcdb --- /dev/null +++ b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd @@ -0,0 +1,328 @@ +--- +title: "E15: where the WIS gains come from" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +In the fan and vintage plots the calibrated intervals look almost the same +as the base ones, yet WIS improves. This notebook asks where that WIS +change comes from. + +**Setup.** The clean `windowed_seasonal` replay for flu (E12: substitutions +off, all 53 locations, the 55 hub rounds, h0–h3), learning from data as +published each week and re-run from scratch each week as prod does, no +burn-in season. Two calibrated configs, both from E12: + +- **REF-op cold**: prod's tracker without its warm start. +- **E11 constant 0.1**: the plain tracker on the rate scale (per 100k) + with a constant learning rate of 0.1. + +Scores are against finalized truth over 2024-25 and 2025-26. "WIS change %" +is `100 × (WIS_base − WIS_cal) / WIS_base`: **positive is better**. +Results lead with states only (US excluded), since US alone is about 45% of +the all-locations WIS. + +**Three ways to cut the WIS change.** + +1. **By quantile level.** WIS is the average pinball loss over the 23 + quantile levels, so the total change splits exactly into one piece per + level. A level's piece is its change in pinball loss as a share of the + base WIS; the pieces add up to the total. +2. **Offset size against the forecast's own spread.** For each forecast, + the offset at a level is `calibrated quantile − base quantile` (in + admissions, after every projection step). It is compared with the base + forecast's 90% interval width (q95 − q05) and 50% interval width + (q75 − q25). A ratio of 0.1 means the quantile moved by a tenth of the + interval width. +3. **Median shift vs width change.** Each calibrated forecast is split into + two parts and each part is scored on its own: + - **median shift only**: every base quantile moves by the same amount, + the calibrated median minus the base median. The base forecast's + shape and widths are kept. + - **width change only**: the calibrated forecast is moved so that its + median sits on the base median. The calibrated widths and shape are + kept, the base centre is kept. + + If the two parts did not interact, their WIS changes would add up to the + full change. Both parts are floored at zero, as the calibrated forecast + is. + +# Findings + +Numbers are from the render of 1 October 2026, states only, h0/h1/h2/h3, +WIS change % with positive better. All-locations numbers point the same way. + +- **The offsets are small, and that is why the fans look unchanged.** The + typical (median) offset is 2–5% of the base forecast's 90% interval width + at every level and horizon, or 7–19% of its 50% interval width. The median + calibrated 90% width is 1.00–1.04 times the base width. Gains of 1–2% of + WIS need only shifts this size, which are hard to see on a fan plot. +- **The offsets nearly always point up.** The median offset is positive at + every level and horizon, and fewer than one forecast in ten is moved down + by more than about 3% of its 90% width. The tracker moves the whole + forecast up a little, which matches a base that under-predicts (E12). +- **The gain is in the centre and shoulders, not the tails.** The six tail + levels (0.01–0.05 and 0.95–0.99) add at most 0.1 point at any horizon. + REF-op's +2.0 at h0 is +1.0 from the centre (0.35–0.65) and +0.9 from the + shoulders (0.1–0.3, 0.7–0.9). +- **The lower half gains; the upper half loses beyond h0.** For REF-op the + levels below the median give +1.4/+0.7/+0.6/+0.6 and those above give + +0.5/−0.1/−0.5/−0.8. Raising the lower quantiles helps when the outcome + comes in above them. At h2–h3 the offsets grow with the level, so the upper + quantiles move up further and stretch the upper side past where outcomes + land. E11 shows the same pattern with a weaker lower-half gain + (+1.0/+0.4/+0.1/0.0) and a larger upper-half loss (−0.3 to −0.9). +- **For REF-op, shifting the median is the whole gain.** Moving every base + quantile by the median offset alone gives +2.3/+0.8/+0.5/+0.4, more than + the full calibration (+2.0/+0.6/+0.1/−0.2). Changing only the widths and + shape, around the base median, costs −0.6/−0.7/−0.9/−1.1. The two parts do + not add up: the full change is 0.3–0.6 points better than their sum. So + REF-op's width changes buy coverage at a WIS cost, and its WIS gain is a + small upward shift. +- **For E11 the median shift helps only at h0** (+0.9, then + −0.4/−1.2/−1.7), and the width change costs −0.2 to −0.8. Its typical + offset is about REF-op's size, but it has a long tail: in one forecast in + ten the offset exceeds 0.26–0.55 of the 90% width (REF-op 0.08–0.20). A + fixed step per 100k is large for a forecast whose interval is narrow, as in + small states or off-season weeks. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +cfgs <- ws_configs(wi$rate_scales, burn_in = FALSE)[c("REF-op cold", "E11 constant 0.1")] +cals <- purrr::map(cfgs, function(cfg) { + do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +}) +pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) +GEOS <- c(states = "states only (US excluded)", all = "all locations") +fc <- purrr::imap(cals, function(cal, nm) cal$forecasts %>% mutate(config = nm)) %>% + bind_rows() %>% + mutate(season = season_of(reference_date), config = factor(config, levels = names(cals))) %>% + filter(season %in% LIVE, !is.na(truth), !is.na(value_base)) +# One copy of each row per location subset. +fc <- bind_rows(fc %>% filter(location != "US") %>% mutate(geos = "states"), fc %>% mutate(geos = "all")) %>% + mutate(geos = factor(geos, levels = names(GEOS), labels = GEOS)) +``` + +# 1. WIS change by quantile level + +Each bar is one level's share of the total WIS change % (positive is +better); the bars in a panel add up to the panel's total, printed in the +strip. + +```{r by-level} +by_level <- fc %>% + group_by(geos, config, horizon, level) %>% + summarize(base = sum(pinball(truth, value_base, level)), cal = sum(pinball(truth, value_cal, level)), .groups = "drop") %>% + group_by(geos, config, horizon) %>% + mutate(contrib = 100 * (base - cal) / sum(base), level_pct = 100 * (base - cal) / base, total = sum(contrib)) %>% + ungroup() +level_plot <- function(g) { + d <- by_level %>% + filter(geos == g) %>% + mutate(panel = sprintf("h%d, total %+.1f%%", horizon, total)) + ggplot(d, aes(factor(level), contrib, fill = config)) + + geom_hline(yintercept = 0, colour = "grey50") + + geom_col(position = position_dodge(width = 0.8), width = 0.75) + + facet_wrap(~ paste(config, panel, sep = ": "), ncol = 4) + + scale_fill_manual(values = c(`REF-op cold` = "#D55E00", `E11 constant 0.1` = "#0072B2"), guide = "none") + + scale_x_discrete(labels = function(x) ifelse(x %in% c("0.01", "0.05", "0.25", "0.5", "0.75", "0.95", "0.99"), x, "")) + + labs(x = "quantile level", y = "share of WIS change, % of base WIS", title = g) + + theme(axis.text.x = element_text(angle = 90, vjust = 0.5, size = 7)) +} +``` + +## States only + +```{r by-level-states, fig.height=6} +level_plot(GEOS[["states"]]) +``` + +## All locations + +```{r by-level-all, fig.height=6} +level_plot(GEOS[["all"]]) +``` + +## Grouped: tails, shoulders, centre + +The same pieces summed over groups of levels: **tails** are the six levels +at or beyond 0.05/0.95, **shoulders** the ten levels 0.1–0.3 and 0.7–0.9, +**centre** the seven levels 0.35–0.65. Below and above the median are +split, to show whether the gain comes from moving the lower or the upper +side. + +```{r by-group} +grouped <- by_level %>% + mutate( + band = case_when(level <= 0.05 | level >= 0.95 ~ "tails", level <= 0.3 | level >= 0.7 ~ "shoulders", TRUE ~ "centre"), + side = case_when(level < 0.5 ~ "lower", level > 0.5 ~ "upper", TRUE ~ "median") + ) +group_tbl <- function(g) { + bind_rows( + grouped %>% group_by(config, horizon, part = band) %>% filter(geos == g) %>% summarize(v = sum(contrib), .groups = "drop"), + grouped %>% group_by(config, horizon, part = paste(side, "side")) %>% filter(geos == g) %>% summarize(v = sum(contrib), .groups = "drop"), + grouped %>% group_by(config, horizon) %>% filter(geos == g) %>% summarize(v = sum(contrib), .groups = "drop") %>% mutate(part = "total") + ) %>% + mutate(part = factor(part, levels = c("tails", "shoulders", "centre", "lower side", "median side", "upper side", "total"), + labels = c("tails", "shoulders", "centre", "below the median", "the median level", "above the median", "total"))) %>% + transmute(config, part, horizon = paste0("h", horizon), v = sprintf("%+.2f", v)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = v) %>% + arrange(config, part) +} +knitr::kable(group_tbl(GEOS[["states"]]), caption = "States only: WIS change % by group of levels (positive is better)") +knitr::kable(group_tbl(GEOS[["all"]]), caption = "All locations: WIS change % by group of levels (positive is better)") +``` + +Change in pinball loss at each level relative to that level's own base +loss (not a share of WIS), states only. Tail levels carry little of the +WIS, so a large relative change there can still be a small piece of the +total. + +```{r level-pct} +knitr::kable( + by_level %>% + filter(geos == GEOS[["states"]], level %in% c(0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99)) %>% + transmute(config, level, horizon = paste0("h", horizon), v = round(level_pct, 1)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = v) %>% + arrange(config, level), + caption = "States only: pinball loss change % at selected levels, vs that level's base loss" +) +``` + +# 2. Offset size against the forecast's spread + +```{r offsets} +widths <- fc %>% + filter(config == names(cals)[1], level %in% c(0.05, 0.25, 0.75, 0.95)) %>% + select(geos, location, horizon, reference_date, level, value_base) %>% + tidyr::pivot_wider(names_from = level, values_from = value_base, names_prefix = "q") %>% + transmute(geos, location, horizon, reference_date, w90 = q0.95 - q0.05, w50 = q0.75 - q0.25) +off <- fc %>% + inner_join(widths, by = c("geos", "location", "horizon", "reference_date")) %>% + filter(w90 > 0, w50 > 0) %>% + mutate(offset = value_cal - value_base, r90 = offset / w90, r50 = offset / w50) +n_dropped <- fc %>% distinct(geos, location, horizon, reference_date) %>% nrow() - off %>% distinct(geos, location, horizon, reference_date) %>% nrow() +``` + +Offsets are signed: positive means the calibrated quantile is above the +base one. The line is the median over forecasts (location × round) and the +band the middle 80%, offset ÷ base 90% interval width, states only. +`r n_dropped` forecast rows with a zero-width base interval are left out. + +```{r offset-plot, fig.height=6} +off_q <- off %>% + filter(geos == GEOS[["states"]]) %>% + group_by(config, horizon, level) %>% + summarize(med = median(r90), lo = quantile(r90, 0.1), hi = quantile(r90, 0.9), .groups = "drop") +ggplot(off_q, aes(level, med, colour = config, fill = config)) + + geom_hline(yintercept = 0, colour = "grey50") + + geom_ribbon(aes(ymin = lo, ymax = hi), alpha = 0.15, colour = NA) + + geom_line() + + geom_point(size = 0.8) + + facet_wrap(~ paste0("h", horizon), nrow = 1) + + scale_colour_manual(values = c(`REF-op cold` = "#D55E00", `E11 constant 0.1` = "#0072B2"), name = NULL) + + scale_fill_manual(values = c(`REF-op cold` = "#D55E00", `E11 constant 0.1` = "#0072B2"), name = NULL) + + labs(x = "quantile level", y = "offset / base (q95 − q05)") + + theme(legend.position = "bottom") +``` + +Typical size, ignoring direction: median of |offset| ÷ the base 90% and +50% interval widths at selected levels, with the share of forecasts whose +offset exceeds a tenth of the 90% width. + +```{r offset-table} +off_tbl <- function(g) { + off %>% + filter(geos == g, level %in% c(0.05, 0.25, 0.5, 0.75, 0.95)) %>% + group_by(config, horizon, level) %>% + summarize( + `|off|/w90 median` = median(abs(r90)), `|off|/w90 p90` = quantile(abs(r90), 0.9), + `|off|/w50 median` = median(abs(r50)), `share |off| > 0.1 w90` = mean(abs(r90) > 0.1), + .groups = "drop" + ) %>% + mutate(horizon = paste0("h", horizon), across(where(is.double) & !level, \(x) round(x, 3))) %>% + arrange(config, horizon, level) +} +knitr::kable(off_tbl(GEOS[["states"]]), caption = "States only") +knitr::kable(off_tbl(GEOS[["all"]]) %>% filter(level %in% c(0.05, 0.5, 0.95)), caption = "All locations, three levels") +``` + +# 3. Median shift vs width change + +```{r split} +med <- fc %>% + filter(level == 0.5) %>% + select(geos, config, location, horizon, reference_date, med_base = value_base, med_cal = value_cal) +split <- fc %>% + left_join(med, by = c("geos", "config", "location", "horizon", "reference_date")) %>% + mutate( + median_only = pmax(value_base + (med_cal - med_base), 0), + width_only = pmax(med_base + (value_cal - med_cal), 0) + ) +split_scores <- split %>% + group_by(geos, config, horizon) %>% + summarize( + base = sum(pinball(truth, value_base, level)), + full = sum(pinball(truth, value_cal, level)), + `median shift only` = sum(pinball(truth, median_only, level)), + `width change only` = sum(pinball(truth, width_only, level)), + .groups = "drop" + ) %>% + tidyr::pivot_longer(c(full, `median shift only`, `width change only`), names_to = "variant", values_to = "wis") %>% + mutate(pct = 100 * (base - wis) / base) +split_tbl <- function(g) { + d <- split_scores %>% filter(geos == g) + bind_rows( + d %>% select(config, variant, horizon, pct), + d %>% + group_by(config, horizon) %>% + summarize(pct = pct[variant == "full"] - pct[variant == "median shift only"] - pct[variant == "width change only"], .groups = "drop") %>% + mutate(variant = "interaction (full − the two parts)") + ) %>% + mutate(variant = factor(variant, levels = c("full", "median shift only", "width change only", "interaction (full − the two parts)"))) %>% + transmute(config, variant, horizon = paste0("h", horizon), pct = sprintf("%+.2f", pct)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = pct) %>% + arrange(config, variant) +} +knitr::kable(split_tbl(GEOS[["states"]]), caption = "States only: WIS change % (positive is better)") +knitr::kable(split_tbl(GEOS[["all"]]), caption = "All locations: WIS change % (positive is better)") +``` + +How the calibrated widths compare with the base: the median over forecasts +of calibrated ÷ base interval width, states only. 1 means unchanged. + +```{r width-ratio} +wr <- split %>% + filter(geos == GEOS[["states"]], level %in% c(0.05, 0.25, 0.75, 0.95)) %>% + select(config, location, horizon, reference_date, level, value_base, value_cal) %>% + tidyr::pivot_wider(names_from = level, values_from = c(value_base, value_cal)) %>% + mutate( + r90 = (value_cal_0.95 - value_cal_0.05) / (value_base_0.95 - value_base_0.05), + r50 = (value_cal_0.75 - value_cal_0.25) / (value_base_0.75 - value_base_0.25) + ) %>% + filter(is.finite(r90), is.finite(r50)) %>% + group_by(config, horizon) %>% + summarize( + `90% width ratio, median` = median(r90), `90% width ratio, middle 80%` = sprintf("%.2f–%.2f", quantile(r90, 0.1), quantile(r90, 0.9)), + `50% width ratio, median` = median(r50), `50% width ratio, middle 80%` = sprintf("%.2f–%.2f", quantile(r50, 0.1), quantile(r50, 0.9)), + .groups = "drop" + ) %>% + mutate(horizon = paste0("h", horizon), across(where(is.double), \(x) round(x, 2))) +knitr::kable(wr) +``` diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index 14733847..93eb42c1 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -42,6 +42,7 @@ for (f in src) { | E10 | Do the flu results carry over to covid? | Only at h−1 (+10% WIS). The covid base is already calibrated at h1–h3, where calibrating is neutral or harmful. | | E11 | How does E09's best constant learning rate compare with the adaptive trackers? | Better coverage than the cold adaptive trackers at less WIS gain near the present, and the most stable across seasons. The warm-started adaptive tracker still wins at h0–h1. | | E14 | On the clean `windowed_seasonal` replay, is there a constant learning rate that is good in both seasons? | Yes, 0.018–0.032 per 100k: better WIS at every horizon in both seasons and lower coverage bias. 2024-25 needs a larger rate for good coverage, which costs WIS. | +| E15 | Why does WIS improve when the calibrated intervals barely move? | The offsets are small (2–5% of the 90% interval width) and point up. The gain comes from shifting the forecast up, mostly at the central levels below the median; width changes cost WIS. | # How to read the results From d78ae89a7ba22756116524afec4c0d0057fda9dc Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:07:04 -0700 Subject: [PATCH 048/107] docs(calibration): record E14 (constant rate by season) and E15 (WIS sources) --- notes/CALIBRATION.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index 8be2da25..7ac85270 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -166,6 +166,25 @@ seasons pooled, states only (US excluded): - US is ~45% of all-locations WIS, but states-only and US-only changes point the same way. Tables: `cache/calibration/ws_replay_scores_burn_in_*.csv`. +## Constant rate by season (E14) and where the WIS gains come from (E15), 2026-10-01 + +Flu, clean `windowed_seasonal` replay, exact, cold, states only. +`e14_lr_by_season.Rmd`, `e15_wis_sources.Rmd`. + +- E14: a constant rate of 0.018–0.032 per 100k improves WIS at every + horizon in both seasons and cuts coverage bias in both (2025-26 reaches + ~0.01 by 0.032; 2024-25 keeps improving up to ~0.3, at a WIS cost past + 0.056). From 0.1 (E11's rate) WIS worsens at h1–h3 in both seasons. The + knee here is lower than E09's 0.05–0.1 (ensemble, burn-in). Neither + REF-op cold nor sqrt adaptive is good in both seasons at every horizon. +- E15: offsets are small (median 2–5% of the base 90% width; width ratio + 1.00–1.04), which is why fan plots look unchanged, and nearly always + upward. Gains come from the center and shoulders, not the tails. For + REF-op, shifting every quantile by the median offset alone gives more WIS + than the full calibration; width/shape changes cost WIS and buy coverage. + E11 0.1 has a long tail of offsets large relative to narrow intervals + (top 10%: 0.26–0.55 of the 90% width vs 0.08–0.20 for REF-op). + ## Data FluSight CMU-TimeSeries submissions: 83 rounds (2023-10-14 … 2026-05-30), 53 From c939e31a54eb1ef2208f0a0fcd8512a893b526f4 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:27:29 -0700 Subject: [PATCH 049/107] feat(calibration): late_decay option, shrink the fast offset each round after a calendar date --- R/calibration/calibrate.R | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/R/calibration/calibrate.R b/R/calibration/calibrate.R index e116522b..16d7a509 100644 --- a/R/calibration/calibrate.R +++ b/R/calibration/calibrate.R @@ -96,6 +96,11 @@ hub_series_matrix <- function(series, round_date, n_levels) { #' still step when revealed inside the off window. The hidden offsets carry #' unchanged into the next season. `"04-01"` switches calibration off for the #' spring tail, where the peak-tuned offsets are out of regime. +#' @param late_decay `NULL`, or `list(from = "MM-DD", factor = )`. At each round +#' whose reference date falls on or after `from` (and before July), the fast +#' offset is multiplied by `factor` in `[0, 1]` before it is played; learning +#' continues as usual. The slow term is not decayed. Mutually exclusive with +#' `off_after`. #' @param lr_seasonal `NULL`, or `list(half_width_weeks = 5)`. Pools into the #' eta residual window the rounds of every earlier season that fall within #' `half_width_weeks` of the same point in the season (same calendar date @@ -160,6 +165,7 @@ calibrate_hub_forecasts <- function( projection = c("isotonic", "sort", "none"), nonneg = TRUE, off_after = NULL, + late_decay = NULL, lr_seasonal = NULL, transform = c("identity", "log1p", "sqrt", "quartic_root"), scales = NULL, @@ -190,6 +196,15 @@ calibrate_hub_forecasts <- function( if (!is.null(off_after) && fast_decay > 0) { cli::cli_abort("{.arg off_after} and {.arg fast_decay} cannot be combined; choose one way of retiring stale offsets.") } + if (!is.null(late_decay)) { + if (!is.null(off_after)) { + cli::cli_abort("{.arg off_after} and {.arg late_decay} cannot be combined.") + } + f <- late_decay$factor + if (!is.character(late_decay$from) || !(is.numeric(f) && length(f) == 1L && f >= 0 && f <= 1)) { + cli::cli_abort("{.arg late_decay} must be {.code list(from = \"MM-DD\", factor = <0..1>)}.") + } + } rounds <- hub_label_seasons(unique(forecasts$reference_date)) round_date <- rounds$round_date @@ -209,6 +224,12 @@ calibrate_hub_forecasts <- function( shrink = shrink_factor ) + if (!is.null(late_decay)) { + mmdd <- format(round_date, "%m-%d") + late <- mmdd >= late_decay$from & mmdd < "07-01" + hidden_scale[late] <- hidden_scale[late] * late_decay$factor + } + # The slow term may train through burn-in; the fast term never does. slow_update_from <- if (burn_in_learns_slow) rep(TRUE, n) else update_from apply <- rep(TRUE, n) From 1310245ed1d0725163e53e41285c6410f6a0c9c7 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:51:17 -0700 Subject: [PATCH 050/107] feat(calibration): E16 notebook, constant learning rate on the sqrt scale and month-by-month view --- .../e16_sqrt_constant_lr.Rmd | 440 ++++++++++++++++++ scripts/calibration/calibration_ws_replay.R | 31 ++ 2 files changed, 471 insertions(+) create mode 100644 reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd diff --git a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd new file mode 100644 index 00000000..df05fd6d --- /dev/null +++ b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd @@ -0,0 +1,440 @@ +--- +title: "E16: constant learning rate on the square-root scale, and month by month" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +E14 found a constant learning rate of 0.018–0.032 per 100k good in both +seasons. E15 found that E11's rate-scale tracker (0.1 per 100k) loses WIS +through a tail of offsets that are large next to narrow intervals: a fixed +step per 100k is big for a small state or an off-season week. Two questions: + +1. **Does the same tracker on the square-root scale do better?** On the + square root of the rate, a fixed step moves the forecast less where the + rate is low. +2. **Do the offsets go stale late in the season?** Offsets learned on the + ramp up point upward; after the peak they may linger and push the + forecast above outcomes that are now falling. + +**Setup.** As E14: the clean `windowed_seasonal` replay for flu (all 53 +locations, the 55 hub rounds, h0–h3), learning from data as published each +week and re-run from scratch each week as prod does, one week of extra +revision delay, no burn-in season. The tracker is the plain one with a +constant learning rate. The new axis is the scale: **sqrt** runs it on the +square root of admissions per 100k, so the learning rate is in +sqrt(per-100k) units and the grid is not comparable number for number with +the rate scale's. The sqrt grid is 11 values from 0.001 to 0.32, a quarter +decade apart, picked after a first look so the knee sits inside it. The rate +curve is E14's runs. + +Reference points: **REF-op cold** (prod's tracker without its warm start) +and the rate-scale rates **0.018** and **0.032** (E14's best range). + +"WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive +is better**. "L1 coverage bias" is the mean over the 23 levels of +|empirical coverage − level|. Each season is scored on its own rounds; the +tracker runs over both, so 2025-26 starts from the offsets 2024-25 left +behind. Results lead with states only (US excluded). + +# Findings + +Numbers are from the render of 1 October 2026, states only, WIS change % +(positive is better) / L1 coverage bias, h0/h1/h2/h3. All-locations numbers +point the same way. + +- **The sqrt scale works, with a knee at 0.01–0.018.** Every sqrt rate from + 0.001 to 0.018 improves WIS at every horizon in both seasons. At 0.032 h3 + of 2024-25 turns negative (−0.3), and from 0.056 up WIS falls fast. The + best rate by E14's criterion (highest mean gain, no loss anywhere) is + **0.018**. +- **sqrt 0.018 vs rate 0.032 (E14's best range).** 2024-25: + +1.7/+0.6/+0.3/+0.1 vs +1.0/+0.5/+0.3/+0.2. 2025-26: +3.0/+1.0/+1.1/+1.5 + vs +1.9/+1.0/+0.8/+0.8. Coverage bias is about the same: 0.099–0.137 vs + 0.098–0.123 in 2024-25, 0.021–0.024 vs 0.010–0.014 in 2025-26. So the sqrt + scale buys more WIS at h0 in both seasons and at every horizon in + 2025-26, for no coverage gain. It does not help h2–h3 in 2024-25. +- **Against REF-op cold**, sqrt 0.018 is ahead in 2025-26 at every horizon + and avoids REF-op's 2024-25 h3 loss (−0.8). REF-op has more WIS at h0–h1 + in 2024-25 (+2.2/+0.9) but keeps 2024-25 coverage bias near the base. +- **The tail shrinks, but not everywhere.** The 90th percentile of + |offset| ÷ base 90% width, over all levels, is 0.092–0.111 for sqrt 0.018 + vs 0.124–0.153 for rate 0.032 (rate 0.1: 0.24–0.34). At the low levels it + is much smaller (0.05 level: 0.008–0.025 vs 0.055–0.064). At the 0.95 level + it is larger (0.115–0.180 vs 0.096–0.152), because a fixed step on the + square root stretches the upper quantiles more where counts are high. + sqrt 0.01 has the smallest tail (0.056–0.071). +- **Yes, offsets go stale after the peak, and March is where it costs.** All + four trackers lose WIS in March of both seasons at h1–h3: rate 0.018 + −1.2 / −10.6 at h2 (2024-25 / 2025-26), rate 0.032 −2.2 / −18.7, sqrt + 0.018 −7.2 / −27.9, REF-op cold −19.8 / −24.3. In March the base + already over-predicts (58% and 67% of outcomes below its median at h2) + and the calibrated median sits higher still (62–77%). March is 4–8% of a + season's base WIS at h2, so a −20% March costs about 1 point of season + WIS. +- **April and May carry little WIS (1–3% of the season) and go both ways.** + In 2024-25 the base under-predicts in May and every tracker gains + (+6 to +18%). In 2025-26 rate 0.032 loses at h3 in May (−14%, 79% of + outcomes below the median). +- **Rate-scale offsets do not come down after the peak.** The mean offset + at the median stays at 0.07–0.2 per 100k from the peak through May, while + the base median falls from 10–12 to under 1 per 100k. On the sqrt scale and + for REF-op the offset falls with the level, but too slowly to avoid March. +- **Season-level coverage hides month-level errors.** In December the base's + monthly coverage bias is 0.25–0.38 and no tracker moves it; in March the + trackers make it worse. Part of the season-level coverage gain comes from + under-prediction on the ramp and over-prediction after the peak cancelling + out. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +RATE_LRS <- signif(10^seq(-2.5, 0.25, by = 0.25), 2) +SQRT_LRS <- signif(10^seq(-3, -0.5, by = 0.25), 2) +all_cfgs <- ws_configs(wi$rate_scales, burn_in = FALSE) +rate_cfg <- function(lr) utils::modifyList(all_cfgs[["E11 constant 0.1"]], list(lr = lr)) +sqrt_cfg <- function(lr) utils::modifyList(all_cfgs[["E11 constant 0.1"]], list(lr = lr, transform = "sqrt")) +# Cached under cache/calibration/experiments/runs; the rate sweep and REF-op cold are E14's runs. +run <- function(cfg) { + do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +} +rate_runs <- purrr::map(RATE_LRS, \(lr) run(rate_cfg(lr))) +sqrt_runs <- purrr::map(SQRT_LRS, \(lr) run(sqrt_cfg(lr))) +refop <- run(all_cfgs[["REF-op cold"]]) +``` + +```{r metrics} +GEOS <- c(states = "states only (US excluded)", all = "all locations") +SEASONS <- c(LIVE, "both") +SEASON_COLS <- c(`2024-2025` = "#0072B2", `2025-2026` = "#D55E00", `both seasons` = "grey45") +score <- function(cal) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) + purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% + bind_rows() %>% + select(geos, season, horizon, which, wis, cal_err, cov50, cov90) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err, cov50, cov90)) %>% + mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) +} +label <- function(df) { + df %>% mutate( + geos = factor(geos, levels = names(GEOS), labels = GEOS), + season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +} +sweep_m <- bind_rows( + purrr::map2(rate_runs, RATE_LRS, \(cal, lr) score(cal) %>% mutate(scale = "rate", lr = lr)), + purrr::map2(sqrt_runs, SQRT_LRS, \(cal, lr) score(cal) %>% mutate(scale = "sqrt", lr = lr)) +) %>% label() +ref_m <- score(refop) %>% mutate(ref = "REF-op cold") %>% label() +# Best sqrt rate: the highest mean WIS change among rates that improve WIS in both seasons at h0–h3 (states only). +best_tbl <- sweep_m %>% + filter(scale == "sqrt", geos == GEOS[["states"]], season %in% LIVE) %>% + group_by(lr) %>% + summarize(worst = min(wis_pct), mean = mean(wis_pct), .groups = "drop") +BEST_SQRT <- best_tbl %>% filter(worst > 0) %>% slice_max(mean, n = 1) %>% pull(lr) +``` + +# 1. Coverage bias vs WIS, by season and scale + +One curve per season and scale, tracing the learning rate from smallest +(near the base, the open circle) to largest. Solid lines are the rate scale +(E14), dashed lines the sqrt scale. Labels mark a few rates on each curve. +Triangles are REF-op cold. The y axis is clipped at −10%. The best sqrt +rate by the rule below is **`r BEST_SQRT`**. + +```{r curve-fn} +LABELLED <- list(rate = c(0.01, 0.032, 0.1, 0.32), sqrt = c(0.0032, 0.01, 0.032, 0.1)) +curve_plot <- function(g) { + d <- sweep_m %>% filter(geos == g, season != "both seasons") %>% arrange(scale, lr) + r <- ref_m %>% filter(geos == g, season != "both seasons") + lab <- d %>% filter((scale == "rate" & lr %in% LABELLED$rate) | (scale == "sqrt" & lr %in% LABELLED$sqrt)) + base <- d %>% distinct(h, season, cal_err_base) + ggplot(d, aes(cal_err_cal, wis_pct, colour = season)) + + geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + + geom_path(aes(linetype = scale, group = interaction(season, scale)), linewidth = 0.7) + + geom_point(aes(shape = scale), size = 1.1) + + geom_text(data = lab, aes(label = paste0(ifelse(scale == "sqrt", "s", "r"), format(lr, drop0trailing = TRUE))), + size = 2.3, vjust = -0.8, show.legend = FALSE) + + geom_point(data = base, aes(cal_err_base, 0), shape = 21, fill = "white", size = 2.2) + + geom_point(data = r, shape = 17, size = 2.6) + + scale_colour_manual(values = SEASON_COLS, name = NULL) + + scale_linetype_manual(values = c(rate = "solid", sqrt = "22"), name = "scale") + + scale_shape_manual(values = c(rate = 16, sqrt = 1), name = "scale") + + facet_wrap(~h, nrow = 1) + + coord_cartesian(ylim = c(max(-10, min(d$wis_pct)), max(c(d$wis_pct, r$wis_pct)) + 0.5)) + + labs(x = "L1 coverage bias", y = "WIS change % (positive is better)", title = g, + caption = "labels: r = rate-scale learning rate (per 100k), s = sqrt-scale learning rate; triangles: REF-op cold") + + theme(legend.position = "bottom") +} +``` + +## States only + +```{r curve-states} +curve_plot(GEOS[["states"]]) +``` + +## All locations + +```{r curve-all} +curve_plot(GEOS[["all"]]) +``` + +## Each metric against the learning rate, sqrt scale + +Top: WIS change %. Bottom: L1 coverage bias, each season's base dotted. +States only. + +```{r vs-lr, fig.height=6.5} +long <- sweep_m %>% + filter(scale == "sqrt", geos == GEOS[["states"]], season != "both seasons") %>% + transmute(h, season, lr, `WIS change %` = wis_pct, `L1 coverage bias` = cal_err_cal) %>% + tidyr::pivot_longer(c(`WIS change %`, `L1 coverage bias`), names_to = "metric") %>% + mutate(metric = factor(metric, levels = c("WIS change %", "L1 coverage bias"))) +base_lines <- sweep_m %>% + filter(scale == "sqrt", geos == GEOS[["states"]], season != "both seasons") %>% + distinct(h, season, value = cal_err_base) %>% + mutate(metric = factor("L1 coverage bias", levels = levels(long$metric))) +zero <- tibble(metric = factor("WIS change %", levels = levels(long$metric)), value = 0) +ggplot(long, aes(lr, value, colour = season)) + + geom_hline(data = zero, aes(yintercept = value), colour = "grey60", linewidth = 0.3) + + geom_hline(data = base_lines, aes(yintercept = value, colour = season), linetype = "dotted") + + geom_line() + + geom_point(size = 1) + + scale_x_log10(breaks = c(0.001, 0.003, 0.01, 0.03, 0.1, 0.3), labels = c("0.001", "0.003", "0.01", "0.03", "0.1", "0.3")) + + scale_colour_manual(values = SEASON_COLS, name = NULL) + + facet_grid(metric ~ h, scales = "free_y") + + labs(x = "constant learning rate (sqrt of per-100k units per step)", y = NULL) + + theme(legend.position = "bottom") +``` + +## Tables + +WIS change % / L1 coverage bias per config (rows) and season × horizon +(columns). The sqrt sweep, then the rate-scale rates around E14's best range +and REF-op cold. + +```{r tables} +season_table <- function(g, rate_keep = c(0.018, 0.032, 0.056, 0.1)) { + rows <- bind_rows( + sweep_m %>% filter(scale == "sqrt", lr == SQRT_LRS[1]) %>% mutate(config = "base", wis_pct = 0, cal_err_cal = cal_err_base), + sweep_m %>% filter(scale == "sqrt") %>% mutate(config = paste("sqrt", format(lr, drop0trailing = TRUE))), + sweep_m %>% filter(scale == "rate", lr %in% rate_keep) %>% mutate(config = paste("rate", format(lr, drop0trailing = TRUE))), + ref_m %>% mutate(config = ref) + ) %>% + filter(geos == g) %>% + mutate(config = factor(config, levels = unique(config))) + rows %>% + transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(config) +} +``` + +### States only + +```{r table-states} +knitr::kable(season_table(GEOS[["states"]])) +``` + +### All locations + +```{r table-all} +knitr::kable(season_table(GEOS[["all"]])) +``` + +### Picking the best sqrt rate + +For each sqrt rate, the worst and the mean WIS change % over the two seasons +and h0–h3 (states only). The best rate is the one with the highest mean +among those whose worst case is still a gain (E14's criterion). + +```{r best} +knitr::kable(best_tbl %>% mutate(across(c(worst, mean), \(x) round(x, 2)))) +``` + +### 50% and 90% interval coverage, states only + +```{r table-cov} +bind_rows( + sweep_m %>% filter(scale == "sqrt", lr == SQRT_LRS[1]) %>% mutate(config = "base", cov50_cal = cov50_base, cov90_cal = cov90_base), + sweep_m %>% filter(scale == "sqrt", lr >= 0.0032, lr <= 0.056) %>% mutate(config = paste("sqrt", format(lr, drop0trailing = TRUE))), + sweep_m %>% filter(scale == "rate", lr %in% c(0.018, 0.032)) %>% mutate(config = paste("rate", format(lr, drop0trailing = TRUE))), + ref_m %>% mutate(config = ref) +) %>% + filter(geos == GEOS[["states"]], h %in% c("h0", "h2"), season != "both seasons") %>% + mutate(config = factor(config, levels = unique(config))) %>% + transmute(config, col = paste(season, h), cell = sprintf("%.2f / %.2f", cov50_cal, cov90_cal)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(config) %>% + knitr::kable() +``` + +# 2. Does the offset tail shrink? + +E15's statistic: for each forecast and level, |offset| ÷ the base 90% +interval width (q95 − q05), and its 90th percentile over forecasts (location +× round, both seasons, states only). "all levels" pools the 23 levels. A +smaller number means fewer offsets that are large next to the interval. + +```{r tail} +pick <- list( + `sqrt best` = sqrt_runs[[which(SQRT_LRS == BEST_SQRT)]], + `rate 0.018` = rate_runs[[which(RATE_LRS == 0.018)]], + `rate 0.032` = rate_runs[[which(RATE_LRS == 0.032)]], + `rate 0.1` = rate_runs[[which(RATE_LRS == 0.1)]], + `REF-op cold` = refop +) +names(pick)[1] <- paste("sqrt", BEST_SQRT) +fc_pick <- purrr::imap(pick, \(cal, nm) cal$forecasts %>% mutate(config = nm)) %>% + bind_rows() %>% + mutate(season = season_of(reference_date), config = factor(config, levels = names(pick))) %>% + filter(season %in% LIVE, location != "US", !is.na(value_base)) +w90 <- fc_pick %>% + filter(config == names(pick)[1], level %in% c(0.05, 0.95)) %>% + select(location, horizon, reference_date, level, value_base) %>% + tidyr::pivot_wider(names_from = level, values_from = value_base, names_prefix = "q") %>% + transmute(location, horizon, reference_date, w90 = q0.95 - q0.05) +ratio <- fc_pick %>% + inner_join(w90, by = c("location", "horizon", "reference_date")) %>% + filter(w90 > 0) %>% + mutate(r = abs(value_cal - value_base) / w90) +tail_tbl <- bind_rows( + ratio %>% filter(level %in% c(0.05, 0.25, 0.5, 0.75, 0.95)) %>% mutate(lv = as.character(level)), + ratio %>% mutate(lv = "all levels") +) %>% + group_by(config, horizon, lv) %>% + summarize(p90 = quantile(r, 0.9), .groups = "drop") %>% + mutate(lv = factor(lv, levels = c("0.05", "0.25", "0.5", "0.75", "0.95", "all levels"))) +knitr::kable( + tail_tbl %>% + transmute(config, lv, h = paste0("h", horizon), p90 = round(p90, 3)) %>% + tidyr::pivot_wider(names_from = h, values_from = p90) %>% + arrange(lv, config), + caption = "States only: 90th percentile of |offset| / base 90% width" +) +``` + +# 3. Month by month + +Five forecasts per state-round are compared: the base, the rate-scale rates +0.018 and 0.032, the best sqrt rate and REF-op cold. Months are calendar +months of the reference date. States only, h0–h3. Each cell holds about +150–260 state forecasts, so single months are noisy, but the March losses +below repeat across seasons and horizons. + +```{r month-data} +MONTH_COLS <- setNames(c("#009E73", "#0072B2", "#CC79A7", "#D55E00"), names(pick)[c(2, 3, 1, 5)]) +month_m <- ws_month_scores(fc_pick %>% filter(config %in% names(MONTH_COLS)) %>% mutate(config = factor(config, levels = names(MONTH_COLS)))) %>% + mutate(h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) +``` + +## WIS change % by month + +Each season a row, each horizon a column. The grey bars (right axis +scaled to the left) are each month's share of the season's base WIS, so a +large loss in a month with a small share costs little overall. The y axis is +clipped at −40% and +30%, which also cuts the tallest bars. + +```{r month-wis, fig.height=6} +share <- month_m %>% filter(config == names(MONTH_COLS)[1]) +ggplot(month_m, aes(month, pct, colour = config, group = config)) + + geom_col(data = share, aes(month, share), inherit.aes = FALSE, fill = "grey85", width = 0.7) + + geom_hline(yintercept = 0, colour = "grey40") + + geom_line() + + geom_point(size = 1.2) + + facet_grid(season ~ h, scales = "free_x", space = "free_x") + + scale_colour_manual(values = MONTH_COLS, name = NULL) + + coord_cartesian(ylim = c(-40, 30)) + + labs(x = "month of reference date", y = "WIS change % (positive is better); grey: % of season base WIS") + + theme(legend.position = "bottom") +``` + +## Coverage by month + +Top: L1 coverage bias per month, base in black. Bottom: share of outcomes +below the median; 0.5 is calibrated, above 0.5 means the median sits too +high (over-prediction). Base in black. + +```{r month-cov, fig.height=8} +cov_long <- bind_rows( + month_m %>% transmute(config = as.character(config), season, h, month, `L1 coverage bias` = cal_err_cal, `share below the median` = below_med_cal), + month_m %>% filter(config == names(MONTH_COLS)[1]) %>% + transmute(config = "base", season, h, month, `L1 coverage bias` = cal_err_base, `share below the median` = below_med_base) +) %>% + tidyr::pivot_longer(c(`L1 coverage bias`, `share below the median`), names_to = "metric") %>% + mutate(config = factor(config, levels = c("base", names(MONTH_COLS)))) +half <- tibble(metric = "share below the median", y = 0.5) +ggplot(cov_long, aes(month, value, colour = config, group = config)) + + geom_hline(data = half, aes(yintercept = y), colour = "grey50", linetype = "dashed") + + geom_line() + + geom_point(size = 1) + + facet_grid(metric + season ~ h, scales = "free", space = "free_x") + + scale_colour_manual(values = c(base = "black", MONTH_COLS), name = NULL) + + labs(x = "month of reference date", y = NULL) + + theme(legend.position = "bottom") +``` + +## Mean offset at the median over time + +The calibrated minus base median, in admissions per 100k, averaged over +states at each round, h0 and h2. The bottom row is the mean base median per +100k at h0, to place the season peak. + +```{r offset-time, fig.height=7} +pop <- wi$rate_scales %>% select(location, scale) +med <- fc_pick %>% + filter(level == 0.5, horizon %in% c(0, 2), config %in% names(MONTH_COLS)) %>% + left_join(pop, by = "location") %>% + group_by(config, season, horizon, reference_date) %>% + summarize(off = mean((value_cal - value_base) / scale), base = mean(value_base / scale), truth = mean(truth / scale, na.rm = TRUE), .groups = "drop") +off_long <- bind_rows( + med %>% transmute(config, season, reference_date, row = paste0("offset, h", horizon), value = off), + med %>% filter(horizon == 0, config == names(MONTH_COLS)[1]) %>% transmute(config = "base median", season, reference_date, row = "base median h0", value = base), + med %>% filter(horizon == 0, config == names(MONTH_COLS)[1]) %>% transmute(config = "truth", season, reference_date, row = "base median h0", value = truth) +) %>% + mutate(config = factor(config, levels = c(names(MONTH_COLS), "base median", "truth")), row = factor(row, levels = c("offset, h0", "offset, h2", "base median h0"))) +ggplot(off_long, aes(reference_date, value, colour = config)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_line() + + geom_point(size = 0.7) + + facet_grid(row ~ season, scales = "free") + + scale_colour_manual(values = c(MONTH_COLS, `base median` = "grey30", truth = "grey65"), name = NULL) + + scale_x_date(date_breaks = "1 month", date_labels = "%b") + + labs(x = "reference date", y = "admissions per 100k (mean over states)") + + theme(legend.position = "bottom") +``` + +## Month table, h2 + +WIS change % per month (states only, h2), with the base's share of the +season's WIS and the share of outcomes below the base and calibrated medians. + +```{r month-table} +month_m %>% + filter(h == "h2") %>% + transmute(season, month, config, cell = sprintf("%+.1f (%.2f)", pct, below_med_cal)) %>% + tidyr::pivot_wider(names_from = config, values_from = cell) %>% + left_join( + month_m %>% filter(h == "h2", config == names(MONTH_COLS)[1]) %>% + transmute(season, month, `base share %` = round(share, 1), `base below median` = round(below_med_base, 2)), + by = c("season", "month") + ) %>% + arrange(season, month) %>% + knitr::kable(caption = "WIS change % (share of outcomes below the calibrated median)") +``` diff --git a/scripts/calibration/calibration_ws_replay.R b/scripts/calibration/calibration_ws_replay.R index 3aea39a7..04cee874 100644 --- a/scripts/calibration/calibration_ws_replay.R +++ b/scripts/calibration/calibration_ws_replay.R @@ -132,6 +132,37 @@ ws_scores <- function(fc) { left_join(above, by = c("season", "horizon")) } +#' V-month per season: WIS change % (positive is better), L1 coverage bias and +#' the share of truth below the median, base vs calibrated, by calendar month of +#' the reference date. `share` is the month's share of the season's base WIS. +#' Groups by `config` too when present. Used by the E16/E17 notebooks. +ws_month_scores <- function(fc) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + by <- intersect(c("config", "season", "horizon"), names(fc)) + fc <- fc %>% + filter(!is.na(truth), !is.na(value_base), season %in% LIVE) %>% + mutate(month = factor(format(reference_date, "%b"), levels = month.abb[c(8:12, 1:7)])) + per_level <- fc %>% + group_by(across(all_of(c(by, "month", "level")))) %>% + summarize( + base = sum(pinball(truth, value_base, level)), cal = sum(pinball(truth, value_cal, level)), + gap_base = mean(truth <= value_base) - first(level), gap_cal = mean(truth <= value_cal) - first(level), + below_base = mean(truth < value_base), below_cal = mean(truth < value_cal), + n = n(), .groups = "drop" + ) + per_level %>% + group_by(across(all_of(c(by, "month")))) %>% + summarize( + wis_base = sum(base), wis_cal = sum(cal), + cal_err_base = mean(abs(gap_base)), cal_err_cal = mean(abs(gap_cal)), + below_med_base = below_base[level == 0.5], below_med_cal = below_cal[level == 0.5], + n = first(n), .groups = "drop" + ) %>% + group_by(across(all_of(by))) %>% + mutate(pct = 100 * (wis_base - wis_cal) / wis_base, share = 100 * wis_base / sum(wis_base)) %>% + ungroup() +} + # Location subsets scored separately: raw-count pooling makes US about half of # the all-locations WIS. WS_GEOS <- list(all = \(loc) rep(TRUE, length(loc)), states = \(loc) loc != "US", us = \(loc) loc == "US") From 32f1c11c0326bda24040087331479f7154dbec40 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:51:17 -0700 Subject: [PATCH 051/107] feat(calibration): E17 notebook, late-season offset decay vs off_after; index rows for E16/E17 --- .../e17_late_decay.Rmd | 221 ++++++++++++++++++ .../calibration_experiments/index.Rmd | 2 + 2 files changed, 223 insertions(+) create mode 100644 reports/writeups/calibration_experiments/e17_late_decay.Rmd diff --git a/reports/writeups/calibration_experiments/e17_late_decay.Rmd b/reports/writeups/calibration_experiments/e17_late_decay.Rmd new file mode 100644 index 00000000..9d2fbe8c --- /dev/null +++ b/reports/writeups/calibration_experiments/e17_late_decay.Rmd @@ -0,0 +1,221 @@ +--- +title: "E17: shrinking the offset late in the season" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +E16 found that the constant-rate trackers lose WIS in March of both seasons: +the offsets learned on the ramp up still point up after the peak, while +outcomes fall below the base median. This notebook asks whether **shrinking +the offset toward zero from a calendar date onward** removes those losses +without giving up the gains elsewhere. + +**Existing options.** None does exactly this: + +- `fast_decay` shrinks the offset every round of the year, so it also + weakens the ramp-up correction. +- `season_policy` (`carry`, `reset`, `shrink`) acts only at the season + boundary in the autumn, after the spring has been scored. +- `off_after` is the abrupt version: from a calendar date to July it plays + the base forecast and stops learning. It is run here as a comparison. + +So this notebook uses a new option, `late_decay = list(from = "MM-DD", +factor = f)`: from that date until July, the offset is multiplied by `f` +at each round before it is played. Learning continues, so new outcomes can +still push the offset. + +**Setup.** As E14/E16: flu, the clean `windowed_seasonal` replay, h0–h3, +learning from data as published each week and re-run from scratch each week, +one week of extra revision delay, no burn-in, states only. Three constant +rates: rate scale 0.018 and 0.032 per 100k (E14) and sqrt scale +0.018 (E16's best). For each, five settings: no decay, `late_decay` from +1 March with factor 0.85, 0.7 or 0.5 per round, and `off_after = "03-01"`. +The tracker carries offsets across seasons, so a decay in spring 2025 also +changes where 2025-26 starts. + +"WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, **positive is +better**; "L1 coverage bias" is the mean over the 23 levels of |empirical +coverage − level|. + +# Findings + +Numbers are from the render of 1 October 2026, states only, WIS change % +(positive is better) / L1 coverage bias, h0/h1/h2/h3. + +- **The decay removes the March loss.** For sqrt 0.018 at h2, March 2025-26 + goes from −27.9% to −10.7 / −6.3 / −2.9% (factor 0.85 / 0.7 / 0.5), and + the calibrated median comes back toward the base's (69–73% of outcomes + below it, base 67%). The rate-scale trackers behave the same way. +- **But it costs more than it saves.** It also removes the April–May gains, + and because the offsets carry across seasons, it hands 2025-26 an offset of + almost zero: the next season starts cold again. 2025-26 at h0 drops from + +3.0 to +1.7–2.1 (sqrt 0.018) and its coverage bias triples (0.021–0.024 + to 0.066–0.078). Pooled over both seasons, every decay setting is worse + than no decay on coverage and no better on WIS for all three rates. +- **`off_after = "03-01"` does better on WIS.** It plays the base from 1 + March and freezes the offsets, which then carry into the next season + unchanged. Pooled WIS for sqrt 0.018 goes from +2.2/+0.8/+0.6/+0.6 to + +2.3/+1.2/+1.2/+1.3, and both seasons gain at h1–h3 (2025-26: + +3.4/+1.6/+1.9/+2.6). Rate 0.032 gains at h2–h3 too (+0.8/+0.9 pooled vs + +0.5/+0.4). The cost is the October 2025 start (about −10% at h2/h3 on a + month with under 1% of the WIS). +- **Any spring switch-off gives back most of the coverage gain.** With + `off_after`, pooled coverage bias for sqrt 0.018 is 0.094–0.116 (no decay + 0.059–0.078, base 0.105–0.127). Spring rounds then play the base, and + the over-prediction after the peak no longer cancels the under-prediction + on the ramp (E16). +- **Answer.** The offsets do go stale after the peak, but shrinking them + toward zero by the calendar is not a free fix. The gradual decay as built + here loses because it also wipes the offset the next season starts from. + `off_after` in March is the better WIS choice and the worse coverage choice. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +base_cfg <- ws_configs(wi$rate_scales, burn_in = FALSE)[["E11 constant 0.1"]] +SQRT_BEST <- 0.018 +RATES <- list( + `rate 0.018` = list(lr = 0.018), + `rate 0.032` = list(lr = 0.032), + sqrt = list(lr = SQRT_BEST, transform = "sqrt") +) +names(RATES)[3] <- paste("sqrt", SQRT_BEST) +DECAYS <- list( + `no decay` = list(), + `decay 0.85` = list(late_decay = list(from = "03-01", factor = 0.85)), + `decay 0.7` = list(late_decay = list(from = "03-01", factor = 0.7)), + `decay 0.5` = list(late_decay = list(from = "03-01", factor = 0.5)), + `off after 1 Mar` = list(off_after = "03-01") +) +grid <- tidyr::expand_grid(rate = names(RATES), decay = names(DECAYS)) +# Cached under cache/calibration/experiments/runs; the no-decay runs are E14/E16's. +cals <- purrr::pmap(grid, function(rate, decay) { + cfg <- c(utils::modifyList(base_cfg, RATES[[rate]]), DECAYS[[decay]]) + do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +}) +fc <- purrr::map2(cals, seq_len(nrow(grid)), function(cal, i) { + cal$forecasts %>% mutate(rate = grid$rate[i], decay = grid$decay[i]) +}) %>% + bind_rows() %>% + mutate(season = season_of(reference_date), rate = factor(rate, levels = names(RATES)), decay = factor(decay, levels = names(DECAYS))) %>% + filter(season %in% LIVE, location != "US") +DECAY_COLS <- setNames(c("black", "#56B4E9", "#0072B2", "#D55E00", "#009E73"), names(DECAYS)) +``` + +# 1. Overall, by season + +WIS change % / L1 coverage bias, states only. + +```{r head} +head_m <- fc %>% + group_by(rate, decay) %>% + group_modify(\(d, k) ws_scores(d)) %>% + ungroup() %>% + select(rate, decay, season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% + mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base, season = factor(season, levels = c(LIVE, "both"), labels = c(LIVE, "both seasons"))) +base_row <- head_m %>% + filter(rate == names(RATES)[1], decay == "no decay") %>% + transmute(config = "base", season, horizon, cell = sprintf("%+.1f / %.3f", 0, cal_err_base)) +bind_rows(base_row, head_m %>% transmute(config = paste(rate, decay, sep = ", "), season, horizon, cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal))) %>% + mutate(config = factor(config, levels = unique(config))) %>% + transmute(config, col = paste0(season, " h", horizon), cell) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + knitr::kable() +``` + +The same WIS changes as a plot, one panel per season and horizon. + +```{r head-plot, fig.height=5.5} +ggplot(head_m, aes(rate, wis_pct, fill = decay)) + + geom_hline(yintercept = 0, colour = "grey50") + + geom_col(position = position_dodge(width = 0.85), width = 0.8) + + facet_grid(season ~ paste0("h", horizon)) + + scale_fill_manual(values = DECAY_COLS, name = NULL) + + labs(x = NULL, y = "WIS change % (positive is better)") + + theme(legend.position = "bottom", axis.text.x = element_text(angle = 20, hjust = 1)) +``` + +# 2. Month by month + +WIS change % by calendar month of the reference date, one figure per +constant rate; rows are seasons, columns horizons. Grey bars are each +month's share of the season's base WIS. Clipped at −40%. + +```{r month, results="asis", fig.height=5.5} +month_m <- ws_month_scores(fc %>% mutate(config = interaction(rate, decay, sep = "|"))) %>% + tidyr::separate(config, c("rate", "decay"), sep = "[|]") %>% + mutate(decay = factor(decay, levels = names(DECAYS)), h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) +for (r in names(RATES)) { + cat("\n\n## ", r, "\n\n", sep = "") + d <- month_m %>% filter(rate == r) + share <- d %>% filter(decay == "no decay") + print( + ggplot(d, aes(month, pct, colour = decay, group = decay)) + + geom_col(data = share, aes(month, share), inherit.aes = FALSE, fill = "grey85", width = 0.7) + + geom_hline(yintercept = 0, colour = "grey40") + + geom_line() + + geom_point(size = 1.1) + + facet_grid(season ~ h, scales = "free_x", space = "free_x") + + scale_colour_manual(values = DECAY_COLS, name = NULL) + + coord_cartesian(ylim = c(-40, 30)) + + labs(x = "month of reference date", y = "WIS change %; grey: % of season base WIS", title = r) + + theme(legend.position = "bottom") + ) +} +``` + +## Share of outcomes below the median, March–May + +0.5 is calibrated; above 0.5 the median is too high. h2, states only. + +```{r month-below} +month_m %>% + filter(h == "h2", month %in% c("Mar", "Apr", "May")) %>% + transmute(rate, season, month, decay, v = sprintf("%+.1f (%.2f)", pct, below_med_cal)) %>% + tidyr::pivot_wider(names_from = decay, values_from = v) %>% + left_join( + month_m %>% filter(h == "h2", decay == "no decay") %>% transmute(rate, season, month, `base below median` = round(below_med_base, 2)), + by = c("rate", "season", "month") + ) %>% + arrange(rate, season, month) %>% + knitr::kable(caption = "h2: WIS change % (share of outcomes below the calibrated median)") +``` + +# 3. Mean offset at the median over time + +Calibrated minus base median in admissions per 100k, averaged over states at +each round, h2. + +```{r offset-time, fig.height=7} +pop <- wi$rate_scales %>% select(location, scale) +fc %>% + filter(level == 0.5, horizon == 2) %>% + left_join(pop, by = "location") %>% + group_by(rate, decay, season, reference_date) %>% + summarize(off = mean((value_cal - value_base) / scale), .groups = "drop") %>% + ggplot(aes(reference_date, off, colour = decay)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_line() + + geom_point(size = 0.6) + + facet_grid(rate ~ season, scales = "free") + + scale_colour_manual(values = DECAY_COLS, name = NULL) + + scale_x_date(date_breaks = "1 month", date_labels = "%b") + + labs(x = "reference date", y = "mean offset at the median, per 100k") + + theme(legend.position = "bottom") +``` diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index 93eb42c1..3b47a6b8 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -43,6 +43,8 @@ for (f in src) { | E11 | How does E09's best constant learning rate compare with the adaptive trackers? | Better coverage than the cold adaptive trackers at less WIS gain near the present, and the most stable across seasons. The warm-started adaptive tracker still wins at h0–h1. | | E14 | On the clean `windowed_seasonal` replay, is there a constant learning rate that is good in both seasons? | Yes, 0.018–0.032 per 100k: better WIS at every horizon in both seasons and lower coverage bias. 2024-25 needs a larger rate for good coverage, which costs WIS. | | E15 | Why does WIS improve when the calibrated intervals barely move? | The offsets are small (2–5% of the 90% interval width) and point up. The gain comes from shifting the forecast up, mostly at the central levels below the median; width changes cost WIS. | +| E16 | Does a constant learning rate work better on the square-root scale, and do the offsets go stale late in the season? | sqrt 0.018 improves WIS at every horizon in both seasons, more than the rate scale at h0 and in 2025-26, with similar coverage. Every tracker loses WIS in March, after the peak, when the base already over-predicts and the offsets still push up. | +| E17 | Does shrinking the offset from 1 March fix the March losses? | It fixes March but wipes the offset the next season starts from, so it is worse overall. Switching calibration off from 1 March (`off_after`) gains WIS at h1–h3 but gives back most of the coverage gain. | # How to read the results From e8b0e5faa8c0432f2c11657d792ffdb25d636e01 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:52:16 -0700 Subject: [PATCH 052/107] =?UTF-8?q?docs(calibration):=20record=20E16/E17;?= =?UTF-8?q?=20ledger=20rows=20E11=E2=80=93E17;=20mark=20E06/E08=20supersed?= =?UTF-8?q?ed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notes/CALIBRATION.md | 27 ++++++++++++++++++++++++++- notes/calibration-ledger.md | 13 ++++++++++--- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index 7ac85270..8909fca8 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -27,7 +27,7 @@ projection. Full repro commands live in `notes/calibration-runbook.md`. Method options beyond the paper: `transform` (power/log space), `scales` (per-location, per-era divisors so rounds from different sources share one tracker), `lr_slow` + `fast_decay` + `burn_in_learns_slow` (the - two-term offset, below), `off_after`, `lr_seasonal`. (`lr_geo_pool`, the + two-term offset, below), `off_after`, `late_decay`, `lr_seasonal`. (`lr_geo_pool`, the geo-pooled eta in the sweep table below, was removed 2026-09-17: it never beat the baseline.) - `scripts/calibration/calibration_ws_backfill.R` — replays flu prod's `windowed_seasonal` @@ -185,6 +185,31 @@ Flu, clean `windowed_seasonal` replay, exact, cold, states only. E11 0.1 has a long tail of offsets large relative to narrow intervals (top 10%: 0.26–0.55 of the 90% width vs 0.08–0.20 for REF-op). +## sqrt constant rate (E16) and late-season decay (E17), 2026-10-01 + +Flu, clean replay, exact, cold, states only. `e16_sqrt_constant_lr.Rmd`, +`e17_late_decay.Rmd`. + +- E16: on the sqrt scale a constant rate of 0.001–0.018 (sqrt per-100k + units) improves WIS at every horizon in both seasons; best is sqrt 0.018. + Against rate 0.032 it gains more at h0 in both seasons and at every + horizon in 2025-26 (+3.0/+1.0/+1.1/+1.5), with similar coverage bias; it + doesn't help h2–h3 in 2024-25. Its offset tail is smaller (90th pct + |offset|/90% width 0.09–0.11 vs 0.12–0.15), though larger at the 0.95 + level. +- Offsets go stale in March in both seasons: every tracker loses WIS at + h1–h3 (sqrt 0.018 at h2: −7 / −28%; REF-op −20 / −24%) because the base + already over-predicts after the peak and the offsets stay positive. + March is 4–8% of season WIS, so this costs about a point. Season-level + coverage partly reflects ramp under-prediction cancelling post-peak + over-prediction. +- E17: `late_decay` (shrink the offset each round from 03-01) fixes March + but loses overall, because it wipes the offset carried into the next + season. `off_after = "03-01"` raises pooled WIS (sqrt 0.018: + +2.3/+1.2/+1.2/+1.3 vs +2.2/+0.8/+0.6/+0.6) at a coverage cost (bias + 0.094–0.116 vs 0.059–0.078). Untested: decaying only the played offset + and keeping the hidden state. + ## Data FluSight CMU-TimeSeries submissions: 83 rounds (2023-10-14 … 2026-05-30), 53 diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 2fc39370..1a9e1b07 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -22,7 +22,7 @@ that step from the reference one axis at a time. | season policy | carry, reset | carry | | burn-in | 2023-24 hub, ILI+, none | 2023-24 hub | | `settle_days` (extra revision delay) | 7, 14, 21, 28 (= 0–3 weeks extra) | 14 | -| offset structure | single term, leak (`fast_decay`), warm start, slow term | see REF-op | +| offset structure | single term, leak (`fast_decay`), late-season decay (`late_decay`), warm start, slow term | see REF-op | | update gating | always, `off_after` date | always | | eta variants | pooled, per-level, geo-pooled, seasonal window | pooled | @@ -73,11 +73,18 @@ base WIS, and pick within strata (season phase × location size). | E03 | which working scale? | count vs sqrt vs rate (log1p, quartic pre-fix only) | REF-paper | flu | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e03_scale.Rmd` | done | | E04 | leak / two-term offset | single, leak 0.1, slow term, slow + leak | REF-paper at sqrt | flu | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e04_offset_structure.Rmd` | done | | E05 | warm start from burn-in | warm vs cold, paired, for single / leak / slow + leak; warm alone | REF-paper at sqrt | flu | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e05_warm_start.Rmd` | done | -| E06 | ILI+ as burn-in | burn-in source | REF-op | flu | post-fix | final | V-head | notes only | invalid (broken replay) | +| E06 | ILI+ as burn-in | burn-in source | REF-op | flu | post-fix | final | V-head | notes only | invalid (broken replay); superseded by E13 (HHS burn-in with real vintages); not rebuilt | | E07 | one forecaster across eras | base forecaster (A ensemble / B `windowed_seasonal`; C invalid) | REF-paper at sqrt | flu | post-fix | final | V-head | `calibration_ws_experiments.R`, findings notebook | current, needs vintage rerun | -| E08 | covid | disease | old REF-paper (count, window 50) | covid | pre-fix | final | V-head, V-state, V-gallery | `calibration_qt_*_covid.Rmd` | stale, replaced by E10 | +| E08 | covid | disease | old REF-paper (count, window 50) | covid | pre-fix | final | V-head, V-state, V-gallery | `calibration_qt_*_covid.Rmd` | superseded by E10; not rebuilt | | E09 | constant lr × revision delay | constant lr grid × `settle_days` {7, 14, 21, 28}; bridge rows below | REF-paper | flu | post-fix | exact (+ vintage, for a stateful tracker) | V-curve, V-head | `e09_lr_delay.Rmd` | done | | E10 | covid at the references | disease (REF-paper, REF-op without warm start as in covid prod) | REF-paper, REF-op | covid | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e10_covid.Rmd` | done | +| E11 | constant vs adaptive lr | rate constant 0.1 vs sqrt adaptive, warm + sqrt adaptive | REF-paper | flu | post-fix | exact | V-head, V-month | `e11_constant_lr.Rmd` | done | +| E12 | calibration on the clean `windowed_seasonal` replay, no burn-in | E11 constant 0.1, REF-op cold | E11, REF-op | flu, covid | post-fix | exact | V-head | `calibration_ws_replay.R` | done | +| E13 | 2023-24 HHS burn-in on the clean replay | burn-in none vs 2023-24 HHS | E12 configs, E05 single | flu, covid | post-fix | exact | V-head (all / states / US) | `calibration_ws_replay.R burn_in` | done | +| E14 | constant lr by season (clean replay) | constant lr grid (rate scale) × season | E11 | flu | post-fix | exact | V-curve, V-head × season | `e14_lr_by_season.Rmd` | done | +| E15 | where the WIS gains come from | diagnostic: per-level pinball, offset/spread, median-only vs width-only | REF-op cold, E11 cold | flu | post-fix | exact | per level | `e15_wis_sources.Rmd` | done | +| E16 | constant lr on the sqrt scale | scale (sqrt vs rate) × constant lr grid | E14 | flu | post-fix | exact | V-curve, V-head, V-month | `e16_sqrt_constant_lr.Rmd` | done | +| E17 | late-season offset handling | none, `late_decay` 0.85/0.7/0.5 from 03-01, `off_after` 03-01 | rate 0.018, rate 0.032, sqrt 0.018 | flu | post-fix | exact | V-head, V-month | `e17_late_decay.Rmd` | done | E09 bridge rows, each one axis from the previous: REF-paper → rate scale → constant lr (sweep) → `settle_days` (sweep). From ab3f57e5b56abb0c04de44a77b5db695828e3f8e Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:31:04 -0700 Subject: [PATCH 053/107] feat(calibration): E07 notebook, ensemble vs windowed_seasonal on the same rows with exact truth --- .../e07_across_eras.Rmd | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 reports/writeups/calibration_experiments/e07_across_eras.Rmd diff --git a/reports/writeups/calibration_experiments/e07_across_eras.Rmd b/reports/writeups/calibration_experiments/e07_across_eras.Rmd new file mode 100644 index 00000000..f302aaa6 --- /dev/null +++ b/reports/writeups/calibration_experiments/e07_across_eras.Rmd @@ -0,0 +1,217 @@ +--- +title: "E07: the submitted ensemble vs one forecaster" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + hub_dir: "../FluSight-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +The hub-era results (E00–E05, E09) calibrate our submitted FluSight +forecasts: an ensemble whose members changed between seasons. **Are the +gains an artifact of that ensemble, or does the tracker help a single, +fixed forecaster the same way?** The fixed forecaster is flu prod's +`windowed_seasonal`. + +This rebuilds the NHSN half of the original E07 +(`scripts/calibration/calibration_ws_experiments.R`, finalized truth, the +2026-09-17 replay with prod's data substitutions). Its third setup, C (the +ILI+ decade), was invalid and is not rebuilt. + +**Setup.** Two forecasters, everything else shared: + +- **A**: the submitted ensemble, read from the hub as in E00 and E03–E05. +- **B**: the clean `windowed_seasonal` replay (E12–E17), with the 2023-24 + HHS backfill (E13) as its burn-in season. + +Both are cut to the same (round, location, horizon) rows: the hub's +submission rounds, all 53 locations, h0–h3 (the ensemble's h−1 is a +different model, so it is left out), and the submissions excluded from the +hub reads are dropped from B too. Both learn from and are scored on the +same NHSN data, as published each week and re-run from scratch each week, +one week of extra revision delay, with 2023-24 as a burn-in. + +The trackers are the original's three, all on the square-root scale with +the adaptive learning rate (mult 0.03, floor 0.001, window 20, carry): + +- **single**: the paper's tracker (REF-paper at sqrt). +- **leaky**: single with the offset shrinking 10% per round (`fast_decay`). +- **operating**: REF-op, prod's tracker: leaky plus a slow term and a warm + start from the burn-in season's errors. + +"WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive +is better**. "L1 coverage bias" is the mean over the 23 levels of +|empirical coverage − level|. Results lead with states only. + +# Findings + +Numbers are from the render of 1 October 2026, states only, WIS change % +(positive is better) / L1 coverage bias, h0/h1/h2/h3. All-locations numbers +point the same way. + +- **The two bases are equally miscalibrated in 2024-25** (coverage bias + 0.13–0.17, 68–77% of outcomes above the median for both). In 2025-26 the + ensemble is better calibrated (0.046–0.078 vs 0.081–0.090). Base WIS is + similar. +- **Calibration does the same thing to both.** The trackers rank the same + way on each: operating gains the most WIS, single gives the best coverage + and loses at h2–h3. Pooled over both seasons: + + | tracker | A ensemble | B `windowed_seasonal` | + |---|---|---| + | single | +2.3/0.0/−1.3/−1.8 | +2.0/0.0/−0.6/−1.0 | + | leaky | +2.2/+0.8/0.0/−0.3 | +1.7/+0.6/+0.4/+0.2 | + | operating | +3.9/+1.9/+1.5/+2.3 | +2.8/+1.1/+0.8/+1.2 | + +- **The ensemble gains about 1 point more with REF-op**, mostly at h2–h3 in + 2024-25 (+2.8/+4.3 vs +1.4/+1.7) and h0 in 2025-26 (+4.3 vs +2.2). The + single tracker loses more on the ensemble in 2025-26 (h2 −3.1 vs −0.7), + where the ensemble was already close to calibrated. +- **Answer: the gains are not an artifact of the ensemble.** On the same + rows and data, one fixed forecaster gains the same way, a little less. +- **Against the old E07** (finalized truth, the replay with prod's data + substitutions, 50 locations, h−1 included): the verdict holds. The old + explanation for B's smaller gains does not: at h0–h3 the gap is about a + point, not up to 11 (that was h−1, a different model), and B is not less + miscalibrated than the ensemble (equal in 2024-25, worse in 2025-26). + B single still has much the best 2025-26 coverage (0.010–0.013). + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +hub_fc <- suppressMessages(hub_read_forecasts(hub_dir = here::here(params$hub_dir), target = HUB_FLU_TARGET)) %>% + filter(horizon %in% 0:3) +ws_fc <- bind_rows(ws_burn_in_forecasts("flu"), wi$fc) +# Keep only (round, location, horizon, level) rows both forecasters have. +key <- c("reference_date", "location", "horizon", "level") +common <- inner_join(distinct(hub_fc, across(all_of(key))), distinct(ws_fc, across(all_of(key))), by = key) +FCS <- list( + `A ensemble` = semi_join(hub_fc, common, by = key), + `B windowed_seasonal` = semi_join(ws_fc, common, by = key) +) +sqrt_single <- list(ref = "paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3), burn_in_seasons = "2023-2024") +TRACKERS <- list( + single = sqrt_single, + leaky = c(sqrt_single, list(fast_decay = 0.1)), + operating = list(ref = "op") +) +grid <- tidyr::expand_grid(forecaster = names(FCS), tracker = names(TRACKERS)) +cals <- purrr::pmap(grid, function(forecaster, tracker) { + do.call(cal_run_cached, c(list(FCS[[forecaster]], wi$truth), TRACKERS[[tracker]], list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +}) +``` + +Rows per forecaster after the cut, by season: + +```{r rows} +purrr::imap(FCS, \(fc, nm) fc %>% mutate(season = season_of(reference_date)) %>% count(season, name = nm)) %>% + purrr::reduce(full_join, by = "season") %>% + left_join(FCS[[1]] %>% mutate(season = season_of(reference_date)) %>% group_by(season) %>% summarize(rounds = n_distinct(reference_date)), by = "season") %>% + knitr::kable() +``` + +```{r metrics} +GEOS <- c(states = "states only (US excluded)", all = "all locations") +SEASONS <- c(LIVE, "both") +score <- function(cal) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) + purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% + bind_rows() %>% + select(geos, season, horizon, which, wis, cal_err, cov50, cov90, above_median) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err, cov50, cov90)) %>% + mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) +} +m <- purrr::map2(cals, seq_len(nrow(grid)), \(cal, i) score(cal) %>% mutate(grid[i, ])) %>% + bind_rows() %>% + mutate( + forecaster = factor(forecaster, levels = names(FCS)), tracker = factor(tracker, levels = names(TRACKERS)), + geos = factor(geos, levels = names(GEOS), labels = GEOS), + season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +FC_COLS <- c(`A ensemble` = "#0072B2", `B windowed_seasonal` = "#D55E00") +``` + +# 1. The two bases + +How miscalibrated each forecaster is before calibration, states only: WIS, +L1 coverage bias, and the share of outcomes above the median (0.5 is +calibrated; above 0.5 means the forecaster under-predicts). + +```{r bases} +m %>% + filter(geos == GEOS[["states"]], tracker == "single") %>% + transmute(forecaster, season, h, cell = sprintf("%.1f / %.3f / %.2f", wis_base, cal_err_base, above_median)) %>% + tidyr::pivot_wider(names_from = h, values_from = cell) %>% + arrange(season, forecaster) %>% + knitr::kable(caption = "Base WIS / L1 coverage bias / share above the median, states only") +``` + +# 2. What calibration does to each + +WIS change % per tracker, one panel per season and horizon, states only. + +```{r head-plot, fig.height=6} +ggplot(m %>% filter(geos == GEOS[["states"]]), aes(tracker, wis_pct, fill = forecaster)) + + geom_hline(yintercept = 0, colour = "grey50") + + geom_col(position = position_dodge(width = 0.85), width = 0.8) + + facet_grid(season ~ h) + + scale_fill_manual(values = FC_COLS, name = NULL) + + labs(x = NULL, y = "WIS change % (positive is better)") + + theme(legend.position = "bottom") +``` + +L1 coverage bias, calibrated (filled) against the base (open), states only. + +```{r cov-plot, fig.height=6} +d <- m %>% filter(geos == GEOS[["states"]]) +ggplot(d, aes(tracker, cal_err_cal, colour = forecaster)) + + geom_point(aes(y = cal_err_base), shape = 1, size = 2.4, position = position_dodge(width = 0.6)) + + geom_point(size = 2.4, position = position_dodge(width = 0.6)) + + facet_grid(season ~ h) + + scale_colour_manual(values = FC_COLS, name = NULL) + + expand_limits(y = 0) + + labs(x = NULL, y = "L1 coverage bias (open: base)") + + theme(legend.position = "bottom") +``` + +WIS change % / L1 coverage bias: + +```{r tables} +season_table <- function(g) { + bind_rows( + m %>% filter(tracker == "single") %>% mutate(tracker = "base", wis_pct = 0, cal_err_cal = cal_err_base), + m + ) %>% + filter(geos == g) %>% + mutate(config = factor(paste(forecaster, tracker, sep = ", "), levels = paste(rep(names(FCS), each = 4), c("base", names(TRACKERS)), sep = ", "))) %>% + transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(config) +} +``` + +## States only + +```{r table-states} +knitr::kable(season_table(GEOS[["states"]])) +``` + +## All locations + +```{r table-all} +knitr::kable(season_table(GEOS[["all"]])) +``` From 3b46e3002a964476eafe89e921ed490630bafed5 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:31:04 -0700 Subject: [PATCH 054/107] feat(calibration): E02 notebook, off_after, per-level eta and seasonal window on the clean replay --- .../e02_eta_variants.Rmd | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 reports/writeups/calibration_experiments/e02_eta_variants.Rmd diff --git a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd new file mode 100644 index 00000000..dd2f5060 --- /dev/null +++ b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd @@ -0,0 +1,244 @@ +--- +title: "E02: switch-off date and learning-rate variants" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Variants of the adaptive tracker, each changed alone from the reference. +**Does any of them beat the reference?** + +- **Switch off in spring** (`off_after`): from a calendar date until July, + play the base forecast and stop learning; the offsets carry unchanged into + the next season. Dates: 1 April, 1 March, 15 February. +- **Per-level learning rate** (`lr_args$per_level`): each of the 23 + quantile levels gets its own eta, instead of one eta pooled over levels. +- **Seasonal window** (`lr_seasonal`): eta is sized from the last 10 rounds + plus the rounds within 5 weeks of the same point in earlier seasons, so the + start of a ramp is sized by last year's ramp. Run with carry and with + reset at the summer. + +This rebuilds the original E02 (`calibration_qt_seasons_flu.Rmd` and the +"Method variants" table in `notes/CALIBRATION.md`: count scale, submitted +ensemble, finalized truth, before the 2026-09-17 fixes). The original also +had a geo-pooled eta (one eta shared across locations, population-weighted). +That option never beat the baseline and was removed from the code on +2026-09-17, so it is not run here. + +**Setup.** As E01: flu, the clean `windowed_seasonal` replay (all 53 +locations, the 55 hub rounds, h0–h3), learning from data as published each +week and re-run from scratch each week, one week of extra revision delay. +The reference is the sqrt single-term tracker with the adaptive learning +rate (mult 0.03, floor 0.001, window 20, carry), no burn-in: E01's center +point. The seasonal window needs an earlier season to look back on, so it +runs with the 2023-24 burn-in (the HHS-stitched backfill from E13), which +only warms eta; the reference is re-run with the same burn-in as a bridge. + +"WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive +is better**. "L1 coverage bias" is the mean over the 23 levels of +|empirical coverage − level|. Results lead with states only. + +# Findings + +Numbers are from the render of 1 October 2026, states only, WIS change % +(positive is better) / L1 coverage bias, h0/h1/h2/h3, both seasons pooled +unless stated. All-locations numbers point the same way. + +- **Switching off in spring is the only variant with a large WIS gain.** + The reference is +1.9/−0.2/−1.1/−1.7. `off_after` 1 March gives + +2.8/+1.7/+2.0/+2.5 and 15 February +2.7/+2.1/+2.5/+3.2, positive at every + horizon in both seasons. 1 April only halves the h2–h3 loss + (+1.6/−0.2/−0.7/−0.8). +- **It costs coverage.** Coverage bias rises from 0.032–0.045 to + 0.080–0.091 (1 March) and 0.086–0.100 (15 February), against the base's + 0.105–0.127. From the switch-off date the base forecast is played, and the + over-prediction after the peak no longer offsets the under-prediction on + the ramp (E16). +- **Where the gain comes from.** It removes the reference's March–April + losses (2024-25 h2: −31% in March, −41% in April). The offsets frozen in + February also carry into autumn 2025 and help November–December (h2 +14% + vs +5–6%) but hurt October (−60%, on under 1% of the season's WIS). +- **Same as E17.** E17 found `off_after = "03-01"` raised WIS and gave back + most of the coverage gain for constant rates (sqrt 0.018: + +2.3/+1.2/+1.2/+1.3, coverage bias 0.094–0.116). The adaptive tracker + gains more from it because it loses more after the peak. +- **Per-level eta: a small gain at h1–h3** (+2.1/+0.5/0.0/−0.3), mostly from + smaller March–April losses, with slightly higher coverage bias + (0.043–0.055). +- **Seasonal window: no gain.** Against the reference with the same burn-in + (+2.0/0.0/−0.6/−1.0), carry gives +2.0/−0.1/−0.8/−1.3 with lower coverage + bias at h2–h3 (0.025–0.032 vs 0.040–0.044); reset gives + +1.4/−0.2/−0.9/−1.6. The burn-in alone (warming eta on 2023-24) is worth + +0.5–0.7 at h2–h3 over the cold reference. +- **Against the old E02** (count scale, ensemble, finalized truth, pre-fix): + the same ranking. Switching off on 1 March or 15 February removes the WIS + cost (15 February best at h1–h3), per-level eta is a modest gain and the + seasonal window does not help. Two changes: reset now trails carry for the + seasonal window (it used to recover some ground), and the coverage cost of + switching off, not reported before, is large. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +fc_burn_in <- bind_rows(ws_burn_in_forecasts("flu"), wi$fc) +# Same builder as E01, so the reference run is shared. +sqrt_cfg <- function(mult = 0.03, window = 20, policy = "carry") { + list( + ref = "paper", transform = "sqrt", lr_args = list(mult = mult, floor = 1e-3), + burn_in_seasons = character(0), slow_init = NULL, lr_window = window, season_policy = policy + ) +} +ref <- sqrt_cfg() +burn <- list(burn_in_seasons = "2023-2024") +seasonal <- list(lr_window = 10, lr_seasonal = list(half_width_weeks = 5)) +CFGS <- list( + reference = list(cfg = ref), + `off after 1 Apr` = list(cfg = c(ref, list(off_after = "04-01"))), + `off after 1 Mar` = list(cfg = c(ref, list(off_after = "03-01"))), + `off after 15 Feb` = list(cfg = c(ref, list(off_after = "02-15"))), + `per-level eta` = list(cfg = utils::modifyList(ref, list(lr_args = list(mult = 0.03, floor = 1e-3, per_level = TRUE)))), + `reference + burn-in` = list(cfg = utils::modifyList(ref, burn), burn_in = TRUE), + `seasonal window, carry` = list(cfg = utils::modifyList(ref, c(burn, seasonal)), burn_in = TRUE), + `seasonal window, reset` = list(cfg = utils::modifyList(ref, c(burn, seasonal, list(season_policy = "reset"))), burn_in = TRUE) +) +run <- function(spec) { + fc <- if (isTRUE(spec$burn_in)) fc_burn_in else wi$fc + do.call(cal_run_cached, c(list(fc, wi$truth), spec$cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +} +cals <- purrr::map(CFGS, run) +``` + +```{r metrics} +GEOS <- c(states = "states only (US excluded)", all = "all locations") +SEASONS <- c(LIVE, "both") +score <- function(cal) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) + purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% + bind_rows() %>% + select(geos, season, horizon, which, wis, cal_err, cov50, cov90) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err, cov50, cov90)) %>% + mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) +} +m <- purrr::imap(cals, \(cal, nm) score(cal) %>% mutate(config = nm)) %>% + bind_rows() %>% + mutate( + config = factor(config, levels = names(CFGS)), + geos = factor(geos, levels = names(GEOS), labels = GEOS), + season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +CFG_COLS <- setNames(c("black", "#56B4E9", "#0072B2", "#003F6B", "#CC79A7", "grey55", "#E69F00", "#D55E00"), names(CFGS)) +``` + +# 1. Overall, by season + +WIS change % per config, one panel per season and horizon, states only. The +seasonal-window bars compare with "reference + burn-in", the others with +"reference". + +```{r head-plot, fig.height=6} +ggplot(m %>% filter(geos == GEOS[["states"]]), aes(config, wis_pct, fill = config)) + + geom_hline(yintercept = 0, colour = "grey50") + + geom_col(width = 0.8) + + facet_grid(season ~ h) + + scale_fill_manual(values = CFG_COLS, name = NULL) + + labs(x = NULL, y = "WIS change % (positive is better)") + + theme(legend.position = "bottom", axis.text.x = element_blank(), axis.ticks.x = element_blank()) +``` + +WIS change % / L1 coverage bias. + +```{r tables} +season_table <- function(g) { + bind_rows( + m %>% filter(config == "reference") %>% mutate(config = "base", wis_pct = 0, cal_err_cal = cal_err_base), + m + ) %>% + filter(geos == g) %>% + mutate(config = factor(config, levels = c("base", names(CFGS)))) %>% + transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(config) +} +``` + +## States only + +```{r table-states} +knitr::kable(season_table(GEOS[["states"]])) +``` + +## All locations + +```{r table-all} +knitr::kable(season_table(GEOS[["all"]])) +``` + +## 50% and 90% interval coverage, states only + +```{r table-cov} +bind_rows( + m %>% filter(config == "reference") %>% mutate(config = "base", cov50_cal = cov50_base, cov90_cal = cov90_base), + m +) %>% + filter(geos == GEOS[["states"]], h %in% c("h0", "h2"), season != "both seasons") %>% + mutate(config = factor(config, levels = c("base", names(CFGS)))) %>% + transmute(config, col = paste(season, h), cell = sprintf("%.2f / %.2f", cov50_cal, cov90_cal)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(config) %>% + knitr::kable() +``` + +# 2. Month by month + +WIS change % by calendar month of the reference date; rows are seasons, +columns horizons. Grey bars are each month's share of the season's base WIS. +States only, clipped at −40%. The burn-in configs are left out except the +seasonal window with reset. + +```{r month, fig.height=6} +MONTH_CFGS <- c("reference", "off after 1 Mar", "off after 15 Feb", "per-level eta", "seasonal window, reset") +fc_m <- purrr::imap(cals[MONTH_CFGS], \(cal, nm) cal$forecasts %>% mutate(config = nm)) %>% + bind_rows() %>% + mutate(season = season_of(reference_date), config = factor(config, levels = MONTH_CFGS)) %>% + filter(season %in% LIVE, location != "US") +month_m <- ws_month_scores(fc_m) %>% mutate(h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) +share <- month_m %>% filter(config == "reference") +ggplot(month_m, aes(month, pct, colour = config, group = config)) + + geom_col(data = share, aes(month, share), inherit.aes = FALSE, fill = "grey85", width = 0.7) + + geom_hline(yintercept = 0, colour = "grey40") + + geom_line() + + geom_point(size = 1.1) + + facet_grid(season ~ h, scales = "free_x", space = "free_x") + + scale_colour_manual(values = CFG_COLS[MONTH_CFGS], name = NULL) + + coord_cartesian(ylim = c(-40, 30)) + + labs(x = "month of reference date", y = "WIS change %; grey: % of season base WIS") + + theme(legend.position = "bottom") +``` + +h2 by month: WIS change % (share of outcomes below the calibrated median). + +```{r month-table} +month_m %>% + filter(h == "h2") %>% + transmute(season, month, config, v = sprintf("%+.1f (%.2f)", pct, below_med_cal)) %>% + tidyr::pivot_wider(names_from = config, values_from = v) %>% + left_join(share %>% filter(h == "h2") %>% transmute(season, month, `base share %` = round(share, 1), `base below` = round(below_med_base, 2)), + by = c("season", "month")) %>% + arrange(season, month) %>% + knitr::kable() +``` From 851c8cc01cffdca172604263e8ba691d20214853 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:31:04 -0700 Subject: [PATCH 055/107] feat(calibration): E01 notebook, adaptive eta multiplier x window x season policy on the sqrt scale --- .../e01_eta_settings.Rmd | 272 ++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100644 reports/writeups/calibration_experiments/e01_eta_settings.Rmd diff --git a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd new file mode 100644 index 00000000..779f89a0 --- /dev/null +++ b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd @@ -0,0 +1,272 @@ +--- +title: "E01: adaptive learning-rate settings" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +The paper's tracker sets its learning rate from recent errors: +`eta = mult × 90th percentile of |outcome − forecast|` over the last +`lr_window` observed rounds. Three settings: the multiplier (`lr_mult`), the +window (`lr_window`), and what happens to the offsets over the summer +(`season_policy`: carry them into the next season, or reset them to zero). +**Which settings should we use, and which of the three matters?** + +This rebuilds the original E01 sweep (`calibration_qt_flu.Rmd`, count scale, +submitted ensemble, finalized truth, before the 2026-09-17 fixes) on the +current setup. + +**Setup.** As E14–E17: flu, the clean `windowed_seasonal` replay (all 53 +locations, the 55 hub rounds, h0–h3), learning from data as published each +week and re-run from scratch each week as prod does, one week of extra +revision delay, no burn-in season. The tracker is the plain single-term one +with the adaptive learning rate (REF-paper), **on the square-root scale** +with an eta floor of 0.001: that is the scale prod's tracker (REF-op) runs +on, and E03 found it better than counts at every horizon. One count-scale +run at the original operating point (mult 0.03, window 20, carry) links to +the old sweep. + +Grid: `lr_mult` {0.3, 0.1, 0.03, 0.01} × `lr_window` {8, 20, Inf} × +`season_policy` {carry, reset}, 24 runs. The old sweep also had window 50, +which it found indistinguishable from 20 and Inf; it is dropped here. With +no burn-in there is one summer to cross (2025 into 2025-26), so the season +policy only changes 2025-26. + +"WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive +is better**. "L1 coverage bias" is the mean over the 23 levels of +|empirical coverage − level|. Each season is scored on its own rounds. +Results lead with states only (US excluded). + +# Findings + +Numbers are from the render of 1 October 2026, states only, WIS change % +(positive is better) / L1 coverage bias, h0/h1/h2/h3, both seasons pooled +unless stated. All-locations numbers point the same way. + +- **The multiplier is the setting that matters.** Changing it moves WIS by + 16–81 points (pooled, averaged over the other settings); changing the + window moves it by 1–2 points and the season policy by 0.3–1.7. +- **On the sqrt scale, 0.1 and 0.3 are far too large.** 0.1 loses WIS at + every horizon (−1.3/−6.8/−11.3/−16.7 at window 20, carry) and 0.3 loses + 14–77%. **0.03** (REF-paper's value) gains at h0 and loses a little at + h1–h3 (+1.9/−0.2/−1.1/−1.7), with coverage bias 0.032–0.045 against the + base's 0.105–0.127. **0.01** is the only multiplier that never loses more + than 0.2 (+1.7/+0.7/+0.5/+0.5), but from a cold start it barely moves + 2024-25's coverage (0.107–0.129 vs base 0.127–0.169). It catches up in + 2025-26 (0.015–0.041). +- **Window: keep 20.** It changes nothing in 2024-25 (under 0.2 points). In + 2025-26, Inf is up to a point worse than 20 at h1–h3 for mult ≤ 0.03; 8 + and 20 are within half a point. +- **Season policy: carry.** Reset only acts at the 2025 summer, and there it + restarts the tracker from zero: at mult 0.03, window 20, 2025-26 goes from + +2.2/−0.6/−0.7/−0.2 (carry) to +0.7/−0.9/−1.3/−1.6 (reset), and coverage + bias from 0.008–0.016 to 0.039–0.050. +- **Count scale** at the same point loses more (−0.4/−2.5/−4.9/−7.0), as E03 + found. +- **Against the old E01** (count scale, ensemble, finalized truth, pre-fix): + the same answer. The multiplier is the knob, the window is immaterial, 0.1 + overpays and 0.01 is near WIS parity but slow. One change: carry vs reset + was "minor" before; here carry is clearly better in 2025-26. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +MULTS <- c(0.3, 0.1, 0.03, 0.01) +WINDOWS <- c(8, 20, Inf) +POLICIES <- c("carry", "reset") +# The sqrt single-term tracker, cold; E02 builds its configs the same way so runs are shared. +sqrt_cfg <- function(mult = 0.03, window = 20, policy = "carry") { + list( + ref = "paper", transform = "sqrt", lr_args = list(mult = mult, floor = 1e-3), + burn_in_seasons = character(0), slow_init = NULL, lr_window = window, season_policy = policy + ) +} +run <- function(cfg) { + do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +} +grid <- tidyr::expand_grid(mult = MULTS, window = WINDOWS, policy = POLICIES) +sweep <- purrr::pmap(grid, \(mult, window, policy) run(sqrt_cfg(mult, window, policy))) +count_ref <- run(list(ref = "paper", burn_in_seasons = character(0), slow_init = NULL)) +``` + +```{r metrics} +GEOS <- c(states = "states only (US excluded)", all = "all locations") +SEASONS <- c(LIVE, "both") +score <- function(cal) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) + purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% + bind_rows() %>% + select(geos, season, horizon, which, wis, cal_err, cov50, cov90) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err, cov50, cov90)) %>% + mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) +} +label <- function(df) { + df %>% mutate( + geos = factor(geos, levels = names(GEOS), labels = GEOS), + season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +} +sweep_m <- purrr::map2(sweep, seq_len(nrow(grid)), \(cal, i) score(cal) %>% mutate(grid[i, ])) %>% + bind_rows() %>% + label() %>% + mutate(window_f = factor(as.character(window), levels = as.character(WINDOWS)), policy = factor(policy, levels = POLICIES)) +count_m <- score(count_ref) %>% label() +WINDOW_COLS <- c(`8` = "#E69F00", `20` = "#0072B2", `Inf` = "#009E73") +``` + +# 1. Coverage bias vs WIS over the multiplier + +One line per window and season policy, tracing the multiplier from 0.01 +(near the base, the open circle) to 0.3. Labels mark the multiplier on the +window-20 carry line. The cross is the count-scale run at mult 0.03, window +20, carry. The y axis is clipped at −15%. + +```{r curve-fn} +curve_plot <- function(g) { + d <- sweep_m %>% filter(geos == g) %>% arrange(mult) + base <- d %>% distinct(h, season, cal_err_base) + ggplot(d, aes(cal_err_cal, wis_pct, colour = window_f)) + + geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + + geom_path(aes(linetype = policy, group = interaction(window_f, policy)), linewidth = 0.7) + + geom_point(size = 1) + + geom_text(data = d %>% filter(window == 20, policy == "carry"), aes(label = mult), + size = 2.4, vjust = -0.8, show.legend = FALSE) + + geom_point(data = base, aes(cal_err_base, 0), colour = "black", shape = 21, fill = "white", size = 2.2) + + geom_point(data = count_m %>% filter(geos == g), aes(cal_err_cal, wis_pct), colour = "black", shape = 4, size = 2.4) + + scale_colour_manual(values = WINDOW_COLS, name = "lr_window") + + scale_linetype_manual(values = c(carry = "solid", reset = "22"), name = "season policy") + + facet_grid(season ~ h) + + coord_cartesian(ylim = c(max(-15, min(d$wis_pct)), max(d$wis_pct) + 0.5)) + + labs(x = "L1 coverage bias", y = "WIS change % (positive is better)", title = g) + + theme(legend.position = "bottom") +} +``` + +## States only + +```{r curve-states, fig.height=8} +curve_plot(GEOS[["states"]]) +``` + +## All locations + +```{r curve-all, fig.height=8} +curve_plot(GEOS[["all"]]) +``` + +# 2. Which setting matters? + +For each season and horizon, how far WIS change % moves when one setting +changes and the other two stay fixed: the range over the four multipliers, +over the three windows, and the carry-vs-reset gap, each averaged over the +settings held fixed. States only. The policy gap is zero in 2024-25 by +construction (no summer crossed yet). + +```{r spread} +st <- sweep_m %>% filter(geos == GEOS[["states"]]) +spread <- bind_rows( + st %>% group_by(season, h, window, policy) %>% summarize(r = diff(range(wis_pct)), .groups = "drop") %>% + group_by(season, h) %>% summarize(v = mean(r), .groups = "drop") %>% mutate(setting = "multiplier"), + st %>% group_by(season, h, mult, policy) %>% summarize(r = diff(range(wis_pct)), .groups = "drop") %>% + group_by(season, h) %>% summarize(v = mean(r), .groups = "drop") %>% mutate(setting = "window"), + st %>% group_by(season, h, mult, window) %>% summarize(r = diff(range(wis_pct)), .groups = "drop") %>% + group_by(season, h) %>% summarize(v = mean(r), .groups = "drop") %>% mutate(setting = "season policy") +) +spread %>% + transmute(setting = factor(setting, levels = c("multiplier", "window", "season policy")), col = paste(season, h), v = round(v, 1)) %>% + tidyr::pivot_wider(names_from = col, values_from = v) %>% + arrange(setting) %>% + knitr::kable(caption = "Mean range of WIS change % (points) when one setting varies, states only") +``` + +# 3. Each metric against the multiplier + +Top: WIS change %, clipped at −30% (mult 0.3 goes far below). Bottom: L1 +coverage bias, each season's base as a thin line. Carry only, states only; +colour is the window. + +```{r vs-mult, fig.height=6.5} +long <- st %>% + filter(policy == "carry", season != "both seasons") %>% + transmute(h, season, mult, window_f, `WIS change %` = pmax(wis_pct, -30), `L1 coverage bias` = cal_err_cal) %>% + tidyr::pivot_longer(c(`WIS change %`, `L1 coverage bias`), names_to = "metric") %>% + mutate(metric = factor(metric, levels = c("WIS change %", "L1 coverage bias"))) +base_lines <- st %>% + filter(season != "both seasons") %>% + distinct(h, season, value = cal_err_base) %>% + mutate(metric = factor("L1 coverage bias", levels = levels(long$metric))) +ggplot(long, aes(mult, value, colour = window_f, linetype = season)) + + geom_hline(data = base_lines, aes(yintercept = value, linetype = season), colour = "grey40", linewidth = 0.3) + + geom_line() + + geom_point(size = 1) + + scale_x_log10(breaks = MULTS) + + scale_colour_manual(values = WINDOW_COLS, name = "lr_window") + + scale_linetype_manual(values = c(`2024-2025` = "solid", `2025-2026` = "22"), name = NULL) + + facet_grid(metric ~ h, scales = "free_y") + + labs(x = "lr_mult", y = NULL) + + theme(legend.position = "bottom") +``` + +# 4. Tables + +WIS change % / L1 coverage bias per config (rows) and season × horizon +(columns). The base row shows the base's coverage bias; the count-scale +run is last. + +```{r tables} +season_table <- function(g) { + rows <- bind_rows( + sweep_m %>% filter(mult == MULTS[1], window == 20, policy == "carry") %>% mutate(config = "base", wis_pct = 0, cal_err_cal = cal_err_base), + sweep_m %>% arrange(desc(mult), window, policy) %>% mutate(config = sprintf("sqrt mult %s, window %s, %s", mult, as.character(window), policy)), + count_m %>% mutate(config = "count mult 0.03, window 20, carry") + ) %>% + filter(geos == g) %>% + mutate(config = factor(config, levels = unique(config))) + rows %>% + transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(config) +} +``` + +## States only + +```{r table-states} +knitr::kable(season_table(GEOS[["states"]])) +``` + +## All locations + +```{r table-all} +knitr::kable(season_table(GEOS[["all"]])) +``` + +## 50% and 90% interval coverage, states only, carry + +```{r table-cov} +bind_rows( + st %>% filter(mult == MULTS[1], window == 20, policy == "carry") %>% mutate(config = "base", cov50_cal = cov50_base, cov90_cal = cov90_base), + st %>% filter(policy == "carry") %>% arrange(desc(mult), window) %>% mutate(config = sprintf("mult %s, window %s", mult, as.character(window))) +) %>% + filter(h %in% c("h0", "h2"), season != "both seasons") %>% + mutate(config = factor(config, levels = unique(config))) %>% + transmute(config, col = paste(season, h), cell = sprintf("%.2f / %.2f", cov50_cal, cov90_cal)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(config) %>% + knitr::kable() +``` From 335b7d086f6b52f9684deb610ddbfa5d85fae743 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:31:04 -0700 Subject: [PATCH 056/107] docs(calibration): index rows for E01, E02, E07 --- reports/writeups/calibration_experiments/index.Rmd | 3 +++ 1 file changed, 3 insertions(+) diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index 3b47a6b8..1cb97dae 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -35,9 +35,12 @@ for (f in src) { | | question | short answer | |---|---|---| | E00 | Should the tracker learn from finalized data or from data as published at the time? | As published, re-run each week as prod does. Finalized data overstated the earlier gains only a little (about 2 points at h−1). | +| E01 | Which adaptive learning-rate settings (multiplier, window, carry or reset over the summer)? | The multiplier is what matters; the window barely does, and carrying offsets beats resetting. On the square-root scale 0.03 gains at h0 and loses a little at h1–h3; 0.01 never loses but corrects coverage slowly from a cold start. | +| E02 | Do a spring switch-off date, a per-level learning rate or a seasonal window improve on the adaptive tracker? | Switching off from 1 March or 15 February gains 2–5 points of WIS over the reference at h1–h3, at the cost of most of the coverage gain. A per-level rate gains a little; a seasonal window does not help. | | E03 | Which scale should the offsets live on: counts, square root, or rate per 100k? | Square root, at every horizon, mostly by avoiding spring losses. Rate equals counts with the adaptive learning rate. | | E04 | Should the offset shrink between updates, or have a second slow component? | Shrinking buys WIS at h1–h3 but nearly triples coverage error. A slow component alone does nothing. | | E05 | Should the offset start from the burn-in season's errors instead of zero? | Yes; it never hurts. With the plain tracker it gives the best-calibrated config so far. | +| E07 | Are the gains an artifact of calibrating the submitted ensemble? | No. On the same rows and data, the single `windowed_seasonal` forecaster gains the same way, about a point less with prod's tracker. | | E09 | With a constant learning rate on the rate scale, how do coverage and WIS trade off, and how much revision delay is best? | An L-shaped trade-off whose knee matches or beats the paper's adaptive rate at every horizon. Under prod's weekly re-run, less delay is better. | | E10 | Do the flu results carry over to covid? | Only at h−1 (+10% WIS). The covid base is already calibrated at h1–h3, where calibrating is neutral or harmful. | | E11 | How does E09's best constant learning rate compare with the adaptive trackers? | Better coverage than the cold adaptive trackers at less WIS gain near the present, and the most stable across seasons. The warm-started adaptive tracker still wins at h0–h1. | From 4ac93b77a3fa5f7396d3195a585e963a8d37f43d Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:32:06 -0700 Subject: [PATCH 057/107] docs(calibration): record the rebuilt E01, E02, E07 and mark the old ones superseded --- notes/CALIBRATION.md | 28 ++++++++++++++++++++++++++++ notes/calibration-ledger.md | 15 +++++++++------ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index 8909fca8..e3b8cf2a 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -210,6 +210,27 @@ Flu, clean replay, exact, cold, states only. `e16_sqrt_constant_lr.Rmd`, 0.094–0.116 vs 0.059–0.078). Untested: decaying only the played offset and keeping the hidden state. +## E01, E02, E07 rebuilt (2026-10-01) + +Flu, clean replay, exact, post-fix, states only. + +- E01 (`e01_eta_settings.Rmd`, sqrt REF-paper, cold): the multiplier is + the only setting that matters (16–81 points of pooled WIS; window 1–2, + season policy 0.3–1.7). 0.1 and 0.3 are too large; 0.03 gains at h0 and + loses a little at h1–h3; 0.01 never loses more than 0.2 but barely fixes + 2024-25 coverage from a cold start. Carry beats reset. Same conclusion + as the old sweep, except carry vs reset now clearly favors carry. +- E02 (`e02_eta_variants.Rmd`): `off_after` 03-01 or 02-15 is the only + large WIS gain (positive at every horizon in both seasons) but gives up + most of the coverage gain (bias 0.080–0.100 vs 0.032–0.045), matching + E17. Per-level eta: small gain at h1–h3. Seasonal window: no help. +- E07 (`e07_across_eras.Rmd`, identical rows, each forecaster with its own + 2023-24 burn-in): the trackers rank the same on the hub ensemble and on + `windowed_seasonal`, so the gains aren't an ensemble artifact. The + ensemble gains about a point more with REF-op (pooled A + +3.9/+1.9/+1.5/+2.3, B +2.8/+1.1/+0.8/+1.2). B is not less + miscalibrated than A (equal in 2024-25, worse in 2025-26). + ## Data FluSight CMU-TimeSeries submissions: 83 rounds (2023-10-14 … 2026-05-30), 53 @@ -232,6 +253,8 @@ and vintages live; the findings notebook uses it. ## Findings (factorial sweep, `calibration_qt_flu.Rmd`; count space, pre-fix) +Superseded by the rebuilt E01 (`e01_eta_settings.Rmd`, 2026-10-01). + Sweep: `season_policy` {carry, reset} × `lr_window` {8, 20, 50, Inf} × `lr_mult` {0.3, 0.1, 0.03, 0.01}; results in `cache/calibration/sweep.rds`. @@ -316,6 +339,8 @@ against the latest NHSN vintage via `nhsn_read_truth()`. ## Method variants (`calibrate_hub_forecasts()` options; 2026-08-27, pre-fix) +Superseded by the rebuilt E02 (`e02_eta_variants.Rmd`, 2026-10-01). + All count space at the original operating point; WIS change vs base, both live seasons, by horizon −1/0/1/2/3. Pre-fix numbers (see above); the transform rows measured on 2026-09-16 were removed, see "Current numbers": @@ -677,6 +702,9 @@ that at the peak bin is 32–38 in count space and 5–10 in sqrt. ### One forecaster across eras (2026-09-17) +Superseded by the rebuilt E07 (`e07_across_eras.Rmd`, 2026-10-01). Its +"less miscalibrated" explanation below is wrong at h0–h3. + Question: are the hub-season gains an artifact of three seasons of one ensemble, and does the tracker behave the same on a decade of the *same* forecaster? `scripts/calibration/calibration_ws_backfill.R` replays flu prod's diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 1a9e1b07..2c7a6e16 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -68,13 +68,13 @@ base WIS, and pick within strata (season phase × location size). | id | question | varies | reference | disease | code | truth | views | where | status | |---|---|---|---|---|---|---|---|---|---| | E00 | how much did final truth flatter results; can vintage stand in for exact? | learning truth {final, vintage, exact} × settle {7, 14} | REF-paper, REF-op | flu | post-fix | all three | V-head | `e00_vintage_backtest.Rmd` | done: exact is the default; vintage is not a stand-in | -| E01 | which eta settings? | season policy × window × mult | REF-paper | flu | pre-fix | final | V-head, V-curve (mult) | `calibration_qt_flu.Rmd` | stale | -| E02 | cutoff and eta variants | `off_after`, per-level, geo-pool, seasonal window (one at a time) | REF-paper | flu | pre-fix | final | V-head, V-month, V-state | `calibration_qt_seasons_flu.Rmd` | stale | +| E01 | which eta settings? | season policy × window × mult | REF-paper | flu | pre-fix | final | V-head, V-curve (mult) | `calibration_qt_flu.Rmd` | superseded by `e01_eta_settings.Rmd` (2026-10-01) | +| E02 | cutoff and eta variants | `off_after`, per-level, geo-pool, seasonal window (one at a time) | REF-paper | flu | pre-fix | final | V-head, V-month, V-state | `calibration_qt_seasons_flu.Rmd` | superseded by `e02_eta_variants.Rmd` (2026-10-01) | | E03 | which working scale? | count vs sqrt vs rate (log1p, quartic pre-fix only) | REF-paper | flu | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e03_scale.Rmd` | done | | E04 | leak / two-term offset | single, leak 0.1, slow term, slow + leak | REF-paper at sqrt | flu | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e04_offset_structure.Rmd` | done | | E05 | warm start from burn-in | warm vs cold, paired, for single / leak / slow + leak; warm alone | REF-paper at sqrt | flu | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e05_warm_start.Rmd` | done | | E06 | ILI+ as burn-in | burn-in source | REF-op | flu | post-fix | final | V-head | notes only | invalid (broken replay); superseded by E13 (HHS burn-in with real vintages); not rebuilt | -| E07 | one forecaster across eras | base forecaster (A ensemble / B `windowed_seasonal`; C invalid) | REF-paper at sqrt | flu | post-fix | final | V-head | `calibration_ws_experiments.R`, findings notebook | current, needs vintage rerun | +| E07 | one forecaster across eras | base forecaster (A ensemble / B `windowed_seasonal`; C invalid) | REF-paper at sqrt | flu | post-fix | final | V-head | `calibration_ws_experiments.R`, findings notebook | superseded by `e07_across_eras.Rmd` (2026-10-01) | | E08 | covid | disease | old REF-paper (count, window 50) | covid | pre-fix | final | V-head, V-state, V-gallery | `calibration_qt_*_covid.Rmd` | superseded by E10; not rebuilt | | E09 | constant lr × revision delay | constant lr grid × `settle_days` {7, 14, 21, 28}; bridge rows below | REF-paper | flu | post-fix | exact (+ vintage, for a stateful tracker) | V-curve, V-head | `e09_lr_delay.Rmd` | done | | E10 | covid at the references | disease (REF-paper, REF-op without warm start as in covid prod) | REF-paper, REF-op | covid | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e10_covid.Rmd` | done | @@ -85,6 +85,9 @@ base WIS, and pick within strata (season phase × location size). | E15 | where the WIS gains come from | diagnostic: per-level pinball, offset/spread, median-only vs width-only | REF-op cold, E11 cold | flu | post-fix | exact | per level | `e15_wis_sources.Rmd` | done | | E16 | constant lr on the sqrt scale | scale (sqrt vs rate) × constant lr grid | E14 | flu | post-fix | exact | V-curve, V-head, V-month | `e16_sqrt_constant_lr.Rmd` | done | | E17 | late-season offset handling | none, `late_decay` 0.85/0.7/0.5 from 03-01, `off_after` 03-01 | rate 0.018, rate 0.032, sqrt 0.018 | flu | post-fix | exact | V-head, V-month | `e17_late_decay.Rmd` | done | +| E01 (rebuilt) | which adaptive-eta settings? | mult {0.3, 0.1, 0.03, 0.01} × window {8, 20, Inf} × season policy; count bridge | REF-paper at sqrt, cold | flu (clean replay) | post-fix | exact | V-curve, V-head | `e01_eta_settings.Rmd` | done | +| E02 (rebuilt) | cutoff and eta variants | `off_after` {04-01, 03-01, 02-15}, per-level eta, seasonal window carry/reset (HHS burn-in) + burn-in bridge; geo-pool removed | E01 center (sqrt, 0.03, 20, carry) | flu (clean replay) | post-fix | exact | V-head, V-month | `e02_eta_variants.Rmd` | done | +| E07 (rebuilt) | one forecaster across eras | forecaster (A hub ensemble / B clean replay) on identical rows, each with its own 2023-24 burn-in | single, leaky, REF-op | flu | post-fix | exact | V-head, base bias | `e07_across_eras.Rmd` | done | E09 bridge rows, each one axis from the previous: REF-paper → rate scale → constant lr (sweep) → `settle_days` (sweep). @@ -100,6 +103,6 @@ bias on NHSN and ILI+. See `notes/CALIBRATION.md`. US rounds; covid: the 2024-11-23 round, 2026-04-25 h−1). E00, E03–E05, E09 and E10 are re-run without them; every older result, and the numbers in `notes/CALIBRATION.md` before that date, include them. -- **Learning truth.** E01, E02 and E06–E08 learned from finalized NHSN (E00 - shows the error is small outside h−1). E03–E05 have been re-run exactly. -- **E07** (ensemble vs `windowed_seasonal`) has not been re-run exactly. +- **Learning truth.** E06 and E08 learned from finalized NHSN (E00 shows + the error is small outside h−1); both are superseded, not re-run. Every + other experiment is exact. From 59300435cfb5585de173e5f6d19f43f3ed0f6bbb Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:51:35 -0700 Subject: [PATCH 058/107] =?UTF-8?q?docs(calibration):=20summary=20of=20E01?= =?UTF-8?q?,=20E02,=20E07=20and=20E11=E2=80=93E17;=20drop=20a=20collaborat?= =?UTF-8?q?or=20mention=20from=20E14?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notes/calibration-summary-2026-10-01.md | 152 ++++++++++++++++++ .../e14_lr_by_season.Rmd | 4 +- 2 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 notes/calibration-summary-2026-10-01.md diff --git a/notes/calibration-summary-2026-10-01.md b/notes/calibration-summary-2026-10-01.md new file mode 100644 index 00000000..7c64951e --- /dev/null +++ b/notes/calibration-summary-2026-10-01.md @@ -0,0 +1,152 @@ +# Calibration: where the experiments stand (2026-10-01) + +A summary of E01, E02, E07 and E11–E17. Each notebook is in +`reports/calibration_experiments/` (`index.html` lists them all). + +## How these were run + +All of these are flu only, except E12 and E13, which cover covid as well. + +- **Forecaster.** Every experiment except E11 calibrates one fixed forecaster, + `windowed_seasonal`. It was replayed weekly from 2024-11-20 with none of + the hand edits we make to prod data (the "clean replay"). The submitted + ensemble changes weights and inputs by hand week to week, and some of its + rounds are known to be spoiled. A fixed forecaster removes both from the + comparison. E11 used the submitted ensemble, as E00–E10 did. +- **Learning truth.** The tracker learns from data as it was published each + week, and is re-run from scratch each week, as prod would run it. Scores + use finalized data. +- **Rounds and seasons.** The 55 hub rounds, 27 in 2024-25 and 28 in + 2025-26. Results are shown per season, because a setting can look good + pooled and still fail in one season. +- **Horizons.** h0–h3 only. h−1 is left to the revision-aware methods. +- **States only.** US is excluded from the headline numbers. WIS is in + counts, so US alone is about 45% of the all-locations total. The + all-locations numbers point the same way throughout. +- **Cold start.** The tracker starts from zero offsets in November 2024 + unless a burn-in is stated. + +"WIS change" is % against the uncalibrated forecast, positive is better. +"Coverage bias" is the mean |coverage − nominal| over quantile levels. + +## Findings + +**1. A constant learning rate on the sqrt scale is the best simple +tracker so far (E14, E16).** + +On the rate scale (per 100k), a constant rate of 0.018–0.032 improves WIS +at every horizon in both seasons (E14). E11's 0.1 is past that range: WIS +gets worse at h1–h3 in both seasons. The sqrt scale does better than the +rate scale (E16). Every sqrt rate from 0.001 to 0.018 improves WIS at +every horizon in both seasons, and 0.018 gains the most. + +| config | 2024-25 h0 | 2024-25 h2 | 2025-26 h0 | 2025-26 h2 | +|---|---|---|---|---| +| sqrt, constant 0.018 | +1.7 | +0.3 | +3.0 | +1.1 | +| rate, constant 0.032 | +1.0 | +0.3 | +1.9 | +0.8 | +| REF-op (current, cold) | +2.2 | 0.0 | +1.8 | +0.3 | + +REF-op gains more at h0–h1 in 2024-25 but loses at h3 there (−0.8). It also barely +moves 2024-25 coverage (bias 0.106–0.124, base 0.127–0.169). Neither REF-op +nor the adaptive sqrt tracker is good in both seasons at every horizon. + +**2. The calibration is a small upward shift, which is why the plots barely +change (E15).** + +- **The offsets are small.** The typical offset is 2–5% of the 90% + interval's width, and calibrated intervals are 1.00–1.04× the base width. +- **They point up.** The base under-predicts, and the tracker moves the + whole forecast up a little. +- **The gain is a median shift.** For REF-op, shifting every quantile by + the median offset alone gives more WIS than the full calibration. The + width and shape changes cost WIS and only buy coverage. +- **The gain is in the center and shoulders, not the tails.** The six tail + levels contribute at most 0.1 point. +- **Why a rate-scale constant step hurts:** a fixed step per 100k is large + for a forecast with a narrow interval, such as a small state or an + off-season week. For 1 forecast in 10 under E11, the offset exceeds + 0.26–0.55 of the 90% width. The sqrt scale shrinks this tail (E16). + +**3. Offsets go stale after the peak, and March is where it costs (E16, +E17, E02).** + +- **The March loss.** Every tracker loses WIS at h1–h3 in March of both + seasons. After the peak the base already over-predicts, and the offsets + are still positive. March is 4–8% of a season's WIS, so this costs about + a point over the season. +- **Season-level coverage hides it.** Under-prediction on the ramp and + over-prediction after the peak partly cancel in the season total. +- **Decaying the offsets toward zero from 1 March fixes March but loses + overall (E17).** The offsets carry into the next season, so the decay + also hands that season a near-zero start. +- **Switching calibration off from 1 March (or 15 February) is the one + variant with a large WIS gain (E02, E17).** It is positive at every + horizon in both seasons, but it gives up most of the coverage gain: + - **sqrt 0.018, pooled WIS:** +2.3/+1.2/+1.2/+1.3 switched off from + March, against +2.2/+0.8/+0.6/+0.6 left on; + - **sqrt 0.018, coverage bias:** 0.094–0.116 switched off, 0.059–0.078 + left on, and 0.105–0.127 for the base. + +**4. On the adaptive tracker, the multiplier is the only setting that +matters (E01, E02).** + +| setting | effect on pooled WIS | +|---|---| +| multiplier | 16–81 points | +| window | 1–2 points | +| carry vs reset | 0.3–1.7 points | + +- **On the sqrt scale, multipliers of 0.1 and 0.3 are too large.** +- **0.03** gains at h0 and loses a little at h1–h3. +- **0.01** never loses more than 0.2 points, but from a cold start it + barely fixes 2024-25 coverage. +- **Carry beats reset.** +- **Per-level eta** gives a small gain at h1–h3; the seasonal window gives + none. +- These agree with the original (pre-fix, finalized-truth) versions of + E01 and E02. + +**5. The gains are not specific to the ensemble (E07).** + +On identical rows, the trackers rank the same way on the submitted ensemble +and on `windowed_seasonal`. The ensemble gains about a point more with +REF-op (pooled +3.9/+1.9/+1.5/+2.3 vs +2.8/+1.1/+0.8/+1.2). The two are +equally miscalibrated in 2024-25. In 2025-26 the ensemble is better +calibrated. The original E07's explanation for the gap, that +`windowed_seasonal` was better calibrated to begin with, does not hold at +h0–h3. + +**6. A 2023-24 burn-in helps the adaptive trackers on flu a little, and +doesn't help a constant rate (E13).** + +- **Burn-in data.** 2023-24 comes from HHS hospital data at its real + weekly versions. It runs about 3% below NHSN for flu and 6% for covid. +- **Flu.** A warm start adds 0.5–1.6 points for REF-op and E05's single + term. The warm configs under-cover the 50% interval (0.40–0.44). +- **Covid.** A warm start is wrong-signed: 2023-24 over-predicted, while + the live seasons under-predict at h0. +- **Constant rate.** A warm start gives no gain on flu, and is badly + harmful on covid (−18% at h3). + +## Where this leaves the operating point + +| candidate | for | against | +|---|---|---| +| sqrt, constant 0.018 | WIS-positive at every horizon in both seasons; small offsets; no learning-rate adaptation to tune | small gains at h2–h3 in 2024-25; loses in March | +| REF-op (current) | largest h0 gain in 2024-25; best with a flu burn-in | loses at h3 in 2024-25; barely improves 2024-25 coverage when cold | +| either, switched off from March | more WIS at h1–h3 | gives up most of the coverage gain | + +The gains are 1–3% of WIS and come from a small upward shift. Whether that's +worth running in prod depends on how much weight goes on coverage against +WIS. + +## Open questions + +- **A decay that keeps the next season warm.** Shrink only the offset + that's applied to the forecast after the peak, and keep the tracker's + learned state, so the next season doesn't start cold. +- **Covid.** Every result here is flu except E12 and E13. Covid under the + sqrt constant rate is untested. The covid results so far (E10, E12, E13) + point to calibrating h0 only. +- **Month-level coverage as a headline metric**, so cancellation within a + season stops flattering the season totals. diff --git a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd index 28991639..71ac8425 100644 --- a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd +++ b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd @@ -20,8 +20,8 @@ ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplo ``` E09 traced how coverage and WIS trade off as the constant learning rate of -the plain tracker changes, pooled over both live seasons. The question -here, from the collaborators: split those curves by season. **Is there a +the plain tracker changes, pooled over both live seasons. This notebook +splits those curves by season. **Is there a range of learning rates that is good in both 2024-25 and 2025-26?** **Setup.** The clean `windowed_seasonal` replay for flu (E12: substitutions From 9e6fddefe31f7486071e925e08c01ab49a34a7bb Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:14:04 -0700 Subject: [PATCH 059/107] docs(calibration): review tracker; E09 WIS axis positive-is-better; E01 marker legend; E01/E02/E07 intros say what changed --- notes/CALIBRATION.md | 6 +- notes/calibration-review.md | 48 +++++++++++++++ .../e01_eta_settings.Rmd | 18 ++++-- .../e02_eta_variants.Rmd | 16 ++++- .../e07_across_eras.Rmd | 11 ++-- .../calibration_experiments/e09_lr_delay.Rmd | 58 +++++++++---------- 6 files changed, 113 insertions(+), 44 deletions(-) create mode 100644 notes/calibration-review.md diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index e3b8cf2a..b2395db7 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -88,10 +88,10 @@ submissions excluded): weeks, exact and stateful learning truth. Pooled over locations: - The coverage/WIS trade-off is L-shaped, with a knee at a learning rate of - about 0.05–0.1 per 100k. At 0.056 and +1 week (re-run): WIS - −3.2/−2.5/−1.1/−0.5/−0.2% (negative is better), coverage bias + about 0.05–0.1 per 100k. At 0.056 and +1 week (re-run): WIS change + +3.2/+2.5/+1.1/+0.5/+0.2% (positive is better), coverage bias 0.040/0.031/0.026/0.027/0.027. That matches or beats REF-paper - (−3.2/−1.9/+2.0/+2.7/+5.3%, 0.044/0.030/0.032/0.034/0.032) at every + (+3.2/+1.9/−2.0/−2.7/−5.3%, 0.044/0.030/0.032/0.034/0.032) at every horizon. - With the adaptive rate, rate scale and count scale give identical results (the adaptive rate rescales with the location), so the rate scale matters diff --git a/notes/calibration-review.md b/notes/calibration-review.md new file mode 100644 index 00000000..2c7601d7 --- /dev/null +++ b/notes/calibration-review.md @@ -0,0 +1,48 @@ +# Calibration review + +Tracks which experiment notebooks Dmitry has read and verified, his +comments, and what was done about them. Notebooks render to +`reports/calibration_experiments/`; sources are in +`reports/writeups/calibration_experiments/`. + +Status: **verified** (read and agreed), **commented** (read, changes asked +for), **not reviewed**. + +## How the experiments build on each other + +1. **E00: learn from data as published.** Finalized data flatters the h−1 + and h0 gains, though not by much. Every experiment from E00 on learns + from each week's published data (the "exact" design), with no further + work needed. +2. **E07: one fixed forecaster is enough.** On identical rows, calibration + gains are similar on the submitted ensemble and on `windowed_seasonal`, + and the trackers rank the same way. So experiments can focus on the + clean `windowed_seasonal` replay, which is free of hand edits and + spoiled submissions. +3. Experiments before E07's re-run (E03–E05, E09–E11) used the submitted + ensemble; E01, E02 (new versions) and E12–E17 use the clean replay. + +## Notebooks + +| exp | forecaster | status | comments | done | +|---|---|---|---|---| +| E00 vintage backtest | ensemble | verified | Finalized data exaggerates h−1/h0 gains a little; use vintages, no more work. | — | +| E01 eta settings | clean replay | commented | (1) Curve axis looked flipped vs E09. (2) x/o markers unlabeled. (3) "Rebuilds the original" is misleading: it re-asks the question on a new setup. | (1) E09 flipped to match; (2) legend added; (3) intro reworded with a what-changed table | +| E02 eta variants | clean replay | not reviewed | | intro reworded (same issue as E01) | +| E03 scale | ensemble | not reviewed | | | +| E04 offset structure | ensemble | not reviewed | | | +| E05 warm start | ensemble | not reviewed | | | +| E07 across eras | both | commented | Needs concrete base WIS numbers, not "similar". Expand into the ensemble vs `windowed_seasonal` comparison: base WIS side by side, base bias by month, every operating-point candidate on both. | intro reworded; expansion in progress | +| E09 lr × delay | ensemble | commented | WIS axis inverted relative to the other notebooks. | flipped to "WIS change %, positive is better" | +| E10 covid | ensemble | not reviewed | | | +| E11 constant lr | ensemble | not reviewed | | | +| E14 lr by season | clean replay | not reviewed | | | +| E15 WIS sources | clean replay | not reviewed | | | +| E16 sqrt constant lr | clean replay | not reviewed | | | +| E17 late decay | clean replay | not reviewed | | | + +E12 and E13 have no notebooks; they live in `notes/CALIBRATION.md` and +`scripts/calibration/calibration_ws_replay.R`. E06 and E08 were superseded +(by E13 and E10) and not rebuilt. + +The summary for sharing is `notes/calibration-summary-2026-10-01.md`. diff --git a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd index 779f89a0..e369a0df 100644 --- a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd +++ b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd @@ -26,9 +26,16 @@ window (`lr_window`), and what happens to the offsets over the summer (`season_policy`: carry them into the next season, or reset them to zero). **Which settings should we use, and which of the three matters?** -This rebuilds the original E01 sweep (`calibration_qt_flu.Rmd`, count scale, -submitted ensemble, finalized truth, before the 2026-09-17 fixes) on the -current setup. +This asks the original E01's question (`calibration_qt_flu.Rmd`) on the +current setup. It is a new experiment, not a reproduction: four things +changed at once, so it can't say which change matters. + +| | original E01 | this notebook | +|---|---|---| +| forecaster | submitted ensemble | clean `windowed_seasonal` replay | +| scale | count | sqrt | +| learning truth | finalized | as published each week | +| tracker code | before the 2026-09-17 fixes | after | **Setup.** As E14–E17: flu, the clean `windowed_seasonal` replay (all 53 locations, the 55 hub rounds, h0–h3), learning from data as published each @@ -146,8 +153,9 @@ curve_plot <- function(g) { geom_point(size = 1) + geom_text(data = d %>% filter(window == 20, policy == "carry"), aes(label = mult), size = 2.4, vjust = -0.8, show.legend = FALSE) + - geom_point(data = base, aes(cal_err_base, 0), colour = "black", shape = 21, fill = "white", size = 2.2) + - geom_point(data = count_m %>% filter(geos == g), aes(cal_err_cal, wis_pct), colour = "black", shape = 4, size = 2.4) + + geom_point(data = base, aes(cal_err_base, 0, shape = "base (uncalibrated)"), colour = "black", fill = "white", size = 2.2) + + geom_point(data = count_m %>% filter(geos == g), aes(cal_err_cal, wis_pct, shape = "count scale, mult 0.03"), colour = "black", size = 2.4) + + scale_shape_manual(values = c(`base (uncalibrated)` = 21, `count scale, mult 0.03` = 4), name = NULL) + scale_colour_manual(values = WINDOW_COLS, name = "lr_window") + scale_linetype_manual(values = c(carry = "solid", reset = "22"), name = "season policy") + facet_grid(season ~ h) + diff --git a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd index dd2f5060..20fb2552 100644 --- a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd +++ b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd @@ -32,9 +32,19 @@ Variants of the adaptive tracker, each changed alone from the reference. start of a ramp is sized by last year's ramp. Run with carry and with reset at the summer. -This rebuilds the original E02 (`calibration_qt_seasons_flu.Rmd` and the -"Method variants" table in `notes/CALIBRATION.md`: count scale, submitted -ensemble, finalized truth, before the 2026-09-17 fixes). The original also +This asks the original E02's question (`calibration_qt_seasons_flu.Rmd` +and the "Method variants" table in `notes/CALIBRATION.md`) on the current +setup. It is a new experiment, not a reproduction: four things changed at +once, so it can't say which change matters. + +| | original E02 | this notebook | +|---|---|---| +| forecaster | submitted ensemble | clean `windowed_seasonal` replay | +| scale | count | sqrt | +| learning truth | finalized | as published each week | +| tracker code | before the 2026-09-17 fixes | after | + +The original also had a geo-pooled eta (one eta shared across locations, population-weighted). That option never beat the baseline and was removed from the code on 2026-09-17, so it is not run here. diff --git a/reports/writeups/calibration_experiments/e07_across_eras.Rmd b/reports/writeups/calibration_experiments/e07_across_eras.Rmd index f302aaa6..310be4ec 100644 --- a/reports/writeups/calibration_experiments/e07_across_eras.Rmd +++ b/reports/writeups/calibration_experiments/e07_across_eras.Rmd @@ -26,10 +26,13 @@ gains an artifact of that ensemble, or does the tracker help a single, fixed forecaster the same way?** The fixed forecaster is flu prod's `windowed_seasonal`. -This rebuilds the NHSN half of the original E07 -(`scripts/calibration/calibration_ws_experiments.R`, finalized truth, the -2026-09-17 replay with prod's data substitutions). Its third setup, C (the -ILI+ decade), was invalid and is not rebuilt. +This asks the NHSN half of the original E07's question +(`scripts/calibration/calibration_ws_experiments.R`) on the current setup. +It is a new experiment, not a reproduction: the original learned from +finalized data and replayed `windowed_seasonal` with prod's data +substitutions; here both learn from data as published each week, and B is +the clean replay. The original's third setup, C (the ILI+ decade), was +invalid and is not repeated. **Setup.** Two forecasters, everything else shared: diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd index 2a2592b5..3fed985d 100644 --- a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -79,8 +79,8 @@ one offset per level, carried across seasons, and a 2023-24 burn-in season whose outcomes take no steps (the adaptive rate uses it to warm up; the constant rate needs no warm-up). Scores are on counts against finalized truth, over the two live seasons (2024-25 and 2025-26), all 53 hub -locations. "WIS inflation %" is `100 × (WIS_cal / WIS_base − 1)`, so negative -is better than the base. "L1 coverage bias" is the mean over the 23 quantile +locations. "WIS change %" is `100 × (1 − WIS_cal / WIS_base)`, so positive +is better than the base, as in the other notebooks. "L1 coverage bias" is the mean over the 23 quantile levels of |empirical coverage − level|. **Reference configs** (marked on the plots): @@ -100,26 +100,26 @@ step. - **The trade-off is L-shaped.** Raising the constant learning rate cuts the coverage bias from about 0.10 to 0.03 at almost no WIS cost, up to a knee - near 0.05–0.1. Past the knee WIS climbs steeply and coverage barely + near 0.05–0.1. Past the knee WIS worsens steeply and coverage barely improves. At the reference delay, h−1 has its best WIS at 0.1 (at 0.32 with no extra delay). The per-location lines spread widely: past 0.3 many - locations inflate WIS by 20% or more. + locations lose 20% of WIS or more. - **On the rate scale, the knee matches or beats the paper's adaptive tracker at every horizon.** At the reference delay (+1 week, re-run - design), the knee gives WIS inflation −3.2/−2.5/−1.1/−0.5/−0.2% and coverage + design), the knee gives WIS change +3.2/+2.5/+1.1/+0.5/+0.2% and coverage bias 0.040/0.031/0.026/0.027/0.027 at h−1…h3. REF-paper gives - −3.2/−1.9/+2.0/+2.7/+5.3% and 0.044/0.030/0.032/0.034/0.032: the same WIS + +3.2/+1.9/−2.0/−2.7/−5.3% and 0.044/0.030/0.032/0.034/0.032: the same WIS at h−1, worse WIS from h0 on, and worse coverage at h−1 and h1–h3. - **The rate scale only matters with a constant learning rate.** "rate, adaptive" matches REF-paper exactly. The adaptive rate is a quantile of the location's own residuals, so dividing a location by its population rescales the steps and the offsets together. -- **REF-op makes a different trade.** Its WIS gain (−5.4/−4.9/−2.4/−1.7/−2.2%) +- **REF-op makes a different trade.** Its WIS gain (+5.4/+4.9/+2.4/+1.7/+2.2%) is larger than any constant rate reaches at h0–h3, but at about twice the coverage bias (0.074/0.054/0.045/0.046/0.041). The leak and the warm start buy WIS; the plain tracker buys coverage. - **Extra delay, re-run design (prod): shorter is better, but it matters only - at h−1/h0.** At a learning rate of 0.1, h−1 WIS inflation is −5.3/−3.6/−1.0% + at h−1/h0.** At a learning rate of 0.1, h−1 WIS change is +5.3/+3.6/+1.0% at +0/+1/+2 weeks. At h1–h3 the curves nearly coincide, and coverage bias barely moves with delay. A value learned early is replaced by its revision in the next week's run, so waiting only delays the signal. @@ -194,7 +194,7 @@ refs <- purrr::pmap(ref_grid, function(learn, settle_days) { ```{r metrics, cache=FALSE} pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) -# Per (location, horizon) and pooled over locations: WIS inflation and L1 +# Per (location, horizon) and pooled over locations: WIS change and L1 # coverage bias, for the calibrated forecast and the base. metrics <- function(cal) { fc <- cal$forecasts %>% filter(!is_burn_in, !is.na(truth), !is.na(value_base)) @@ -209,7 +209,7 @@ metrics <- function(cal) { group_by(across(all_of(c(by, "horizon")))) %>% summarize( bias_base = mean(abs(cov_base - level)), bias_cal = mean(abs(cov_cal - level)), - wis_infl = 100 * (sum(wis_cal) / sum(wis_base) - 1), + wis_pct = 100 * (1 - sum(wis_cal) / sum(wis_base)), .groups = "drop" ) } @@ -244,7 +244,7 @@ REF-op is for orientation only; it differs in several settings. best_const <- sweep_m %>% filter(location == "pooled", learn == "exact", settle_days == 14L) %>% group_by(horizon) %>% - slice_min(wis_infl, n = 1, with_ties = FALSE) %>% + slice_max(wis_pct, n = 1, with_ties = FALSE) %>% ungroup() %>% mutate(ref = "rate, constant lr (best WIS per horizon)") bridge <- bind_rows( @@ -252,11 +252,11 @@ bridge <- bind_rows( best_const ) %>% mutate(ref = factor(ref, levels = c("REF-paper (count, adaptive)", "rate, adaptive", "rate, constant lr (best WIS per horizon)", "REF-op"))) -base_row <- bridge %>% filter(ref == "REF-paper (count, adaptive)") %>% mutate(ref = "base", wis_infl = 0, bias_cal = bias_base) +base_row <- bridge %>% filter(ref == "REF-paper (count, adaptive)") %>% mutate(ref = "base", wis_pct = 0, bias_cal = bias_base) tbl <- bind_rows(base_row, bridge) %>% - transmute(config = ref, horizon, cell = sprintf("%+.1f%% / %.3f", wis_infl, bias_cal)) %>% + transmute(config = ref, horizon, cell = sprintf("%+.1f%% / %.3f", wis_pct, bias_cal)) %>% tidyr::pivot_wider(names_from = horizon, values_from = cell) -knitr::kable(tbl, caption = "WIS inflation % / L1 coverage bias, pooled over locations") +knitr::kable(tbl, caption = "WIS change % (positive is better) / L1 coverage bias, pooled over locations") knitr::kable( best_const %>% select(horizon, lr) %>% mutate(lr = signif(lr, 2)) %>% tidyr::pivot_wider(names_from = horizon, values_from = lr), caption = "Constant learning rate with the lowest pooled WIS (rate units per step)" @@ -266,8 +266,8 @@ knitr::kable( # Coverage bias vs WIS, per location Each thin grey line is one location traced across learning rates; the bold -line is the pooled result, labelled with the learning rate at each decade. Learning rates whose pooled WIS inflation exceeds -`r params$wis_max`% in a panel are dropped from that panel; the axes are +line is the pooled result, labelled with the learning rate at each decade. Learning rates whose pooled WIS is more than +`r params$wis_max`% worse than the base in a panel are dropped from that panel; the axes are clipped to the bulk of the per-location points, so a few location lines run off the edge. The open circle is the base forecast (no calibration) and the triangles are the reference configs at the same delay. @@ -282,7 +282,7 @@ ACCENT <- "#1f5fa8" curve_plot <- function(which_learn) { d <- sweep_m %>% filter(learn == which_learn) keep <- d %>% - filter(location == "pooled", wis_infl <= params$wis_max) %>% + filter(location == "pooled", wis_pct >= -params$wis_max) %>% select(horizon, extra, lr) d <- d %>% semi_join(keep, by = c("horizon", "extra", "lr")) %>% arrange(lr) pooled <- d %>% filter(location == "pooled") @@ -291,20 +291,20 @@ curve_plot <- function(which_learn) { marks <- ref_m %>% filter(learn == which_learn, ref %in% c("REF-paper (count, adaptive)", "REF-op")) lab <- pooled %>% filter(abs(log10(lr) - round(log10(lr))) < 1e-9) xlim <- c(0, max(stats::quantile(locs$bias_cal, 0.97), base$bias_base) * 1.05) - ylim <- range(stats::quantile(locs$wis_infl, c(0.03, 0.97)), pooled$wis_infl, marks$wis_infl, 0) + ylim <- range(stats::quantile(locs$wis_pct, c(0.03, 0.97)), pooled$wis_pct, marks$wis_pct, 0) ggplot() + geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + - geom_path(data = locs, aes(bias_cal, wis_infl, group = location), colour = "grey45", alpha = 0.25, linewidth = 0.3) + - geom_path(data = pooled, aes(bias_cal, wis_infl), colour = ACCENT, linewidth = 1.1) + - geom_point(data = pooled, aes(bias_cal, wis_infl), colour = ACCENT, size = 1.2) + - geom_text(data = lab, aes(bias_cal, wis_infl, label = format(lr, scientific = FALSE, drop0trailing = TRUE)), + geom_path(data = locs, aes(bias_cal, wis_pct, group = location), colour = "grey45", alpha = 0.25, linewidth = 0.3) + + geom_path(data = pooled, aes(bias_cal, wis_pct), colour = ACCENT, linewidth = 1.1) + + geom_point(data = pooled, aes(bias_cal, wis_pct), colour = ACCENT, size = 1.2) + + geom_text(data = lab, aes(bias_cal, wis_pct, label = format(lr, scientific = FALSE, drop0trailing = TRUE)), colour = "grey15", size = 2.6, vjust = -0.9) + geom_point(data = base, aes(bias_base, 0), shape = 21, fill = "white", colour = "black", size = 2.2) + - geom_point(data = marks, aes(bias_cal, wis_infl, shape = ref), colour = "black", size = 2) + + geom_point(data = marks, aes(bias_cal, wis_pct, shape = ref), colour = "black", size = 2) + scale_shape_manual(values = c(`REF-paper (count, adaptive)` = 2, `REF-op` = 17), name = NULL) + facet_grid(horizon ~ extra) + coord_cartesian(xlim = xlim, ylim = ylim) + - labs(x = "L1 coverage bias", y = "WIS inflation % (negative = better than base)") + + labs(x = "L1 coverage bias", y = "WIS change % (positive is better)") + theme(legend.position = "bottom") } ``` @@ -331,11 +331,11 @@ stateful one. DELAY_COLS <- setNames(RColorBrewer::brewer.pal(9, "Blues")[c(4, 6, 8, 9)], levels(sweep_m$extra)) pooled_all <- sweep_m %>% filter(location == "pooled") %>% - semi_join(sweep_m %>% filter(location == "pooled", wis_infl <= params$wis_max) %>% select(horizon, extra, lr, learn), + semi_join(sweep_m %>% filter(location == "pooled", wis_pct >= -params$wis_max) %>% select(horizon, extra, lr, learn), by = c("horizon", "extra", "lr", "learn")) %>% arrange(lr) %>% mutate(design = factor(LEARN_LABELS[learn], levels = LEARN_LABELS)) -ggplot(pooled_all, aes(bias_cal, wis_infl, colour = extra, linetype = design, group = interaction(extra, design))) + +ggplot(pooled_all, aes(bias_cal, wis_pct, colour = extra, linetype = design, group = interaction(extra, design))) + geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + geom_path(linewidth = 0.8) + geom_point(data = pooled_all %>% distinct(horizon, bias_base), aes(bias_base, 0), inherit.aes = FALSE, @@ -343,20 +343,20 @@ ggplot(pooled_all, aes(bias_cal, wis_infl, colour = extra, linetype = design, gr scale_colour_manual(values = DELAY_COLS, name = NULL) + scale_linetype_manual(values = c(`re-run (prod)` = "solid", stateful = "22"), name = NULL) + facet_wrap(~horizon, nrow = 1) + - labs(x = "L1 coverage bias", y = "WIS inflation %") + + labs(x = "L1 coverage bias", y = "WIS change % (positive is better)") + theme(legend.position = "bottom", legend.box = "vertical") ``` # Tables -Pooled WIS inflation % / L1 coverage bias for every learning rate, by horizon +Pooled WIS change % (positive is better) / L1 coverage bias for every learning rate, by horizon (columns) and extra delay (row blocks). ```{r table-fn} sweep_table <- function(which_learn) { sweep_m %>% filter(location == "pooled", learn == which_learn) %>% - transmute(extra, lr = signif(lr, 2), horizon, cell = sprintf("%+.1f / %.3f", wis_infl, bias_cal)) %>% + transmute(extra, lr = signif(lr, 2), horizon, cell = sprintf("%+.1f / %.3f", wis_pct, bias_cal)) %>% tidyr::pivot_wider(names_from = horizon, values_from = cell) %>% arrange(extra, lr) %>% knitr::kable() From 86cbe3fd24c58ff1f56f7720f11efb31ac6d141a Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:29:42 -0700 Subject: [PATCH 060/107] docs(calibration): E07 adds base WIS, month bias and operating-point candidates on both forecasters --- .../e07_across_eras.Rmd | 267 +++++++++++++++++- 1 file changed, 265 insertions(+), 2 deletions(-) diff --git a/reports/writeups/calibration_experiments/e07_across_eras.Rmd b/reports/writeups/calibration_experiments/e07_across_eras.Rmd index 310be4ec..196b00dc 100644 --- a/reports/writeups/calibration_experiments/e07_across_eras.Rmd +++ b/reports/writeups/calibration_experiments/e07_across_eras.Rmd @@ -55,6 +55,9 @@ the adaptive learning rate (mult 0.03, floor 0.001, window 20, carry): - **operating**: REF-op, prod's tracker: leaky plus a slow term and a warm start from the burn-in season's errors. +Section 3 also runs the operating-point candidates (E14, E16, E17) on both +forecasters. + "WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive is better**. "L1 coverage bias" is the mean over the 23 levels of |empirical coverage − level|. Results lead with states only. @@ -65,10 +68,49 @@ Numbers are from the render of 1 October 2026, states only, WIS change % (positive is better) / L1 coverage bias, h0/h1/h2/h3. All-locations numbers point the same way. +- **Base WIS: equal in 2024-25, B worse in 2025-26.** B/A base WIS is + 0.93/0.99/1.01/1.01 in 2024-25 and 1.33/1.22/1.11/1.05 in 2025-26 + (1.06/1.07/1.05/1.02 pooled). Per state, pooled over h0–h3, the median + ratio is 0.99 in 2024-25 (B better in 29 of 52 states) and 1.12 in + 2025-26 (B better in 7 of 52). So in 2025-26 the ensemble beats B almost + everywhere, by about 10%, most at h0–h1. - **The two bases are equally miscalibrated in 2024-25** (coverage bias 0.13–0.17, 68–77% of outcomes above the median for both). In 2025-26 the - ensemble is better calibrated (0.046–0.078 vs 0.081–0.090). Base WIS is - similar. + ensemble is better calibrated (0.046–0.078 vs 0.081–0.090). +- **By month, both bases have the same shape.** Both under-predict on the + ramp (December: 96–97% of outcomes above the median in 2024-25, 86–94% in + 2025-26, coverage bias 0.24–0.34) and over-predict after the peak (March + 2025-26: 32% above for A, 38% for B). B's ramp bias is larger in 2025-26 + (November–December coverage bias 0.19/0.31 vs 0.14/0.24). The one place + they differ is spring 2024-25: A still under-predicts in March–April (59% + and 75% above the median), while B is close to calibrated (45% and 56%). +- **The operating-point candidates rank mostly the same on both, with one + exception.** Pooled over both seasons, h0/h1/h2/h3 WIS change %: + + | candidate | A ensemble | B `windowed_seasonal` | + |---|---|---| + | REF-op warm | +3.9/+1.9/+1.5/+2.3 | +2.8/+1.1/+0.8/+1.2 | + | REF-op cold | +2.8/+0.7/−0.3/−0.7 | +2.0/+0.6/+0.1/−0.2 | + | sqrt 0.018 | +2.4/+0.6/+0.1/0.0 | +2.2/+0.8/+0.6/+0.6 | + | rate 0.032 | +1.5/+0.7/+0.5/+0.2 | +1.3/+0.7/+0.5/+0.4 | + | sqrt 0.018, off 1 Mar | +1.7/+0.6/+0.4/+0.6 | +2.3/+1.2/+1.2/+1.3 | + + Ordered by the mean over h0–h3, sqrt 0.018 > rate 0.032 > REF-op cold on + both. REF-op warm is first on A (+2.4 mean) and tied first with + off-1-March on B (both +1.5). Coverage bias ranks identically on both, + pooled: REF-op warm best (A 0.035–0.047, B 0.037–0.061), then rate 0.032, + sqrt 0.018, REF-op cold, and off-1-March worst (A 0.084–0.092, B + 0.094–0.116). +- **The exception is the March switch-off, and it follows the base's spring + bias.** In 2024-25, off-1-March is last of the five on A (+0.6/+0.2/0.0/+0.1) + but second on B (+1.6/+0.8/+0.7/+0.5): A was still under-predicting that + spring, so the offsets it switches off were still helping. In 2025-26, + when both over-predict in March, it is first on both. So whether to + switch off in spring depends on the forecaster, and B's result for it + should not be carried over to the ensemble without checking. +- **sqrt 0.018 is WIS-positive at every horizon in both seasons only on B.** + On A it is −0.3/−0.1 at h2/h3 in 2025-26, where the ensemble was already + close to calibrated. - **Calibration does the same thing to both.** The trackers rank the same way on each: operating gains the most WIS, single gives the best coverage and loses at h2–h3. Pooled over both seasons: @@ -85,6 +127,10 @@ point the same way. where the ensemble was already close to calibrated. - **Answer: the gains are not an artifact of the ensemble.** On the same rows and data, one fixed forecaster gains the same way, a little less. + B's base is about 5% worse on WIS pooled and its bias has the same + monthly shape, so experiments on B transfer to the ensemble for the + choice of tracker. They do not transfer for date-based switches, which + depend on where each base is biased in spring. - **Against the old E07** (finalized truth, the replay with prod's data substitutions, 50 locations, h−1 included): the verdict holds. The old explanation for B's smaller gains does not: at h0–h3 the gap is about a @@ -115,6 +161,22 @@ grid <- tidyr::expand_grid(forecaster = names(FCS), tracker = names(TRACKERS)) cals <- purrr::pmap(grid, function(forecaster, tracker) { do.call(cal_run_cached, c(list(FCS[[forecaster]], wi$truth), TRACKERS[[tracker]], list(learn = "exact", vintages = wi$vintages, workers = params$workers))) }) +# Operating-point candidates (E14, E16, E17). Cold ones see only the live seasons, so they start from zero offsets. +const_cfg <- ws_configs(wi$rate_scales, burn_in = FALSE)[["E11 constant 0.1"]] +sqrt_018 <- utils::modifyList(const_cfg, list(lr = 0.018, transform = "sqrt")) +CANDS <- list( + `REF-op warm` = list(cfg = list(ref = "op"), warm = TRUE), + `REF-op cold` = list(cfg = list(ref = "op", burn_in_seasons = character(0), slow_init = NULL), warm = FALSE), + `sqrt 0.018` = list(cfg = sqrt_018, warm = FALSE), + `rate 0.032` = list(cfg = utils::modifyList(const_cfg, list(lr = 0.032)), warm = FALSE), + `sqrt 0.018, off 1 Mar` = list(cfg = c(sqrt_018, list(off_after = "03-01")), warm = FALSE) +) +FCS_LIVE <- purrr::map(FCS, \(fc) filter(fc, season_of(reference_date) %in% LIVE)) +cgrid <- tidyr::expand_grid(forecaster = names(FCS), cand = names(CANDS)) +ccals <- purrr::pmap(cgrid, function(forecaster, cand) { + fc <- if (CANDS[[cand]]$warm) FCS[[forecaster]] else FCS_LIVE[[forecaster]] + do.call(cal_run_cached, c(list(fc, wi$truth), CANDS[[cand]]$cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +}) ``` Rows per forecaster after the cut, by season: @@ -150,6 +212,57 @@ FC_COLS <- c(`A ensemble` = "#0072B2", `B windowed_seasonal` = "#D55E00") # 1. The two bases +## Base WIS + +Uncalibrated WIS (2 × mean pinball, in admissions) on identical rows, and +the ratio B/A (below 1: `windowed_seasonal` is better). + +```{r base-wis} +m %>% + filter(tracker == "single") %>% + select(geos, season, h, forecaster, wis_base) %>% + tidyr::pivot_wider(names_from = forecaster, values_from = wis_base) %>% + transmute(geos, season, h, A = .data[[names(FCS)[1]]], B = .data[[names(FCS)[2]]], `B/A` = B / A) %>% + arrange(geos, season, h) %>% + knitr::kable(digits = c(0, 0, 0, 1, 1, 3), caption = "Base WIS, A ensemble vs B windowed_seasonal") +``` + +Per state: base WIS pooled over h0–h3, A against B (log scales; below the +line, B is better). + +```{r base-wis-state, fig.height=5} +base_fc <- purrr::map(which(grid$tracker == "single"), \(i) cals[[i]]$forecasts %>% mutate(forecaster = grid$forecaster[i])) %>% + bind_rows() %>% + mutate(season = season_of(reference_date)) %>% + filter(season %in% LIVE, !is.na(truth), !is.na(value_base)) +pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) +loc_wis <- base_fc %>% + filter(location != "US") %>% + group_by(forecaster, season, location) %>% + summarize(wis = 2 * mean(pinball(truth, value_base, level)), .groups = "drop") %>% + tidyr::pivot_wider(names_from = forecaster, values_from = wis) %>% + transmute(season, location, A = .data[[names(FCS)[1]]], B = .data[[names(FCS)[2]]], ratio = B / A) +ggplot(loc_wis, aes(A, B)) + + geom_abline(slope = 1, intercept = 0, colour = "grey50") + + geom_point(alpha = 0.8) + + facet_wrap(~season) + + scale_x_log10() + + scale_y_log10() + + labs(x = "A ensemble base WIS", y = "B windowed_seasonal base WIS") +``` + +```{r base-wis-ratio} +loc_wis %>% + group_by(season) %>% + summarize( + states = n(), `B better` = sum(ratio < 1), q10 = quantile(ratio, 0.1), q25 = quantile(ratio, 0.25), + median = median(ratio), q75 = quantile(ratio, 0.75), q90 = quantile(ratio, 0.9) + ) %>% + knitr::kable(digits = 2, caption = "Per-state B/A base WIS ratio, h0–h3 pooled") +``` + +## Base miscalibration + How miscalibrated each forecaster is before calibration, states only: WIS, L1 coverage bias, and the share of outcomes above the median (0.5 is calibrated; above 0.5 means the forecaster under-predicts). @@ -163,6 +276,47 @@ m %>% knitr::kable(caption = "Base WIS / L1 coverage bias / share above the median, states only") ``` +## Base bias by month + +Share of outcomes above the base median and L1 coverage bias, by calendar +month of the reference date, states only. + +```{r base-month, fig.height=7} +month_base <- function(d) { + d %>% + group_by(across(c(any_of(c("forecaster", "season", "horizon", "month")), level))) %>% + summarize(gap = mean(truth <= value_base) - first(level), above = mean(truth > value_base), n = n(), .groups = "drop") %>% + group_by(across(any_of(c("forecaster", "season", "horizon", "month")))) %>% + summarize(above = above[level == 0.5], cal_err = mean(abs(gap)), n = first(n), .groups = "drop") +} +base_fc_states <- base_fc %>% + filter(location != "US") %>% + mutate(month = factor(format(reference_date, "%b"), levels = month.abb[c(8:12, 1:7)])) +mb <- month_base(base_fc_states) %>% mutate(h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) +mb %>% + tidyr::pivot_longer(c(above, cal_err), names_to = "metric") %>% + mutate(metric = factor(metric, levels = c("above", "cal_err"), labels = c("share above median", "L1 coverage bias"))) %>% + ggplot(aes(month, value, colour = forecaster, group = forecaster)) + + geom_hline(data = data.frame(metric = factor("share above median", levels = c("share above median", "L1 coverage bias")), y = 0.5), aes(yintercept = y), colour = "grey50") + + geom_line() + + geom_point(size = 1.1) + + facet_grid(metric + season ~ h, scales = "free", space = "free_x") + + scale_colour_manual(values = FC_COLS, name = NULL) + + labs(x = "month of reference date", y = NULL) + + theme(legend.position = "bottom") +``` + +Pooled over h0–h3: share above the base median / L1 coverage bias, A vs B. + +```{r base-month-table} +month_base(base_fc_states %>% select(-horizon)) %>% + transmute(season, month, forecaster, cell = sprintf("%.2f / %.3f", above, cal_err)) %>% + tidyr::pivot_wider(names_from = forecaster, values_from = cell) %>% + left_join(base_fc_states %>% filter(level == 0.5) %>% count(season, month, forecaster) %>% filter(forecaster == names(FCS)[1]) %>% select(season, month, forecasts = n), by = c("season", "month")) %>% + arrange(season, month) %>% + knitr::kable(caption = "Share above base median / L1 coverage bias, states only, h0–h3 pooled") +``` + # 2. What calibration does to each WIS change % per tracker, one panel per season and horizon, states only. @@ -218,3 +372,112 @@ knitr::kable(season_table(GEOS[["states"]])) ```{r table-all} knitr::kable(season_table(GEOS[["all"]])) ``` + +# 3. Operating-point candidates on both + +The candidates from E14, E16 and E17, each run on both forecasters on the +same rows: REF-op warm (as in section 2) and cold, sqrt-scale constant +0.018 (E16), rate-scale constant 0.032 per 100k (E14), and sqrt 0.018 +switched off from 1 March (`off_after = "03-01"`, E17). The cold ones learn +from the live seasons only. + +```{r cand-metrics} +cm <- purrr::map2(ccals, seq_len(nrow(cgrid)), \(cal, i) score(cal) %>% mutate(cgrid[i, ])) %>% + bind_rows() %>% + mutate( + forecaster = factor(forecaster, levels = names(FCS)), cand = factor(cand, levels = names(CANDS)), + geos = factor(geos, levels = names(GEOS), labels = GEOS), + season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +``` + +WIS change % per candidate, states only. + +```{r cand-plot, fig.height=6.5} +ggplot(cm %>% filter(geos == GEOS[["states"]]), aes(cand, wis_pct, fill = forecaster)) + + geom_hline(yintercept = 0, colour = "grey50") + + geom_col(position = position_dodge(width = 0.85), width = 0.8) + + facet_grid(season ~ h) + + scale_fill_manual(values = FC_COLS, name = NULL) + + labs(x = NULL, y = "WIS change % (positive is better)") + + theme(legend.position = "bottom", axis.text.x = element_text(angle = 30, hjust = 1)) +``` + +L1 coverage bias, calibrated (filled) against the base (open), states only. + +```{r cand-cov, fig.height=6.5} +ggplot(cm %>% filter(geos == GEOS[["states"]]), aes(cand, cal_err_cal, colour = forecaster)) + + geom_point(aes(y = cal_err_base), shape = 1, size = 2.4, position = position_dodge(width = 0.6)) + + geom_point(size = 2.4, position = position_dodge(width = 0.6)) + + facet_grid(season ~ h) + + scale_colour_manual(values = FC_COLS, name = NULL) + + expand_limits(y = 0) + + labs(x = NULL, y = "L1 coverage bias (open: base)") + + theme(legend.position = "bottom", axis.text.x = element_text(angle = 30, hjust = 1)) +``` + +## Rankings on A and B + +Candidates ordered by mean WIS change % over h0–h3, separately on each +forecaster, states only. `rank A` / `rank B` are the positions (1 is the +largest gain); Spearman ρ compares the two orders per season, over all +(candidate, horizon) cells. + +```{r cand-rank} +rk <- cm %>% + filter(geos == GEOS[["states"]]) %>% + group_by(forecaster, season, cand) %>% + summarize(wis = mean(wis_pct), cov = mean(cal_err_cal), .groups = "drop") %>% + group_by(forecaster, season) %>% + mutate(rank = rank(-wis), cov_rank = rank(cov)) %>% + ungroup() %>% + mutate(fc = ifelse(forecaster == names(FCS)[1], "A", "B")) %>% + select(season, cand, fc, wis, rank, cov, cov_rank) %>% + tidyr::pivot_wider(names_from = fc, values_from = c(wis, rank, cov, cov_rank)) %>% + arrange(season, rank_A) +rk %>% + transmute(season, cand, `WIS % A` = wis_A, `rank A` = rank_A, `WIS % B` = wis_B, `rank B` = rank_B, + `cov bias A` = cov_A, `cov rank A` = cov_rank_A, `cov bias B` = cov_B, `cov rank B` = cov_rank_B) %>% + knitr::kable(digits = c(0, 0, 2, 0, 2, 0, 3, 0, 3, 0), caption = "Mean over h0–h3, states only") +cm %>% + filter(geos == GEOS[["states"]]) %>% + select(season, cand, h, forecaster, wis_pct, cal_err_cal) %>% + tidyr::pivot_wider(names_from = forecaster, values_from = c(wis_pct, cal_err_cal)) %>% + group_by(season) %>% + summarize( + `ρ WIS cells` = cor(.data[[paste0("wis_pct_", names(FCS)[1])]], .data[[paste0("wis_pct_", names(FCS)[2])]], method = "spearman"), + `ρ coverage cells` = cor(.data[[paste0("cal_err_cal_", names(FCS)[1])]], .data[[paste0("cal_err_cal_", names(FCS)[2])]], method = "spearman") + ) %>% + knitr::kable(digits = 2, caption = "Spearman correlation of A vs B over (candidate, horizon) cells") +``` + +## Tables + +WIS change % / L1 coverage bias. + +```{r cand-tables} +cand_table <- function(g) { + bind_rows( + cm %>% filter(cand == names(CANDS)[1]) %>% mutate(cand = "base", wis_pct = 0, cal_err_cal = cal_err_base), + cm + ) %>% + filter(geos == g) %>% + mutate(config = factor(paste(forecaster, cand, sep = ", "), levels = paste(rep(names(FCS), each = length(CANDS) + 1), c("base", names(CANDS)), sep = ", "))) %>% + transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(config) +} +``` + +### States only + +```{r cand-table-states} +knitr::kable(cand_table(GEOS[["states"]])) +``` + +### All locations + +```{r cand-table-all} +knitr::kable(cand_table(GEOS[["all"]])) +``` From f196fb6b9346e52285022a813070237c3d3f9a50 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:30:21 -0700 Subject: [PATCH 061/107] docs(calibration): record the expanded E07 in the review tracker, summary and notes --- notes/CALIBRATION.md | 6 ++++++ notes/calibration-review.md | 7 +++++-- notes/calibration-summary-2026-10-01.md | 12 +++++++++--- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index b2395db7..cca84426 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -230,6 +230,12 @@ Flu, clean replay, exact, post-fix, states only. ensemble gains about a point more with REF-op (pooled A +3.9/+1.9/+1.5/+2.3, B +2.8/+1.1/+0.8/+1.2). B is not less miscalibrated than A (equal in 2024-25, worse in 2025-26). + Expanded: base WIS B/A (states) 0.93–1.01 in 2024-25, 1.05–1.33 in + 2025-26. Same monthly bias shape on both; the one difference is spring + 2024-25, where A still under-predicts. Candidates (states, pooled WIS + mean over h0–h3) rank sqrt 0.018 > rate 0.032 > REF-op cold on both, and + coverage ranks identically; `off_after` 03-01 is last on A in 2024-25 but + second on B, so date-based switches don't transfer between forecasters. ## Data diff --git a/notes/calibration-review.md b/notes/calibration-review.md index 2c7601d7..9975e149 100644 --- a/notes/calibration-review.md +++ b/notes/calibration-review.md @@ -18,7 +18,10 @@ for), **not reviewed**. gains are similar on the submitted ensemble and on `windowed_seasonal`, and the trackers rank the same way. So experiments can focus on the clean `windowed_seasonal` replay, which is free of hand edits and - spoiled submissions. + spoiled submissions. Caveat: the tracker ranking transfers, but a + date-based switch-off (`off_after`) does not; it depends on each + forecaster's spring bias (last on the ensemble in 2024-25, second on + `windowed_seasonal`). 3. Experiments before E07's re-run (E03–E05, E09–E11) used the submitted ensemble; E01, E02 (new versions) and E12–E17 use the clean replay. @@ -32,7 +35,7 @@ for), **not reviewed**. | E03 scale | ensemble | not reviewed | | | | E04 offset structure | ensemble | not reviewed | | | | E05 warm start | ensemble | not reviewed | | | -| E07 across eras | both | commented | Needs concrete base WIS numbers, not "similar". Expand into the ensemble vs `windowed_seasonal` comparison: base WIS side by side, base bias by month, every operating-point candidate on both. | intro reworded; expansion in progress | +| E07 across eras | both | commented | Needs concrete base WIS numbers, not "similar". Expand into the ensemble vs `windowed_seasonal` comparison: base WIS side by side, base bias by month, every operating-point candidate on both. | intro reworded; expanded with base WIS side by side (B/A 0.93–1.01 in 2024-25, 1.05–1.33 in 2025-26), base bias by month, and five candidates on both forecasters | | E09 lr × delay | ensemble | commented | WIS axis inverted relative to the other notebooks. | flipped to "WIS change %, positive is better" | | E10 covid | ensemble | not reviewed | | | | E11 constant lr | ensemble | not reviewed | | | diff --git a/notes/calibration-summary-2026-10-01.md b/notes/calibration-summary-2026-10-01.md index 7c64951e..7808096f 100644 --- a/notes/calibration-summary-2026-10-01.md +++ b/notes/calibration-summary-2026-10-01.md @@ -80,7 +80,9 @@ E17, E02).** overall (E17).** The offsets carry into the next season, so the decay also hands that season a near-zero start. - **Switching calibration off from 1 March (or 15 February) is the one - variant with a large WIS gain (E02, E17).** It is positive at every + variant with a large WIS gain on `windowed_seasonal` (E02, E17).** It + doesn't transfer to the ensemble: in 2024-25 the ensemble was still + under-predicting in spring, and the switch-off ranks last there (E07). It is positive at every horizon in both seasons, but it gives up most of the coverage gain: - **sqrt 0.018, pooled WIS:** +2.3/+1.2/+1.2/+1.3 switched off from March, against +2.2/+0.8/+0.6/+0.6 left on; @@ -109,7 +111,11 @@ matters (E01, E02).** **5. The gains are not specific to the ensemble (E07).** On identical rows, the trackers rank the same way on the submitted ensemble -and on `windowed_seasonal`. The ensemble gains about a point more with +and on `windowed_seasonal`. Base WIS (states, B/A) is 0.93–1.01 in 2024-25 +and 1.05–1.33 in 2025-26 (worst at h0); per state, `windowed_seasonal` is +better in 29 of 52 states in 2024-25 and 7 of 52 in 2025-26. Both have the +same monthly bias shape: under-prediction on the ramp, over-prediction +after the peak. The ensemble gains about a point more with REF-op (pooled +3.9/+1.9/+1.5/+2.3 vs +2.8/+1.1/+0.8/+1.2). The two are equally miscalibrated in 2024-25. In 2025-26 the ensemble is better calibrated. The original E07's explanation for the gap, that @@ -132,7 +138,7 @@ doesn't help a constant rate (E13).** | candidate | for | against | |---|---|---| -| sqrt, constant 0.018 | WIS-positive at every horizon in both seasons; small offsets; no learning-rate adaptation to tune | small gains at h2–h3 in 2024-25; loses in March | +| sqrt, constant 0.018 | WIS-positive at every horizon in both seasons on `windowed_seasonal` (on the ensemble, −0.3/−0.1 at h2/h3 in 2025-26); small offsets; no learning-rate adaptation to tune | small gains at h2–h3 in 2024-25; loses in March | | REF-op (current) | largest h0 gain in 2024-25; best with a flu burn-in | loses at h3 in 2024-25; barely improves 2024-25 coverage when cold | | either, switched off from March | more WIS at h1–h3 | gives up most of the coverage gain | From 8d6dd2a196286424cabe5149d9bba640902c6612 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:25:09 -0700 Subject: [PATCH 062/107] docs(calibration): review tracker, E01 verified, E02 comments --- notes/calibration-review.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notes/calibration-review.md b/notes/calibration-review.md index 9975e149..76971db8 100644 --- a/notes/calibration-review.md +++ b/notes/calibration-review.md @@ -30,8 +30,8 @@ for), **not reviewed**. | exp | forecaster | status | comments | done | |---|---|---|---|---| | E00 vintage backtest | ensemble | verified | Finalized data exaggerates h−1/h0 gains a little; use vintages, no more work. | — | -| E01 eta settings | clean replay | commented | (1) Curve axis looked flipped vs E09. (2) x/o markers unlabeled. (3) "Rebuilds the original" is misleading: it re-asks the question on a new setup. | (1) E09 flipped to match; (2) legend added; (3) intro reworded with a what-changed table | -| E02 eta variants | clean replay | not reviewed | | intro reworded (same issue as E01) | +| E01 eta settings | clean replay | verified | Recap of the August sweep: carry slightly better, window 20 as good as larger, the multiplier has an elbow. Note: on this replay 0.03 is the coverage elbow and 0.01 the WIS-safe choice. Earlier: (1) Curve axis looked flipped vs E09. (2) x/o markers unlabeled. (3) "Rebuilds the original" is misleading: it re-asks the question on a new setup. | (1) E09 flipped to match; (2) legend added; (3) intro reworded with a what-changed table | +| E02 eta variants | clean replay | commented | Headline: switching off in Feb/Mar recovers WIS, loses coverage. (1) Coverage numbers hard to find in the WIS / bias cells. (2) Month plot: unclear whether positive % is good; the sign convention must be labeled explicitly in every notebook. | intro reworded; (1)–(2) in progress | | E03 scale | ensemble | not reviewed | | | | E04 offset structure | ensemble | not reviewed | | | | E05 warm start | ensemble | not reviewed | | | From a3059199c9325c2e09de416202e7a842db905991 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:28:07 -0700 Subject: [PATCH 063/107] docs(calibration): label WIS % as "WIS reduction % (positive = calibrated WIS lower than base)" and coverage bias as lower-is-better everywhere --- R/calibration/views.R | 6 +-- .../_standard_views.Rmd | 16 +++--- .../e00_vintage_backtest.Rmd | 13 ++--- .../e01_eta_settings.Rmd | 34 ++++++------ .../e02_eta_variants.Rmd | 22 ++++---- .../calibration_experiments/e03_scale.Rmd | 5 +- .../e04_offset_structure.Rmd | 5 +- .../e05_warm_start.Rmd | 5 +- .../e07_across_eras.Rmd | 53 ++++++++++--------- .../calibration_experiments/e09_lr_delay.Rmd | 23 ++++---- .../calibration_experiments/e10_covid.Rmd | 5 +- .../e11_constant_lr.Rmd | 5 +- .../e14_lr_by_season.Rmd | 35 ++++++------ .../e15_wis_sources.Rmd | 28 +++++----- .../e16_sqrt_constant_lr.Rmd | 52 +++++++++--------- .../e17_late_decay.Rmd | 22 ++++---- .../calibration_experiments/index.Rmd | 9 ++-- scripts/calibration/calibration_ws_replay.R | 6 +-- 18 files changed, 178 insertions(+), 166 deletions(-) diff --git a/R/calibration/views.R b/R/calibration/views.R index 530c0cf0..f8c96ca3 100644 --- a/R/calibration/views.R +++ b/R/calibration/views.R @@ -50,7 +50,7 @@ cal_run <- function(forecasts, truth, ref = "paper", ..., learn = c("final", "vi } -#' V-head: WIS change % and L1 coverage bias by horizon, one row per +#' V-head: WIS reduction % and L1 coverage bias by horizon, one row per #' (variant, `by`, horizon). #' #' @param cals named list of [calibrate_hub_forecasts()] outputs. @@ -82,7 +82,7 @@ cal_wide <- function(tbl, col, digits = 1, id = "variant") { } -#' V-month: WIS change % by month of reference date, with each month's share +#' V-month: WIS reduction % by month of reference date, with each month's share #' of the base WIS, per horizon. #' @export cal_by_month <- function(cal) { @@ -222,7 +222,7 @@ cal_state_panel <- function(cal, loc, season, truth, vintages, title = NULL) { #' Ranking by a single metric fills a gallery with small states late in the #' season, where a stale offset on a tiny base inflates relative error, and #' ranking by WIS per 100k favours small places too (their rates are noisier). -#' Here each forecast is scored by its WIS change as a share of its own +#' Here each forecast is scored by its WIS reduction as a share of its own #' location-season's base WIS, so a late-season forecast on a tiny base only #' ranks high if it moved a real part of that location's season error. Small #' places still vary more, so slots are split by size (population above or diff --git a/reports/writeups/calibration_experiments/_standard_views.Rmd b/reports/writeups/calibration_experiments/_standard_views.Rmd index 2803bdb7..a0c24fed 100644 --- a/reports/writeups/calibration_experiments/_standard_views.Rmd +++ b/reports/writeups/calibration_experiments/_standard_views.Rmd @@ -7,14 +7,14 @@ hub locations, against finalized truth; burn-in rounds are excluded. ## WIS and coverage -WIS change % (positive is better than the base): +WIS reduction % (positive = calibrated WIS lower than base): ```{r v-head} head_all <- cal_headline(cals) knitr::kable(cal_wide(head_all, "pct_improvement")) ``` -L1 coverage bias (mean over levels of |coverage − level|), with the base first: +L1 coverage bias (lower is better; mean over levels of |coverage − level|), with the base first: ```{r v-head-cov} knitr::kable(bind_rows( @@ -23,7 +23,7 @@ knitr::kable(bind_rows( )) ``` -WIS change % by season: +WIS reduction % by season (positive = calibrated WIS lower than base): ```{r v-head-season} knitr::kable( @@ -34,7 +34,7 @@ knitr::kable( ## Absolute error of the median -Summed |truth − median|, change % vs base (positive is better): +Summed |truth − median|, reduction % vs base (positive = calibrated absolute error lower than base): ```{r v-ae} knitr::kable(cal_wide(cal_ae(cals), "ae_pct")) @@ -42,7 +42,7 @@ knitr::kable(cal_wide(cal_ae(cals), "ae_pct")) ## Where in the season -WIS change % by month of the reference date (clipped at −60), h0 and h3, +WIS reduction % (positive = calibrated WIS lower than base) by month of the reference date (clipped at −60), h0 and h3, and each month's share of the base WIS at h3. Months run July to June; only months with forecasts are shown. @@ -57,7 +57,7 @@ ggplot(month_tbl %>% filter(horizon %in% c(0, 3)), aes(month, pct, colour = vari facet_wrap(~ paste0("h", horizon)) + scale_colour_brewer(palette = "Dark2", name = NULL) + coord_cartesian(ylim = c(-60, 30)) + - labs(x = "month of reference date", y = "WIS change %") + + labs(x = "month of reference date", y = "WIS reduction % (positive = calibrated\nWIS lower than base)") + theme(legend.position = "bottom", legend.direction = "vertical") knitr::kable( month_tbl %>% @@ -100,7 +100,7 @@ for (nm in names(CAL_STATES)) { For each season phase and location size (population above or below the median), the forecast with the largest WIS gain and the one with the largest loss, each from a different location. Forecasts are ranked by their -WIS change as a share of their own location-season's base WIS, so small states late in the season, where the base is +WIS reduction as a share of their own location-season's base WIS, so small states late in the season, where the base is tiny, do not crowd the gallery. Ribbons are the 50% and 90% bands, the dashed line is the forecast date. @@ -110,7 +110,7 @@ abbrev <- setNames(toupper(get_population_data()$state_id), get_population_data( for (i in seq_len(nrow(picks))) { p <- picks[i, ] if (i == 1L || p$phase != picks$phase[i - 1L]) cat("\n\n### ", as.character(p$phase), "\n\n", sep = "") - sub <- sprintf("%s (%s), %s, %s: WIS %+.1f%% (%+.1f%% of its season's base WIS)", abbrev[[p$location]], p$size, p$reference_date, p$kind, p$wis_pct, p$share) + sub <- sprintf("%s (%s), %s, %s: WIS reduction %+.1f%% (%+.1f%% of its season's base WIS; positive = calibrated WIS lower)", abbrev[[p$location]], p$size, p$reference_date, p$kind, p$wis_pct, p$share) print(cal_gallery_panel(cals[[focus]], p$location, p$reference_date, truth, vintages, subtitle = sub)) } ``` diff --git a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd index d9351c5a..d422b435 100644 --- a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd +++ b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd @@ -43,8 +43,9 @@ learns from, at both reference configs (REF-paper and REF-op; see the index): A round with reference date `d` sees data published by `d - 3` (the Wednesday submission). Vintages start 2024-11-19; earlier as-of dates use that first vintage. Scores are always against finalized truth, over both live seasons, -all hub locations, with burn-in rounds excluded. "WIS change %" is positive -when calibration improves on the base. +all hub locations, with burn-in rounds excluded. "WIS reduction %" is +100 × (base WIS − calibrated WIS) / base WIS, so positive = calibrated WIS +lower than base. "L1 coverage bias" is lower-is-better. Questions: (1) how much did learning from finalized values flatter the earlier results, and (2) is the cheap mode close enough to exact to use in @@ -55,7 +56,7 @@ sweeps? Numbers below are from the render of 30 September 2026. - **Finalized truth flattered the earlier results only a little.** At REF-op - with one week of extra delay, exact vs final WIS change is +5.4 vs +7.6 at + with one week of extra delay, exact vs final WIS reduction is +5.4 vs +7.6 at h−1, within 0.5 points at h0–h3, and the h−1 coverage bias is 0.074 vs 0.067. At REF-paper, exact is within 0.6 points of final at every horizon: slightly better at h2/h3, slightly worse at h1. Earlier conclusions stand; @@ -107,13 +108,13 @@ burn-in rounds use that vintage in place of their own. # Headline -WIS change % by horizon: +WIS reduction % by horizon (positive = calibrated WIS lower than base): ```{r wis-table} knitr::kable(cal_wide(head_tbl, "pct_improvement", id = c("ref", "settle_days", "learn")) %>% arrange(ref, settle_days, learn)) ``` -L1 coverage bias (mean over levels of |coverage − level|); the `base` row is +L1 coverage bias (lower is better; mean over levels of |coverage − level|); the `base` row is the uncalibrated forecast: ```{r cal-table} @@ -125,7 +126,7 @@ knitr::kable( ) ``` -The same by season: +WIS reduction % by season (positive = calibrated WIS lower than base): ```{r season-table} knitr::kable( diff --git a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd index e369a0df..ac8e9a03 100644 --- a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd +++ b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd @@ -53,15 +53,15 @@ which it found indistinguishable from 20 and Inf; it is dropped here. With no burn-in there is one summer to cross (2025 into 2025-26), so the season policy only changes 2025-26. -"WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive -is better**. "L1 coverage bias" is the mean over the 23 levels of -|empirical coverage − level|. Each season is scored on its own rounds. +"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so +**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is +better) is the mean over the 23 levels of |empirical coverage − level|. Each season is scored on its own rounds. Results lead with states only (US excluded). # Findings -Numbers are from the render of 1 October 2026, states only, WIS change % -(positive is better) / L1 coverage bias, h0/h1/h2/h3, both seasons pooled +Numbers are from the render of 1 October 2026, states only, WIS reduction % +(positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), h0/h1/h2/h3, both seasons pooled unless stated. All-locations numbers point the same way. - **The multiplier is the setting that matters.** Changing it moves WIS by @@ -160,7 +160,7 @@ curve_plot <- function(g) { scale_linetype_manual(values = c(carry = "solid", reset = "22"), name = "season policy") + facet_grid(season ~ h) + coord_cartesian(ylim = c(max(-15, min(d$wis_pct)), max(d$wis_pct) + 0.5)) + - labs(x = "L1 coverage bias", y = "WIS change % (positive is better)", title = g) + + labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)", title = g) + theme(legend.position = "bottom") } ``` @@ -179,7 +179,7 @@ curve_plot(GEOS[["all"]]) # 2. Which setting matters? -For each season and horizon, how far WIS change % moves when one setting +For each season and horizon, how far WIS reduction % moves when one setting changes and the other two stay fixed: the range over the four multipliers, over the three windows, and the carry-vs-reset gap, each averaged over the settings held fixed. States only. The policy gap is zero in 2024-25 by @@ -199,25 +199,25 @@ spread %>% transmute(setting = factor(setting, levels = c("multiplier", "window", "season policy")), col = paste(season, h), v = round(v, 1)) %>% tidyr::pivot_wider(names_from = col, values_from = v) %>% arrange(setting) %>% - knitr::kable(caption = "Mean range of WIS change % (points) when one setting varies, states only") + knitr::kable(caption = "Mean range of WIS reduction % (points; positive = calibrated WIS lower than base) when one setting varies, states only") ``` # 3. Each metric against the multiplier -Top: WIS change %, clipped at −30% (mult 0.3 goes far below). Bottom: L1 -coverage bias, each season's base as a thin line. Carry only, states only; +Top: WIS reduction % (positive = calibrated WIS lower than base), clipped at +−30% (mult 0.3 goes far below). Bottom: L1 coverage bias (lower is better), each season's base as a thin line. Carry only, states only; colour is the window. ```{r vs-mult, fig.height=6.5} long <- st %>% filter(policy == "carry", season != "both seasons") %>% - transmute(h, season, mult, window_f, `WIS change %` = pmax(wis_pct, -30), `L1 coverage bias` = cal_err_cal) %>% - tidyr::pivot_longer(c(`WIS change %`, `L1 coverage bias`), names_to = "metric") %>% - mutate(metric = factor(metric, levels = c("WIS change %", "L1 coverage bias"))) + transmute(h, season, mult, window_f, `WIS reduction % (positive = calibrated WIS lower than base)` = pmax(wis_pct, -30), `L1 coverage bias (lower is better)` = cal_err_cal) %>% + tidyr::pivot_longer(c(`WIS reduction % (positive = calibrated WIS lower than base)`, `L1 coverage bias (lower is better)`), names_to = "metric") %>% + mutate(metric = factor(metric, levels = c("WIS reduction % (positive = calibrated WIS lower than base)", "L1 coverage bias (lower is better)"))) base_lines <- st %>% filter(season != "both seasons") %>% distinct(h, season, value = cal_err_base) %>% - mutate(metric = factor("L1 coverage bias", levels = levels(long$metric))) + mutate(metric = factor("L1 coverage bias (lower is better)", levels = levels(long$metric))) ggplot(long, aes(mult, value, colour = window_f, linetype = season)) + geom_hline(data = base_lines, aes(yintercept = value, linetype = season), colour = "grey40", linewidth = 0.3) + geom_line() + @@ -225,15 +225,15 @@ ggplot(long, aes(mult, value, colour = window_f, linetype = season)) + scale_x_log10(breaks = MULTS) + scale_colour_manual(values = WINDOW_COLS, name = "lr_window") + scale_linetype_manual(values = c(`2024-2025` = "solid", `2025-2026` = "22"), name = NULL) + - facet_grid(metric ~ h, scales = "free_y") + + facet_grid(metric ~ h, scales = "free_y", labeller = labeller(metric = label_wrap_gen(28))) + labs(x = "lr_mult", y = NULL) + theme(legend.position = "bottom") ``` # 4. Tables -WIS change % / L1 coverage bias per config (rows) and season × horizon -(columns). The base row shows the base's coverage bias; the count-scale +WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage +bias (lower is better) per config (rows) and season × horizon (columns). The base row shows the base's coverage bias; the count-scale run is last. ```{r tables} diff --git a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd index 20fb2552..953bd644 100644 --- a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd +++ b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd @@ -58,14 +58,14 @@ point. The seasonal window needs an earlier season to look back on, so it runs with the 2023-24 burn-in (the HHS-stitched backfill from E13), which only warms eta; the reference is re-run with the same burn-in as a bridge. -"WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive -is better**. "L1 coverage bias" is the mean over the 23 levels of -|empirical coverage − level|. Results lead with states only. +"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so +**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is +better) is the mean over the 23 levels of |empirical coverage − level|. Results lead with states only. # Findings -Numbers are from the render of 1 October 2026, states only, WIS change % -(positive is better) / L1 coverage bias, h0/h1/h2/h3, both seasons pooled +Numbers are from the render of 1 October 2026, states only, WIS reduction % +(positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), h0/h1/h2/h3, both seasons pooled unless stated. All-locations numbers point the same way. - **Switching off in spring is the only variant with a large WIS gain.** @@ -156,7 +156,7 @@ CFG_COLS <- setNames(c("black", "#56B4E9", "#0072B2", "#003F6B", "#CC79A7", "gre # 1. Overall, by season -WIS change % per config, one panel per season and horizon, states only. The +WIS reduction % (positive = calibrated WIS lower than base) per config, one panel per season and horizon, states only. The seasonal-window bars compare with "reference + burn-in", the others with "reference". @@ -166,11 +166,11 @@ ggplot(m %>% filter(geos == GEOS[["states"]]), aes(config, wis_pct, fill = confi geom_col(width = 0.8) + facet_grid(season ~ h) + scale_fill_manual(values = CFG_COLS, name = NULL) + - labs(x = NULL, y = "WIS change % (positive is better)") + + labs(x = NULL, y = "WIS reduction % (positive = calibrated WIS lower than base)") + theme(legend.position = "bottom", axis.text.x = element_blank(), axis.ticks.x = element_blank()) ``` -WIS change % / L1 coverage bias. +WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better). ```{r tables} season_table <- function(g) { @@ -215,7 +215,7 @@ bind_rows( # 2. Month by month -WIS change % by calendar month of the reference date; rows are seasons, +WIS reduction % (positive = calibrated WIS lower than base) by calendar month of the reference date; rows are seasons, columns horizons. Grey bars are each month's share of the season's base WIS. States only, clipped at −40%. The burn-in configs are left out except the seasonal window with reset. @@ -236,11 +236,11 @@ ggplot(month_m, aes(month, pct, colour = config, group = config)) + facet_grid(season ~ h, scales = "free_x", space = "free_x") + scale_colour_manual(values = CFG_COLS[MONTH_CFGS], name = NULL) + coord_cartesian(ylim = c(-40, 30)) + - labs(x = "month of reference date", y = "WIS change %; grey: % of season base WIS") + + labs(x = "month of reference date", y = "WIS reduction % (positive = calibrated\nWIS lower than base); grey: % of season base WIS") + theme(legend.position = "bottom") ``` -h2 by month: WIS change % (share of outcomes below the calibrated median). +h2 by month: WIS reduction % (positive = calibrated WIS lower than base) (share of outcomes below the calibrated median). ```{r month-table} month_m %>% diff --git a/reports/writeups/calibration_experiments/e03_scale.Rmd b/reports/writeups/calibration_experiments/e03_scale.Rmd index 7d254686..d58b374f 100644 --- a/reports/writeups/calibration_experiments/e03_scale.Rmd +++ b/reports/writeups/calibration_experiments/e03_scale.Rmd @@ -41,8 +41,9 @@ change goes with the scale. # Findings Numbers are from the render of 30 September 2026, pooled over both live -seasons and all locations; WIS and absolute-error changes are % vs base, -positive is better. +seasons and all locations. WIS reduction % is 100 × (base − calibrated) / base, +so positive = calibrated WIS lower than base; the absolute-error change of the +median is defined the same way. L1 coverage bias: lower is better. - **sqrt beats count at every horizon.** WIS +5.1/+4.0/+0.7/−1.3/−1.4 vs +3.2/+1.9/−2.0/−2.7/−5.3 at h−1…h3, with similar coverage bias diff --git a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd index fcc01567..938abf3c 100644 --- a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd +++ b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd @@ -38,8 +38,9 @@ E03 sqrt variant), learning from vintages the way prod does (E00). # Findings Numbers are from the render of 30 September 2026, pooled over both live -seasons and all locations; WIS and absolute-error changes are % vs base, -positive is better. +seasons and all locations. WIS reduction % is 100 × (base − calibrated) / base, +so positive = calibrated WIS lower than base; the absolute-error change of the +median is defined the same way. L1 coverage bias: lower is better. - **The leak buys WIS at h1–h3 with coverage.** `leak 0.1` moves WIS from +0.7/−1.3/−1.4 to +1.6/+0.8/+0.6 at h1–h3, but coverage bias nearly diff --git a/reports/writeups/calibration_experiments/e05_warm_start.Rmd b/reports/writeups/calibration_experiments/e05_warm_start.Rmd index bc271f58..2aae3385 100644 --- a/reports/writeups/calibration_experiments/e05_warm_start.Rmd +++ b/reports/writeups/calibration_experiments/e05_warm_start.Rmd @@ -38,8 +38,9 @@ from E04: # Findings Numbers are from the render of 30 September 2026, pooled over both live -seasons and all locations; WIS and absolute-error changes are % vs base, -positive is better. +seasons and all locations. WIS reduction % is 100 × (base − calibrated) / base, +so positive = calibrated WIS lower than base; the absolute-error change of the +median is defined the same way. L1 coverage bias: lower is better. - **The warm start helps every tracker, and never hurts.** Each warm run is at least as good as its cold twin on WIS at every horizon, and better on diff --git a/reports/writeups/calibration_experiments/e07_across_eras.Rmd b/reports/writeups/calibration_experiments/e07_across_eras.Rmd index 196b00dc..3bda4326 100644 --- a/reports/writeups/calibration_experiments/e07_across_eras.Rmd +++ b/reports/writeups/calibration_experiments/e07_across_eras.Rmd @@ -58,14 +58,15 @@ the adaptive learning rate (mult 0.03, floor 0.001, window 20, carry): Section 3 also runs the operating-point candidates (E14, E16, E17) on both forecasters. -"WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive -is better**. "L1 coverage bias" is the mean over the 23 levels of -|empirical coverage − level|. Results lead with states only. +"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so +**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is +better) is the mean over the 23 levels of |empirical coverage − level|. Results lead with states only. # Findings -Numbers are from the render of 1 October 2026, states only, WIS change % -(positive is better) / L1 coverage bias, h0/h1/h2/h3. All-locations numbers +Numbers are from the render of 1 October 2026, states only, WIS reduction % +(positive = calibrated WIS lower than base) / L1 coverage bias (lower is +better), h0/h1/h2/h3. All-locations numbers point the same way. - **Base WIS: equal in 2024-25, B worse in 2025-26.** B/A base WIS is @@ -85,7 +86,7 @@ point the same way. they differ is spring 2024-25: A still under-predicts in March–April (59% and 75% above the median), while B is close to calibrated (45% and 56%). - **The operating-point candidates rank mostly the same on both, with one - exception.** Pooled over both seasons, h0/h1/h2/h3 WIS change %: + exception.** Pooled over both seasons, h0/h1/h2/h3 WIS reduction %: | candidate | A ensemble | B `windowed_seasonal` | |---|---|---| @@ -273,7 +274,7 @@ m %>% transmute(forecaster, season, h, cell = sprintf("%.1f / %.3f / %.2f", wis_base, cal_err_base, above_median)) %>% tidyr::pivot_wider(names_from = h, values_from = cell) %>% arrange(season, forecaster) %>% - knitr::kable(caption = "Base WIS / L1 coverage bias / share above the median, states only") + knitr::kable(caption = "Base WIS / L1 coverage bias (lower is better) / share above the median, states only") ``` ## Base bias by month @@ -295,18 +296,18 @@ base_fc_states <- base_fc %>% mb <- month_base(base_fc_states) %>% mutate(h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) mb %>% tidyr::pivot_longer(c(above, cal_err), names_to = "metric") %>% - mutate(metric = factor(metric, levels = c("above", "cal_err"), labels = c("share above median", "L1 coverage bias"))) %>% + mutate(metric = factor(metric, levels = c("above", "cal_err"), labels = c("share above median", "L1 coverage bias (lower is better)"))) %>% ggplot(aes(month, value, colour = forecaster, group = forecaster)) + - geom_hline(data = data.frame(metric = factor("share above median", levels = c("share above median", "L1 coverage bias")), y = 0.5), aes(yintercept = y), colour = "grey50") + + geom_hline(data = data.frame(metric = factor("share above median", levels = c("share above median", "L1 coverage bias (lower is better)")), y = 0.5), aes(yintercept = y), colour = "grey50") + geom_line() + geom_point(size = 1.1) + - facet_grid(metric + season ~ h, scales = "free", space = "free_x") + + facet_grid(metric + season ~ h, scales = "free", space = "free_x", labeller = labeller(metric = label_wrap_gen(20))) + scale_colour_manual(values = FC_COLS, name = NULL) + labs(x = "month of reference date", y = NULL) + theme(legend.position = "bottom") ``` -Pooled over h0–h3: share above the base median / L1 coverage bias, A vs B. +Pooled over h0–h3: share above the base median / L1 coverage bias (lower is better), A vs B. ```{r base-month-table} month_base(base_fc_states %>% select(-horizon)) %>% @@ -314,12 +315,12 @@ month_base(base_fc_states %>% select(-horizon)) %>% tidyr::pivot_wider(names_from = forecaster, values_from = cell) %>% left_join(base_fc_states %>% filter(level == 0.5) %>% count(season, month, forecaster) %>% filter(forecaster == names(FCS)[1]) %>% select(season, month, forecasts = n), by = c("season", "month")) %>% arrange(season, month) %>% - knitr::kable(caption = "Share above base median / L1 coverage bias, states only, h0–h3 pooled") + knitr::kable(caption = "Share above base median / L1 coverage bias (lower is better), states only, h0–h3 pooled") ``` # 2. What calibration does to each -WIS change % per tracker, one panel per season and horizon, states only. +WIS reduction % (positive = calibrated WIS lower than base) per tracker, one panel per season and horizon, states only. ```{r head-plot, fig.height=6} ggplot(m %>% filter(geos == GEOS[["states"]]), aes(tracker, wis_pct, fill = forecaster)) + @@ -327,11 +328,11 @@ ggplot(m %>% filter(geos == GEOS[["states"]]), aes(tracker, wis_pct, fill = fore geom_col(position = position_dodge(width = 0.85), width = 0.8) + facet_grid(season ~ h) + scale_fill_manual(values = FC_COLS, name = NULL) + - labs(x = NULL, y = "WIS change % (positive is better)") + + labs(x = NULL, y = "WIS reduction % (positive = calibrated WIS lower than base)") + theme(legend.position = "bottom") ``` -L1 coverage bias, calibrated (filled) against the base (open), states only. +L1 coverage bias (lower is better), calibrated (filled) against the base (open), states only. ```{r cov-plot, fig.height=6} d <- m %>% filter(geos == GEOS[["states"]]) @@ -341,11 +342,11 @@ ggplot(d, aes(tracker, cal_err_cal, colour = forecaster)) + facet_grid(season ~ h) + scale_colour_manual(values = FC_COLS, name = NULL) + expand_limits(y = 0) + - labs(x = NULL, y = "L1 coverage bias (open: base)") + + labs(x = NULL, y = "L1 coverage bias (lower is better; open: base)") + theme(legend.position = "bottom") ``` -WIS change % / L1 coverage bias: +WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better): ```{r tables} season_table <- function(g) { @@ -392,7 +393,7 @@ cm <- purrr::map2(ccals, seq_len(nrow(cgrid)), \(cal, i) score(cal) %>% mutate(c ) ``` -WIS change % per candidate, states only. +WIS reduction % (positive = calibrated WIS lower than base) per candidate, states only. ```{r cand-plot, fig.height=6.5} ggplot(cm %>% filter(geos == GEOS[["states"]]), aes(cand, wis_pct, fill = forecaster)) + @@ -400,11 +401,11 @@ ggplot(cm %>% filter(geos == GEOS[["states"]]), aes(cand, wis_pct, fill = foreca geom_col(position = position_dodge(width = 0.85), width = 0.8) + facet_grid(season ~ h) + scale_fill_manual(values = FC_COLS, name = NULL) + - labs(x = NULL, y = "WIS change % (positive is better)") + + labs(x = NULL, y = "WIS reduction % (positive = calibrated WIS lower than base)") + theme(legend.position = "bottom", axis.text.x = element_text(angle = 30, hjust = 1)) ``` -L1 coverage bias, calibrated (filled) against the base (open), states only. +L1 coverage bias (lower is better), calibrated (filled) against the base (open), states only. ```{r cand-cov, fig.height=6.5} ggplot(cm %>% filter(geos == GEOS[["states"]]), aes(cand, cal_err_cal, colour = forecaster)) + @@ -413,15 +414,15 @@ ggplot(cm %>% filter(geos == GEOS[["states"]]), aes(cand, cal_err_cal, colour = facet_grid(season ~ h) + scale_colour_manual(values = FC_COLS, name = NULL) + expand_limits(y = 0) + - labs(x = NULL, y = "L1 coverage bias (open: base)") + + labs(x = NULL, y = "L1 coverage bias (lower is better; open: base)") + theme(legend.position = "bottom", axis.text.x = element_text(angle = 30, hjust = 1)) ``` ## Rankings on A and B -Candidates ordered by mean WIS change % over h0–h3, separately on each +Candidates ordered by mean WIS reduction % (positive = calibrated WIS lower than base) over h0–h3, separately on each forecaster, states only. `rank A` / `rank B` are the positions (1 is the -largest gain); Spearman ρ compares the two orders per season, over all +largest WIS reduction; `cov rank` 1 is the lowest L1 coverage bias); Spearman ρ compares the two orders per season, over all (candidate, horizon) cells. ```{r cand-rank} @@ -437,9 +438,9 @@ rk <- cm %>% tidyr::pivot_wider(names_from = fc, values_from = c(wis, rank, cov, cov_rank)) %>% arrange(season, rank_A) rk %>% - transmute(season, cand, `WIS % A` = wis_A, `rank A` = rank_A, `WIS % B` = wis_B, `rank B` = rank_B, + transmute(season, cand, `WIS reduction % A` = wis_A, `rank A` = rank_A, `WIS reduction % B` = wis_B, `rank B` = rank_B, `cov bias A` = cov_A, `cov rank A` = cov_rank_A, `cov bias B` = cov_B, `cov rank B` = cov_rank_B) %>% - knitr::kable(digits = c(0, 0, 2, 0, 2, 0, 3, 0, 3, 0), caption = "Mean over h0–h3, states only") + knitr::kable(digits = c(0, 0, 2, 0, 2, 0, 3, 0, 3, 0), caption = "Mean over h0–h3, states only. WIS reduction %: positive = calibrated WIS lower than base. Coverage bias is L1 coverage bias (lower is better)") cm %>% filter(geos == GEOS[["states"]]) %>% select(season, cand, h, forecaster, wis_pct, cal_err_cal) %>% @@ -454,7 +455,7 @@ cm %>% ## Tables -WIS change % / L1 coverage bias. +WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better). ```{r cand-tables} cand_table <- function(g) { diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd index 3fed985d..560d8c86 100644 --- a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -79,9 +79,10 @@ one offset per level, carried across seasons, and a 2023-24 burn-in season whose outcomes take no steps (the adaptive rate uses it to warm up; the constant rate needs no warm-up). Scores are on counts against finalized truth, over the two live seasons (2024-25 and 2025-26), all 53 hub -locations. "WIS change %" is `100 × (1 − WIS_cal / WIS_base)`, so positive -is better than the base, as in the other notebooks. "L1 coverage bias" is the mean over the 23 quantile -levels of |empirical coverage − level|. +locations. "WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so +positive = calibrated WIS lower than base, as in the other notebooks. "L1 +coverage bias" (lower is better) is the mean over the 23 quantile levels of +|empirical coverage − level|. **Reference configs** (marked on the plots): @@ -95,6 +96,8 @@ levels of |empirical coverage − level|. # Findings Numbers are from the render of 30 September 2026, pooled over locations. +WIS reduction %: positive = calibrated WIS lower than base. L1 coverage bias: +lower is better. "Knee" below means a constant learning rate of 0.056 admissions per 100k per step. @@ -106,7 +109,7 @@ step. locations lose 20% of WIS or more. - **On the rate scale, the knee matches or beats the paper's adaptive tracker at every horizon.** At the reference delay (+1 week, re-run - design), the knee gives WIS change +3.2/+2.5/+1.1/+0.5/+0.2% and coverage + design), the knee gives WIS reduction +3.2/+2.5/+1.1/+0.5/+0.2% and coverage bias 0.040/0.031/0.026/0.027/0.027 at h−1…h3. REF-paper gives +3.2/+1.9/−2.0/−2.7/−5.3% and 0.044/0.030/0.032/0.034/0.032: the same WIS at h−1, worse WIS from h0 on, and worse coverage at h−1 and h1–h3. @@ -119,7 +122,7 @@ step. coverage bias (0.074/0.054/0.045/0.046/0.041). The leak and the warm start buy WIS; the plain tracker buys coverage. - **Extra delay, re-run design (prod): shorter is better, but it matters only - at h−1/h0.** At a learning rate of 0.1, h−1 WIS change is +5.3/+3.6/+1.0% + at h−1/h0.** At a learning rate of 0.1, h−1 WIS reduction is +5.3/+3.6/+1.0% at +0/+1/+2 weeks. At h1–h3 the curves nearly coincide, and coverage bias barely moves with delay. A value learned early is replaced by its revision in the next week's run, so waiting only delays the signal. @@ -194,7 +197,7 @@ refs <- purrr::pmap(ref_grid, function(learn, settle_days) { ```{r metrics, cache=FALSE} pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) -# Per (location, horizon) and pooled over locations: WIS change and L1 +# Per (location, horizon) and pooled over locations: WIS reduction % and L1 # coverage bias, for the calibrated forecast and the base. metrics <- function(cal) { fc <- cal$forecasts %>% filter(!is_burn_in, !is.na(truth), !is.na(value_base)) @@ -256,7 +259,7 @@ base_row <- bridge %>% filter(ref == "REF-paper (count, adaptive)") %>% mutate(r tbl <- bind_rows(base_row, bridge) %>% transmute(config = ref, horizon, cell = sprintf("%+.1f%% / %.3f", wis_pct, bias_cal)) %>% tidyr::pivot_wider(names_from = horizon, values_from = cell) -knitr::kable(tbl, caption = "WIS change % (positive is better) / L1 coverage bias, pooled over locations") +knitr::kable(tbl, caption = "WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), pooled over locations") knitr::kable( best_const %>% select(horizon, lr) %>% mutate(lr = signif(lr, 2)) %>% tidyr::pivot_wider(names_from = horizon, values_from = lr), caption = "Constant learning rate with the lowest pooled WIS (rate units per step)" @@ -304,7 +307,7 @@ curve_plot <- function(which_learn) { scale_shape_manual(values = c(`REF-paper (count, adaptive)` = 2, `REF-op` = 17), name = NULL) + facet_grid(horizon ~ extra) + coord_cartesian(xlim = xlim, ylim = ylim) + - labs(x = "L1 coverage bias", y = "WIS change % (positive is better)") + + labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)") + theme(legend.position = "bottom") } ``` @@ -343,13 +346,13 @@ ggplot(pooled_all, aes(bias_cal, wis_pct, colour = extra, linetype = design, gro scale_colour_manual(values = DELAY_COLS, name = NULL) + scale_linetype_manual(values = c(`re-run (prod)` = "solid", stateful = "22"), name = NULL) + facet_wrap(~horizon, nrow = 1) + - labs(x = "L1 coverage bias", y = "WIS change % (positive is better)") + + labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)") + theme(legend.position = "bottom", legend.box = "vertical") ``` # Tables -Pooled WIS change % (positive is better) / L1 coverage bias for every learning rate, by horizon +Pooled WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better) for every learning rate, by horizon (columns) and extra delay (row blocks). ```{r table-fn} diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd index 1025b7da..c7b4f948 100644 --- a/reports/writeups/calibration_experiments/e10_covid.Rmd +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -40,8 +40,9 @@ seasons. For the per-season views below, seasons are August–July years. # Findings Numbers are from the render of 30 September 2026, pooled over all rounds -(2024-11-30 to 2026-09-19) and locations; WIS changes are % vs base, positive -is better. The first covid round (2024-11-23) is excluded: it was made from a +(2024-11-30 to 2026-09-19) and locations. WIS reduction % is 100 × (base − +calibrated) / base, so positive = calibrated WIS lower than base. L1 coverage +bias: lower is better. The first covid round (2024-11-23) is excluded: it was made from a bad NHSN release that was corrected the next day. - **The covid base is already calibrated at h1–h3,** so there is little to diff --git a/reports/writeups/calibration_experiments/e11_constant_lr.Rmd b/reports/writeups/calibration_experiments/e11_constant_lr.Rmd index 5e2954c8..8948c8b5 100644 --- a/reports/writeups/calibration_experiments/e11_constant_lr.Rmd +++ b/reports/writeups/calibration_experiments/e11_constant_lr.Rmd @@ -42,8 +42,9 @@ prod does (re-run each week; E00). # Findings Numbers are from the render of 1 October 2026, pooled over both live -seasons and all locations; WIS and absolute-error changes are % vs base, -positive is better. The constant run matches E09's table at 0.1, +1 week, +seasons and all locations. WIS reduction % is 100 × (base − calibrated) / base, +so positive = calibrated WIS lower than base; the absolute-error change of the +median is defined the same way. L1 coverage bias: lower is better. The constant run matches E09's table at 0.1, +1 week, re-run. - **Against the cold adaptive trackers, the constant rate trades WIS at diff --git a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd index 71ac8425..ed88c394 100644 --- a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd +++ b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd @@ -39,17 +39,17 @@ Reference points, run on the same data: - **sqrt, adaptive**: the plain tracker on the square-root scale with the adaptive learning rate (E03's winner; E13's "E05 single cold"). -"WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive -is better** (the opposite sign of E09's "inflation"). "L1 coverage bias" is -the mean over the 23 levels of |empirical coverage − level|. Each season is +"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so +**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is +better) is the mean over the 23 levels of |empirical coverage − level|. Each season is scored on its own rounds; the tracker still runs over both seasons, so 2025-26 starts from the offsets 2024-25 left behind. Results lead with states only (US excluded), since US alone is about 45% of the all-locations WIS. # Findings -Numbers are from the render of 1 October 2026, states only, WIS change % -(positive is better) / L1 coverage bias. All-locations numbers point the +Numbers are from the render of 1 October 2026, states only, WIS reduction % +(positive = calibrated WIS lower than base) / L1 coverage bias (lower is better). All-locations numbers point the same way. - **Yes: 0.018–0.032 per 100k is good in both seasons.** At those rates WIS @@ -96,7 +96,7 @@ refs <- purrr::map(ref_cfgs, run) ```{r metrics} GEOS <- c(states = "states only (US excluded)", all = "all locations") SEASONS <- c(LIVE, "both") -# ws_scores() per location subset; wis_pct is the change vs base, positive is better. +# ws_scores() per location subset; wis_pct is the WIS reduction % vs base (positive = calibrated WIS lower). score <- function(cal) { fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% @@ -144,7 +144,7 @@ curve_plot <- function(g) { scale_shape_manual(values = c(`REF-op cold` = 17, `sqrt, adaptive` = 15), name = NULL) + facet_wrap(~h, nrow = 1) + coord_cartesian(ylim = c(max(-10, min(d$wis_pct)), max(c(d$wis_pct, r$wis_pct)) + 0.5)) + - labs(x = "L1 coverage bias", y = "WIS change % (positive is better)", title = g) + + labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)", title = g) + theme(legend.position = "bottom") } ``` @@ -164,20 +164,20 @@ curve_plot(GEOS[["all"]]) # Each metric against the learning rate The same runs with the learning rate on the x axis, so the seasons can be -compared at the same rate. Top: WIS change % (positive is better). Bottom: -L1 coverage bias, with each season's base as a dotted line. States only. +compared at the same rate. Top: WIS reduction % (positive = calibrated WIS lower than +base). Bottom: L1 coverage bias (lower is better), with each season's base as a dotted line. States only. ```{r vs-lr, fig.height=6.5} long <- sweep_m %>% filter(geos == GEOS[["states"]], season != "both seasons") %>% - transmute(h, season, lr, `WIS change %` = wis_pct, `L1 coverage bias` = cal_err_cal) %>% - tidyr::pivot_longer(c(`WIS change %`, `L1 coverage bias`), names_to = "metric") %>% - mutate(metric = factor(metric, levels = c("WIS change %", "L1 coverage bias"))) + transmute(h, season, lr, `WIS reduction % (positive = calibrated WIS lower than base)` = wis_pct, `L1 coverage bias (lower is better)` = cal_err_cal) %>% + tidyr::pivot_longer(c(`WIS reduction % (positive = calibrated WIS lower than base)`, `L1 coverage bias (lower is better)`), names_to = "metric") %>% + mutate(metric = factor(metric, levels = c("WIS reduction % (positive = calibrated WIS lower than base)", "L1 coverage bias (lower is better)"))) base_lines <- sweep_m %>% filter(geos == GEOS[["states"]], season != "both seasons") %>% distinct(h, season, value = cal_err_base) %>% - mutate(metric = factor("L1 coverage bias", levels = levels(long$metric))) -zero <- tibble(metric = factor("WIS change %", levels = levels(long$metric)), value = 0) + mutate(metric = factor("L1 coverage bias (lower is better)", levels = levels(long$metric))) +zero <- tibble(metric = factor("WIS reduction % (positive = calibrated WIS lower than base)", levels = levels(long$metric)), value = 0) ggplot(long, aes(lr, value, colour = season)) + geom_hline(data = zero, aes(yintercept = value), colour = "grey60", linewidth = 0.3) + geom_hline(data = base_lines, aes(yintercept = value, colour = season), linetype = "dotted") + @@ -185,15 +185,16 @@ ggplot(long, aes(lr, value, colour = season)) + geom_point(size = 1) + scale_x_log10(breaks = c(0.003, 0.01, 0.03, 0.1, 0.3, 1), labels = c("0.003", "0.01", "0.03", "0.1", "0.3", "1")) + scale_colour_manual(values = SEASON_COLS, name = NULL) + - facet_grid(metric ~ h, scales = "free_y") + + facet_grid(metric ~ h, scales = "free_y", labeller = labeller(metric = label_wrap_gen(28))) + labs(x = "constant learning rate (per 100k per step)", y = NULL) + theme(legend.position = "bottom") ``` # Tables -WIS change % / L1 coverage bias, per learning rate (rows) and season × -horizon (columns). The base row shows the base's coverage bias. The +WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage +bias (lower is better), per learning rate (rows) and season × horizon +(columns). The base row shows the base's coverage bias. The references follow the sweep. ```{r tables} diff --git a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd index b4c3fcdb..9dfdc3b9 100644 --- a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd +++ b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd @@ -32,12 +32,12 @@ burn-in season. Two calibrated configs, both from E12: - **E11 constant 0.1**: the plain tracker on the rate scale (per 100k) with a constant learning rate of 0.1. -Scores are against finalized truth over 2024-25 and 2025-26. "WIS change %" -is `100 × (WIS_base − WIS_cal) / WIS_base`: **positive is better**. +Scores are against finalized truth over 2024-25 and 2025-26. "WIS reduction %" +is `100 × (WIS_base − WIS_cal) / WIS_base`: **positive = calibrated WIS lower than base**. Results lead with states only (US excluded), since US alone is about 45% of the all-locations WIS. -**Three ways to cut the WIS change.** +**Three ways to cut the WIS reduction.** 1. **By quantile level.** WIS is the average pinball loss over the 23 quantile levels, so the total change splits exactly into one piece per @@ -58,14 +58,14 @@ the all-locations WIS. median sits on the base median. The calibrated widths and shape are kept, the base centre is kept. - If the two parts did not interact, their WIS changes would add up to the + If the two parts did not interact, their WIS reductions would add up to the full change. Both parts are floored at zero, as the calibrated forecast is. # Findings Numbers are from the render of 1 October 2026, states only, h0/h1/h2/h3, -WIS change % with positive better. All-locations numbers point the same way. +WIS reduction % (positive = calibrated WIS lower than base). All-locations numbers point the same way. - **The offsets are small, and that is why the fans look unchanged.** The typical (median) offset is 2–5% of the base forecast's 90% interval width @@ -119,10 +119,10 @@ fc <- bind_rows(fc %>% filter(location != "US") %>% mutate(geos = "states"), fc mutate(geos = factor(geos, levels = names(GEOS), labels = GEOS)) ``` -# 1. WIS change by quantile level +# 1. WIS reduction by quantile level -Each bar is one level's share of the total WIS change % (positive is -better); the bars in a panel add up to the panel's total, printed in the +Each bar is one level's share of the total WIS reduction % (positive = +calibrated WIS lower than base); the bars in a panel add up to the panel's total, printed in the strip. ```{r by-level} @@ -142,7 +142,7 @@ level_plot <- function(g) { facet_wrap(~ paste(config, panel, sep = ": "), ncol = 4) + scale_fill_manual(values = c(`REF-op cold` = "#D55E00", `E11 constant 0.1` = "#0072B2"), guide = "none") + scale_x_discrete(labels = function(x) ifelse(x %in% c("0.01", "0.05", "0.25", "0.5", "0.75", "0.95", "0.99"), x, "")) + - labs(x = "quantile level", y = "share of WIS change, % of base WIS", title = g) + + labs(x = "quantile level", y = "share of WIS reduction, % of base WIS\n(positive = calibrated WIS lower than base)", title = g) + theme(axis.text.x = element_text(angle = 90, vjust = 0.5, size = 7)) } ``` @@ -185,8 +185,8 @@ group_tbl <- function(g) { tidyr::pivot_wider(names_from = horizon, values_from = v) %>% arrange(config, part) } -knitr::kable(group_tbl(GEOS[["states"]]), caption = "States only: WIS change % by group of levels (positive is better)") -knitr::kable(group_tbl(GEOS[["all"]]), caption = "All locations: WIS change % by group of levels (positive is better)") +knitr::kable(group_tbl(GEOS[["states"]]), caption = "States only: WIS reduction % by group of levels (positive = calibrated WIS lower than base)") +knitr::kable(group_tbl(GEOS[["all"]]), caption = "All locations: WIS reduction % by group of levels (positive = calibrated WIS lower than base)") ``` Change in pinball loss at each level relative to that level's own base @@ -201,7 +201,7 @@ knitr::kable( transmute(config, level, horizon = paste0("h", horizon), v = round(level_pct, 1)) %>% tidyr::pivot_wider(names_from = horizon, values_from = v) %>% arrange(config, level), - caption = "States only: pinball loss change % at selected levels, vs that level's base loss" + caption = "States only: pinball loss reduction % at selected levels, vs that level's base loss (positive = calibrated loss lower than base)" ) ``` @@ -300,8 +300,8 @@ split_tbl <- function(g) { tidyr::pivot_wider(names_from = horizon, values_from = pct) %>% arrange(config, variant) } -knitr::kable(split_tbl(GEOS[["states"]]), caption = "States only: WIS change % (positive is better)") -knitr::kable(split_tbl(GEOS[["all"]]), caption = "All locations: WIS change % (positive is better)") +knitr::kable(split_tbl(GEOS[["states"]]), caption = "States only: WIS reduction % (positive = calibrated WIS lower than base)") +knitr::kable(split_tbl(GEOS[["all"]]), caption = "All locations: WIS reduction % (positive = calibrated WIS lower than base)") ``` How the calibrated widths compare with the base: the median over forecasts diff --git a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd index df05fd6d..ebdf2915 100644 --- a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd +++ b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd @@ -45,16 +45,16 @@ curve is E14's runs. Reference points: **REF-op cold** (prod's tracker without its warm start) and the rate-scale rates **0.018** and **0.032** (E14's best range). -"WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive -is better**. "L1 coverage bias" is the mean over the 23 levels of -|empirical coverage − level|. Each season is scored on its own rounds; the +"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so +**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is +better) is the mean over the 23 levels of |empirical coverage − level|. Each season is scored on its own rounds; the tracker runs over both, so 2025-26 starts from the offsets 2024-25 left behind. Results lead with states only (US excluded). # Findings -Numbers are from the render of 1 October 2026, states only, WIS change % -(positive is better) / L1 coverage bias, h0/h1/h2/h3. All-locations numbers +Numbers are from the render of 1 October 2026, states only, WIS reduction % +(positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), h0/h1/h2/h3. All-locations numbers point the same way. - **The sqrt scale works, with a knee at 0.01–0.018.** Every sqrt rate from @@ -141,7 +141,7 @@ sweep_m <- bind_rows( purrr::map2(sqrt_runs, SQRT_LRS, \(cal, lr) score(cal) %>% mutate(scale = "sqrt", lr = lr)) ) %>% label() ref_m <- score(refop) %>% mutate(ref = "REF-op cold") %>% label() -# Best sqrt rate: the highest mean WIS change among rates that improve WIS in both seasons at h0–h3 (states only). +# Best sqrt rate: the highest mean WIS reduction among rates that improve WIS in both seasons at h0–h3 (states only). best_tbl <- sweep_m %>% filter(scale == "sqrt", geos == GEOS[["states"]], season %in% LIVE) %>% group_by(lr) %>% @@ -177,7 +177,7 @@ curve_plot <- function(g) { scale_shape_manual(values = c(rate = 16, sqrt = 1), name = "scale") + facet_wrap(~h, nrow = 1) + coord_cartesian(ylim = c(max(-10, min(d$wis_pct)), max(c(d$wis_pct, r$wis_pct)) + 0.5)) + - labs(x = "L1 coverage bias", y = "WIS change % (positive is better)", title = g, + labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)", title = g, caption = "labels: r = rate-scale learning rate (per 100k), s = sqrt-scale learning rate; triangles: REF-op cold") + theme(legend.position = "bottom") } @@ -197,20 +197,20 @@ curve_plot(GEOS[["all"]]) ## Each metric against the learning rate, sqrt scale -Top: WIS change %. Bottom: L1 coverage bias, each season's base dotted. +Top: WIS reduction % (positive = calibrated WIS lower than base). Bottom: L1 coverage bias (lower is better), each season's base dotted. States only. ```{r vs-lr, fig.height=6.5} long <- sweep_m %>% filter(scale == "sqrt", geos == GEOS[["states"]], season != "both seasons") %>% - transmute(h, season, lr, `WIS change %` = wis_pct, `L1 coverage bias` = cal_err_cal) %>% - tidyr::pivot_longer(c(`WIS change %`, `L1 coverage bias`), names_to = "metric") %>% - mutate(metric = factor(metric, levels = c("WIS change %", "L1 coverage bias"))) + transmute(h, season, lr, `WIS reduction % (positive = calibrated WIS lower than base)` = wis_pct, `L1 coverage bias (lower is better)` = cal_err_cal) %>% + tidyr::pivot_longer(c(`WIS reduction % (positive = calibrated WIS lower than base)`, `L1 coverage bias (lower is better)`), names_to = "metric") %>% + mutate(metric = factor(metric, levels = c("WIS reduction % (positive = calibrated WIS lower than base)", "L1 coverage bias (lower is better)"))) base_lines <- sweep_m %>% filter(scale == "sqrt", geos == GEOS[["states"]], season != "both seasons") %>% distinct(h, season, value = cal_err_base) %>% - mutate(metric = factor("L1 coverage bias", levels = levels(long$metric))) -zero <- tibble(metric = factor("WIS change %", levels = levels(long$metric)), value = 0) + mutate(metric = factor("L1 coverage bias (lower is better)", levels = levels(long$metric))) +zero <- tibble(metric = factor("WIS reduction % (positive = calibrated WIS lower than base)", levels = levels(long$metric)), value = 0) ggplot(long, aes(lr, value, colour = season)) + geom_hline(data = zero, aes(yintercept = value), colour = "grey60", linewidth = 0.3) + geom_hline(data = base_lines, aes(yintercept = value, colour = season), linetype = "dotted") + @@ -218,14 +218,14 @@ ggplot(long, aes(lr, value, colour = season)) + geom_point(size = 1) + scale_x_log10(breaks = c(0.001, 0.003, 0.01, 0.03, 0.1, 0.3), labels = c("0.001", "0.003", "0.01", "0.03", "0.1", "0.3")) + scale_colour_manual(values = SEASON_COLS, name = NULL) + - facet_grid(metric ~ h, scales = "free_y") + + facet_grid(metric ~ h, scales = "free_y", labeller = labeller(metric = label_wrap_gen(28))) + labs(x = "constant learning rate (sqrt of per-100k units per step)", y = NULL) + theme(legend.position = "bottom") ``` ## Tables -WIS change % / L1 coverage bias per config (rows) and season × horizon +WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better) per config (rows) and season × horizon (columns). The sqrt sweep, then the rate-scale rates around E14's best range and REF-op cold. @@ -260,7 +260,7 @@ knitr::kable(season_table(GEOS[["all"]])) ### Picking the best sqrt rate -For each sqrt rate, the worst and the mean WIS change % over the two seasons +For each sqrt rate, the worst and the mean WIS reduction % (positive = calibrated WIS lower than base) over the two seasons and h0–h3 (states only). The best rate is the one with the highest mean among those whose worst case is still a gain (E14's criterion). @@ -344,9 +344,9 @@ month_m <- ws_month_scores(fc_pick %>% filter(config %in% names(MONTH_COLS)) %>% mutate(h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) ``` -## WIS change % by month +## WIS reduction % by month -Each season a row, each horizon a column. The grey bars (right axis +WIS reduction % (positive = calibrated WIS lower than base). Each season a row, each horizon a column. The grey bars (right axis scaled to the left) are each month's share of the season's base WIS, so a large loss in a month with a small share costs little overall. The y axis is clipped at −40% and +30%, which also cuts the tallest bars. @@ -361,30 +361,30 @@ ggplot(month_m, aes(month, pct, colour = config, group = config)) + facet_grid(season ~ h, scales = "free_x", space = "free_x") + scale_colour_manual(values = MONTH_COLS, name = NULL) + coord_cartesian(ylim = c(-40, 30)) + - labs(x = "month of reference date", y = "WIS change % (positive is better); grey: % of season base WIS") + + labs(x = "month of reference date", y = "WIS reduction % (positive = calibrated\nWIS lower than base); grey: % of season base WIS") + theme(legend.position = "bottom") ``` ## Coverage by month -Top: L1 coverage bias per month, base in black. Bottom: share of outcomes +Top: L1 coverage bias (lower is better) per month, base in black. Bottom: share of outcomes below the median; 0.5 is calibrated, above 0.5 means the median sits too high (over-prediction). Base in black. ```{r month-cov, fig.height=8} cov_long <- bind_rows( - month_m %>% transmute(config = as.character(config), season, h, month, `L1 coverage bias` = cal_err_cal, `share below the median` = below_med_cal), + month_m %>% transmute(config = as.character(config), season, h, month, `L1 coverage bias (lower is better)` = cal_err_cal, `share below the median` = below_med_cal), month_m %>% filter(config == names(MONTH_COLS)[1]) %>% - transmute(config = "base", season, h, month, `L1 coverage bias` = cal_err_base, `share below the median` = below_med_base) + transmute(config = "base", season, h, month, `L1 coverage bias (lower is better)` = cal_err_base, `share below the median` = below_med_base) ) %>% - tidyr::pivot_longer(c(`L1 coverage bias`, `share below the median`), names_to = "metric") %>% + tidyr::pivot_longer(c(`L1 coverage bias (lower is better)`, `share below the median`), names_to = "metric") %>% mutate(config = factor(config, levels = c("base", names(MONTH_COLS)))) half <- tibble(metric = "share below the median", y = 0.5) ggplot(cov_long, aes(month, value, colour = config, group = config)) + geom_hline(data = half, aes(yintercept = y), colour = "grey50", linetype = "dashed") + geom_line() + geom_point(size = 1) + - facet_grid(metric + season ~ h, scales = "free", space = "free_x") + + facet_grid(metric + season ~ h, scales = "free", space = "free_x", labeller = labeller(metric = label_wrap_gen(20))) + scale_colour_manual(values = c(base = "black", MONTH_COLS), name = NULL) + labs(x = "month of reference date", y = NULL) + theme(legend.position = "bottom") @@ -422,7 +422,7 @@ ggplot(off_long, aes(reference_date, value, colour = config)) + ## Month table, h2 -WIS change % per month (states only, h2), with the base's share of the +WIS reduction % (positive = calibrated WIS lower than base) per month (states only, h2), with the base's share of the season's WIS and the share of outcomes below the base and calibrated medians. ```{r month-table} @@ -436,5 +436,5 @@ month_m %>% by = c("season", "month") ) %>% arrange(season, month) %>% - knitr::kable(caption = "WIS change % (share of outcomes below the calibrated median)") + knitr::kable(caption = "WIS reduction %, positive = calibrated WIS lower than base (share of outcomes below the calibrated median)") ``` diff --git a/reports/writeups/calibration_experiments/e17_late_decay.Rmd b/reports/writeups/calibration_experiments/e17_late_decay.Rmd index 9d2fbe8c..79a12e25 100644 --- a/reports/writeups/calibration_experiments/e17_late_decay.Rmd +++ b/reports/writeups/calibration_experiments/e17_late_decay.Rmd @@ -48,14 +48,14 @@ rates: rate scale 0.018 and 0.032 per 100k (E14) and sqrt scale The tracker carries offsets across seasons, so a decay in spring 2025 also changes where 2025-26 starts. -"WIS change %" is `100 × (WIS_base − WIS_cal) / WIS_base`, **positive is -better**; "L1 coverage bias" is the mean over the 23 levels of |empirical -coverage − level|. +"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive += calibrated WIS lower than base**; "L1 coverage bias" (lower is better) is +the mean over the 23 levels of |empirical coverage − level|. # Findings -Numbers are from the render of 1 October 2026, states only, WIS change % -(positive is better) / L1 coverage bias, h0/h1/h2/h3. +Numbers are from the render of 1 October 2026, states only, WIS reduction % +(positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), h0/h1/h2/h3. - **The decay removes the March loss.** For sqrt 0.018 at h2, March 2025-26 goes from −27.9% to −10.7 / −6.3 / −2.9% (factor 0.85 / 0.7 / 0.5), and @@ -119,7 +119,7 @@ DECAY_COLS <- setNames(c("black", "#56B4E9", "#0072B2", "#D55E00", "#009E73"), n # 1. Overall, by season -WIS change % / L1 coverage bias, states only. +WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), states only. ```{r head} head_m <- fc %>% @@ -139,7 +139,7 @@ bind_rows(base_row, head_m %>% transmute(config = paste(rate, decay, sep = ", ") knitr::kable() ``` -The same WIS changes as a plot, one panel per season and horizon. +The same WIS reductions as a plot, one panel per season and horizon. ```{r head-plot, fig.height=5.5} ggplot(head_m, aes(rate, wis_pct, fill = decay)) + @@ -147,13 +147,13 @@ ggplot(head_m, aes(rate, wis_pct, fill = decay)) + geom_col(position = position_dodge(width = 0.85), width = 0.8) + facet_grid(season ~ paste0("h", horizon)) + scale_fill_manual(values = DECAY_COLS, name = NULL) + - labs(x = NULL, y = "WIS change % (positive is better)") + + labs(x = NULL, y = "WIS reduction % (positive = calibrated WIS lower than base)") + theme(legend.position = "bottom", axis.text.x = element_text(angle = 20, hjust = 1)) ``` # 2. Month by month -WIS change % by calendar month of the reference date, one figure per +WIS reduction % (positive = calibrated WIS lower than base) by calendar month of the reference date, one figure per constant rate; rows are seasons, columns horizons. Grey bars are each month's share of the season's base WIS. Clipped at −40%. @@ -174,7 +174,7 @@ for (r in names(RATES)) { facet_grid(season ~ h, scales = "free_x", space = "free_x") + scale_colour_manual(values = DECAY_COLS, name = NULL) + coord_cartesian(ylim = c(-40, 30)) + - labs(x = "month of reference date", y = "WIS change %; grey: % of season base WIS", title = r) + + labs(x = "month of reference date", y = "WIS reduction % (positive = calibrated\nWIS lower than base); grey: % of season base WIS", title = r) + theme(legend.position = "bottom") ) } @@ -194,7 +194,7 @@ month_m %>% by = c("rate", "season", "month") ) %>% arrange(rate, season, month) %>% - knitr::kable(caption = "h2: WIS change % (share of outcomes below the calibrated median)") + knitr::kable(caption = "h2: WIS reduction %, positive = calibrated WIS lower than base (share of outcomes below the calibrated median)") ``` # 3. Mean offset at the median over time diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index 1cb97dae..fd155c0d 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -54,10 +54,11 @@ for (f in src) { - **Horizons** h−1 to h3 are weeks relative to the forecast's reference date. h−1 is the latest week NHSN has reported, still open to revision; h0 and later have not been reported yet. -- **WIS change %** compares the calibrated forecast's weighted interval score - with the uncalibrated submission's, pooled over locations. The sign - convention is stated in each notebook. -- **L1 coverage bias** is the mean over the 23 quantile levels of +- **WIS reduction %** is `100 × (WIS_base − WIS_cal) / WIS_base`, the + calibrated forecast's weighted interval score against the uncalibrated + submission's, pooled over locations. Positive = calibrated WIS lower than + base. Every notebook uses this sign. +- **L1 coverage bias** (lower is better) is the mean over the 23 quantile levels of |empirical coverage − level|. 0 is perfect; the uncalibrated flu submission sits near 0.1. - **Scoring.** Every score is against finalized NHSN data, over the live diff --git a/scripts/calibration/calibration_ws_replay.R b/scripts/calibration/calibration_ws_replay.R index 04cee874..43a6eb49 100644 --- a/scripts/calibration/calibration_ws_replay.R +++ b/scripts/calibration/calibration_ws_replay.R @@ -132,7 +132,7 @@ ws_scores <- function(fc) { left_join(above, by = c("season", "horizon")) } -#' V-month per season: WIS change % (positive is better), L1 coverage bias and +#' V-month per season: WIS reduction % (positive = calibrated WIS lower than base), L1 coverage bias and #' the share of truth below the median, base vs calibrated, by calendar month of #' the reference date. `share` is the month's share of the season's base WIS. #' Groups by `config` too when present. Used by the E16/E17 notebooks. @@ -194,9 +194,9 @@ ws_report <- function(disease, res) { for (g in names(WS_GEOS)) { cat("\n\n## ", disease, ", ", g, " locations\n", sep = "") for (spec in list( - c("wis", "WIS (2 x mean pinball)", 1), c("wis_pct", "WIS change % vs base (positive is better)", 1), + c("wis", "WIS (2 x mean pinball)", 1), c("wis_pct", "WIS reduction % (positive = calibrated WIS lower than base)", 1), c("cov50", "50% interval coverage", 3), c("cov90", "90% interval coverage", 3), - c("cal_err", "L1 coverage bias", 3) + c("cal_err", "L1 coverage bias (lower is better)", 3) )) { cat("\n### ", spec[2], "\n\n", sep = "") print(knitr::kable(wide(spec[1], as.integer(spec[3]), g))) From ed58f020cd4ce68834abcb6115afd772f5d2d439 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:29:33 -0700 Subject: [PATCH 064/107] docs(calibration): E02 splits season tables into WIS reduction and coverage bias, adds a WIS vs coverage scatter --- .../e02_eta_variants.Rmd | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd index 953bd644..86914bd3 100644 --- a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd +++ b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd @@ -170,32 +170,62 @@ ggplot(m %>% filter(geos == GEOS[["states"]]), aes(config, wis_pct, fill = confi theme(legend.position = "bottom", axis.text.x = element_blank(), axis.ticks.x = element_blank()) ``` -WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better). +WIS reduction % against L1 coverage bias, one point per config, states only. +The open circle is the base (at its coverage bias and zero WIS reduction); up +and to the left is better. The seasonal-window configs compare with +"reference + burn-in", the others with "reference". + +```{r scatter, fig.height=7} +d <- m %>% filter(geos == GEOS[["states"]]) +base <- d %>% filter(config == "reference") %>% distinct(season, h, cal_err_base) +ggplot(d, aes(cal_err_cal, wis_pct, colour = config)) + + geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + + geom_point(data = base, aes(cal_err_base, 0, shape = "base (uncalibrated)"), colour = "black", fill = "white", size = 2.2, inherit.aes = FALSE) + + geom_point(size = 2) + + facet_grid(season ~ h) + + scale_colour_manual(values = CFG_COLS, name = NULL) + + scale_shape_manual(values = c(`base (uncalibrated)` = 21), name = NULL) + + expand_limits(x = 0) + + labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)") + + theme(legend.position = "bottom") +``` + +Per config (rows) and season × horizon (columns): first WIS reduction % +(positive = calibrated WIS lower than base), then L1 coverage bias (lower is +better), with the base's coverage bias in the base row. ```{r tables} -season_table <- function(g) { +season_table <- function(g, value, fmt) { bind_rows( m %>% filter(config == "reference") %>% mutate(config = "base", wis_pct = 0, cal_err_cal = cal_err_base), m ) %>% filter(geos == g) %>% mutate(config = factor(config, levels = c("base", names(CFGS)))) %>% - transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% + transmute(config, col = paste(season, h), cell = sprintf(fmt, .data[[value]])) %>% tidyr::pivot_wider(names_from = col, values_from = cell) %>% arrange(config) } +wis_table <- function(g) { + knitr::kable(season_table(g, "wis_pct", "%+.1f"), caption = paste0("WIS reduction %, ", g, "; positive = calibrated WIS lower than base")) +} +cov_table <- function(g) { + knitr::kable(season_table(g, "cal_err_cal", "%.3f"), caption = paste0("L1 coverage bias (lower is better), ", g)) +} ``` ## States only ```{r table-states} -knitr::kable(season_table(GEOS[["states"]])) +wis_table(GEOS[["states"]]) +cov_table(GEOS[["states"]]) ``` ## All locations ```{r table-all} -knitr::kable(season_table(GEOS[["all"]])) +wis_table(GEOS[["all"]]) +cov_table(GEOS[["all"]]) ``` ## 50% and 90% interval coverage, states only From 1a76a7d6c28e81899890aaaba82f834e7da87a14 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:37:26 -0700 Subject: [PATCH 065/107] docs(calibration): review tracker, E02 fixes done --- notes/calibration-review.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notes/calibration-review.md b/notes/calibration-review.md index 76971db8..92473eca 100644 --- a/notes/calibration-review.md +++ b/notes/calibration-review.md @@ -31,7 +31,7 @@ for), **not reviewed**. |---|---|---|---|---| | E00 vintage backtest | ensemble | verified | Finalized data exaggerates h−1/h0 gains a little; use vintages, no more work. | — | | E01 eta settings | clean replay | verified | Recap of the August sweep: carry slightly better, window 20 as good as larger, the multiplier has an elbow. Note: on this replay 0.03 is the coverage elbow and 0.01 the WIS-safe choice. Earlier: (1) Curve axis looked flipped vs E09. (2) x/o markers unlabeled. (3) "Rebuilds the original" is misleading: it re-asks the question on a new setup. | (1) E09 flipped to match; (2) legend added; (3) intro reworded with a what-changed table | -| E02 eta variants | clean replay | commented | Headline: switching off in Feb/Mar recovers WIS, loses coverage. (1) Coverage numbers hard to find in the WIS / bias cells. (2) Month plot: unclear whether positive % is good; the sign convention must be labeled explicitly in every notebook. | intro reworded; (1)–(2) in progress | +| E02 eta variants | clean replay | commented | Headline: switching off in Feb/Mar recovers WIS, loses coverage. (1) Coverage numbers hard to find in the WIS / bias cells. (2) Month plot: unclear whether positive % is good; the sign convention must be labeled explicitly in every notebook. | intro reworded; (1) tables split into WIS and coverage, plus a WIS vs coverage scatter; (2) every notebook now labels "WIS reduction % (positive = calibrated WIS lower than base)" and "L1 coverage bias (lower is better)" (all computations already used that sign). Open: season-level coverage lets ramp and post-peak errors cancel, so the switch-off's coverage cost may be mostly an artifact; month-level coverage would settle it. | | E03 scale | ensemble | not reviewed | | | | E04 offset structure | ensemble | not reviewed | | | | E05 warm start | ensemble | not reviewed | | | From dae0bfc1b0b5b465ec9b17ca39a69f5cb1806adc Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:47:39 -0700 Subject: [PATCH 066/107] feat(calibration): shared month view helper (WIS, within-month coverage bias, median AE) --- R/calibration/views.R | 176 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/R/calibration/views.R b/R/calibration/views.R index f8c96ca3..a2873e4c 100644 --- a/R/calibration/views.R +++ b/R/calibration/views.R @@ -310,3 +310,179 @@ cal_gallery_panel <- function(cal, loc, ref_date, truth, vintages, subtitle = NU ggplot2::scale_fill_manual(values = CAL_FAN_COLS, aesthetics = c("fill", "colour")) + ggplot2::labs(x = NULL, y = "admissions", colour = NULL, fill = NULL, subtitle = subtitle) } + + +#' V-month3: the month view. Per config × season × calendar month × horizon: +#' WIS reduction %, L1 coverage bias within the month (base and calibrated), +#' and the absolute error of the median, reduction % vs base. +#' +#' Seasons are August–July years of the reference date, so hub runs and the +#' clean replay are cut the same way. Burn-in rounds are dropped. +#' +#' @param cals named list of runs (each with `$forecasts`) or of forecast +#' frames with `location`, `horizon`, `reference_date`, `level`, `truth`, +#' `value_base`, `value_cal` and optionally `is_burn_in`. +#' @param horizons horizons to keep; `NULL` keeps all. +#' @param states_only drop the US national series. +#' @return list of `month` (one row per config, season, month, horizon) and +#' `cov` (season-pooled and month-average L1 coverage bias per config, +#' season and horizon; season `"both"` pools the seasons). +#' @export +cal_month_view <- function(cals, seasons = c("2024-2025", "2025-2026"), horizons = 0:3, states_only = TRUE) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + per_level <- purrr::imap(cals, function(x, nm) { + fc <- if (is.data.frame(x)) x else x$forecasts + if (!"is_burn_in" %in% names(fc)) fc$is_burn_in <- FALSE + if (!is.null(horizons)) fc <- fc %>% filter(.data$horizon %in% horizons) + if (states_only) fc <- fc %>% filter(.data$location != "US") + yr <- as.integer(format(fc$reference_date, "%Y")) - (as.integer(format(fc$reference_date, "%m")) < 8L) + fc %>% + mutate(season = paste0(yr, "-", yr + 1L)) %>% + filter(!.data$is_burn_in, !is.na(.data$truth), !is.na(.data$value_base), .data$season %in% seasons) %>% + mutate(month = factor(format(.data$reference_date, "%b"), levels = month.abb[c(8:12, 1:7)])) %>% + group_by(.data$season, .data$month, .data$horizon, .data$level) %>% + summarize( + wis_base = sum(pinball(.data$truth, .data$value_base, .data$level)), + wis_cal = sum(pinball(.data$truth, .data$value_cal, .data$level)), + ae_base = sum(abs(.data$truth - .data$value_base)), ae_cal = sum(abs(.data$truth - .data$value_cal)), + below_base = sum(.data$truth <= .data$value_base), below_cal = sum(.data$truth <= .data$value_cal), + n = n(), rounds = n_distinct(.data$reference_date), .groups = "drop" + ) %>% + mutate(config = nm, .before = 1) + }) %>% + bind_rows() %>% + mutate(config = factor(.data$config, levels = names(cals))) + month <- per_level %>% + group_by(.data$config, .data$season, .data$month, .data$horizon) %>% + summarize( + wis_base = sum(.data$wis_base), wis_cal = sum(.data$wis_cal), + cov_base = mean(abs(.data$below_base / .data$n - .data$level)), + cov_cal = mean(abs(.data$below_cal / .data$n - .data$level)), + ae_base = .data$ae_base[.data$level == 0.5], ae_cal = .data$ae_cal[.data$level == 0.5], + n = first(.data$n), rounds = first(.data$rounds), .groups = "drop" + ) %>% + group_by(.data$config, .data$season, .data$horizon) %>% + mutate( + wis_pct = 100 * (.data$wis_base - .data$wis_cal) / .data$wis_base, + ae_pct = 100 * (.data$ae_base - .data$ae_cal) / .data$ae_base, + share = 100 * .data$wis_base / sum(.data$wis_base) + ) %>% + ungroup() + both <- function(d) bind_rows(d, d %>% mutate(season = "both")) + pooled <- both(per_level) %>% + group_by(.data$config, .data$season, .data$horizon, .data$level) %>% + summarize(b = sum(.data$below_base) / sum(.data$n), c = sum(.data$below_cal) / sum(.data$n), .groups = "drop") %>% + group_by(.data$config, .data$season, .data$horizon) %>% + summarize(pooled_base = mean(abs(.data$b - .data$level)), pooled_cal = mean(abs(.data$c - .data$level)), .groups = "drop") + mavg <- both(month) %>% + group_by(.data$config, .data$season, .data$horizon) %>% + summarize(mavg_base = mean(.data$cov_base), mavg_cal = mean(.data$cov_cal), months = n(), .groups = "drop") + cov <- left_join(pooled, mavg, by = c("config", "season", "horizon")) %>% + mutate(season = factor(.data$season, levels = c(seasons, "both"))) + list(month = month, cov = cov) +} + + +#' Caption for [cal_month_cov_table()]. +#' @export +CAL_MONTH_COV_CAPTION <- paste( + "L1 coverage bias (lower is better), states only, as season-pooled / month-avg.", + "Season-pooled scores all of a season's rounds together. Month-avg is the mean over calendar months", + "of the bias computed within each month, every month weighted equally ('both': every season-month weighted equally)." +) + + +#' Season-pooled / month-average L1 coverage bias as a wide table: configs as +#' rows (base first; one base row per distinct base), `season hX` as columns. +#' @export +cal_month_cov_table <- function(mv, seasons = NULL) { + cov <- mv$cov + if (!is.null(seasons)) cov <- cov %>% filter(.data$season %in% seasons) + cell <- function(p, m) sprintf("%.3f / %.3f", p, m) + wide <- function(d) { + d %>% + arrange(.data$season, .data$horizon) %>% + transmute(.data$config, col = paste0(.data$season, " h", .data$horizon), cell = .data$cell) %>% + tidyr::pivot_wider(names_from = "col", values_from = "cell") + } + base <- wide(cov %>% mutate(cell = cell(.data$pooled_base, .data$mavg_base))) + base <- base[!duplicated(base[-1]), ] + base$config <- if (nrow(base) == 1L) "base" else paste0("base (", base$config, ")") + cal <- wide(cov %>% mutate(cell = cell(.data$pooled_cal, .data$mavg_cal))) %>% mutate(config = as.character(.data$config)) + bind_rows(base, cal) +} + + +#' The month view plot for one season: rows are WIS reduction %, L1 coverage +#' bias and median absolute-error reduction %; columns are horizons; one line +#' per config. The base's coverage bias is grey (dashed unless `linetype` is +#' given), and grey bars are each month's share of the season's base WIS. +#' +#' @param month_tbl the `month` element of [cal_month_view()]. +#' @param colours named colours, keyed by the `colour_by` values; defaults to +#' a fixed palette. +#' @param colour_by column of `month_tbl` mapped to colour; each config is +#' still its own line. +#' @param linetype optional column of `month` mapped to linetype (e.g. the +#' forecaster); the base line is then drawn once per linetype group. +#' @param clip WIS and AE reductions are squished into this range; clipped +#' points are drawn open. +#' @export +cal_month_plot <- function(month_tbl, season, colours = NULL, linetype = NULL, colour_by = "config", clip = c(-40, 40)) { + m_wis <- "WIS reduction %\n(positive = calibrated\nWIS lower than base)" + m_cov <- "L1 coverage bias\n(lower is better)" + m_ae <- "median AE reduction %\n(positive = calibrated\nAE lower than base)" + lv <- c(m_wis, m_cov, m_ae) + d <- month_tbl %>% + filter(.data$season == !!season) %>% + mutate(month = droplevels(.data$month), h = factor(paste0("h", .data$horizon), levels = paste0("h", sort(unique(month_tbl$horizon))))) + d$grp <- if (is.null(linetype)) "base" else as.character(d[[linetype]]) + d$col <- d[[colour_by]] + configs <- levels(droplevels(d$config)) + if (is.null(colours)) { + keys <- if (is.factor(d$col)) levels(droplevels(d$col)) else unique(d$col) + pal <- c("black", "#0072B2", "#D55E00", "#009E73", "#CC79A7", "#E69F00", "#56B4E9", "#999933") + colours <- setNames(rep_len(pal, length(keys)), keys) + } + sq <- function(x) pmin(pmax(x, clip[1]), clip[2]) + long <- bind_rows( + d %>% transmute(.data$config, .data$col, .data$grp, .data$month, .data$h, metric = m_wis, value = sq(.data$wis_pct), clipped = .data$wis_pct != value), + d %>% transmute(.data$config, .data$col, .data$grp, .data$month, .data$h, metric = m_cov, value = .data$cov_cal, clipped = FALSE), + d %>% transmute(.data$config, .data$col, .data$grp, .data$month, .data$h, metric = m_ae, value = sq(.data$ae_pct), clipped = .data$ae_pct != value) + ) %>% mutate(metric = factor(.data$metric, levels = lv)) + first_cfg <- d %>% distinct(.data$grp, .data$config) %>% group_by(.data$grp) %>% slice(1) %>% ungroup() + base <- d %>% + semi_join(first_cfg, by = c("grp", "config")) %>% + transmute(.data$grp, .data$month, .data$h, metric = factor(m_cov, levels = lv), value = .data$cov_base) + share <- d %>% + filter(.data$config == configs[1]) %>% + transmute(.data$month, .data$h, metric = factor(m_wis, levels = lv), value = sq(.data$share)) + zero <- tibble(metric = factor(c(m_wis, m_ae), levels = lv), y = 0) + p <- ggplot2::ggplot(long, ggplot2::aes(.data$month, .data$value)) + + ggplot2::geom_col(data = share, fill = "grey85", width = 0.7) + + ggplot2::geom_hline(data = zero, ggplot2::aes(yintercept = .data$y), colour = "grey40", linewidth = 0.3) + p <- if (is.null(linetype)) { + p + ggplot2::geom_line(data = base, ggplot2::aes(group = .data$grp), colour = "grey45", linetype = "22", linewidth = 0.6) + + ggplot2::geom_line(ggplot2::aes(colour = .data$col, group = .data$config)) + } else { + p + ggplot2::geom_line(data = base, ggplot2::aes(group = .data$grp, linetype = .data$grp), colour = "grey55", linewidth = 0.6) + + ggplot2::geom_line(ggplot2::aes(colour = .data$col, group = .data$config, linetype = .data$grp)) + + ggplot2::scale_linetype_manual(values = c("solid", "22", "13")[seq_along(unique(d$grp))], name = NULL) + } + p + + ggplot2::geom_point(ggplot2::aes(colour = .data$col, shape = .data$clipped), size = 1.2) + + ggplot2::scale_shape_manual(values = c(`FALSE` = 16, `TRUE` = 1), guide = "none") + + ggplot2::scale_colour_manual(values = colours, name = NULL) + + ggplot2::facet_grid(metric ~ h, scales = "free_y", switch = "y") + + ggplot2::labs( + x = "month of reference date", y = NULL, title = season, + caption = sprintf( + "States only. Grey bars: month's share of the season's base WIS (%%). Grey %sline: base coverage bias. Reductions clipped to [%d, %d]; open points are clipped.", + if (is.null(linetype)) "dashed " else "", clip[1], clip[2] + ) + ) + + ggplot2::theme( + legend.position = "bottom", legend.box = "vertical", strip.placement = "outside", strip.text.y.left = ggplot2::element_text(angle = 0), + axis.text.x = ggplot2::element_text(angle = 45, hjust = 1) + ) +} From a4e6363759bc027dfa9201a70818fb251d443128 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:49:11 -0700 Subject: [PATCH 067/107] docs(calibration): month view section (WIS, coverage, median error) in every experiment notebook --- .../calibration_experiments/_month_view.Rmd | 43 +++++++++++++++++ .../_standard_views.Rmd | 14 +++++- .../e00_vintage_backtest.Rmd | 17 +++++++ .../e01_eta_settings.Rmd | 23 ++++++++++ .../e02_eta_variants.Rmd | 40 ++++++++-------- .../e05_warm_start.Rmd | 2 + .../e07_across_eras.Rmd | 28 +++++++++++ .../calibration_experiments/e09_lr_delay.Rmd | 35 ++++++++++++++ .../e14_lr_by_season.Rmd | 16 +++++++ .../e15_wis_sources.Rmd | 7 +++ .../e16_sqrt_constant_lr.Rmd | 46 +++++++++++-------- .../e17_late_decay.Rmd | 17 +++++++ 12 files changed, 248 insertions(+), 40 deletions(-) create mode 100644 reports/writeups/calibration_experiments/_month_view.Rmd diff --git a/reports/writeups/calibration_experiments/_month_view.Rmd b/reports/writeups/calibration_experiments/_month_view.Rmd new file mode 100644 index 00000000..1d5b7849 --- /dev/null +++ b/reports/writeups/calibration_experiments/_month_view.Rmd @@ -0,0 +1,43 @@ +# Month view: WIS, coverage, median error + + + +The same scores split by calendar month of the reference date, one figure +per season, **states only (US excluded)**. Rows are the three metrics, +columns the horizons: + +- **WIS reduction %**: `100 × (WIS_base − WIS_cal) / WIS_base` within the + month, so positive = calibrated WIS lower than base. Grey bars are the + month's share of the season's base WIS, so a large swing in a month with a + small bar costs little overall. +- **L1 coverage bias** (lower is better), computed within the month: the + mean over the 23 levels of |share of outcomes at or below the quantile − + level|. Calibrated in colour, the base in grey. A season-pooled bias lets + under-prediction on the ramp cancel over-prediction after the peak; the + monthly one does not. +- **Median AE reduction %**: the same as WIS reduction % for the absolute + error of the 0.5 quantile, so positive = calibrated median closer to the + outcome. + +A month holds one to five rounds of about 52 states, and outcomes in the +same round move together, so a single month's coverage bias is noisy and +sits above zero even for a calibrated forecaster. Compare the calibrated +line with the base, not with zero. + +```{r mview-plots, results="asis", fig.height=7.5} +mv_plot_month <- mv$month +if (!is.null(get0("mv_plot_cfgs"))) { + mv_plot_month <- mv_plot_month %>% filter(config %in% mv_plot_cfgs) %>% mutate(config = droplevels(config)) +} +for (s in sort(unique(mv_plot_month$season))) { + cat("\n\n## ", s, "\n\n", sep = "") + print(cal_month_plot(mv_plot_month, s, colours = get0("mv_cols"), linetype = get0("mv_linetype"), colour_by = get0("mv_colour_by") %||% "config")) +} +``` + +```{r mview-table, results="asis"} +if (!isFALSE(get0("mv_show_table"))) { + cat("\n\n## Coverage bias: season-pooled and month-average\n\n") + print(knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION)) +} +``` diff --git a/reports/writeups/calibration_experiments/_standard_views.Rmd b/reports/writeups/calibration_experiments/_standard_views.Rmd index a0c24fed..678bd4ec 100644 --- a/reports/writeups/calibration_experiments/_standard_views.Rmd +++ b/reports/writeups/calibration_experiments/_standard_views.Rmd @@ -1,6 +1,6 @@ # Standard views - + Every score is over both live seasons, all hub locations, against finalized truth; burn-in rounds are excluded. @@ -23,6 +23,15 @@ knitr::kable(bind_rows( )) ``` +The same bias for states only, season-pooled against month-average (see +"Month view" below), per season and for both seasons: + +```{r v-head-cov-month} +mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) +mv_show_table <- FALSE +knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) +``` + WIS reduction % by season (positive = calibrated WIS lower than base): ```{r v-head-season} @@ -114,3 +123,6 @@ for (i in seq_len(nrow(picks))) { print(cal_gallery_panel(cals[[focus]], p$location, p$reference_date, truth, vintages, subtitle = sub)) } ``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd index d422b435..230034f1 100644 --- a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd +++ b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd @@ -126,6 +126,15 @@ knitr::kable( ) ``` +The same bias for states only, both seasons, season-pooled against +month-average (see "Month view" below). Configs are `ref/settle_days/learn`: + +```{r cal-table-month} +mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) +mv_show_table <- FALSE +knitr::kable(cal_month_cov_table(mv, seasons = "both"), caption = CAL_MONTH_COV_CAPTION) +``` + WIS reduction % by season (positive = calibrated WIS lower than base): ```{r season-table} @@ -163,3 +172,11 @@ ggplot(drift, aes(month, 100 * rel, colour = factor(horizon), group = horizon)) facet_wrap(~config) + labs(x = NULL, y = "median |cheap − exact| / base median (%)", colour = "horizon") ``` + +```{r month-view-cfgs} +# The runs the Findings compare: final vs exact at one week of delay, and the cheap (vintage) mode vs exact with none. +mv_plot_cfgs <- c("op/14/final", "op/14/exact", "op/7/vintage", "op/7/exact", "paper/14/final", "paper/14/exact") +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd index ac8e9a03..414fed65 100644 --- a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd +++ b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd @@ -258,6 +258,18 @@ season_table <- function(g) { knitr::kable(season_table(GEOS[["states"]])) ``` +L1 coverage bias, season-pooled against month-average (see "Month view" below): + +```{r table-states-month} +mv_cals <- c( + setNames(sweep, with(grid, sprintf("sqrt mult %s, window %s, %s", mult, as.character(window), policy))), + list(`count mult 0.03, window 20, carry` = count_ref) +) +mv <- cal_month_view(mv_cals, seasons = LIVE) +mv_show_table <- FALSE +knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) +``` + ## All locations ```{r table-all} @@ -278,3 +290,14 @@ bind_rows( arrange(config) %>% knitr::kable() ``` + +```{r month-view-cfgs} +# The multipliers and the season policy the Findings compare, at window 20, plus the count-scale run. +mv_plot_cfgs <- c( + "sqrt mult 0.1, window 20, carry", "sqrt mult 0.03, window 20, carry", "sqrt mult 0.01, window 20, carry", + "sqrt mult 0.03, window 20, reset", "count mult 0.03, window 20, carry" +) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd index 86914bd3..850fe6c5 100644 --- a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd +++ b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd @@ -221,6 +221,14 @@ wis_table(GEOS[["states"]]) cov_table(GEOS[["states"]]) ``` +L1 coverage bias, season-pooled against month-average (see "Month view" below): + +```{r table-states-month} +mv <- cal_month_view(cals, seasons = LIVE) +mv_show_table <- FALSE +knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) +``` + ## All locations ```{r table-all} @@ -245,12 +253,12 @@ bind_rows( # 2. Month by month -WIS reduction % (positive = calibrated WIS lower than base) by calendar month of the reference date; rows are seasons, -columns horizons. Grey bars are each month's share of the season's base WIS. -States only, clipped at −40%. The burn-in configs are left out except the +The month plots are in "Month view" below. h2 by month, states only: WIS +reduction % (positive = calibrated WIS lower than base) (share of outcomes +below the calibrated median). The burn-in configs are left out except the seasonal window with reset. -```{r month, fig.height=6} +```{r month-table} MONTH_CFGS <- c("reference", "off after 1 Mar", "off after 15 Feb", "per-level eta", "seasonal window, reset") fc_m <- purrr::imap(cals[MONTH_CFGS], \(cal, nm) cal$forecasts %>% mutate(config = nm)) %>% bind_rows() %>% @@ -258,21 +266,6 @@ fc_m <- purrr::imap(cals[MONTH_CFGS], \(cal, nm) cal$forecasts %>% mutate(config filter(season %in% LIVE, location != "US") month_m <- ws_month_scores(fc_m) %>% mutate(h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) share <- month_m %>% filter(config == "reference") -ggplot(month_m, aes(month, pct, colour = config, group = config)) + - geom_col(data = share, aes(month, share), inherit.aes = FALSE, fill = "grey85", width = 0.7) + - geom_hline(yintercept = 0, colour = "grey40") + - geom_line() + - geom_point(size = 1.1) + - facet_grid(season ~ h, scales = "free_x", space = "free_x") + - scale_colour_manual(values = CFG_COLS[MONTH_CFGS], name = NULL) + - coord_cartesian(ylim = c(-40, 30)) + - labs(x = "month of reference date", y = "WIS reduction % (positive = calibrated\nWIS lower than base); grey: % of season base WIS") + - theme(legend.position = "bottom") -``` - -h2 by month: WIS reduction % (positive = calibrated WIS lower than base) (share of outcomes below the calibrated median). - -```{r month-table} month_m %>% filter(h == "h2") %>% transmute(season, month, config, v = sprintf("%+.1f (%.2f)", pct, below_med_cal)) %>% @@ -282,3 +275,12 @@ month_m %>% arrange(season, month) %>% knitr::kable() ``` + +```{r month-view-cfgs} +# Same configs as the h2 table. +mv_plot_cfgs <- MONTH_CFGS +mv_cols <- CFG_COLS +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e05_warm_start.Rmd b/reports/writeups/calibration_experiments/e05_warm_start.Rmd index 2aae3385..0aec3faf 100644 --- a/reports/writeups/calibration_experiments/e05_warm_start.Rmd +++ b/reports/writeups/calibration_experiments/e05_warm_start.Rmd @@ -76,6 +76,8 @@ cals <- list( `warm + slow + leak (REF-op)` = run("op") ) focus <- "warm + slow + leak (REF-op)" +# The month view drops the two leak-only runs to keep it readable. +mv_plot_cfgs <- c("warm only", "single", "warm + single", "slow + leak", "warm + slow + leak (REF-op)") ``` ```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} diff --git a/reports/writeups/calibration_experiments/e07_across_eras.Rmd b/reports/writeups/calibration_experiments/e07_across_eras.Rmd index 3bda4326..96f046c5 100644 --- a/reports/writeups/calibration_experiments/e07_across_eras.Rmd +++ b/reports/writeups/calibration_experiments/e07_across_eras.Rmd @@ -368,6 +368,13 @@ season_table <- function(g) { knitr::kable(season_table(GEOS[["states"]])) ``` +L1 coverage bias, season-pooled against month-average (see "Month view" below): + +```{r table-states-month} +mv_trk <- cal_month_view(setNames(cals, paste(grid$forecaster, grid$tracker, sep = ", ")), seasons = LIVE) +knitr::kable(cal_month_cov_table(mv_trk), caption = CAL_MONTH_COV_CAPTION) +``` + ## All locations ```{r table-all} @@ -477,8 +484,29 @@ cand_table <- function(g) { knitr::kable(cand_table(GEOS[["states"]])) ``` +L1 coverage bias, season-pooled against month-average (see "Month view" below): + +```{r cand-table-states-month} +mv <- cal_month_view(setNames(ccals, paste(cgrid$forecaster, cgrid$cand, sep = ", ")), seasons = LIVE) +mv_show_table <- FALSE +knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) +``` + ### All locations ```{r cand-table-all} knitr::kable(cand_table(GEOS[["all"]])) ``` + +```{r month-view-cfgs} +# Three candidates on both forecasters: colour is the candidate, linetype the forecaster (also for the grey base lines). +MV_CANDS <- c(`REF-op warm` = "black", `sqrt 0.018` = "#0072B2", `sqrt 0.018, off 1 Mar` = "#D55E00") +mv_plot_cfgs <- paste(rep(names(FCS), each = length(MV_CANDS)), names(MV_CANDS), sep = ", ") +mv_cols <- MV_CANDS +mv$month <- mv$month %>% mutate(forecaster = sub(",.*", "", config), cand = factor(sub("^[^,]*, ", "", config), levels = names(CANDS))) +mv_colour_by <- "cand" +mv_linetype <- "forecaster" +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd index 560d8c86..159f7abc 100644 --- a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -266,6 +266,30 @@ knitr::kable( ) ``` +L1 coverage bias for states only, both seasons, season-pooled against +month-average (see "Month view" below), for the reference configs, the knee +(0.056) and 0.1 at several delays and both designs: + +```{r bridge-month} +sweep_run <- function(learn, settle_days, lr) { + sweep[[which(sweep_grid$learn == learn & sweep_grid$settle_days == settle_days & abs(log10(sweep_grid$lr / lr)) < 1e-6)]] +} +ref_run <- function(learn, settle_days, nm) refs[[which(ref_grid$learn == learn & ref_grid$settle_days == settle_days)]][[nm]] +mv_cals <- list( + `REF-paper, re-run +1 wk` = ref_run("exact", 14L, "REF-paper (count, adaptive)"), + `REF-op, re-run +1 wk` = ref_run("exact", 14L, "REF-op"), + `knee 0.056, re-run +0 wk` = sweep_run("exact", 7L, 10^-1.25), + `knee 0.056, re-run +1 wk` = sweep_run("exact", 14L, 10^-1.25), + `knee 0.056, re-run +2 wk` = sweep_run("exact", 21L, 10^-1.25), + `0.1, re-run +1 wk` = sweep_run("exact", 14L, 10^-1), + `knee 0.056, stateful +0 wk` = sweep_run("vintage", 7L, 10^-1.25), + `knee 0.056, stateful +1 wk` = sweep_run("vintage", 14L, 10^-1.25) +) +mv <- cal_month_view(mv_cals, seasons = c("2024-2025", "2025-2026"), horizons = NULL) +mv_show_table <- FALSE +knitr::kable(cal_month_cov_table(mv, seasons = "both"), caption = CAL_MONTH_COV_CAPTION) +``` + # Coverage bias vs WIS, per location Each thin grey line is one location traced across learning rates; the bold @@ -377,3 +401,14 @@ sweep_table("exact") ```{r table-vintage} sweep_table("vintage") ``` + +```{r month-view-cfgs} +# The configs the Findings compare: the references and the knee at one week of delay, and the knee in the stateful design. +mv_plot_cfgs <- c( + "REF-paper, re-run +1 wk", "REF-op, re-run +1 wk", "knee 0.056, re-run +1 wk", "0.1, re-run +1 wk", + "knee 0.056, stateful +0 wk", "knee 0.056, stateful +1 wk" +) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd index ed88c394..ba29a090 100644 --- a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd +++ b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd @@ -219,6 +219,14 @@ season_table <- function(g) { knitr::kable(season_table(GEOS[["states"]])) ``` +L1 coverage bias, season-pooled against month-average (see "Month view" below): + +```{r table-states-month} +mv <- cal_month_view(c(setNames(sweep, paste("rate", format(LRS, drop0trailing = TRUE))), refs), seasons = LIVE) +mv_show_table <- FALSE +knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) +``` + ## All locations ```{r table-all} @@ -243,3 +251,11 @@ bind_rows( arrange(config) %>% knitr::kable() ``` + +```{r month-view-cfgs} +# The good range (0.018–0.032), the knee and E11's 0.1, and the two references. +mv_plot_cfgs <- c("rate 0.018", "rate 0.032", "rate 0.056", "rate 0.1", "REF-op cold", "sqrt, adaptive") +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd index 9dfdc3b9..572cf804 100644 --- a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd +++ b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd @@ -326,3 +326,10 @@ wr <- split %>% mutate(horizon = paste0("h", horizon), across(where(is.double), \(x) round(x, 2))) knitr::kable(wr) ``` + +```{r month-view-cfgs} +mv <- cal_month_view(cals, seasons = LIVE) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd index ebdf2915..dd2839a1 100644 --- a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd +++ b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd @@ -252,6 +252,20 @@ season_table <- function(g, rate_keep = c(0.018, 0.032, 0.056, 0.1)) { knitr::kable(season_table(GEOS[["states"]])) ``` +L1 coverage bias, season-pooled against month-average (see "Month view" below): + +```{r table-states-month} +lr_lab <- function(lr) vapply(lr, \(x) format(x, drop0trailing = TRUE), character(1)) +rate_keep <- RATE_LRS %in% c(0.018, 0.032, 0.056, 0.1) +mv <- cal_month_view(c( + setNames(sqrt_runs, paste("sqrt", lr_lab(SQRT_LRS))), + setNames(rate_runs[rate_keep], paste("rate", lr_lab(RATE_LRS[rate_keep]))), + list(`REF-op cold` = refop) +), seasons = LIVE) +mv_show_table <- FALSE +knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) +``` + ### All locations ```{r table-all} @@ -344,26 +358,9 @@ month_m <- ws_month_scores(fc_pick %>% filter(config %in% names(MONTH_COLS)) %>% mutate(h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) ``` -## WIS reduction % by month - -WIS reduction % (positive = calibrated WIS lower than base). Each season a row, each horizon a column. The grey bars (right axis -scaled to the left) are each month's share of the season's base WIS, so a -large loss in a month with a small share costs little overall. The y axis is -clipped at −40% and +30%, which also cuts the tallest bars. - -```{r month-wis, fig.height=6} -share <- month_m %>% filter(config == names(MONTH_COLS)[1]) -ggplot(month_m, aes(month, pct, colour = config, group = config)) + - geom_col(data = share, aes(month, share), inherit.aes = FALSE, fill = "grey85", width = 0.7) + - geom_hline(yintercept = 0, colour = "grey40") + - geom_line() + - geom_point(size = 1.2) + - facet_grid(season ~ h, scales = "free_x", space = "free_x") + - scale_colour_manual(values = MONTH_COLS, name = NULL) + - coord_cartesian(ylim = c(-40, 30)) + - labs(x = "month of reference date", y = "WIS reduction % (positive = calibrated\nWIS lower than base); grey: % of season base WIS") + - theme(legend.position = "bottom") -``` +WIS, coverage bias and median error by month for these four trackers are in +"Month view" at the end. Below: the share of outcomes below the median, the +offset over time and an h2 table. ## Coverage by month @@ -438,3 +435,12 @@ month_m %>% arrange(season, month) %>% knitr::kable(caption = "WIS reduction %, positive = calibrated WIS lower than base (share of outcomes below the calibrated median)") ``` + +```{r month-view-cfgs} +# The four trackers of section 3. +mv_plot_cfgs <- names(MONTH_COLS) +mv_cols <- MONTH_COLS +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e17_late_decay.Rmd b/reports/writeups/calibration_experiments/e17_late_decay.Rmd index 79a12e25..c2fbc059 100644 --- a/reports/writeups/calibration_experiments/e17_late_decay.Rmd +++ b/reports/writeups/calibration_experiments/e17_late_decay.Rmd @@ -139,6 +139,14 @@ bind_rows(base_row, head_m %>% transmute(config = paste(rate, decay, sep = ", ") knitr::kable() ``` +L1 coverage bias, season-pooled against month-average (see "Month view" below): + +```{r head-month} +mv <- cal_month_view(setNames(cals, paste(grid$rate, grid$decay, sep = ", ")), seasons = LIVE) +mv_show_table <- FALSE +knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) +``` + The same WIS reductions as a plot, one panel per season and horizon. ```{r head-plot, fig.height=5.5} @@ -219,3 +227,12 @@ fc %>% labs(x = "reference date", y = "mean offset at the median, per 100k") + theme(legend.position = "bottom") ``` + +```{r month-view-cfgs} +# sqrt 0.018 with each decay setting; section 2 has the WIS months for the rate-scale trackers. +mv_plot_cfgs <- paste(names(RATES)[3], names(DECAYS), sep = ", ") +mv_cols <- setNames(DECAY_COLS, mv_plot_cfgs) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` From 982acfd3171c5aa1b648d3fc423fc79110f5ec9d Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:07:21 -0700 Subject: [PATCH 068/107] docs(calibration): month-view check of each notebook's coverage findings --- .../calibration_experiments/e01_eta_settings.Rmd | 9 +++++++++ .../calibration_experiments/e02_eta_variants.Rmd | 14 ++++++++++++++ .../writeups/calibration_experiments/e03_scale.Rmd | 5 +++++ .../e04_offset_structure.Rmd | 7 +++++++ .../calibration_experiments/e05_warm_start.Rmd | 9 +++++++++ .../calibration_experiments/e07_across_eras.Rmd | 12 ++++++++++++ .../calibration_experiments/e09_lr_delay.Rmd | 10 ++++++++++ .../writeups/calibration_experiments/e10_covid.Rmd | 8 ++++++++ .../calibration_experiments/e11_constant_lr.Rmd | 9 +++++++++ .../calibration_experiments/e14_lr_by_season.Rmd | 9 +++++++++ .../e16_sqrt_constant_lr.Rmd | 11 +++++++++++ .../calibration_experiments/e17_late_decay.Rmd | 9 +++++++++ 12 files changed, 112 insertions(+) diff --git a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd index 414fed65..1970e2d2 100644 --- a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd +++ b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd @@ -88,6 +88,15 @@ unless stated. All-locations numbers point the same way. the same answer. The multiplier is the knob, the window is immaterial, 0.1 overpays and 0.01 is near WIS parity but slow. One change: carry vs reset was "minor" before; here carry is clearly better in 2025-26. +- **Month view (2026-10-01):** scored within each month, the coverage + ranking changes. Month-avg L1 coverage bias (both seasons, h0/h1/h2/h3, + window 20, carry): base 0.112/0.127/0.153/0.177, mult 0.01 + 0.093/0.114/0.142/0.168, 0.03 0.095/0.120/0.150/0.176, 0.1 + 0.097/0.130/0.161/0.190. So 0.01, not 0.03, has the lowest month-level + bias, and at h2–h3 0.03 is level with the base: most of its season-pooled + coverage gain is ramp and post-peak errors cancelling. Carry vs reset + flips slightly in 2025-26: reset 0.076/0.103/0.122/0.138, carry + 0.078/0.105/0.130/0.143. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) diff --git a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd index 850fe6c5..2fcd0a08 100644 --- a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd +++ b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd @@ -100,6 +100,20 @@ unless stated. All-locations numbers point the same way. seasonal window does not help. Two changes: reset now trails carry for the seasonal window (it used to recover some ground), and the coverage cost of switching off, not reported before, is large. +- **Month view (2026-10-01):** the coverage cost of switching off is + mostly an artifact of pooling the season. Month-avg L1 coverage bias + (h0/h1/h2/h3): reference 0.095/0.120/0.150/0.176, off 1 March + 0.104/0.124/0.153/0.170, off 15 February 0.106/0.126/0.153/0.169, base + 0.112/0.127/0.153/0.177. The reference itself gains only 0.001–0.017 over + the base by month, so switching off gives up at most 0.01 and is better at + h3. In March 2025-26 switching off lowers the monthly bias (h2: 0.094 vs + 0.196 for the reference). Per-level eta is slightly better than the + reference by month (0.092/0.117/0.147/0.173), not worse, and the seasonal + window with carry is worse than the burn-in reference at h2–h3 + (0.156/0.183 vs 0.149/0.174). +- **Month view (2026-10-01), median error:** in March the reference's median loses more + than its WIS (2025-26 h0–h3: absolute error −34/−54/−86/−109%, WIS + −22/−38/−60/−81%); switching off removes it. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) diff --git a/reports/writeups/calibration_experiments/e03_scale.Rmd b/reports/writeups/calibration_experiments/e03_scale.Rmd index d58b374f..5445c3b6 100644 --- a/reports/writeups/calibration_experiments/e03_scale.Rmd +++ b/reports/writeups/calibration_experiments/e03_scale.Rmd @@ -62,6 +62,11 @@ median is defined the same way. L1 coverage bias: lower is better. with a constant learning rate (E09). - **2025-26 is harder for both.** sqrt loses 1.8–4.6% at h1–h3 there (count 6.9–7.9%), against gains in 2024-25. +- **Month view (2026-10-01):** scored within each month (states only), + sqrt's coverage edge is clearer than "similar": month-avg L1 coverage bias + 0.108/0.102/0.117/0.130/0.150 at h−1…h3 vs count + 0.115/0.102/0.120/0.143/0.168, base 0.134/0.116/0.123/0.137/0.151. Count + is worse than the base at h2–h3 month by month. ```{r runs} cals <- list( diff --git a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd index 938abf3c..bcd96b32 100644 --- a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd +++ b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd @@ -59,6 +59,13 @@ median is defined the same way. L1 coverage bias: lower is better. - The median follows WIS: the leak improves absolute error at h1–h3 (+3.4/+2.8/+3.4 vs +2.4/−0.6/+1.2) and costs 1.1 points at h−1 and 0.3 at h0. +- **Month view (2026-10-01):** scored within each month (states only), the + leak costs no coverage. Month-avg L1 coverage bias at h−1…h3: single + 0.108/0.102/0.117/0.130/0.150, leak 0.1 0.115/0.102/0.115/0.132/0.143, + base 0.134/0.116/0.123/0.137/0.151. The near-tripling of the + season-pooled bias is the leak no longer letting ramp and post-peak errors + cancel; month by month the two are level at h0–h2, the leak is better at + h3 and slightly worse at h−1. ```{r runs} sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) diff --git a/reports/writeups/calibration_experiments/e05_warm_start.Rmd b/reports/writeups/calibration_experiments/e05_warm_start.Rmd index 0aec3faf..f938b461 100644 --- a/reports/writeups/calibration_experiments/e05_warm_start.Rmd +++ b/reports/writeups/calibration_experiments/e05_warm_start.Rmd @@ -62,6 +62,15 @@ median is defined the same way. L1 coverage bias: lower is better. - **Where REF-op loses** is unchanged from the finalized-truth runs: at h3, October (−30%) and March (−29%), 6% of the base WIS between them; December–January (71% of it) gain about 5%. +- **Month view (2026-10-01):** scored within each month (states only), + `warm + single` is not the best-calibrated config. Month-avg L1 coverage + bias at h−1…h3: `warm + single` 0.107/0.100/0.116/0.128/0.146, REF-op + 0.107/0.091/0.111/0.125/0.136, base 0.134/0.116/0.123/0.137/0.151. REF-op + is lowest at h0–h3, so the two do not mark opposite ends of a coverage + trade-off; `warm + single`'s low season-pooled bias is largely ramp and + post-peak errors cancelling (in March 2025-26 it loses 71/83% of WIS at + h2/h3 with bias 0.25/0.27 vs the base's 0.15/0.18). The warm start alone + cuts month-avg bias by 5–15% at h0–h3, not a third. ```{r runs} sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) diff --git a/reports/writeups/calibration_experiments/e07_across_eras.Rmd b/reports/writeups/calibration_experiments/e07_across_eras.Rmd index 96f046c5..5fa19a85 100644 --- a/reports/writeups/calibration_experiments/e07_across_eras.Rmd +++ b/reports/writeups/calibration_experiments/e07_across_eras.Rmd @@ -138,6 +138,18 @@ point the same way. point, not up to 11 (that was h−1, a different model), and B is not less miscalibrated than the ensemble (equal in 2024-25, worse in 2025-26). B single still has much the best 2025-26 coverage (0.010–0.013). +- **Month view (2026-10-01):** the coverage rankings do not survive + scoring within each month. Month-avg L1 coverage bias, both seasons, mean + over h0–h3: on A, REF-op warm 0.115 < REF-op cold 0.120 < sqrt 0.018 + 0.124 < off 1 Mar 0.126 < rate 0.032 0.133 (base 0.132); on B, sqrt 0.018 + 0.130 < REF-op cold 0.133 < REF-op warm 0.135 < off 1 Mar 0.136 < rate + 0.032 0.142 (base 0.142). So off-1-March is not the worst, rate 0.032 is + (no better than the base), and the order differs between A and B. Among + the trackers, operating has the lowest month-avg bias on A + (0.090/0.110/0.124/0.138 vs single 0.101/0.117/0.130/0.150) and the three + tie on B; B single's 2025-26 lead disappears (0.078/0.105/0.128/0.142 vs + operating 0.078/0.111/0.128/0.145). The base comparison holds: in + 2025-26 A's month-avg bias is 0.089–0.117 against B's 0.097–0.146. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd index 159f7abc..584d9f0b 100644 --- a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -132,6 +132,16 @@ step. the uncalibrated 0.111, and h−1 WIS is 0.9% worse than the base. +1 week recovers most of it (0.066); +2 and +3 weeks change little at h1–h3 and lose WIS at h−1. +- **Month view (2026-10-01):** scored within each month (states only, +1 + week, re-run), the knee does not beat REF-paper on coverage and REF-op has + the lowest bias, not twice it. Month-avg L1 coverage bias at h−1…h3: knee + 0.122/0.117/0.129/0.142/0.164, 0.1 0.118/0.114/0.127/0.147/0.172, + REF-paper 0.115/0.102/0.120/0.143/0.168, REF-op + 0.107/0.091/0.111/0.125/0.136, base 0.134/0.116/0.123/0.137/0.151. At + h1–h3 the knee and 0.1 are worse than the base month by month, so the + L-shaped drop in season-pooled bias is ramp and post-peak errors + cancelling. The delay results hold: stateful +0 weeks is worse than the + base at h−1 (0.151 vs 0.134) and +1 week recovers (0.118). ```{r libs, cache=FALSE} suppressPackageStartupMessages(source(here::here("R/load_all.R"))) diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd index c7b4f948..14a3a913 100644 --- a/reports/writeups/calibration_experiments/e10_covid.Rmd +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -67,6 +67,14 @@ bad NHSN release that was corrected the next day. - **Implication for prod:** for covid, calibrating h−1 (and perhaps h0) and passing h1–h3 through would keep the gain and drop the losses. The covid prod pipeline calibrates every horizon today. +- **Month view (2026-10-01):** the covid base is calibrated only + season-pooled. Scored within each month (states only, August–July + seasons 2024-25 and 2025-26), its L1 coverage bias is + 0.148/0.102/0.125/0.143/0.157 at h−1…h3, about the flu base's level + (0.134/0.116/0.123/0.137/0.151). Neither config improves on it beyond + h−1/h0: REF-op + 0.123/0.096/0.126/0.147/0.161, REF-paper 0.111/0.094/0.126/0.155/0.171. + The h−1 gain and the h2–h3 coverage cost hold. ```{r runs} no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) diff --git a/reports/writeups/calibration_experiments/e11_constant_lr.Rmd b/reports/writeups/calibration_experiments/e11_constant_lr.Rmd index 8948c8b5..502d3809 100644 --- a/reports/writeups/calibration_experiments/e11_constant_lr.Rmd +++ b/reports/writeups/calibration_experiments/e11_constant_lr.Rmd @@ -69,6 +69,15 @@ re-run. - **Untested:** a warm start on the constant-rate tracker. The warm start is the largest single gain for the adaptive tracker (E05), and the constant rate keeps coverage without a leak. +- **Month view (2026-10-01):** scored within each month (states only), the + constant rate has the worst coverage, not the best. Month-avg L1 coverage + bias at h−1…h3: constant 0.1 0.118/0.114/0.127/0.147/0.172, `sqrt, + adaptive` 0.108/0.102/0.117/0.130/0.150, REF-paper + 0.115/0.102/0.120/0.143/0.168, `warm + sqrt, adaptive` + 0.107/0.100/0.116/0.128/0.146, REF-op 0.107/0.091/0.111/0.125/0.136, base + 0.134/0.116/0.123/0.137/0.151. It is highest of the five at every horizon + and worse than the base at h1–h3; its h−1 edge over the warm tracker + disappears. ```{r runs} sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) diff --git a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd index ba29a090..f6ba4a4d 100644 --- a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd +++ b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd @@ -77,6 +77,15 @@ same way. - **The 50% interval.** At 0.018–0.032 the 50% interval covers 0.47–0.51 at h0/h2, close to nominal; 0.1 and above under-cover it in 2024-25 (0.39–0.45 at h2). +- **Month view (2026-10-01):** scored within each month, coverage bias + does not keep falling with the rate. Month-avg L1 coverage bias (both + seasons, h0/h1/h2/h3) is lowest near 0.01 (0.099/0.115/0.144/0.170; base + 0.112/0.127/0.153/0.177) and is above the base at h1–h3 from 0.056 + (0.112/0.135/0.164/0.191) up (0.32: 0.113/0.145/0.178/0.207). In 2024-25 + it bottoms at 0.032 at h1–h3 (0.131/0.164/0.196), not 0.3; in 2025-26 at + 0.0056–0.01, and 0.032 is already worse than the base there + (0.104/0.123/0.147/0.161 vs 0.097/0.112/0.130/0.146). Above about 0.03, + the season-pooled coverage gain is ramp and post-peak errors cancelling. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) diff --git a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd index dd2839a1..0084fb80 100644 --- a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd +++ b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd @@ -99,6 +99,17 @@ point the same way. trackers make it worse. Part of the season-level coverage gain comes from under-prediction on the ramp and over-prediction after the peak cancelling out. +- **Month view (2026-10-01):** scored within each month, sqrt 0.018 is + better calibrated than rate 0.032, not equal. Month-avg L1 coverage bias in + 2025-26 (h0/h1/h2/h3): sqrt 0.018 0.075/0.098/0.119/0.138, rate 0.032 + 0.104/0.123/0.147/0.161, base 0.097/0.112/0.130/0.146; both seasons + 0.094/0.116/0.143/0.169 vs 0.108/0.127/0.155/0.177 (base + 0.112/0.127/0.153/0.177). Rate 0.032's low 2025-26 season-pooled bias is + cancellation; by month it is worse than the base. Larger sqrt rates rise + above the base at h1–h3 too (0.1: 0.095/0.129/0.157/0.182). +- **Month view (2026-10-01), median error:** the March losses are about twice as large + on the median as on WIS (h2, 2025-26: sqrt 0.018 −52% vs −28%, REF-op cold + −52% vs −24%). ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) diff --git a/reports/writeups/calibration_experiments/e17_late_decay.Rmd b/reports/writeups/calibration_experiments/e17_late_decay.Rmd index c2fbc059..7c623b5d 100644 --- a/reports/writeups/calibration_experiments/e17_late_decay.Rmd +++ b/reports/writeups/calibration_experiments/e17_late_decay.Rmd @@ -83,6 +83,15 @@ Numbers are from the render of 1 October 2026, states only, WIS reduction % toward zero by the calendar is not a free fix. The gradual decay as built here loses because it also wipes the offset the next season starts from. `off_after` in March is the better WIS choice and the worse coverage choice. +- **Month view (2026-10-01):** scored within each month, the coverage + costs nearly vanish. Month-avg L1 coverage bias, both seasons, + h0/h1/h2/h3, sqrt 0.018: no decay 0.094/0.116/0.143/0.169, decay 0.85 + 0.103/0.123/0.149/0.173, off after 1 Mar 0.104/0.121/0.150/0.170, base + 0.112/0.127/0.153/0.177. No decay is still lowest, but by 0.001–0.010, + not the season-pooled 0.03–0.04. In 2025-26 the decays raise it by + 0.002–0.015 (decay 0.85: 0.084/0.108/0.127/0.140 vs + 0.075/0.098/0.119/0.138), not threefold. For rate 0.032, off after 1 Mar + ties no decay (0.112/0.128/0.156/0.173 vs 0.108/0.127/0.155/0.177). ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) From bcf432b9fc11b4f562f0d3387c87d04693c698d9 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:19:21 -0700 Subject: [PATCH 069/107] docs(calibration): month-view coverage verdicts in the review tracker and summary --- notes/calibration-review.md | 29 +++++++++++++++++++++++++ notes/calibration-summary-2026-10-01.md | 28 ++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/notes/calibration-review.md b/notes/calibration-review.md index 92473eca..e9718adb 100644 --- a/notes/calibration-review.md +++ b/notes/calibration-review.md @@ -25,6 +25,35 @@ for), **not reviewed**. 3. Experiments before E07's re-run (E03–E05, E09–E11) used the submitted ensemble; E01, E02 (new versions) and E12–E17 use the clean replay. +## Month view (added 2026-10-01) + +Every notebook now has a "Month view: WIS, coverage, median error" section +(WIS reduction %, within-month L1 coverage bias, median absolute-error +reduction %, per month and horizon, states only) and a month-average +coverage column next to its season-pooled one. Season-pooled coverage +claims mostly don't survive: where one changed, the notebook's Findings end +with a "**Month view (2026-10-01):**" line. Verdicts: + +| notebook | coverage claim | verdict | +|---|---|---| +| E00 | exact ≈ final; vintage +0 weeks worse at h−1 | holds | +| E01 | 0.03 is the coverage elbow; carry beats reset | reversed: 0.01 lowest month by month; carry ≈ reset | +| E02 | switching off costs coverage | weakened: mostly an artifact; better at h3 | +| E02 | per-level eta, seasonal window coverage | reversed (small differences) | +| E03 | sqrt ≈ count on coverage | holds; sqrt better at h2–h3 | +| E04 | the leak nearly triples coverage bias | reversed: no difference month by month | +| E05 | warm + single best calibrated; warm start cuts bias a third | reversed / weakened: REF-op lowest; warm start −5 to −15% | +| E07 | coverage ranks the same on A and B | reversed: rankings differ by month | +| E09 | knee beats REF-paper on coverage | reversed: REF-op lowest; knee worse than base at h1–h3 | +| E10 | covid base already calibrated at h1–h3 | weakened | +| E11 | constant rate better coverage everywhere | reversed: worst of five, worse than base at h1–h3 | +| E14 | coverage keeps improving with lr | reversed: minimum near 0.01–0.032; ≥0.056 worse than base at h1–h3 | +| E15 | REF-op width changes buy coverage | holds, smaller | +| E16 | sqrt 0.018 ≈ rate 0.032 on coverage | reversed: sqrt better; rate worse than base in 2025-26 | +| E17 | decay / off-after give back coverage | weakened: small effect month by month | + +Differences under about 0.01 are within the month-level noise floor. + ## Notebooks | exp | forecaster | status | comments | done | diff --git a/notes/calibration-summary-2026-10-01.md b/notes/calibration-summary-2026-10-01.md index 7808096f..8337e2ba 100644 --- a/notes/calibration-summary-2026-10-01.md +++ b/notes/calibration-summary-2026-10-01.md @@ -29,6 +29,34 @@ All of these are flu only, except E12 and E13, which cover covid as well. "WIS change" is % against the uncalibrated forecast, positive is better. "Coverage bias" is the mean |coverage − nominal| over quantile levels. +## Read this first: coverage claims and the month view + +Coverage bias pooled over a whole season lets under-prediction on the ramp +cancel over-prediction after the peak. Scored within each month and then +averaged over months, the coverage gains from calibration mostly disappear: +calibrated configs that beat the base by 0.03 vs 0.10 season-pooled come +within 0-0.02 of the base month by month, and some are worse than the base. +A month holds only 1-5 rounds, so month-level bias has a noise floor above +zero; differences under about 0.01 between configs shouldn't be read as +real. + +So every coverage statement below that comes from season-pooled numbers is +weaker than it looks. WIS and median-error statements are unaffected. What +holds month by month: + +- REF-op (warm, leaky) has the lowest month-level coverage bias of the + hub-ensemble configs. +- Constant rates above about 0.03 per 100k are worse than the base at + h1-h3 month by month; sqrt 0.018 beats rate 0.032. +- Switching calibration off from March costs little coverage month by + month (and helps at h3), so its coverage cost was mostly an artifact. +- In March 2025-26 the median-error losses are about twice the WIS losses + (sqrt 0.018 at h2: −52% vs −28%). + +Each notebook now has a "Month view: WIS, coverage, median error" section +and, where a coverage claim changed, a "Month view (2026-10-01)" line at the +end of its Findings. + ## Findings **1. A constant learning rate on the sqrt scale is the best simple From becc39c27d8084b307e2559476b236987dd45100 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:15:45 -0700 Subject: [PATCH 070/107] docs(calibration): E18 bridge from REF-op to sqrt constant 0.018, one setting per step --- .../e18_ref_op_bridge.Rmd | 250 ++++++++++++++++++ .../calibration_experiments/index.Rmd | 1 + 2 files changed, 251 insertions(+) create mode 100644 reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd diff --git a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd new file mode 100644 index 00000000..7533ab2d --- /dev/null +++ b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd @@ -0,0 +1,250 @@ +--- +title: "E18: a bridge from REF-op to sqrt constant 0.018" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +REF-op (prod's tracker) and sqrt constant 0.018 (E16's best) differ in +several settings at once. This notebook walks from one to the other, +changing one setting per step, to see which piece of REF-op does the work. + +**Setup.** As E13/E16: the clean `windowed_seasonal` replay for flu (53 +locations, the 55 hub rounds, h0–h3), learning from data as published each +week and re-run from scratch each week as prod does, one week of extra +revision delay. Scores are over the two live seasons; results are states +only (US excluded). + +**The steps.** Each row changes one setting from the row above. + +| step | change | config after the change | +|---|---|---| +| 0 REF-op | — | sqrt, adaptive mult 0.03 floor 1e-3, window 20, carry; slow term (mult 0.003) warm-started from the 2023-24 burn-in; leak 0.1 on the fast term | +| 1 no warm start | `slow_init = NULL` | the slow term starts at zero | +| 2 no slow term | `lr_slow = NULL` | single fast term with the leak | +| 3 no leak | `fast_decay = 0` | sqrt, adaptive, mult 0.03, window 20, carry, with the 2023-24 season present | +| 4 no burn-in season | drop the 2023-24 rounds | as 3, cold: the eta pool starts empty | +| 5 per-100k units | `scales` = population / 100k | as 4, on the square root of admissions per 100k | +| 6 constant 0.018 | `lr = 0.018` | E16's best: sqrt per 100k, constant 0.018 | + +**Why steps 4 and 5.** The 2023-24 burn-in season does two things. With +`slow_init` it sets the slow term's starting value (the warm start). Without +it, its residuals still fill the adaptive learning rate's window, so eta is +sized from the first live round instead of growing from an empty pool; the +offsets themselves stay at zero, since burn-in outcomes never step. So the +burn-in season stays present through step 3, and dropping it is step 4: step +1 isolates the warm start, step 4 the eta warm-up (E13 follow-up 2). E16's +constant rate runs on the square root of the per-100k rate. The adaptive +rate is almost unchanged by that rescaling (it scales with the residuals; +only the 1e-3 floor does not), but it is still a change, so it is step 5, +and step 6 changes only the learning rate. + +Steps 0, 3, 4 and 6 are the cached runs of E13 (REF-op warm), E02 +("reference + burn-in"), E01/E02 ("reference") and E16 (sqrt 0.018). Step 3 +is written as E02 writes it (REF-paper with sqrt and the 1e-3 floor), which +equals REF-op with `slow_init`, `lr_slow` and the leak removed. + +"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so +**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is +better) is the mean over the 23 levels of |empirical coverage − level|. +"Month-avg" coverage bias is computed within each calendar month and then +averaged, so errors on the ramp and after the peak cannot cancel. + +# Findings + +Numbers are from the render of 1 October 2026, states only, both seasons +pooled unless named, h0/h1/h2/h3. WIS is WIS reduction % (positive = +calibrated WIS lower than base); coverage is L1 coverage bias (lower is +better). + +- **Endpoints.** REF-op +2.8/+1.1/+0.9/+1.1, sqrt constant 0.018 + +2.2/+0.8/+0.6/+0.6. REF-op is ahead overall because of 2024-25 + (+3.2/+1.9/+1.6/+1.6 vs +1.7/+0.6/+0.3/+0.1); sqrt 0.018 is ahead in + 2025-26 (+3.0/+1.0/+1.1/+1.5 vs +2.2/−0.1/−0.0/+0.2). Month-avg coverage + bias is about the same: 0.095/0.121/0.150/0.175 vs 0.094/0.116/0.143/0.169. +- **WIS: the learning rate moves it most at h1–h3, the warm start at h0.** + Switching the adaptive rate to constant 0.018 (step 6) adds + +0.3/+1.0/+1.7/+2.3. Inside REF-op, three pieces each add about a point: + the warm start (step 1 costs −0.9/−0.5/−0.6/−1.0, nearly all of it in + 2024-25: h3 goes from +1.6 to −0.4), the leak (step 3 costs + −0.6/−1.0/−1.2 at h1–h3 and gains +0.4 at h0) and, smaller, the burn-in + season's eta warm-up (step 4 costs −0.1/−0.2/−0.5/−0.7, again in + 2024-25). Without the warm start, the slow term does nothing (step 2: + within ±0.2). +- **Warm start vs eta warm-up (E13 follow-up 2).** Both matter only in + 2024-25, the first live season. The warm start is worth about twice as + much WIS as the eta warm-up and shows from the first rounds (the month + view gains at h2–h3 in Nov–Feb 2024-25). The eta warm-up is about half a + point at h2–h3 and nothing in 2025-26. +- **Month-level coverage: no step moves it beyond noise.** Every step + changes month-avg coverage bias by 0.008 or less, under the ~0.01 noise + floor. The largest are dropping the warm start at h3 (−0.008) and the + constant rate at h1–h3 (−0.005/−0.007/−0.007). +- **Season-pooled coverage moves a lot, and it is cancellation.** Dropping + the warm start worsens pooled bias by +0.024/+0.032/+0.044/+0.052, + dropping the leak improves it by −0.040/−0.042/−0.051/−0.060, and the + constant rate worsens it by +0.014/+0.023/+0.034/+0.047. None of these + shows up month by month, so they come from ramp and post-peak errors + cancelling differently, not from better-calibrated months. +- **Where the WIS steps act: March and April.** The month view shows the + WIS steps mostly as spring losses at h2–h3. In 2024-25 at h2, March costs + about −13% with the leak and no slow term (step 2), −23% without the leak + (step 3) and −31% without the burn-in season (steps 4–5); constant 0.018 + costs about −7%. April is worse for the adaptive trackers without the leak + (down to the −40% clip). In 2025-26 every step loses 27–40% in March at + h1–h2, constant 0.018 slightly less. +- **Per-100k units change nothing (step 5).** The adaptive rate scales with + the residuals, so on the square-root scale only the 1e-3 floor sees the + units. Calibrated values differ by at most 1.6 admissions and every score + matches step 4 to the reported digits. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +fc_burn_in <- bind_rows(ws_burn_in_forecasts("flu"), wi$fc) +# Each config is written as the experiment that first ran it writes it, so cached runs are reused. +refop_warm <- ws_configs(wi$rate_scales, burn_in = TRUE)[["REF-op warm"]] +sqrt_adaptive <- list( + ref = "paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3), + burn_in_seasons = character(0), slow_init = NULL, lr_window = 20, season_policy = "carry" +) +const_cfg <- ws_configs(wi$rate_scales, burn_in = FALSE)[["E11 constant 0.1"]] +STEPS <- list( + `0 REF-op` = list(cfg = refop_warm, burn_in = TRUE), + `1 no warm start` = list(cfg = list(ref = "op", burn_in_seasons = "2023-2024", slow_init = NULL), burn_in = TRUE), + `2 no slow term` = list(cfg = list(ref = "op", burn_in_seasons = "2023-2024", slow_init = NULL, lr_slow = NULL), burn_in = TRUE), + `3 no leak` = list(cfg = utils::modifyList(sqrt_adaptive, list(burn_in_seasons = "2023-2024")), burn_in = TRUE), + `4 no burn-in season` = list(cfg = sqrt_adaptive, burn_in = FALSE), + `5 per-100k units` = list(cfg = c(sqrt_adaptive, list(scales = wi$rate_scales)), burn_in = FALSE), + `6 constant 0.018` = list(cfg = utils::modifyList(const_cfg, list(lr = 0.018, transform = "sqrt")), burn_in = FALSE) +) +run <- function(spec) { + fc <- if (spec$burn_in) fc_burn_in else wi$fc + do.call(cal_run_cached, c(list(fc, wi$truth), spec$cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +} +cals <- purrr::map(STEPS, run) +``` + +```{r metrics} +SEASONS <- c(LIVE, "both") +STEP_COLS <- setNames(c("#000000", "#E69F00", "#56B4E9", "#009E73", "#CC79A7", "#0072B2", "#D55E00"), names(STEPS)) +step_m <- purrr::imap(cals, function(cal, nm) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) %>% filter(location != "US") + ws_scores(fc) %>% + select(season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% + mutate(step = nm, wis_pct = 100 * (wis_base - wis_cal) / wis_base) +}) %>% + bind_rows() %>% + mutate( + step = factor(step, levels = names(STEPS)), + season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +mv <- cal_month_view(cals, seasons = LIVE) +step_table <- function(col, fmt, base_col = NULL) { + rows <- step_m %>% mutate(step = as.character(step), v = .data[[col]]) + if (!is.null(base_col)) { + rows <- bind_rows(step_m %>% filter(step == names(STEPS)[1]) %>% mutate(step = "base", v = .data[[base_col]]), rows) + } + rows %>% + mutate(step = factor(step, levels = unique(step))) %>% + arrange(season, h) %>% + transmute(step, col = paste(season, h), cell = sprintf(fmt, v)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(step) +} +``` + +# 1. Season scores per step + +States only. Each season is scored on its own rounds; "both seasons" pools +them. The tracker runs over both seasons, so 2025-26 starts from the offsets +2024-25 left behind. + +## WIS reduction % (positive = calibrated WIS lower than base) + +```{r table-wis} +knitr::kable(step_table("wis_pct", "%+.1f"), caption = "WIS reduction % (positive = calibrated WIS lower than base), states only") +``` + +## L1 coverage bias (lower is better), season-pooled + +```{r table-cov} +knitr::kable(step_table("cal_err_cal", "%.3f", base_col = "cal_err_base"), caption = "L1 coverage bias (lower is better), season-pooled, states only") +``` + +## L1 coverage bias (lower is better), season-pooled and month-average + +```{r table-month-cov} +mv_show_table <- FALSE +knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) +``` + +# 2. What each step changes + +The change each step makes to the step before it, both seasons pooled, +states only. WIS: change in WIS reduction % (positive = the step lowers +WIS). Coverage: change in L1 coverage bias, season-pooled and month-average +(negative = the step improves coverage). + +```{r deltas} +wis_both <- step_m %>% + filter(season == "both seasons") %>% + transmute(step, h, v = wis_pct) +cov_both <- mv$cov %>% + filter(season == "both") %>% + transmute(step = config, h = factor(paste0("h", horizon), levels = paste0("h", 0:3)), pooled = pooled_cal, mavg = mavg_cal) +delta <- wis_both %>% + left_join(cov_both, by = c("step", "h")) %>% + arrange(h, step) %>% + group_by(h) %>% + mutate(d_wis = v - lag(v), d_pooled = pooled - lag(pooled), d_mavg = mavg - lag(mavg)) %>% + ungroup() %>% + filter(step != names(STEPS)[1]) +knitr::kable( + delta %>% + transmute(step, h, cell = sprintf("%+.1f / %+.3f / %+.3f", d_wis, d_pooled, d_mavg)) %>% + tidyr::pivot_wider(names_from = h, values_from = cell) %>% + arrange(step), + caption = "Both seasons, states only: change from the previous step in WIS reduction % / season-pooled L1 coverage bias / month-avg L1 coverage bias" +) +``` + +```{r delta-plot, fig.height=6.5} +delta_long <- delta %>% + transmute(step, h, + `WIS reduction %, change\n(positive = step lowers WIS)` = d_wis, + `month-avg L1 coverage bias,\nchange (negative = better)` = d_mavg, + `season-pooled L1 coverage bias,\nchange (negative = better)` = d_pooled + ) %>% + tidyr::pivot_longer(-c(step, h), names_to = "metric") %>% + mutate(metric = factor(metric, levels = unique(metric)), step = factor(step, levels = rev(names(STEPS)[-1]))) +ggplot(delta_long, aes(value, step, fill = step)) + + geom_vline(xintercept = 0, colour = "grey50", linewidth = 0.3) + + geom_col(width = 0.7, show.legend = FALSE) + + scale_fill_manual(values = STEP_COLS) + + facet_grid(h ~ metric, scales = "free_x") + + labs(x = "change from the previous step (both seasons, states only)", y = NULL) +``` + +```{r month-view-cfgs} +mv_cols <- STEP_COLS +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index fd155c0d..7afd2eb0 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -48,6 +48,7 @@ for (f in src) { | E15 | Why does WIS improve when the calibrated intervals barely move? | The offsets are small (2–5% of the 90% interval width) and point up. The gain comes from shifting the forecast up, mostly at the central levels below the median; width changes cost WIS. | | E16 | Does a constant learning rate work better on the square-root scale, and do the offsets go stale late in the season? | sqrt 0.018 improves WIS at every horizon in both seasons, more than the rate scale at h0 and in 2025-26, with similar coverage. Every tracker loses WIS in March, after the peak, when the base already over-predicts and the offsets still push up. | | E17 | Does shrinking the offset from 1 March fix the March losses? | It fixes March but wipes the offset the next season starts from, so it is worse overall. Switching calibration off from 1 March (`off_after`) gains WIS at h1–h3 but gives back most of the coverage gain. | +| E18 | Which part of prod's tracker (REF-op) does the work, stepping one setting at a time to E16's sqrt constant 0.018? | The warm start from the 2023-24 season adds 0.5–1 point of WIS (mostly in 2024-25) and the leak about a point at h1–h3 (both seasons); the burn-in season's learning-rate warm-up adds about half a point at h2–h3, the slow term nothing on its own. Switching to the constant rate adds 1–2 points at h1–h3. No step changes month-level coverage beyond noise. | # How to read the results From d156e67255dc3bbf61080b5bf30065cfab69a3a7 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:22:19 -0700 Subject: [PATCH 071/107] docs(calibration): record E18 in the ledger, review tracker and notes --- notes/CALIBRATION.md | 20 ++++++++++++++++++++ notes/calibration-ledger.md | 1 + notes/calibration-review.md | 2 ++ 3 files changed, 23 insertions(+) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md index cca84426..403a9ccc 100644 --- a/notes/CALIBRATION.md +++ b/notes/CALIBRATION.md @@ -237,6 +237,26 @@ Flu, clean replay, exact, post-fix, states only. coverage ranks identically; `off_after` 03-01 is last on A in 2024-25 but second on B, so date-based switches don't transfer between forecasters. +## REF-op bridge (E18, 2026-10-01) + +`e18_ref_op_bridge.Rmd`: flu, clean replay with the 2023-24 HHS season, +exact, states only, one setting per step from REF-op to sqrt constant +0.018. Pooled WIS reduction h0/h1/h2/h3: REF-op +2.8/+1.1/+0.9/+1.1, +sqrt 0.018 +2.2/+0.8/+0.6/+0.6; REF-op leads in 2024-25, sqrt 0.018 in +2025-26. + +- REF-op's edge comes from the warm start (0.5–1 point, 2024-25 only), the + leak (~1 point at h1–h3, both seasons) and the eta warm-up from the + burn-in season (~0.5 point at h2–h3, 2024-25 only). The slow term adds + nothing without the warm start. Per-100k units are a no-op for adaptive + sqrt. +- Over cold adaptive sqrt, the constant rate is worth +1.0 to +2.3 at h1–h3. +- No step moves month-averaged coverage bias by more than 0.008 (noise + floor ~0.01); the large season-pooled coverage moves are cancellation. +- E13 follow-up answered: the warm start is worth about twice the eta + warm-up; both matter only in the first live season. +- Untested: sqrt 0.018 with a warm start, the one REF-op piece it lacks. + ## Data FluSight CMU-TimeSeries submissions: 83 rounds (2023-10-14 … 2026-05-30), 53 diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 2c7a6e16..e29a23c5 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -88,6 +88,7 @@ base WIS, and pick within strata (season phase × location size). | E01 (rebuilt) | which adaptive-eta settings? | mult {0.3, 0.1, 0.03, 0.01} × window {8, 20, Inf} × season policy; count bridge | REF-paper at sqrt, cold | flu (clean replay) | post-fix | exact | V-curve, V-head | `e01_eta_settings.Rmd` | done | | E02 (rebuilt) | cutoff and eta variants | `off_after` {04-01, 03-01, 02-15}, per-level eta, seasonal window carry/reset (HHS burn-in) + burn-in bridge; geo-pool removed | E01 center (sqrt, 0.03, 20, carry) | flu (clean replay) | post-fix | exact | V-head, V-month | `e02_eta_variants.Rmd` | done | | E07 (rebuilt) | one forecaster across eras | forecaster (A hub ensemble / B clean replay) on identical rows, each with its own 2023-24 burn-in | single, leaky, REF-op | flu | post-fix | exact | V-head, base bias | `e07_across_eras.Rmd` | done | +| E18 | which piece of REF-op does the work | bridge, one setting per step: −warm start, −slow term, −leak, −burn-in season, per-100k units, constant 0.018 | REF-op → E16 sqrt 0.018 | flu (clean replay) | post-fix | exact | V-head, V-month | `e18_ref_op_bridge.Rmd` | done | E09 bridge rows, each one axis from the previous: REF-paper → rate scale → constant lr (sweep) → `settle_days` (sweep). diff --git a/notes/calibration-review.md b/notes/calibration-review.md index e9718adb..f862e822 100644 --- a/notes/calibration-review.md +++ b/notes/calibration-review.md @@ -51,6 +51,7 @@ with a "**Month view (2026-10-01):**" line. Verdicts: | E15 | REF-op width changes buy coverage | holds, smaller | | E16 | sqrt 0.018 ≈ rate 0.032 on coverage | reversed: sqrt better; rate worse than base in 2025-26 | | E17 | decay / off-after give back coverage | weakened: small effect month by month | +| E18 | warm start, leak, constant rate move coverage | reversed: no step moves month-avg coverage by more than 0.008 | Differences under about 0.01 are within the month-level noise floor. @@ -72,6 +73,7 @@ Differences under about 0.01 are within the month-level noise floor. | E15 WIS sources | clean replay | not reviewed | | | | E16 sqrt constant lr | clean replay | not reviewed | | | | E17 late decay | clean replay | not reviewed | | | +| E18 REF-op bridge | clean replay | not reviewed | | | E12 and E13 have no notebooks; they live in `notes/CALIBRATION.md` and `scripts/calibration/calibration_ws_replay.R`. E06 and E08 were superseded From ae032dca945f06b9cfd2b2145389de979f07e349 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:42:46 -0700 Subject: [PATCH 072/107] feat(calibration): month view shows coverage as bias reduction % vs base, plus an absolute monthly table --- R/calibration/views.R | 21 +++++-------- .../calibration_experiments/_month_view.Rmd | 30 +++++++++++++++---- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/R/calibration/views.R b/R/calibration/views.R index a2873e4c..3de1f4fb 100644 --- a/R/calibration/views.R +++ b/R/calibration/views.R @@ -430,7 +430,7 @@ cal_month_cov_table <- function(mv, seasons = NULL) { #' @export cal_month_plot <- function(month_tbl, season, colours = NULL, linetype = NULL, colour_by = "config", clip = c(-40, 40)) { m_wis <- "WIS reduction %\n(positive = calibrated\nWIS lower than base)" - m_cov <- "L1 coverage bias\n(lower is better)" + m_cov <- "L1 coverage bias\nreduction %\n(positive = calibrated\nbias lower than base)" m_ae <- "median AE reduction %\n(positive = calibrated\nAE lower than base)" lv <- c(m_wis, m_cov, m_ae) d <- month_tbl %>% @@ -447,26 +447,21 @@ cal_month_plot <- function(month_tbl, season, colours = NULL, linetype = NULL, c sq <- function(x) pmin(pmax(x, clip[1]), clip[2]) long <- bind_rows( d %>% transmute(.data$config, .data$col, .data$grp, .data$month, .data$h, metric = m_wis, value = sq(.data$wis_pct), clipped = .data$wis_pct != value), - d %>% transmute(.data$config, .data$col, .data$grp, .data$month, .data$h, metric = m_cov, value = .data$cov_cal, clipped = FALSE), + d %>% mutate(cov_pct = 100 * (.data$cov_base - .data$cov_cal) / .data$cov_base) %>% + transmute(.data$config, .data$col, .data$grp, .data$month, .data$h, metric = m_cov, value = sq(.data$cov_pct), clipped = .data$cov_pct != value), d %>% transmute(.data$config, .data$col, .data$grp, .data$month, .data$h, metric = m_ae, value = sq(.data$ae_pct), clipped = .data$ae_pct != value) ) %>% mutate(metric = factor(.data$metric, levels = lv)) - first_cfg <- d %>% distinct(.data$grp, .data$config) %>% group_by(.data$grp) %>% slice(1) %>% ungroup() - base <- d %>% - semi_join(first_cfg, by = c("grp", "config")) %>% - transmute(.data$grp, .data$month, .data$h, metric = factor(m_cov, levels = lv), value = .data$cov_base) share <- d %>% filter(.data$config == configs[1]) %>% transmute(.data$month, .data$h, metric = factor(m_wis, levels = lv), value = sq(.data$share)) - zero <- tibble(metric = factor(c(m_wis, m_ae), levels = lv), y = 0) + zero <- tibble(metric = factor(lv, levels = lv), y = 0) p <- ggplot2::ggplot(long, ggplot2::aes(.data$month, .data$value)) + ggplot2::geom_col(data = share, fill = "grey85", width = 0.7) + ggplot2::geom_hline(data = zero, ggplot2::aes(yintercept = .data$y), colour = "grey40", linewidth = 0.3) p <- if (is.null(linetype)) { - p + ggplot2::geom_line(data = base, ggplot2::aes(group = .data$grp), colour = "grey45", linetype = "22", linewidth = 0.6) + - ggplot2::geom_line(ggplot2::aes(colour = .data$col, group = .data$config)) + p + ggplot2::geom_line(ggplot2::aes(colour = .data$col, group = .data$config)) } else { - p + ggplot2::geom_line(data = base, ggplot2::aes(group = .data$grp, linetype = .data$grp), colour = "grey55", linewidth = 0.6) + - ggplot2::geom_line(ggplot2::aes(colour = .data$col, group = .data$config, linetype = .data$grp)) + + p + ggplot2::geom_line(ggplot2::aes(colour = .data$col, group = .data$config, linetype = .data$grp)) + ggplot2::scale_linetype_manual(values = c("solid", "22", "13")[seq_along(unique(d$grp))], name = NULL) } p + @@ -477,8 +472,8 @@ cal_month_plot <- function(month_tbl, season, colours = NULL, linetype = NULL, c ggplot2::labs( x = "month of reference date", y = NULL, title = season, caption = sprintf( - "States only. Grey bars: month's share of the season's base WIS (%%). Grey %sline: base coverage bias. Reductions clipped to [%d, %d]; open points are clipped.", - if (is.null(linetype)) "dashed " else "", clip[1], clip[2] + "States only. All rows: positive = better than the uncalibrated base. Grey bars: month's share of the season's base WIS (%%). Reductions clipped to [%d, %d]; open points are clipped.", + clip[1], clip[2] ) ) + ggplot2::theme( diff --git a/reports/writeups/calibration_experiments/_month_view.Rmd b/reports/writeups/calibration_experiments/_month_view.Rmd index 1d5b7849..107f29b1 100644 --- a/reports/writeups/calibration_experiments/_month_view.Rmd +++ b/reports/writeups/calibration_experiments/_month_view.Rmd @@ -10,19 +10,22 @@ columns the horizons: month, so positive = calibrated WIS lower than base. Grey bars are the month's share of the season's base WIS, so a large swing in a month with a small bar costs little overall. -- **L1 coverage bias** (lower is better), computed within the month: the - mean over the 23 levels of |share of outcomes at or below the quantile − - level|. Calibrated in colour, the base in grey. A season-pooled bias lets +- **L1 coverage bias reduction %**: `100 × (bias_base − bias_cal) / + bias_base`, so positive = calibrated coverage bias lower than base. The + bias is computed within the month: the mean over the 23 levels of |share + of outcomes at or below the quantile − level|. A season-pooled bias lets under-prediction on the ramp cancel over-prediction after the peak; the - monthly one does not. + monthly one does not. The absolute monthly values are in the table below + the plots: when the base's bias is already small, a small absolute change + shows as a large percentage. - **Median AE reduction %**: the same as WIS reduction % for the absolute error of the 0.5 quantile, so positive = calibrated median closer to the outcome. A month holds one to five rounds of about 52 states, and outcomes in the same round move together, so a single month's coverage bias is noisy and -sits above zero even for a calibrated forecaster. Compare the calibrated -line with the base, not with zero. +sits above zero even for a calibrated forecaster. That's why the plot shows +it relative to the base. ```{r mview-plots, results="asis", fig.height=7.5} mv_plot_month <- mv$month @@ -41,3 +44,18 @@ if (!isFALSE(get0("mv_show_table"))) { print(knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION)) } ``` + +```{r mview-abs, results="asis"} +# Absolute within-month coverage bias behind the plot's reduction %, at h0 and h2. +mv_abs <- mv_plot_month %>% filter(horizon %in% c(0, 2)) +mv_abs_base <- mv_abs %>% distinct(season, month, horizon, cov_base) %>% group_by(season, month, horizon) %>% slice(1) %>% ungroup() +cat("\n\n## Monthly L1 coverage bias, absolute (h0 and h2)\n\n") +print(knitr::kable( + mv_abs %>% + transmute(season, month, h = paste0("h", horizon), config, v = sprintf("%.3f", cov_cal)) %>% + tidyr::pivot_wider(names_from = config, values_from = v) %>% + left_join(mv_abs_base %>% transmute(season, month, h = paste0("h", horizon), base = sprintf("%.3f", cov_base)), by = c("season", "month", "h")) %>% + arrange(h, season, month), + caption = "Within-month L1 coverage bias (lower is better), calibrated configs and the uncalibrated base." +)) +``` From b773e4efebc1a93a48337e1f1d1df51a47612723 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:55:48 -0700 Subject: [PATCH 073/107] docs(calibration): E18 adds rate-scale constant references (E14 0.032, E09 knee 0.056) --- .../e18_ref_op_bridge.Rmd | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd index 7533ab2d..36027e5f 100644 --- a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd +++ b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd @@ -111,6 +111,16 @@ better). the residuals, so on the square-root scale only the 1e-3 floor sees the units. Calibrated values differ by at most 1.6 admissions and every score matches step 4 to the reported digits. +- **Rate-scale constant rates (references, outside the chain).** Rate + 0.032 (E14's best) and 0.056 (E09's knee) give +1.3/+0.7/+0.5/+0.4 and + +1.3/+0.5/+0.3/0.0, below sqrt 0.018 (+2.2/+0.8/+0.6/+0.6) and REF-op + (+2.8/+1.1/+0.9/+1.1). Month-avg coverage bias is about the base's + (0.108–0.177 and 0.112–0.191 vs base 0.112–0.177; 0.056 is worse at + h2–h3). Both sit at zero through the ramp. On the ensemble (E09) the knee + looked good after the 2024-25 peak because the ensemble was still + under-predicting that spring (E07); `windowed_seasonal` over-predicts + after the peak, so here the same upward offsets cost WIS and coverage in + March–April. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) @@ -132,16 +142,22 @@ STEPS <- list( `5 per-100k units` = list(cfg = c(sqrt_adaptive, list(scales = wi$rate_scales)), burn_in = FALSE), `6 constant 0.018` = list(cfg = utils::modifyList(const_cfg, list(lr = 0.018, transform = "sqrt")), burn_in = FALSE) ) +# Rate-scale constant rates for comparison, outside the chain: E14's best and E09's knee, as E14 runs them. +e14_const <- ws_configs(wi$rate_scales, burn_in = TRUE)[["E11 constant 0.1"]] +REFS <- list( + `ref: rate 0.032 (E14)` = list(cfg = utils::modifyList(e14_const, list(lr = 0.032)), burn_in = FALSE), + `ref: rate 0.056 (E09 knee)` = list(cfg = utils::modifyList(e14_const, list(lr = 0.056)), burn_in = FALSE) +) run <- function(spec) { fc <- if (spec$burn_in) fc_burn_in else wi$fc do.call(cal_run_cached, c(list(fc, wi$truth), spec$cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) } -cals <- purrr::map(STEPS, run) +cals <- purrr::map(c(STEPS, REFS), run) ``` ```{r metrics} SEASONS <- c(LIVE, "both") -STEP_COLS <- setNames(c("#000000", "#E69F00", "#56B4E9", "#009E73", "#CC79A7", "#0072B2", "#D55E00"), names(STEPS)) +STEP_COLS <- setNames(c("#000000", "#E69F00", "#56B4E9", "#009E73", "#CC79A7", "#0072B2", "#D55E00", "#999999", "#882255"), names(c(STEPS, REFS))) step_m <- purrr::imap(cals, function(cal, nm) { fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) %>% filter(location != "US") ws_scores(fc) %>% @@ -151,7 +167,7 @@ step_m <- purrr::imap(cals, function(cal, nm) { }) %>% bind_rows() %>% mutate( - step = factor(step, levels = names(STEPS)), + step = factor(step, levels = names(c(STEPS, REFS))), season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) ) @@ -204,7 +220,7 @@ WIS). Coverage: change in L1 coverage bias, season-pooled and month-average ```{r deltas} wis_both <- step_m %>% - filter(season == "both seasons") %>% + filter(season == "both seasons", step %in% names(STEPS)) %>% transmute(step, h, v = wis_pct) cov_both <- mv$cov %>% filter(season == "both") %>% From e801a98abe4b06c450743e6ec3379fd0c68d35f6 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Fri, 2 Oct 2026 01:03:35 -0700 Subject: [PATCH 074/107] docs(calibration): E18 labels step 6 as sqrt constant 0.018 --- .../writeups/calibration_experiments/e18_ref_op_bridge.Rmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd index 36027e5f..dff2f897 100644 --- a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd +++ b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd @@ -39,7 +39,7 @@ only (US excluded). | 3 no leak | `fast_decay = 0` | sqrt, adaptive, mult 0.03, window 20, carry, with the 2023-24 season present | | 4 no burn-in season | drop the 2023-24 rounds | as 3, cold: the eta pool starts empty | | 5 per-100k units | `scales` = population / 100k | as 4, on the square root of admissions per 100k | -| 6 constant 0.018 | `lr = 0.018` | E16's best: sqrt per 100k, constant 0.018 | +| 6 sqrt constant 0.018 (E16) | `lr = 0.018` | E16's best: sqrt per 100k, constant 0.018 | **Why steps 4 and 5.** The 2023-24 burn-in season does two things. With `slow_init` it sets the slow term's starting value (the warm start). Without @@ -140,7 +140,7 @@ STEPS <- list( `3 no leak` = list(cfg = utils::modifyList(sqrt_adaptive, list(burn_in_seasons = "2023-2024")), burn_in = TRUE), `4 no burn-in season` = list(cfg = sqrt_adaptive, burn_in = FALSE), `5 per-100k units` = list(cfg = c(sqrt_adaptive, list(scales = wi$rate_scales)), burn_in = FALSE), - `6 constant 0.018` = list(cfg = utils::modifyList(const_cfg, list(lr = 0.018, transform = "sqrt")), burn_in = FALSE) + `6 sqrt constant 0.018 (E16)` = list(cfg = utils::modifyList(const_cfg, list(lr = 0.018, transform = "sqrt")), burn_in = FALSE) ) # Rate-scale constant rates for comparison, outside the chain: E14's best and E09's knee, as E14 runs them. e14_const <- ws_configs(wi$rate_scales, burn_in = TRUE)[["E11 constant 0.1"]] From 1a0469316e1e978bbeb56021f400b11a889630f8 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Fri, 2 Oct 2026 01:04:05 -0700 Subject: [PATCH 075/107] docs(calibration): review tracker catches up on E02, E09, E18 and queued work --- notes/calibration-review.md | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/notes/calibration-review.md b/notes/calibration-review.md index f862e822..c68ee66b 100644 --- a/notes/calibration-review.md +++ b/notes/calibration-review.md @@ -23,13 +23,21 @@ for), **not reviewed**. forecaster's spring bias (last on the ensemble in 2024-25, second on `windowed_seasonal`). 3. Experiments before E07's re-run (E03–E05, E09–E11) used the submitted - ensemble; E01, E02 (new versions) and E12–E17 use the clean replay. + ensemble; E01, E02 (new versions) and E12–E18 use the clean replay. +4. **Anything that acts mainly after the peak depends on the forecaster.** + The knee constant rate looks good after the 2024-25 peak in E09 because + the ensemble was still under-predicting that spring; on + `windowed_seasonal`, which over-predicts after the peak, the same + offsets cost WIS and coverage (E18). Same for switching off in March + (E07). ## Month view (added 2026-10-01) Every notebook now has a "Month view: WIS, coverage, median error" section (WIS reduction %, within-month L1 coverage bias, median absolute-error -reduction %, per month and horizon, states only) and a month-average +reduction %, per month and horizon, states only; all three rows read +"positive = better than the uncalibrated base", with absolute monthly +coverage bias in a table under the plots) and a month-average coverage column next to its season-pooled one. Season-pooled coverage claims mostly don't survive: where one changed, the notebook's Findings end with a "**Month view (2026-10-01):**" line. Verdicts: @@ -61,22 +69,30 @@ Differences under about 0.01 are within the month-level noise floor. |---|---|---|---|---| | E00 vintage backtest | ensemble | verified | Finalized data exaggerates h−1/h0 gains a little; use vintages, no more work. | — | | E01 eta settings | clean replay | verified | Recap of the August sweep: carry slightly better, window 20 as good as larger, the multiplier has an elbow. Note: on this replay 0.03 is the coverage elbow and 0.01 the WIS-safe choice. Earlier: (1) Curve axis looked flipped vs E09. (2) x/o markers unlabeled. (3) "Rebuilds the original" is misleading: it re-asks the question on a new setup. | (1) E09 flipped to match; (2) legend added; (3) intro reworded with a what-changed table | -| E02 eta variants | clean replay | commented | Headline: switching off in Feb/Mar recovers WIS, loses coverage. (1) Coverage numbers hard to find in the WIS / bias cells. (2) Month plot: unclear whether positive % is good; the sign convention must be labeled explicitly in every notebook. | intro reworded; (1) tables split into WIS and coverage, plus a WIS vs coverage scatter; (2) every notebook now labels "WIS reduction % (positive = calibrated WIS lower than base)" and "L1 coverage bias (lower is better)" (all computations already used that sign). Open: season-level coverage lets ramp and post-peak errors cancel, so the switch-off's coverage cost may be mostly an artifact; month-level coverage would settle it. | +| E02 eta variants | clean replay | commented | Headline: switching off in Feb/Mar recovers WIS, loses coverage. (1) Coverage numbers hard to find in the WIS / bias cells. (2) Month plot: unclear whether positive % is good; the sign convention must be labeled explicitly in every notebook. | intro reworded; (1) tables split into WIS and coverage, plus a WIS vs coverage scatter; (2) every notebook now labels "WIS reduction % (positive = calibrated WIS lower than base)" and "L1 coverage bias (lower is better)" (all computations already used that sign). Month view settled the open question: the switch-off's coverage cost is mostly an artifact of season pooling (better at h3 month by month). | | E03 scale | ensemble | not reviewed | | | | E04 offset structure | ensemble | not reviewed | | | | E05 warm start | ensemble | not reviewed | | | | E07 across eras | both | commented | Needs concrete base WIS numbers, not "similar". Expand into the ensemble vs `windowed_seasonal` comparison: base WIS side by side, base bias by month, every operating-point candidate on both. | intro reworded; expanded with base WIS side by side (B/A 0.93–1.01 in 2024-25, 1.05–1.33 in 2025-26), base bias by month, and five candidates on both forecasters | -| E09 lr × delay | ensemble | commented | WIS axis inverted relative to the other notebooks. | flipped to "WIS change %, positive is better" | +| E09 lr × delay | ensemble | commented | (1) WIS axis inverted relative to the other notebooks. (2) Month view: dashed base coverage line hard to see. (3) Why does the knee look so good in 2024-25? | (1) flipped; now "WIS reduction % (positive = calibrated WIS lower than base)"; (2) coverage row is now "bias reduction % vs base" in every notebook, absolute values in a table; (3) its post-peak gains come from the ensemble's spring 2024-25 under-prediction, and its ramp months are flat; on `windowed_seasonal` it is the weakest candidate (E18) | | E10 covid | ensemble | not reviewed | | | | E11 constant lr | ensemble | not reviewed | | | | E14 lr by season | clean replay | not reviewed | | | | E15 WIS sources | clean replay | not reviewed | | | | E16 sqrt constant lr | clean replay | not reviewed | | | | E17 late decay | clean replay | not reviewed | | | -| E18 REF-op bridge | clean replay | not reviewed | | | +| E18 REF-op bridge | clean replay | not reviewed | | added rate 0.032 (E14) and 0.056 (E09 knee) as reference rows; step 6 labelled "sqrt constant 0.018 (E16)" | E12 and E13 have no notebooks; they live in `notes/CALIBRATION.md` and `scripts/calibration/calibration_ws_replay.R`. E06 and E08 were superseded (by E13 and E10) and not rebuilt. +## Queued (not started) + +- Headline split: report the ramp (up to the peak) and post-peak months + separately, instead of season-pooled or equally weighted month averages. +- sqrt constant 0.018 with a warm start: the one REF-op piece it lacks (E18). +- Possibly widen the month plot's clip on the coverage row (its ceiling is + +100%), if the +40% saturation hides differences. + The summary for sharing is `notes/calibration-summary-2026-10-01.md`. From 5f29bda5de212faad186bdf0e1100f27999f04a8 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Fri, 2 Oct 2026 01:15:06 -0700 Subject: [PATCH 076/107] docs(calibration): fold calibration notes into one ledger rebuilt from rendered notebook numbers --- R/calibration/calibrate.R | 2 +- notes/CALIBRATION.md | 911 ------------------ notes/ROADMAP.md | 16 +- notes/calibration-ledger.md | 342 +++++-- notes/calibration-review.md | 98 -- notes/calibration-runbook.md | 6 +- notes/calibration-summary-2026-10-01.md | 186 ---- pipelines/covid_hosp_prod.R | 2 +- pipelines/flu_hosp_prod.R | 2 +- .../calibration/calibration_findings_flu.Rmd | 4 +- .../calibration_qt_gallery_covid.Rmd | 2 +- .../calibration_qt_gallery_flu.Rmd | 2 +- .../calibration_qt_seasons_flu.Rmd | 2 +- .../e02_eta_variants.Rmd | 4 +- .../calibration/calibration_ili_backfill.R | 4 +- .../calibration/calibration_ws_experiments.R | 2 +- scripts/calibration/calibration_ws_replay.R | 4 +- 17 files changed, 262 insertions(+), 1327 deletions(-) delete mode 100644 notes/CALIBRATION.md delete mode 100644 notes/calibration-review.md delete mode 100644 notes/calibration-summary-2026-10-01.md diff --git a/R/calibration/calibrate.R b/R/calibration/calibrate.R index 16d7a509..48d7a2c9 100644 --- a/R/calibration/calibrate.R +++ b/R/calibration/calibrate.R @@ -75,7 +75,7 @@ hub_series_matrix <- function(series, round_date, n_levels) { #' first version use the first version. The warm start then uses the vintage #' as of the first live round. A table without `version` is used as-is for #' learning, which is how [calibrate_hub_forecasts_exact()] passes one -#' snapshot per round. See "Learning truth" in `notes/CALIBRATION.md`. +#' snapshot per round. See E00 in `notes/calibration-ledger.md`. #' @param settle_days how long after `target_end_date` the outcome is trusted. #' 14 gives exactly `horizon + 2` rounds of delay on hub coordinates. #' @param burn_in_seasons season labels (as produced by [hub_label_seasons()]) diff --git a/notes/CALIBRATION.md b/notes/CALIBRATION.md deleted file mode 100644 index 403a9ccc..00000000 --- a/notes/CALIBRATION.md +++ /dev/null @@ -1,911 +0,0 @@ -# Online quantile calibration (MultiQT) - -Post-hoc online calibration of our submitted forecasts, following MultiQT -(Ding, Gibbs & Tibshirani 2025): per (location, horizon) series, an additive -offset vector over the 23 hub quantile levels is updated by online gradient -descent on pinball loss, with an adaptive learning rate and isotonic -projection. Full repro commands live in `notes/calibration-runbook.md`. - -# State - -## Code - -- `R/calibration/qt.R` — the tracker (`qt_track`, `qt_learning_rate`, - `qt_project`, `qt_delay_from_dates`). Bit-exact against the authors' Python - on the `delphi-fixes` branch of `~/repos/delphi/multiQT` (their published - code had several bugs we fixed and reported; see that branch's commit - message). Oracle fixtures in `tests/testthat/test-qt.R`, plus a real-series - cross-check (`scripts/calibration/calibration_export_series.R`). -- `R/calibration/hub_data.R` — FluSight hub adapters: `hub_read_forecasts()` - (CMU-TimeSeries submissions from the sparse hub checkout), - `nhsn_read_truth()` (latest NHSN vintage from `get_nhsn_data_archive()`, on - the hub's Saturday grid; replaced the hub target-data CSV on 2026-09-23), - `hub_label_seasons()`. -- `R/calibration/calibrate.R` — `calibrate_hub_forecasts()` driver (burn-in, - season policy, ~5 s for all 265 series) and metrics: `hub_coverage()`, - `hub_coverage_summary()`, `hub_quantile_loss()`, `hub_rolling_tradeoff()`. - Method options beyond the paper: `transform` (power/log space), - `scales` (per-location, per-era divisors so rounds from different sources - share one tracker), `lr_slow` + `fast_decay` + `burn_in_learns_slow` (the - two-term offset, below), `off_after`, `late_decay`, `lr_seasonal`. (`lr_geo_pool`, the - geo-pooled eta in the sweep table below, was removed 2026-09-17: it never - beat the baseline.) -- `scripts/calibration/calibration_ws_backfill.R` — replays flu prod's `windowed_seasonal` - over the NHSN seasons (hub schema, cached to - `cache/calibration/ws_pseudo_hub_forecasts.parquet`); `scripts/calibration/calibration_ws_experiments.R` - runs the one-forecaster-across-eras comparison (below). -- `scripts/calibration/calibration_ili_backfill.R` — runs flu prod's `windowed_seasonal` - forecaster on the ILI+ state history (2010–2024, Wednesday labels, snapshot - truncated one week to mimic reporting lag) and writes a pseudo-hub - `forecasts`/`truth` pair in hub schema to `cache/calibration/ili_pseudo_hub_*.parquet` - (~30 min). Rebuilt 2026-09-23 after the latency fix described in "The ILI+ - replay was broken" below; ILI+ results older than that are invalid. -- `scripts/calibration/calibration_harness.R` — targets-based harness for calibrating our - own forecasters outside the hub path (covid - `windowed_seasonal_extra_sources`, aheads 0:4, `covid_hosp_evaluation` - store). WIP. - -The index of experiments (what was varied against what, and which results -are stale) is `notes/calibration-ledger.md`. - -## Learning truth (2026-09-30) - -Until 2026-09-30, `calibrate_hub_forecasts()` learned only from `truth`, the -latest NHSN vintage: the gradients, the adaptive eta pool and the -`burn_in_quantile` warm start all saw finalized values, and only the timing of -each reveal (`settle_days`) was realistic. Every number in this file below -that date was measured that way. - -`learn_truth` now selects what the tracker learns from (scores stay on -finalized truth). A round with reference date `d` sees data published by -`d - 3`. Modes, as `cal_run(learn = )`: - -- `"final"`: the old behavior. -- `"exact"`: what prod does. Prod re-runs the tracker from scratch each week on - that week's data, so at round `t` every revealed outcome has its value as - of `t`. `calibrate_hub_forecasts_exact()` replays this with one run per - round (about 1.2 min per config with 12 forked workers). -- `"vintage"`: one run in which each outcome keeps its value at the round - that revealed it: a stateful tracker that never revisits a step. - -Findings (E00, `e00_vintage_backtest.Rmd`, flu, both live seasons, spoiled -submissions excluded): - -- Final vs exact is small except at h−1: REF-op at settle 14 is - +7.6/+5.4/+2.4/+1.8/+2.2 final vs +5.4/+4.9/+2.4/+1.7/+2.2 exact, coverage - bias 0.067–0.040 vs 0.074–0.041. REF-paper is within 0.6 points. -- Vintage mode is not a stand-in for exact: at settle 7 it learns once from - the under-reported first report (REF-op h−1 0.0, coverage bias 0.121). -- Under the exact (prod) design, settle 7 beats or ties settle 14 at every - horizon at both references (REF-op +6.1/+5.1/+2.6/+1.8/+2.2), because next - week's run corrects what an early reveal got wrong. A stateful tracker would - prefer the longer delay. The right revision delay depends on the design. - -## Constant learning rate × revision delay (E09, 2026-09-30) - -`e09_lr_delay.Rmd`: the plain tracker (REF-paper) on the rate scale (per -100k) with a constant learning rate, swept over 0.001–3.2 × extra delay 0–3 -weeks, exact and stateful learning truth. Pooled over locations: - -- The coverage/WIS trade-off is L-shaped, with a knee at a learning rate of - about 0.05–0.1 per 100k. At 0.056 and +1 week (re-run): WIS change - +3.2/+2.5/+1.1/+0.5/+0.2% (positive is better), coverage bias - 0.040/0.031/0.026/0.027/0.027. That matches or beats REF-paper - (+3.2/+1.9/−2.0/−2.7/−5.3%, 0.044/0.030/0.032/0.034/0.032) at every - horizon. -- With the adaptive rate, rate scale and count scale give identical results - (the adaptive rate rescales with the location), so the rate scale matters - only with a constant rate. -- REF-op gains more WIS than any constant rate at h0–h3, at about twice the - coverage bias. -- Re-run design (prod): shorter delay is better, mostly at h−1/h0. Stateful - design: zero extra delay makes h−1 coverage worse than base (learning from - the under-reported first report); one extra week fixes it. - -## Covid at the references (E10, 2026-09-30) - -`e10_covid.Rmd`, exact learning truth, REF-op without the warm start (as in -covid prod), rounds 2024-11-30 to 2026-09-19 (the spoiled 2024-11-23 round -excluded). The covid base is already calibrated at h1–h3 (coverage bias -0.046/0.036/0.028), so calibration there is neutral to slightly harmful -(REF-op 0.0/−0.6/−1.3% WIS) or harmful (REF-paper −4.5/−7.3/−10.3%). h−1, which over -E10's rounds is 100% the `climate_linear` ensemble (`revision_aware` took -over h−1 only on 2026-09-23), gains +10.2% WIS (REF-paper +11.4%). REF-op's losses come from 2025-26 and from -September–October. Candidate change: calibrate covid at h−1 (maybe h0) only. - -## Clean `windowed_seasonal` replay, no burn-in (E12, 2026-10-01) - -`scripts/calibration/calibration_ws_replay.R` with `ch_use("{flu,covid}_windowed_seasonal")`, -forecasts read from the evaluation store's clean replay (substitutions off, all -53 geos; harness backfill matches it exactly), h0–h3 (h−1 is left to the -revision-aware methods), exact learning -truth, hub rounds only (flu 55, covid 89). This changes several things at -once compared with E07-B (no burn-in, clean replay, store truth, all geos), -so it is not a one-axis comparison. Pooled WIS change %, h0..h3: - -| disease | config | h0 | h1 | h2 | h3 | -|---|---|---|---|---|---| -| flu | E11 constant rate | +1.1 | −0.4 | −0.9 | −1.2 | -| flu | REF-op cold | +2.9 | +1.1 | +0.5 | 0.0 | -| covid | E11 constant rate | +0.5 | −4.9 | −6.9 | −5.1 | -| covid | REF-op cold | +2.8 | −1.3 | −2.6 | −2.3 | - -- Flu base bias matches E07-B's `windowed_seasonal` (share of truth above - the median to within 0.01), so the substitutions barely move the - aggregate bias. -- Flu E11 cuts coverage bias 4–5× (L1 ~0.02) at a ~1% WIS cost, but - under-covers the 50% interval (0.45–0.48 at h0–h3). Check that before - adopting it cold. -- Covid: gains at h0, losses at h1–h3, worst in 2025-26, the same - season pattern as E10. Supports calibrating covid at h0 only. -- Per-season tables: `cache/calibration/ws_replay_scores_{flu,covid}.csv`. - Rerun with `burn_in_seasons = "2023-2024"` once the HHS burn-in exists - (ROADMAP 1b). - -## Clean replay with a 2023-24 HHS burn-in (E13, 2026-10-01) - -E12's setup plus 2023-24 as a burn-in (HHS data at real versions, back to -2020, stitched before NHSN in `ch_inputs_with_burn_in()`; run -`calibration_ws_replay.R 12 burn_in`). WIS change %, h0/h1/h2/h3, both -seasons pooled, states only (US excluded): - -| disease | config | cold | warm | -|---|---|---|---| -| flu | REF-op | 2.0/0.6/0.1/−0.2 | 2.8/1.1/0.9/1.1 | -| flu | E05 single | 1.9/−0.2/−1.1/−1.7 | 2.7/0.4/−0.1/−0.1 | -| flu | E11 constant 0.1 | 0.7/−0.2/−0.7/−1.0 | 1.0/0.0/−0.7/−1.1 | -| covid | REF-op | 1.3/−1.1/−1.7/−1.6 | −0.1/−1.8/−0.4/2.8 | -| covid | E11 constant 0.1 | −0.9/−4.3/−4.8/−3.4 | −2.9/−9.2/−13.2/−17.9 | - -- Flu: the warm start adds 0.5–1.6 points for REF-op and E05, but the - warm configs under-cover the 50% interval (0.40–0.44). -- Covid: 2023-24 over-predicted while the live seasons under-predict at - h0, so the warm start is wrong-signed there. E11 warm is badly harmful. -- Open thread 1 answered: a warm start doesn't help E11's constant rate - (flu) and hurts it (covid). -- US is ~45% of all-locations WIS, but states-only and US-only changes - point the same way. Tables: `cache/calibration/ws_replay_scores_burn_in_*.csv`. - -## Constant rate by season (E14) and where the WIS gains come from (E15), 2026-10-01 - -Flu, clean `windowed_seasonal` replay, exact, cold, states only. -`e14_lr_by_season.Rmd`, `e15_wis_sources.Rmd`. - -- E14: a constant rate of 0.018–0.032 per 100k improves WIS at every - horizon in both seasons and cuts coverage bias in both (2025-26 reaches - ~0.01 by 0.032; 2024-25 keeps improving up to ~0.3, at a WIS cost past - 0.056). From 0.1 (E11's rate) WIS worsens at h1–h3 in both seasons. The - knee here is lower than E09's 0.05–0.1 (ensemble, burn-in). Neither - REF-op cold nor sqrt adaptive is good in both seasons at every horizon. -- E15: offsets are small (median 2–5% of the base 90% width; width ratio - 1.00–1.04), which is why fan plots look unchanged, and nearly always - upward. Gains come from the center and shoulders, not the tails. For - REF-op, shifting every quantile by the median offset alone gives more WIS - than the full calibration; width/shape changes cost WIS and buy coverage. - E11 0.1 has a long tail of offsets large relative to narrow intervals - (top 10%: 0.26–0.55 of the 90% width vs 0.08–0.20 for REF-op). - -## sqrt constant rate (E16) and late-season decay (E17), 2026-10-01 - -Flu, clean replay, exact, cold, states only. `e16_sqrt_constant_lr.Rmd`, -`e17_late_decay.Rmd`. - -- E16: on the sqrt scale a constant rate of 0.001–0.018 (sqrt per-100k - units) improves WIS at every horizon in both seasons; best is sqrt 0.018. - Against rate 0.032 it gains more at h0 in both seasons and at every - horizon in 2025-26 (+3.0/+1.0/+1.1/+1.5), with similar coverage bias; it - doesn't help h2–h3 in 2024-25. Its offset tail is smaller (90th pct - |offset|/90% width 0.09–0.11 vs 0.12–0.15), though larger at the 0.95 - level. -- Offsets go stale in March in both seasons: every tracker loses WIS at - h1–h3 (sqrt 0.018 at h2: −7 / −28%; REF-op −20 / −24%) because the base - already over-predicts after the peak and the offsets stay positive. - March is 4–8% of season WIS, so this costs about a point. Season-level - coverage partly reflects ramp under-prediction cancelling post-peak - over-prediction. -- E17: `late_decay` (shrink the offset each round from 03-01) fixes March - but loses overall, because it wipes the offset carried into the next - season. `off_after = "03-01"` raises pooled WIS (sqrt 0.018: - +2.3/+1.2/+1.2/+1.3 vs +2.2/+0.8/+0.6/+0.6) at a coverage cost (bias - 0.094–0.116 vs 0.059–0.078). Untested: decaying only the played offset - and keeping the hidden state. - -## E01, E02, E07 rebuilt (2026-10-01) - -Flu, clean replay, exact, post-fix, states only. - -- E01 (`e01_eta_settings.Rmd`, sqrt REF-paper, cold): the multiplier is - the only setting that matters (16–81 points of pooled WIS; window 1–2, - season policy 0.3–1.7). 0.1 and 0.3 are too large; 0.03 gains at h0 and - loses a little at h1–h3; 0.01 never loses more than 0.2 but barely fixes - 2024-25 coverage from a cold start. Carry beats reset. Same conclusion - as the old sweep, except carry vs reset now clearly favors carry. -- E02 (`e02_eta_variants.Rmd`): `off_after` 03-01 or 02-15 is the only - large WIS gain (positive at every horizon in both seasons) but gives up - most of the coverage gain (bias 0.080–0.100 vs 0.032–0.045), matching - E17. Per-level eta: small gain at h1–h3. Seasonal window: no help. -- E07 (`e07_across_eras.Rmd`, identical rows, each forecaster with its own - 2023-24 burn-in): the trackers rank the same on the hub ensemble and on - `windowed_seasonal`, so the gains aren't an ensemble artifact. The - ensemble gains about a point more with REF-op (pooled A - +3.9/+1.9/+1.5/+2.3, B +2.8/+1.1/+0.8/+1.2). B is not less - miscalibrated than A (equal in 2024-25, worse in 2025-26). - Expanded: base WIS B/A (states) 0.93–1.01 in 2024-25, 1.05–1.33 in - 2025-26. Same monthly bias shape on both; the one difference is spring - 2024-25, where A still under-predicts. Candidates (states, pooled WIS - mean over h0–h3) rank sqrt 0.018 > rate 0.032 > REF-op cold on both, and - coverage ranks identically; `off_after` 03-01 is last on A in 2024-25 but - second on B, so date-based switches don't transfer between forecasters. - -## REF-op bridge (E18, 2026-10-01) - -`e18_ref_op_bridge.Rmd`: flu, clean replay with the 2023-24 HHS season, -exact, states only, one setting per step from REF-op to sqrt constant -0.018. Pooled WIS reduction h0/h1/h2/h3: REF-op +2.8/+1.1/+0.9/+1.1, -sqrt 0.018 +2.2/+0.8/+0.6/+0.6; REF-op leads in 2024-25, sqrt 0.018 in -2025-26. - -- REF-op's edge comes from the warm start (0.5–1 point, 2024-25 only), the - leak (~1 point at h1–h3, both seasons) and the eta warm-up from the - burn-in season (~0.5 point at h2–h3, 2024-25 only). The slow term adds - nothing without the warm start. Per-100k units are a no-op for adaptive - sqrt. -- Over cold adaptive sqrt, the constant rate is worth +1.0 to +2.3 at h1–h3. -- No step moves month-averaged coverage bias by more than 0.008 (noise - floor ~0.01); the large season-pooled coverage moves are cancellation. -- E13 follow-up answered: the warm start is worth about twice the eta - warm-up; both matter only in the first live season. -- Untested: sqrt 0.018 with a warm start, the one REF-op piece it lacks. - -## Data - -FluSight CMU-TimeSeries submissions: 83 rounds (2023-10-14 … 2026-05-30), 53 -locations, horizons −1…3, 23 quantile levels. Season 2023-24 is a burn-in -(incomplete horizons/locations): its residuals warm the learning rate but no -gradient steps are taken. Waiting 14 days for NHSN to settle makes the reveal -delay exactly `horizon + 2` rounds, so the offset at h3 is always 5 rounds -stale. - -As-of NHSN vintages for the gallery come from -`cache/calibration/nhsn_archive_flu.parquet`, copied from the oracle capture -`cache/oracle/flu_hosp_prod/today-main-prmpwlvz/nhsn_archive_data.parquet` -(versions 2024-11-19 … 2026-07-22, so both live seasons are covered; no -vintages exist before 2024-11-19 anywhere). Note: the S3 -`nhsn_data_archive.parquet` object stalled at version 2026-01-30 — the -polling job appears to have stopped; refresh the copy from a newer oracle -capture if one exists. Since 2026-09-23 `get_nhsn_data_archive("flu")` (the -cast API, versions 2024-11-19 onward) serves both truth (`nhsn_read_truth()`) -and vintages live; the findings notebook uses it. - -## Findings (factorial sweep, `calibration_qt_flu.Rmd`; count space, pre-fix) - -Superseded by the rebuilt E01 (`e01_eta_settings.Rmd`, 2026-10-01). - -Sweep: `season_policy` {carry, reset} × `lr_window` {8, 20, 50, Inf} × -`lr_mult` {0.3, 0.1, 0.03, 0.01}; results in `cache/calibration/sweep.rds`. - -- **The multiplier is the control knob; the window barely matters.** Per - horizon, WIS-improvement spread across multipliers is 8–63 points; across - windows 1.5–5. Windows 20/50/Inf are nearly indistinguishable at this data - size. The expanding-window learning-rate ratchet is real but immaterial. -- **The paper's `mult = 0.1` overpays for calibration**: calibration error - improves 70–85% but WIS degrades −3.6% (h0) to −22.9% (h3). `mult = 0.03` - gets equal-or-better calibration error at h ≥ 1 at a fraction of the WIS - cost (−2 to −6%). `mult = 0.01` is near WIS-parity but converges too slowly - (bad first-live-season calibration error, 0.07–0.10). The real tradeoff is - convergence speed vs steady-state sharpness cost. -- **Horizon −1 calibration improves WIS outright (+4%)** — calibration is - free there. -- **Carry vs reset across the off-season is minor** (carry slightly better at - h3, otherwise a wash). -- The WIS cost grows with horizon, consistent with `h + 2` staleness: a stale - additive offset does the most damage where the seasonal ramp is steep. -- Residual scale swings 1–2 orders of magnitude within a season, so an - additive offset learned near the peak is badly sized for the trough. - -**Original operating point (count space): `lr_mult = 0.03`, `lr_window = 20`, -`season_policy = "carry"`** (20 over the untested 15 because it is a -validated grid point and the window is immaterial). At this multiplier the -tracker needs most of a season to warm up, which makes burn-in/state-carrying -the central integration design issue. - -**Current operating point: `transform = "sqrt"`, `lr_mult = 0.03`, -`floor = 1e-3`, `lr_window = 20`, carry, `slow_init = "burn_in_quantile"`, -`lr_slow = list(mult = 0.003)`, `fast_decay = 0.1`.** Re-measured 2026-09-23 -(after the correctness fixes, NHSN truth, all 53 hub locations): WIS change -+8.9/+5.4/+2.6/+1.9/+1.9 at h−1…3 over both live seasons, calibration error -0.040–0.073 against 0.098–0.118 uncalibrated. Current numbers for every -variant are in `reports/writeups/calibration/calibration_findings_flu.Rmd` -(section "Current numbers" below). With exact learning truth and spoiled -submissions excluded (2026-09-30): +5.4/+4.9/+2.4/+1.7/+2.2, coverage bias -0.041–0.074 against 0.095–0.111 uncalibrated. - -**Numbers dated before 2026-09-17 predate the two correctness fixes** -("Correctness fixes" below) and used the hub target-data CSV as truth. The -sqrt-space tables from 2026-09-16 were also hit by the unclamped sqrt -inverse, so they have been replaced; the count-space and 2026-08-27 tables are -kept as a record of how the design was reached, not as current numbers. - -## Notebooks - -All in `reports/writeups/calibration/`, rendered into `rendered_reports/`. -Each takes a `hub_dir` param (the hub checkout, relative to the repo root; -defaults `../FluSight-forecast-hub` and `../covid19-forecast-hub`) and scores -against the latest NHSN vintage via `nhsn_read_truth()`. - -- `calibration_findings_flu.Rmd` — the current summary: base bias on NHSN, - the same forecaster's bias on ILI+, staleness (lagged correlation of played vs needed - shift), the leak by month, count vs sqrt, ensemble vs `windowed_seasonal`. - All numbers are post-fix. -- `calibration_qt_flu.Rmd` — the parameter-sweep EDA. Frozen as - the sweep record. -- `calibration_qt_seasons_flu.Rmd` — whole-season views of four - states × two live seasons, one panel per (state, season): every-other-round - fans (80% band) for h 0–3, the NHSN vintage each round saw painted over its - fortnight, finalized truth, and an eta strip; plus collapsible internals - tables at h2 (eta, pre-PAVA hidden offset, post-PAVA offset, base, - calibrated at 5 levels). The states are picked from the baseline run: best - and worst by WIS change and best and worst by median absolute-error change, - pooled over horizons and both live seasons, deduplicated (as of 2026-09-16: - AK best on both, so HI is the best-AE panel; NY worst WIS; CT worst AE). - Repeated for six method variants (baseline, off after March 1 / February 15, - per-level eta, geo-pooled eta, seasonal window) with a headline - WIS/calibration-error comparison at the top, plus a by-month breakdown of the - baseline (WIS change, share of base WIS, median shift) that shows where in - the season the method gains and loses. -- `calibration_qt_gallery_flu.Rmd` — fixed operating point. Slim - headline table (WIS + calibration error per horizon per season), then a - ranked per-forecast gallery: top-N (location, round) panels ordered by mean - absolute quantile displacement relative to the base median. Each panel: - as-of NHSN snapshot (data the forecaster saw), finalized truth, base and - calibrated fans (50% + 90% ribbons, horizons −1…3), captioned with MAD, WIS - delta, and whether truth escaped the base 90% band. Seasons 2024-25 and - 2025-26 only, flu only. A dynamic/paginated app version was considered and - rejected for now in favor of static top-N HTML. - -## Method variants (`calibrate_hub_forecasts()` options; 2026-08-27, pre-fix) - -Superseded by the rebuilt E02 (`e02_eta_variants.Rmd`, 2026-10-01). - -All count space at the original operating point; WIS change vs base, both -live seasons, by horizon −1/0/1/2/3. Pre-fix numbers (see above); the -transform rows measured on 2026-09-16 were removed, see "Current numbers": - -| variant | option | h−1 | h0 | h1 | h2 | h3 | -|---|---|---|---|---|---|---| -| baseline | — | +4.3 | +1.8 | −1.9 | −5.3 | −6.7 | -| off after April 1 | `off_after = "04-01"` | +4.9 | +1.4 | −1.3 | −2.8 | −3.9 | -| off after March 1 | `off_after = "03-01"` | +4.5 | +2.0 | +0.4 | 0.0 | +0.2 | -| off after February 15 | `off_after = "02-15"` | +3.0 | +2.5 | +1.3 | +0.8 | +0.9 | -| per-level eta | `lr_args$per_level = TRUE` | +4.2 | +1.8 | −1.3 | −3.4 | −5.4 | -| geo-pooled eta (option since removed) | `lr_geo_pool = ` | +4.4 | +2.0 | −2.2 | −5.5 | −7.7 | -| seasonal eta window (carry) | `lr_window = 10, lr_seasonal = list(half_width_weeks = 5)` | +2.9 | +0.1 | −3.0 | −5.5 | −7.5 | -| seasonal eta window (reset) | same + `season_policy = "reset"` | +3.2 | +1.1 | −2.1 | −4.7 | −7.0 | - -- Eta was never pooled across horizons: each (location, horizon) series has its - own tracker and its own eta (pooled over the 23 levels and the window). -- **Switching off after April 1** (play base, no updates, offsets carry to next - season) roughly halves the WIS cost at h2/h3 — the spring tail, where - peak-tuned additive offsets are out of regime, is where most of the damage - was. -- **Switching off after March 1 removes the WIS cost entirely** (h1–h3 within - ±0.4 of base, h−1/h0 still positive) — the out-of-regime tail starts around - March, not April. First variant that is WIS-neutral-or-better at every - horizon. -- **February 15 is better still at h0–h3** (+2.5/+1.3/+0.8/+0.9, - WIS-*positive* everywhere) at the cost of ~1.5 points at h−1 vs March 1 — - the post-peak descent is already out of regime, not just the spring tail. - The notebook now runs the March-1 and Feb-15 cutoffs (April 1 dropped). -- **Per-level eta** is a modest gain at h ≥ 1: the pooled 0.9 quantile of - |residual| is dominated by the outer levels and over-steps the median. -- **Geo-pooled eta** gives visibly smoother eta trajectories but no WIS gain. -- **Seasonal window** is worse everywhere on the headline. In the panels its - eta at season start is *smaller* than baseline's (last year's same weeks were - a slow ramp), so the carried-over spring offsets take longer to unwind. - Reset instead of carry recovers some of that (h2 −4.7) but still trails - baseline; 2024-25 is fine (+5.7/+2.8/+0.1 at h−1/0/1), 2025-26 is uniformly - worse. The notebook's seasonal section now runs the reset variant. -- **Transforms** (`transform = "log1p"` / `"sqrt"` / `"quartic_root"`, the - last being the forecasters' whitening map `(x + 0.01)^0.25`). Y and all 23 - base quantiles are mapped, residuals/eta/offsets/PAVA all live in the mapped - space, the played quantiles are mapped back and clamped at zero, and scoring - is on counts. Coverage indicators are invariant under a monotone map, so the - gradient is identical to count space; only the step geometry changes, and - multipliers are not comparable across spaces. The 2026-09-16 measurements - of these variants predate the correctness fixes (the sqrt ones were also hit - by the unclamped inverse) and were removed. Re-measured since: sqrt vs - count only, in "Current numbers" below. Log space and fourth root have not - been re-run; before the fixes log space was worse than count space at h ≥ 1 - and fourth root at mult 0.03 under-stepped, but neither result should be - relied on. If scale-awareness is retried beyond sqrt, normalize by a smooth - scale proxy (trailing 4-week truth, floored) rather than a log. - -## Where the WIS moves (2026-09-16, count-space baseline, pre-fix) - -By calendar month of the reference date, both live seasons pooled. These are -pre-fix count-space numbers; the mechanism (the spring and October losses, -the stale wrong-signed offset) is re-confirmed post-fix for the sqrt variants -in the findings notebook, with the numbers in "Current numbers" below. - -- **The mass is in December–February.** At h3, Dec + Jan carry ~75% of the - season's base WIS and Mar–May ~7%. Calibration is −2% in Dec/Jan at h2/h3 - and −33/−120/−84% in Mar/Apr/May. Of the −6.7% headline at h3, roughly 56% - comes from the spring on tiny mass and 44% from small losses on the huge - Dec–Feb mass. The gains are Feb at h−1/h0 (+7.6/+4.1%) and Mar at h−1 - (+11.5%). -- **The base median is biased low almost everywhere**, by 10–60% of truth - (Dec: 37–59% at h0–h3, truth above the base median in 90% of December - forecasts). The direction of the needed correction is stable; only its - magnitude swings, by two orders of magnitude within a season. -- **The tracker gets the sign wrong at every turn**, because the offset is a - 5-round-stale count. NY h3, 2025-26: Oct 18 opens with hidden = +159 carried - from spring; the five May outcomes revealed that round (where the played - forecast had over-covered) knock it to +15, then −9 by Nov 22. Then - Nov 29 – Dec 20 reveal *nothing* (the 5-week Oct/Nov gap plus the 5-round - delay), so the offset sits at −9 while base under-predicts by 500–1900. - January pushes it to +148 chasing December, when truth is already falling; - February over-corrects back to −13; spring climbs to +281 on a base of - 26–47 with eta stuck at 85 (the 20-round window still holds January - residuals in the hundreds). The calibrated h3 median is *below* base in 90% - of December forecasts across all states. -- **Eta is not the lever.** The gradient is a coverage indicator whose sign is - set by outcomes 5 rounds old; eta only scales how fast the offset chases - it. Any eta large enough to unwind a stale offset quickly is large enough - to have built it. That is why the seasonal-eta variants lose (small eta at - season start slows the unwind of last spring's offsets) and why the - cutoff variants win (they stop the chase where it is wrong-signed and, - with carry, hand the next season a mid-February offset instead of a May - one). The multiplier sweep in `calibration_qt_flu.Rmd` said the same thing - from the other side: mult trades convergence speed against overshoot with - no setting that wins both. - -### Two-term offset: slow + fast (2026-09-16) - -`qt_track()` now carries `hidden = slow + fast`. Both take the same coverage -gradient; `fast` is the paper's iterate (short eta window, obeys the season -policy, optional per-round leak `fast_decay`), `slow` has its own long window -and small multiplier, never decays, ignores the season policy, and may train -through burn-in (`burn_in_learns_slow`). Without `lr_slow` and with -`fast_decay = 0` the output is bit-identical to before (tests pin this). - -The 2026-09-16 comparison table for this section (sqrt space) predated the -correctness fixes and was removed; the post-fix numbers for the variants that -were re-run (single term, leak 0.1, and the warm-start combinations) are in -"Current numbers" below. What still holds: - -- **The leak is what fixes the spring and h2/h3.** Re-measured: decay 0.1 - alone takes h3 from −1.4% to +0.5% and cuts the h3 March/October losses from - about −47% to −17% and −48% to −14%, but gives back most of the coverage - gain (calibration error 0.022–0.043 single-term vs 0.073–0.089 leaky): a - correction that must be continually refreshed cannot hold coverage through - a 5-round reveal lag. -- **A slow gradient term on its own** (no warm start) was measured only - before the fixes: at mult 0.01 it barely moved over one burn-in plus one - live season, and at mult 0.03 it acted like a second fast term. Not re-run; - the warm start below replaced it as the way to hold the persistent level. - -### Is a learned seasonal offset viable? - -Checked on the only pair of live seasons with the same base forecaster -(2024-25 → 2025-26, 53 locations × 27 season-rounds, aligned by -`season_round`), base-median residual relative to the base median: - -| horizon | cor(rel residual, same season_round, YoY) | Spearman | -|---|---|---| -| −1 | −0.01 | 0.02 | -| 0 | 0.05 | 0.13 | -| 1 | 0.13 | 0.21 | -| 2 | 0.32 | 0.25 | -| 3 | 0.30 | 0.26 | - -Raw count residuals correlate at 0.4–0.7, but that is geography (big states -have big residuals every year), not seasonality. Smoothing last year's -residual over ±2…8 season-rounds does not raise the correlation (it drifts -down slightly), and aligning seasons on each location's peak round instead of -the calendar changes nothing, because the two seasons peaked within 3 rounds -of each other (median season_round 11 vs 8) so the two axes coincide. -Pooling the prior nationally helps a little (0.45 at h2). In fourth-root -residuals the correlations are higher (0.14–0.19 at h0, 0.36–0.44 at -h2/h3), since that is a variance-stabilized quantity. So: weak-to-moderate -year-over-year signal at long horizons, none at short, from one season pair -whose phases happened to line up; the test of whether phase-alignment matters -needs a season that peaks late. The learned hidden offsets -themselves correlate YoY at 0.11 (h2 median). Pooled nationally by 4-round -bin, the relative residual is positive in nearly every bin both years but -its size differs by year (bin 5: +56% vs +20%; bin 17: +23% vs −29%), and -the 2025-26 axis is shifted by the 5-week Oct/Nov gap. Conclusion: a -per-(location, horizon, season-week) offset learned from one prior season is -noise at h ≤ 1 and weak at h2/h3; the stable, learnable component is a -*constant* low bias, not a seasonal profile. Revisit after more seasons of a -stable base forecaster, and align on peak-relative rather than calendar -weeks if so. - -### Warm start from the burn-in season (2026-09-16) - -`slow_init = "burn_in_quantile"` sets the slow term per (horizon, level) to the -conformal offset implied by the burn-in rounds pooled over locations: the -`level`-quantile of `Y − Yhat[level]` in working units. The tracker then runs -on top of it. Same setup as the two-term table (sqrt, mult 0.03, window 20, -carry, hub 2023-24 burn-in). The 2026-09-16 table was pre-fix and was removed; -re-measured numbers are in "Current numbers" below. - -- **One season of batch conformal offsets is already WIS-positive at every - horizon** (+0.4 to +1.9, re-measured) while removing a third of the - coverage error: the persistent low bias is real and cheap to correct. -- **Warm start + leaky fast tracker is WIS-positive at every horizon** and - halves the coverage error (+8.3/+5.1/+2.6/+2.0/+2.1, calibration error - 0.046–0.080 vs 0.098–0.118 base). Its h3 gain comes from December–January - (+4% each, the months holding ~75% of the h3 base WIS); its spring losses - are only partly contained by the leak (h3 March −28%), because the warm - start lives in the slow term, which does not decay. -- Adding a slow gradient term on top of the warm start was, before the fixes, - a small dial between WIS (mult 0.003) and coverage (mult 0.01); the chosen - point is 0.003 with decay 0.1. Post-fix only 0.003 was re-run. -- The h3 warm-start offsets at the outer levels are large (0.95 level: +2.9 - scaled-sqrt units for CA) because the 2023-24 ensemble was badly - under-dispersed at long horizons; it is the correct conformal answer for - that season but a reminder that a one-season warm start inherits that - season's base forecaster. In prod the warm start would come from the - previous live season(s) of the *current* forecaster. - -### ILI+ burn-in (2026-09-16): invalid, measured on the broken replay - -The idea: the prod flu forecasters already train on ILI+ (percent positive × -percent ILI, state level, 2010–2024) as faux-versioned augmentation rows, so -*forecast* the ILI+ seasons with the same `windowed_seasonal` forecaster and -use those residuals as a multi-season burn-in (13 usable seasons instead of -one). The pseudo-hub tables are on the ILI+ percent scale, so the tracker runs -in scaled units: `scales` divides each location's rounds by an era-specific -divisor (90th percentile of in-season truth) before the sqrt transform, so -offsets and eta learned on ILI+ carry over as fractions of a typical -season-peak level. - -The result recorded here on 2026-09-16 (ILI+ residuals carry no information -about the live bias; an ILI+-trained slow term costs 130–500% WIS) was -measured on a broken replay (see "The ILI+ replay was broken" below) and has -been removed. The `scales` machinery is fine and stays. - -The useful implication is the opposite one: the hub's own 2023-24 burn-in -*does* share the live bias, so the slow term should be warm-started from it -directly (`slow_init = "burn_in_quantile"`: the conformal offset per horizon -and level, pooled over locations) rather than accumulated by gradient steps. - -### Correctness fixes (2026-09-17) - -A review of the calibration code found two defects that affect every table -above dated 2026-09-16; the fixes are on `ds/calibrate` as separate commits. - -1. **`sqrt` inverse was `x^2`, not `pmax(x, 0)^2`.** The isotonic projection - runs on the sqrt scale, so a played vector with negative low quantiles is - monotone there but squaring folds the negative part back up and the - count-space quantiles cross. On the hub run this hit 578 of 20,393 forecast - sets in the single-term baseline and 1,708 at the operating point, almost - all at levels 0.01–0.05 in small geos at the trough. `quartic_root` already - clamped; `sqrt` now does the same, and `tests/testthat/test-calibrate.R` - exercises the negative regime. -2. **Gating was by reveal round, not issue round.** `update_from[t]` gated the - step *taken at* round `t`, but that step applies gradients of the rounds in - `delay[[t]]`, which at the first live round are the last `h + 2` burn-in - rounds (revealed in a burst after the off-season gap). So the live season - opened with several stale spring gradients, and `off_after` rounds played at - base were learned from the following October. Both masks are now indexed by - the round a forecast was issued at; burn-in and switched-off outcomes still - feed the eta pool but never step. - -Re-measured after both fixes (sqrt, mult 0.03, floor 1e-3, window 20, carry, -hub 2023-24 burn-in), WIS change vs base by horizon −1…3 and calibration error -range. The issue-round gating is what removes most of the h2/h3 loss of the -single-term tracker: the stale burst at season start was a large part of it. - -| variant | h−1 | h0 | h1 | h2 | h3 | cal err | -|---|---|---|---|---|---|---| -| sqrt, single term (was +8.0/+4.1/−0.4/−3.1/−3.4) | +8.0 | +4.5 | +0.8 | −0.6 | −1.4 | 0.022–0.043 | -| operating point: warm start + slow 0.003 + fast decay 0.1 (was +9.0/+5.3/+1.9/+0.8/+0.5) | +9.0 | +5.6 | +2.7 | +1.9 | +1.9 | 0.040–0.072 | - -Also in the same set of commits: `lr_geo_pool` removed (never beat baseline); -`lr_slow` aborts with a constant `lr` or per-level eta (it silently ignored its -own `mult` before); `off_after` and `fast_decay` abort when combined (the leak -would run through the off window, so "frozen offset" would be false); the -hub reader aborts on duplicate (round, level) rows; the accidentally committed -`covid_hosp_prod/workspaces/` binaries were dropped from history and the -pattern is now ignored. The older notebooks have not been re-rendered since; -`calibration_findings_flu.Rmd` has the post-fix numbers. - -### The ILI+ replay was broken (fixed 2026-09-23) - -Symptom: `windowed_seasonal`'s ILI+ pseudo-hub forecasts did not follow their -input. Log-log slope of median on truth was 0.40 at h0 (0.29 at h3). Indiana -on 2018-01-03 had a last observed value of 2.28 and an h0 median of 0.14. - -Cause: `adjust_latency = "extend_lags"` (the `default_args_list()` default) -computes one latency per column as `forecast_date - min over geos of that -geo's last time_value`, i.e. the *worst* state, and adds it to every lag. The -ILI+ archive has states whose series end years before the forecast date (DC -stops 2015-04-08; `hhs > 1e-4` also drops trailing zero weeks). On 2018-01-03 -the latency was 1001 days, so every predictor was about three years old. -Every round from 2011 on was affected, at every horizon, with latencies from -weeks to about nine years. The NHSN replay was not affected: all NHSN states -are current, and the ILI+/flusurv augmentation rows are in `keys_to_ignore`. -The same trap applies to any archive in which one geo stops reporting, NHSN -included (a single late state would stretch every state's lags). - -Fix: `ili_forecast_one()` passes the states whose last observation predates -the snapshot's latest week as `keys_to_ignore = list(list("geo_value", ...))`, -so they are left out of the latency check. They still contribute training -rows; they get no forecast that round (forecast locations per round are -unchanged: min 26, median 45, max 48). Indiana's h0 median is now 2.44. - -Checks on the rebuilt replay (all seasons): log-log slope 0.73/0.86/0.78/ -0.69/0.59 at h−1..h3. The overall slope is pulled down by the 43% of points -near the trough (truth under 10% of the location's 90th percentile, where -the zero clip and whitening flatten the median); above that the slope is -0.92/0.98/0.90/0.81/0.70. - -h−1 is not an echo of the last observed week, in this replay or in prod. -`filter_minus_one_ahead()` drops the week at `as_of + ahead`, because -otherwise the target column equals the extended lag-7 predictor and -`quantile_reg` freezes. With that week gone, the latency is 14 days and h−1 -is a one-week forecast from the week before. - -Still invalid, not re-run on the fixed replay: the ILI+ burn-in result above -and setup C in "One forecaster across eras" below. The bias on ILI+ is in -section 1.2 of the findings notebook: about level (43–46% of truth above the -median; summed level within ±4% at every horizon), with the same peak -shortfall at h3 as NHSN. So the "bias flips sign with the data source" -reading does not hold. On ILI+ the base is about level on average, not -biased high. - -### Current numbers (2026-09-23; superseded by the exact re-runs below) - -Re-run 2026-09-30 with exact learning truth (the prod re-run design) and -spoiled submissions excluded (`notes/spoiled-submissions.md`) in -`e03_scale.Rmd`, `e04_offset_structure.Rmd`, `e05_warm_start.Rmd`. WIS change -% (positive is better) and coverage bias, same setup (base -0.111/0.105/0.095/0.098/0.102): - -| variant | h−1 | h0 | h1 | h2 | h3 | cal err h−1…h3 | -|---|---|---|---|---|---|---| -| count space, single term | +3.2 | +1.9 | −2.0 | −2.7 | −5.3 | 0.044/0.030/0.032/0.034/0.032 | -| sqrt, single term | +5.1 | +4.0 | +0.7 | −1.3 | −1.4 | 0.049/0.032/0.026/0.028/0.026 | -| sqrt, leak 0.1 | +4.8 | +3.7 | +1.6 | +0.8 | +0.6 | 0.090/0.082/0.075/0.079/0.081 | -| warm start only | +0.5 | +1.3 | +0.9 | +1.2 | +2.1 | 0.098/0.075/0.062/0.065/0.062 | -| warm start + single term | +5.1 | +4.7 | +1.3 | −0.2 | +0.3 | 0.046/0.020/0.013/0.016/0.014 | -| warm start + leak 0.1 | +5.0 | +4.5 | +2.3 | +1.8 | +2.4 | 0.080/0.060/0.050/0.052/0.047 | -| operating point | +5.4 | +4.9 | +2.4 | +1.7 | +2.2 | 0.074/0.054/0.045/0.046/0.041 | - -The ordering of the variants is unchanged from the finalized-truth table. -Warm start + single term (not run before) is the best-calibrated variant, and -with REF-op it brackets the WIS/coverage trade-off. At h3, sqrt single term -loses more than count in October (−35% vs −27%) and March (−42% vs −26%); -the leak brings those to −12% and −14%. The finalized-truth numbers follow for -the record. - -From `reports/writeups/calibration/calibration_findings_flu.Rmd`: post-fix, -truth from `nhsn_read_truth()`, all 53 hub locations, the submitted ensemble, -sqrt unless stated, mult 0.03, floor 1e-3, window 20, carry, 2023-24 burn-in, -both live seasons pooled. WIS change % vs base, then calibration error -(base 0.118/0.105/0.095/0.098/0.102): - -| variant | h−1 | h0 | h1 | h2 | h3 | cal err h−1…h3 | -|---|---|---|---|---|---|---| -| count space, single term (the paper) | +4.2 | +2.4 | −0.8 | −3.4 | −5.6 | 0.033/0.020/0.025/0.027/0.029 | -| count space, leak 0.1 | +6.0 | +2.9 | +0.6 | +0.3 | −0.8 | 0.079/0.062/0.061/0.066/0.065 | -| sqrt, single term | +7.9 | +4.4 | +0.7 | −0.5 | −1.4 | 0.043/0.027/0.022/0.023/0.022 | -| sqrt, leak 0.1 | +8.1 | +4.4 | +2.0 | +1.1 | +0.5 | 0.089/0.078/0.073/0.078/0.080 | -| warm start only, no tracking | +0.4 | +1.1 | +0.8 | +1.1 | +1.9 | 0.105/0.076/0.063/0.066/0.062 | -| warm start + leak 0.1 | +8.3 | +5.1 | +2.6 | +2.0 | +2.1 | 0.080/0.057/0.049/0.050/0.046 | -| operating point (warm start + slow 0.003 + leak 0.1) | +8.9 | +5.4 | +2.6 | +1.9 | +1.9 | 0.073/0.051/0.042/0.044/0.040 | - -h3 WIS change % by month of the reference date, and each month's share of the -h3 base WIS: - -| variant | Oct | Nov | Dec | Jan | Feb | Mar | Apr | May | -|---|---|---|---|---|---|---|---|---| -| sqrt, single term | −48.4 | +6.5 | +1.9 | +0.6 | −1.9 | −47.3 | −36.2 | +20.1 | -| sqrt, leak 0.1 | −13.5 | +5.2 | +0.6 | +1.2 | +1.3 | −17.2 | +8.5 | +6.8 | -| warm start + leak 0.1 | −26.9 | +5.7 | +4.1 | +4.1 | +0.6 | −27.5 | −4.7 | +3.3 | -| operating point | −28.9 | +6.4 | +4.3 | +4.2 | 0.0 | −33.2 | −6.6 | +5.4 | -| share of base WIS % | 0.2 | 6.1 | 41.5 | 33.5 | 11.8 | 4.7 | 1.1 | 1.1 | - -Staleness, measured directly: correlate the played median shift at round `t` -with the needed shift (`sqrt(truth) − sqrt(base median)`) at round `t − k`, -per series, averaged over locations (single-term sqrt tracker). At `k = 0` the -correlation is negative from h0 up (−0.22/−0.25/−0.31/−0.35 at h0…h3): the -offset currently played pushes the wrong way on average. It peaks at -`k = 4/6/7/9` for h0…h3, i.e. the reveal lag `h + 2` plus two to four rounds of -build-up. By direction alone (share of >1% moves with the right sign), no -tracker beats the static warm start at h0–h3 (67–69%); the single-term count -tracker is worst (56–62%). - -Base bias on NHSN (share of forecasts with truth above the median, h−1…h3): -ensemble 69/65/63/63/63, `windowed_seasonal` 58/63/64/67/70; summed median vs -summed truth −11…−38% and −8…−46%. At h3 the bias concentrates at the peak -(median 68–69% low when truth is above the location's 90th-percentile level); -at h0 it is a fairly even 5–20% low. - -Count vs sqrt: the ratio of the relative median shift at the trough bin to -that at the peak bin is 32–38 in count space and 5–10 in sqrt. - -### One forecaster across eras (2026-09-17) - -Superseded by the rebuilt E07 (`e07_across_eras.Rmd`, 2026-10-01). Its -"less miscalibrated" explanation below is wrong at h0–h3. - -Question: are the hub-season gains an artifact of three seasons of one -ensemble, and does the tracker behave the same on a decade of the *same* -forecaster? `scripts/calibration/calibration_ws_backfill.R` replays flu prod's -`windowed_seasonal` (prod archive, seed, substitutions) over the NHSN seasons -in hub schema: 2024-25 and 2025-26 as honest as-of replays, 2023-24 with the -`"cheating"` policy (NHSN vintages start 2024-11-19). With the ILI+ pseudo-hub -(`scripts/calibration/calibration_ili_backfill.R`) this is one forecaster from 2011 to -2026. `scripts/calibration/calibration_ws_experiments.R` runs three setups, all sqrt, -mult 0.03, window 20, carry, on the 50 locations common to all sources and -on the hub's submission rounds only: - -- **A** submitted ensemble, burn-in 2023-24 (the reference). -- **B** `windowed_seasonal` on NHSN, burn-in 2023-24. -- **C** `windowed_seasonal` on the ILI+ decade (2011-12 burn-in, 2012-2020 and - 2022-23 tracked live) then NHSN 2023-26, whitened per location and era by - the 90th percentile of in-season truth (`scales`), offsets carried across - the source switch. **C is invalid**: its ILI+ half comes from the broken - replay (above). Its rows and the ILI+ numbers were removed. - -Variants: `single` (paper's tracker), `leaky` (`fast_decay = 0.1`), -`operating` (warm start + slow 0.003 + leaky fast). WIS change % vs base -(post-fix, hub target-data truth, 50 locations; the findings notebook re-runs -A and B on NHSN truth, with the same pattern): - -| setup / variant | 2024-25 h−1…h3 | 2025-26 h−1…h3 | -|---|---|---| -| A single | +11.7 / +2.6 / +1.1 / −0.1 / −1.3 | −2.6 / +2.0 / −2.1 / −3.4 / −2.8 | -| A operating | +11.2 / +3.8 / +2.6 / +2.5 / +3.8 | +0.4 / +3.9 / +0.2 / −0.5 / −0.7 | -| B single | +0.7 / +2.0 / +0.3 / −0.7 / −1.5 | −1.4 / +2.3 / −0.5 / −0.7 / −0.3 | -| B leaky | +1.4 / +2.1 / +0.8 / +0.3 / −0.3 | −0.3 / +1.7 / +0.4 / +0.8 / +1.3 | -| B operating | +2.1 / +2.7 / +1.1 / +0.7 / +0.8 | −0.1 / +2.3 / +0.4 / +0.5 / +0.6 | - -Calibration error (mean over levels; base first) for B: -2024-25 base 0.074–0.168, single 0.053–0.088, operating 0.048–0.093; -2025-26 base 0.044–0.088, single 0.007–0.016, operating 0.016–0.042. - -Fraction of truth above the base median (the sign of the bias): - -| base | 2023-24 h−1…h3 | 2024-25 | 2025-26 | -|---|---|---|---| -| ensemble | 0.56 / 0.55 / 0.55 / 0.53 / 0.52 | 0.83 / 0.74 / 0.73 / 0.74 / 0.76 | 0.58 / 0.63 / 0.59 / 0.60 / 0.60 | -| windowed_seasonal on NHSN | 0.55 / 0.54 / 0.58 / 0.62 / 0.68 | 0.59 / 0.68 / 0.68 / 0.72 / 0.76 | 0.58 / 0.66 / 0.65 / 0.65 / 0.65 | - -Reading: - -- **The low bias is not the ensemble's alone.** `windowed_seasonal` on NHSN - under-predicts in all three seasons (0.55–0.76), growing with horizon, so - there is something to correct; the gains are smaller than the ensemble's - (+0.4 to +2.7% at the operating point vs up to +11%) because the component - is less miscalibrated to begin with (base cal err 0.07–0.17 vs the - ensemble's 0.13–0.19 in 2024-25) and because the ensemble's h−1 is a - different, worse model (climate_linear). -- **On NHSN with one forecaster (B)** the single-term and operating variants - are close, with `operating` slightly ahead on WIS and `single` far ahead on - coverage in 2025-26; the leaky variant gives small, consistent WIS gains - with the weakest coverage gains. -- **Overfitting verdict (NHSN only).** The tracker is not an artifact of the - ensemble: it helps `windowed_seasonal` the same way, with smaller gains. - Whether the warm start transfers to a different data source is open (the - ILI+ evidence for "it does not" came from the broken replay). A prod design - should warm-start from the *current* forecaster's most recent NHSN season, - or use the leaky tracker with no warm start when that season is - unavailable. - -### Directions this points to, ranked - -1. **Split the offset into a slow level term and a fast tracker.** Done - (two-term tracker + conformal warm start, above); this is the current - operating point. Open: warm-starting from more than one season, and a - relative (log-space) slow term so the warm start transfers across the - scale swing more evenly than a sqrt-space constant does. -2. **Attack the staleness directly, not eta.** h−1 is free because its lag is - one round; h3 loses because its lag is five. Short-horizon coverage - indicators for the same target weeks are known ~4 rounds before h3's - own, and the base forecaster's bias direction is shared across horizons - (all horizons under-predict in the ramp). Feeding the fresh h−1/h0 - gradient into h2/h3's update (a cross-horizon proxy signal) is the only - idea on the list that changes *when* the offset turns rather than how fast. - The lagged-correlation measurement in "Current numbers" is the baseline to - beat: a fix should move the best-matching lag toward `k = 0`. -3. **Phase-gate updates on data, not the calendar.** The Feb-15 cutoff works - because both live seasons peaked around the turn of the year; a - late-peaking season would break it. Gating on the base median's trailing - trend (post-peak ⇒ stop updating, or shrink) expresses the same rule - without a hard-coded date. -4. **Scale-aware offsets via a smooth proxy**, if sqrt is not enough (the - log-space result is pre-fix and was not re-run). -5. **Eta variants** (seasonal, per-level, pooled): parked. None can fix a - wrong-signed step. - -## Production integration (2026-09-21) - -Calibrated forecasts are now threaded into both flu and covid production pipelines as a secondary submission, `CMU-TimeSeries-Calibrated`, leaving the primary `CMU-TimeSeries` ensemble unchanged. - -**Architecture:** three new targets added to each prod script, executed after the ensemble: - -- `calibrated_ensemble_nhsn` (`cue = tar_cue("always")`) — reads all past `CMU-TimeSeries` submissions from the hub checkout via `hub_read_forecasts()`, appends the current round's `ensemble_mix` output converted to hub schema via `internal_to_hub_forecasts()` if not already present, reads finalized truth, and calls `calibrate_hub_forecasts()`. - Skips (returns `NULL`) when `g_submission_directory == "cache"` (no hub checkout set). -- `make_calibrated_submission_csv` — writes the calibrated quantiles to `model-output/CMU-TimeSeries-Calibrated/-CMU-TimeSeries-Calibrated.csv` in the hub checkout. -- `local_calibrated_scores_nhsn` — scores the calibrated forecasts against `nhsn_latest_data` via `score_forecasts()`; fed into the ongoing score report alongside the base forecasters. - -**Operating point (flu):** `transform = "sqrt"`, `lr_args = list(mult = 0.03, floor = 1e-3)`, `lr_window = 20`, `season_policy = "carry"`, `slow_init = "burn_in_quantile"`, `lr_slow = list(mult = 0.003)`, `fast_decay = 0.1`, `burn_in_seasons = "2023-2024"`. -Flu gets the 2023-24 burn-in for free: the hub checkout already contains all past CMU-TimeSeries flu submissions. - -**Operating point (covid):** same except `burn_in_seasons = character(0)`, `slow_init = NULL` (no hub burn-in available; covid submissions begin 2024-11-23). - -**Notebooks:** `CMU-TimeSeries-Calibrated` is added to `ongoing_score_report.Rmd` (`our_forecasters`, score table styling, line width). -The per-date `forecast_report.Rmd` notebook receives calibrated forecasts for the current round's reference date, converted back to internal schema (state abbr, Wednesday `forecast_date`, count-space quantile values) via the `notebook` target in `prod_shared.R`. - -**What the covid submission actually is:** the submitted covid forecast is `ensemble_mix`, whose components are `windowed_seasonal`, `windowed_seasonal_extra_sources` and `revision_aware` (`pipelines/covid_hosp_prod.R`). Its raw weights come from `pipelines/covid_geo_exclusions.csv`, and `ensemble_weighted()` renormalizes them per (geo, ahead). The 2026-09-23 block sets `windowed_seasonal = 0.5`, `windowed_seasonal_extra_sources = 3` and `revision_aware = 1` at aheads −1 and 0 only. The effective mix is therefore: -- aheads 1–3: ~86% `windowed_seasonal_extra_sources`, ~14% `windowed_seasonal`; -- ahead 0: ~67% / ~11% / ~22% `revision_aware`; -- ahead −1: 100% `revision_aware`, because `drop_negative_aheads` strips the two AR components there. - -Before 2026-09-23 (all of E10's rounds), the mix had no `revision_aware`: ahead −1 was 100% the `climate_linear` ensemble, since the AR components were filtered out of negative aheads. - -`windowed_seasonal_extra_sources` excludes mo and wy, so those states fall back to `windowed_seasonal`. Blocks before 2026-09-16 used `windowed_seasonal = 0.05`, which made the split ~98% / ~2%. The `climate_linear` rows in the CSV are not `ensemble_mix` components. So covid forecast-quality work, calibration included, is mostly about `windowed_seasonal_extra_sources`. The CSV is edited by hand each week, so re-derive the weights before relying on these numbers. - -**Model metadata:** `CMU-TimeSeries-Calibrated.yml` created in both `../FluSight-forecast-hub/model-metadata/` and `../covid19-forecast-hub/model-metadata/` (`designated_model: false`). - -**Bug fixed en route:** `nssp_archive` in both `flu_data_targets.R` and `covid_data_targets.R` had a duplicate-key error (pre-existing since August 2026): `compactify = TRUE` can retain multiple rows per `(geo_value, time_value)` when the value changed across issues, and the subsequent `mutate(version = time_value + 7)` flattened all to the same version, producing duplicate keys in `as_epi_archive()`. -Fix: `group_by(geo_value, time_value) %>% slice_max(version, n = 1L, with_ties = FALSE) %>% ungroup()` before the version clobber. - -# Open threads - -Calibration-specific open items, roughly in priority order. Repo-wide items -(evaluation testbed, NSSP-target backtesting, revision-aware backtests) are -in `notes/ROADMAP.md`. - -1. **Warm start on the constant-rate tracker.** E11 (constant 0.1 per 100k, - rate scale, +1 week, re-run) beats the cold adaptive trackers on coverage - and is the most stable across seasons, but `warm + sqrt, adaptive` still - wins at h0–h1. Try E11's constant rate with - `slow_init = "burn_in_quantile"`. -2. **Calibrate h0 and above only.** Revisit h−1 calibration once prod's h−1 - component is chosen (`notes/ROADMAP.md`, revision-method backtests); the - harness h−1 history predates the 2026-09-23 switch to `revision_aware`. -3. **Covid per-horizon calibration** (calibrate h−1 and perhaps h0, pass - h1–h3 through; E10): untested. -4. **Re-run E07, E01 and E02 on vintages** (still marked stale in the - ledger). E03–E05 already use the re-run design. -5. **Re-run the ILI+ experiments on the fixed replay**: the ILI+ burn-in and - setup C (does a warm start learned on ILI+ transfer to NHSN?). -6. **Sweep `windowed_seasonal`, the harness proxy, for spoiled - submissions.** Needs a variant of the sweep script that reads the harness - forecasts (`notes/spoiled-submissions.md`). -7. **`windowed_seasonal_extra_sources` retrospective** via - `scripts/calibration/calibration_harness.R` (covid, evaluation store): does - calibration help our best component forecaster, not just the submitted - ensemble? -8. **Review the ranked gallery.** Does the WIS cost concentrate at turning - points (the `h + 2` staleness prediction)? Are the biggest offsets fixing - real miscalibration or chasing data-revision artifacts? -9. **Method extensions**, in the order argued under "Directions this points - to": slow relative level term + fast tracker; cross-horizon proxy - gradients for staleness; data-driven phase gating; scale proxy. Pooling - offsets across locations and per-level learning rates are lower priority. -10. **Speed up exact tracker runs**, if more exact sweeps are coming. An - exact config replays prod's weekly re-run, about 55 full tracker runs, - so cost grows with the square of the number of rounds (E09: about 72 - configs, about 1.5 h on 14 cores). - - Quick wins, all exact, about 2–2.5x: sort-based "data as of round t" - lookup instead of a grouped slice_max (about 4–5 s per round); - validate `arg_match` once outside the loop (about 15%); skip isoreg - when quantiles are already ordered (about 8%); plain vectors instead - of tibble/glue in the per-series loop (about 15%). Check bit-for-bit - against cached runs. - - Checkpointing, another 5–10x: start round t's run from round t−1's - tracker state at the first round whose data differ. Needs - `qt_track()` to save and resume offsets, the learning-rate window and - the slow term; test for exact equality with the full re-run. -11. **Send the collaborator email** (sweep findings; h−1 free win; carry vs - reset a wash). -12. **Agentic triage** of gallery panels against a fixed rubric (data - revision visible? base missed the turn? calibration helped/hurt?) if - manual skimming of the ranked gallery proves insufficient. -13. **Dynamic gallery app** (Shiny/Posit or a served page with dynamic data - fetch) if the static top-N format becomes limiting. diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index c4df841f..526a4f7c 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -2,14 +2,14 @@ Repo-wide tasks, TODOs and tech debt, roughly in priority order within each section. Calibration-specific threads live under "Open threads" in -`notes/CALIBRATION.md`; refactor designs (E2 ensemble sweep, snapshot +`notes/calibration-ledger.md`; refactor designs (E2 ensemble sweep, snapshot validator, simplification inventory) live in `notes/refactor-ideas.md`. ## Forecast evaluation 1. **Use the evaluation project as the calibration testbed** (parked 2026-10-01 as validation for after the operating point is chosen; the - operating-point work runs on the hub harness, `notes/CALIBRATION.md`). + operating-point work runs on the hub harness, `notes/calibration-ledger.md`). Done so far: a1–a4, b1–b4 (E12, E13). Calibrating needs full quantile forecasts over many rounds, not just scores. The evaluation project (`flu_hosp_evaluation` / `covid_hosp_evaluation`) replays the current prod components and @@ -55,7 +55,7 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. `scripts/calibration/calibration_harness.R` reads forecasts from the clean store (decision: read the store, backfill only for spec changes; the two paths match exactly on 5 dates per disease). - Results are E12 in `notes/CALIBRATION.md`. The two diseases' + Results are E12 in `notes/calibration-ledger.md`. The two diseases' replays differ on 2024-11-20: flu generates on 11-21, covid on 11-20 (NHSN's bad release; the hub-round filter drops that round). Decision (2026-10-01): calibrate h0 and up only. h−1 belongs to @@ -109,7 +109,7 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. - Done 2026-10-01 in the harness, not the pipeline: HHS back to 2020-08 (issues before 2023-07 collapse to one version), stitched before NHSN's first version; a 2024-12-04 control matches the - store exactly. Results: E13 in `notes/CALIBRATION.md`. + store exactly. Results: E13 in `notes/calibration-ledger.md`. c. *The ensemble with fixed weights.* The geo-exclusions CSVs carry the ensemble weights as well as exclusions, so "no hand edits" for `ensemble_mix` means one fixed weight block for every date. Decide @@ -190,7 +190,7 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. 3. **Revision-method backtests.** `revision_aware` against the `revision_ratio` baseline at h−1 (explore h−1 families plus `nowcast_notebook`, both diseases), then choose prod's h−1 component. - Calibrating h−1 waits on this (`notes/CALIBRATION.md`). + Calibrating h−1 waits on this (`notes/calibration-ledger.md`). 4. **Flu explore has no current scores for the revision families.** In the flu explore store on S3 (`joined_scores_2024_2026`, 2026-09-19), none of `revision_aware`, `revision_aware_augmented`, `revision_aware_nssp`, @@ -226,13 +226,13 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. ## Parked (2026-10-01, not on the calibration path) -- Covid `windowed_seasonal_extra_sources` calibration (CALIBRATION open - thread 7). It is ~86% of covid `ensemble_mix` but has no 2023-24 NSSP, so +- Covid `windowed_seasonal_extra_sources` calibration (calibration-ledger open + threads). It is ~86% of covid `ensemble_mix` but has no 2023-24 NSSP, so no burn-in. - CMU-climate_baseline US forecast is too low (item 2b). - `us_method` spec column and removing the `usa` alias (item 2b). - Calibration WIS summaries pool raw counts over all locations, so US is - about half of every "WIS change %" in CALIBRATION.md. b4 reports states + about half of every "WIS change %" in calibration-ledger.md. b4 reports states only as well; re-reading the older ledger waits. ## Tech debt diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index e29a23c5..84e7dbd8 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -1,109 +1,239 @@ -# Calibration experiment ledger - -Index of every calibration comparison: what was varied, against what, and -whether the result still stands. Findings and numbers live in -`notes/CALIBRATION.md`; this file only tracks the design of each comparison. - -Rule: each experiment names a reference config and varies one axis from it. -When an experiment has to move several axes (E09), it includes bridge rows -that step from the reference one axis at a time. - -## Axes - -| axis | values seen | reference value | -|---|---|---| -| disease | flu, covid | flu | -| base forecaster | submitted ensemble, `windowed_seasonal` | submitted ensemble | -| working scale | count, rate (per 100k), sqrt, log1p, quartic root | sqrt | -| learning truth | final (latest vintage), vintage (value at reveal; a stateful tracker), exact (prod's weekly re-run) | exact (E00 on; E01–E08 used final) | -| learning rate type | adaptive+ (`mult`, `floor`, `lr_window`), constant (0.001–3.2 per 100k, E09) | adaptive+ | -| `lr_mult` | 0.3, 0.1, 0.03, 0.01 | 0.03 | -| `lr_window` | 8, 20, 50, Inf | 20 | -| season policy | carry, reset | carry | -| burn-in | 2023-24 hub, ILI+, none | 2023-24 hub | -| `settle_days` (extra revision delay) | 7, 14, 21, 28 (= 0–3 weeks extra) | 14 | -| offset structure | single term, leak (`fast_decay`), late-season decay (`late_decay`), warm start, slow term | see REF-op | -| update gating | always, `off_after` date | always | -| eta variants | pooled, per-level, geo-pooled, seasonal window | pooled | - -Reference configs: - -- **REF-paper**: flu, ensemble, count, adaptive+ mult 0.03, window 20, - carry, 2023-24 burn-in, settle 14, single term. -- **REF-op** (current operating point, in prod): REF-paper with sqrt, floor - 1e-3, warm start `burn_in_quantile`, slow mult 0.003, `fast_decay` 0.1. - -Both are `cal_ref_args()` in `R/calibration/views.R`; notebooks run them -through `cal_run()`, which also picks the learning truth. Experiment notebooks -live in `reports/writeups/calibration_experiments/` and render to -`reports/calibration_experiments/` (with a reader-facing `index.html`) via -`just calibration-experiments [notebook ...]`. - -## Views - -Standard outputs an experiment can report. The goal is for every live config -to have all of them. - -| id | view | -|---|---| -| V-head | WIS % change and L1 coverage bias by horizon (× season) | -| V-month | WIS % change by month of reference date, with base-WIS share | -| V-state | per-state season panels with as-of vintages (a few states) | -| V-gallery | a few ranked forecasts (see the note below) | -| V-ae | absolute error of the median, change vs base | -| V-curve | coverage bias vs WIS curve over a swept parameter, per location + aggregate | - -V-gallery selection: ranking best/worst n by a single metric filled the old -gallery with small states at the end of the season. There, a mid-season -upward offset that no longer applies inflates the error on a tiny base. A -gallery should show only a few panels and pick them so that one regime -cannot take over. For example, weight the ranking by the forecast's share of -base WIS, and pick within strata (season phase × location size). +# Calibration ledger + +Post-hoc online calibration of our hub forecasts with MultiQT (Ding, Gibbs & +Tibshirani 2025). Each (location, horizon) series gets an additive offset over +the 23 quantile levels, updated by online gradient descent on pinball loss +and projected to be monotone. Repro commands are in +`notes/calibration-runbook.md`; dropped hub rounds are in +`notes/spoiled-submissions.md`. + +Every number below was re-read from the rendered notebook (or, for E12/E13, +the score CSV) on 2026-10-02. Older notes, pre-fix tables and the +summary-of-reviews layers were removed; they are in VCS history at change +`uxposooz` (`notes/CALIBRATION.md`, `notes/calibration-review.md`, +`notes/calibration-summary-2026-10-01.md`). + +## How to read the numbers + +- **WIS change %** is against the uncalibrated base; positive is better. + **Coverage bias** is mean |coverage − nominal| over levels; lower is + better. +- **There are no uncertainty estimates.** Two live seasons, 27 and 28 + rounds. This file calls a gap under 1 WIS point "no clear difference". + That is a reading convention, not a test. +- **Use month-avg coverage, not season-pooled.** Pooling over a season (or + two) lets ramp under-prediction cancel post-peak over-prediction, and lets + one season's error cancel the other's. Season-pooled gains of 2–5× mostly + vanish month by month. Month-level noise is about 0.01. +- **Report each season.** Most rankings flip between 2024-25 and 2025-26. + Pooled numbers hide that. +- **Cold starts dilute 2024-25.** Burn-in rounds warm the learning rate but + never step the offset, so every config without a warm start equals the + base through Nov–Dec 2024 (35–40% of that season's WIS). Warm-vs-cold in + 2024-25 is "some offset vs none". +- **Tuning is in-sample.** The constant rates (0.018, 0.032) and the + `off_after` dates were picked on the same two seasons they are scored on. +- **Two base forecasters.** E00, E03–E05 and E09–E11 calibrate the + submitted ensemble (h−1…h3, all locations in headline tables). E01, E02, + E07 and E12–E18 calibrate the clean `windowed_seasonal` replay (h0–h3, + states only, except E12). + +## What the numbers support + +Large and consistent (several points, both seasons): + +- **Too-large steps hurt.** Adaptive multiplier 0.1 on sqrt costs 7–17 WIS + points at h1–h3, 0.3 costs 17–81 (E01). A constant per-100k rate above + about 0.1 loses steeply: −3 to −4 at h3 by 0.18 (E14), −12 to −23 at 1.0 + (E09). +- **sqrt beats count.** 2–5 points at h1–h3 on the clean replay (E01), + 1.4–3.9 pooled on the ensemble, mostly from 2025-26 (E03). +- **h−1 is where calibration pays.** Covid +10 to +11% WIS and about +20% + median absolute error, both seasons, either config (E10); flu ensemble +3 + to +6 (E00, E03). +- **Stateful learning needs a delay.** A tracker that learns once from the + first report loses all of REF-op's h−1 gain (0.0 vs +6.1) and makes h−1 + coverage worse than base; one extra week fixes it (E00, E09). Prod re-runs + from scratch each week, so this does not apply to prod. +- **Some covid configs are harmful.** REF-paper loses 10–19 points at h1–h3 + in 2025-26 (E10); constant 0.1 with a warm start loses 8–27 (E13). + +Small, or dependent on season (0–3 points): + +- **Among reasonable trackers nothing clearly wins.** REF-op (warm or cold), + sqrt constant 0.018, rate constant 0.018–0.032 and sqrt adaptive 0.01 are + within about 1 point of each other pooled, and the order flips by season. + Configs with a warm start lead in 2024-25; the others lead in 2025-26 + (E07, E16, E18). +- **Warm start: first live season only.** +1 to +3 points at h2–h3 in + 2024-25, −0.7 to +0.4 in 2025-26 (E05, E13, E18). Warm configs under-cover + the 50% interval (0.39–0.44, E13). +- **Leak (`fast_decay` 0.1): season-dependent.** On the ensemble it avoids + a 3–5 point loss at h1–h3 in 2025-26 and costs 0.4–1.3 in 2024-25 (E04, + E05). On the clean replay it adds 0.5–1.3 at h1–h3 in both seasons and + costs 0.4 at h0 (E18). +- **Switching off from 1 Mar or 15 Feb.** On the clean replay: +2 to +4 at + h1–h3 over sqrt adaptive 0.03 (E02), +0.4 to +0.7 over sqrt constant 0.018 + (E17). The date was chosen after seeing the March losses. About half of + the 2025-26 gain at h2 is offsets frozen in February 2025 carried into + December, not a spring fix. On the ensemble it is the weakest candidate in + 2024-25 (E07). +- **Per-level eta:** about +1 point at h1–h3 (E02). +- **March costs every tracker.** Up to −28% WIS within March at h2, about 1 + point of the season (E16). The base already over-predicts after the peak + and the offsets are still positive. +- **The correction is a small upward shift.** Offsets are a few percent of + the 90% interval width; the width ratio is 1.00–1.04. REF-op's gain comes + from raising the lower half; the upper half loses beyond h0 (E15). + +Coverage: + +- **Month-avg, no flu config moves coverage much.** Every config is within + about 0.025 of base, most within 0.01. REF-op and the warm configs on the + ensemble are 0.012–0.025 better than base at h1–h3 (E05, E09). Constant + rates are no better than base, or slightly worse at h3 (E09, E14). +- **Season-pooled coverage gains are cancellation.** For example, E11's + "4–5× lower bias" is about 2× per season, and nothing month by month. + +Operating point: nothing tested clearly beats the prod config (REF-op). +Choosing between the candidates above needs uncertainty estimates and more +seasons, not more single-run comparisons. + +## Reference configs + +- **REF-paper**: count scale, adaptive eta (mult 0.03, window 20), carry, + 2023-24 burn-in, `settle_days` 14 (one extra week of revision delay), + single offset term. +- **REF-op** (prod): REF-paper with sqrt, eta floor 1e-3, warm start + `slow_init = "burn_in_quantile"`, slow term mult 0.003, leak + `fast_decay = 0.1`. "REF-op cold" drops the warm start and burn-in. + +Both are `cal_ref_args()` in `R/calibration/views.R`; notebooks run them via +`cal_run()`, which also picks the learning truth (`"exact"`: re-run from +scratch each round on that round's data, as prod does). Notebooks live in +`reports/writeups/calibration_experiments/` and render to +`reports/calibration_experiments/` with `just calibration-experiments`. + +Standard views (`_standard_views.Rmd`, `_month_view.Rmd`): V-head (WIS and +coverage by horizon × season), V-month (by month of reference date, with +month-avg coverage), V-state (per-state season panels), V-gallery (a few +forecasts picked by `cal_gallery_pick()`, stratified by season phase × +location size), V-ae (median absolute error), V-curve (coverage vs WIS over a +swept parameter). ## Experiments -"Code" is pre-fix or post-fix relative to the 2026-09-17 correctness fixes -(sqrt clamp, issue-round gating). - -| id | question | varies | reference | disease | code | truth | views | where | status | -|---|---|---|---|---|---|---|---|---|---| -| E00 | how much did final truth flatter results; can vintage stand in for exact? | learning truth {final, vintage, exact} × settle {7, 14} | REF-paper, REF-op | flu | post-fix | all three | V-head | `e00_vintage_backtest.Rmd` | done: exact is the default; vintage is not a stand-in | -| E01 | which eta settings? | season policy × window × mult | REF-paper | flu | pre-fix | final | V-head, V-curve (mult) | `calibration_qt_flu.Rmd` | superseded by `e01_eta_settings.Rmd` (2026-10-01) | -| E02 | cutoff and eta variants | `off_after`, per-level, geo-pool, seasonal window (one at a time) | REF-paper | flu | pre-fix | final | V-head, V-month, V-state | `calibration_qt_seasons_flu.Rmd` | superseded by `e02_eta_variants.Rmd` (2026-10-01) | -| E03 | which working scale? | count vs sqrt vs rate (log1p, quartic pre-fix only) | REF-paper | flu | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e03_scale.Rmd` | done | -| E04 | leak / two-term offset | single, leak 0.1, slow term, slow + leak | REF-paper at sqrt | flu | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e04_offset_structure.Rmd` | done | -| E05 | warm start from burn-in | warm vs cold, paired, for single / leak / slow + leak; warm alone | REF-paper at sqrt | flu | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e05_warm_start.Rmd` | done | -| E06 | ILI+ as burn-in | burn-in source | REF-op | flu | post-fix | final | V-head | notes only | invalid (broken replay); superseded by E13 (HHS burn-in with real vintages); not rebuilt | -| E07 | one forecaster across eras | base forecaster (A ensemble / B `windowed_seasonal`; C invalid) | REF-paper at sqrt | flu | post-fix | final | V-head | `calibration_ws_experiments.R`, findings notebook | superseded by `e07_across_eras.Rmd` (2026-10-01) | -| E08 | covid | disease | old REF-paper (count, window 50) | covid | pre-fix | final | V-head, V-state, V-gallery | `calibration_qt_*_covid.Rmd` | superseded by E10; not rebuilt | -| E09 | constant lr × revision delay | constant lr grid × `settle_days` {7, 14, 21, 28}; bridge rows below | REF-paper | flu | post-fix | exact (+ vintage, for a stateful tracker) | V-curve, V-head | `e09_lr_delay.Rmd` | done | -| E10 | covid at the references | disease (REF-paper, REF-op without warm start as in covid prod) | REF-paper, REF-op | covid | post-fix | exact | V-head, V-ae, V-month, V-state, V-gallery | `e10_covid.Rmd` | done | -| E11 | constant vs adaptive lr | rate constant 0.1 vs sqrt adaptive, warm + sqrt adaptive | REF-paper | flu | post-fix | exact | V-head, V-month | `e11_constant_lr.Rmd` | done | -| E12 | calibration on the clean `windowed_seasonal` replay, no burn-in | E11 constant 0.1, REF-op cold | E11, REF-op | flu, covid | post-fix | exact | V-head | `calibration_ws_replay.R` | done | -| E13 | 2023-24 HHS burn-in on the clean replay | burn-in none vs 2023-24 HHS | E12 configs, E05 single | flu, covid | post-fix | exact | V-head (all / states / US) | `calibration_ws_replay.R burn_in` | done | -| E14 | constant lr by season (clean replay) | constant lr grid (rate scale) × season | E11 | flu | post-fix | exact | V-curve, V-head × season | `e14_lr_by_season.Rmd` | done | -| E15 | where the WIS gains come from | diagnostic: per-level pinball, offset/spread, median-only vs width-only | REF-op cold, E11 cold | flu | post-fix | exact | per level | `e15_wis_sources.Rmd` | done | -| E16 | constant lr on the sqrt scale | scale (sqrt vs rate) × constant lr grid | E14 | flu | post-fix | exact | V-curve, V-head, V-month | `e16_sqrt_constant_lr.Rmd` | done | -| E17 | late-season offset handling | none, `late_decay` 0.85/0.7/0.5 from 03-01, `off_after` 03-01 | rate 0.018, rate 0.032, sqrt 0.018 | flu | post-fix | exact | V-head, V-month | `e17_late_decay.Rmd` | done | -| E01 (rebuilt) | which adaptive-eta settings? | mult {0.3, 0.1, 0.03, 0.01} × window {8, 20, Inf} × season policy; count bridge | REF-paper at sqrt, cold | flu (clean replay) | post-fix | exact | V-curve, V-head | `e01_eta_settings.Rmd` | done | -| E02 (rebuilt) | cutoff and eta variants | `off_after` {04-01, 03-01, 02-15}, per-level eta, seasonal window carry/reset (HHS burn-in) + burn-in bridge; geo-pool removed | E01 center (sqrt, 0.03, 20, carry) | flu (clean replay) | post-fix | exact | V-head, V-month | `e02_eta_variants.Rmd` | done | -| E07 (rebuilt) | one forecaster across eras | forecaster (A hub ensemble / B clean replay) on identical rows, each with its own 2023-24 burn-in | single, leaky, REF-op | flu | post-fix | exact | V-head, base bias | `e07_across_eras.Rmd` | done | -| E18 | which piece of REF-op does the work | bridge, one setting per step: −warm start, −slow term, −leak, −burn-in season, per-100k units, constant 0.018 | REF-op → E16 sqrt 0.018 | flu (clean replay) | post-fix | exact | V-head, V-month | `e18_ref_op_bridge.Rmd` | done | - -E09 bridge rows, each one axis from the previous: REF-paper → rate scale → -constant lr (sweep) → `settle_days` (sweep). - -Diagnostics that are analyses rather than configs (all flu, finalized truth): -year-over-year seasonal offset viability, staleness lagged correlation, base -bias on NHSN and ILI+. See `notes/CALIBRATION.md`. - -## Gaps - -- **Spoiled submissions.** From 2026-09-30, hub reads drop the submissions - listed in `notes/spoiled-submissions.md` (flu: 2025-11-22 h−1, three halved - US rounds; covid: the 2024-11-23 round, 2026-04-25 h−1). E00, E03–E05, E09 - and E10 are re-run without them; every older result, and the numbers in - `notes/CALIBRATION.md` before that date, include them. -- **Learning truth.** E06 and E08 learned from finalized NHSN (E00 shows - the error is small outside h−1); both are superseded, not re-run. Every - other experiment is exact. +All flu unless stated; all post-fix (2026-09-17 sqrt clamp and issue-round +gating), exact learning truth, spoiled submissions excluded. + +| id | notebook | base | question | what the numbers show | caveats | +|---|---|---|---|---|---| +| E00 | `e00_vintage_backtest` | ensemble | learn from final, vintage or exact data? | exact ≈ final within 0.6 points except REF-op h−1 (2.2, 2024-25 only). Vintage at settle 7 loses the h−1 gain. Settle 7 vs 14: REF-paper +1 to +3 at every horizon; REF-op tie (flips by season) | | +| E01 | `e01_eta_settings` | replay, cold | adaptive eta settings | mult ≥ 0.1 costs 7–81 at h1–h3. 0.01 beats 0.03 by 0.5–2.4 at h1–h3 in both seasons; 0.03 better at h0 in 2024-25 (0.8). Window and carry vs reset: ≤ 2.4, mostly < 1. Count is 2–5 worse than sqrt | month-avg coverage of 0.01 vs 0.03 within noise | +| E02 | `e02_eta_variants` | replay, cold | `off_after`, per-level eta, seasonal window | off 1 Mar / 15 Feb +2 to +4 at h1–h3 over the 0.03 reference, both seasons, no month-avg coverage cost. Per-level +1. Seasonal window, burn-in, off 1 Apr: < 1 | dates in-sample; half the 2025-26 gain is December carry-over; "seasonal window" also changes `lr_window` 20 → 10 | +| E03 | `e03_scale` | ensemble | count vs sqrt vs rate | sqrt +1.4 to +3.9 over count pooled, mostly 2025-26. Coverage: neither better (sqrt worse per season pooled, better month-avg only in 2024-25) | rate identical to count in every cell (see Known issues) | +| E04 | `e04_offset_structure` | ensemble | leak, slow term | leak +2.5 to +5.6 at h1–h3 in 2025-26, −0.4 to −1.3 in 2024-25; month-avg coverage flips the same way (±0.015). Slow term without warm start: ≤ 0.5 | | +| E05 | `e05_warm_start` | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | +| E07 | `e07_across_eras` | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | month-view absolute table shows A's base for B rows (bug) | +| E09 | `e09_lr_delay` | ensemble | constant rate × revision delay | knee 0.056: +3 to +5.5 over REF-paper at h1–h3; REF-op +1.2 to +2.4 over knee. Extra delay costs about 1 h−1 point per week. Above 0.1 loses steeply | month-avg: no constant rate beats base at h1–h3 | +| E10 | `e10_covid` | covid ensemble | REF-paper and REF-op on covid | h−1 +10 to +11. REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 at h1–h3 in 2025-26 | headline and month tables include Aug–Sep 2026 rounds (see Known issues) | +| E11 | `e11_constant_lr` | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | +| E12 | none (`calibration_ws_replay.R`) | replay, cold, flu + covid | first look at the clean replay | flu: REF-op cold leads constant 0.1 by 0–2.5. Covid: REF-op cold h0 +1.3 states, losses at h1–h3 in 2025-26 | **all locations** (US is 47% of WIS); covid h0 gains are mostly US | +| E13 | none (`calibration_ws_replay.R burn_in`) | replay, flu + covid | 2023-24 HHS burn-in | flu warm +1 to +2.6 in 2024-25, ≤ 0 in 2025-26. Covid: warm worse at h0–h1, better at h2–h3 (+1.3, +4.4); constant 0.1 warm −8 to −27 | HHS used as-is (3–9% below NHSN); coverage is season-pooled only | +| E14 | `e14_lr_by_season` | replay, cold | constant rate grid by season | every rate 0.0032–0.056 is ≥ 0 in both seasons and within 1.2 of each other; 0.018 vs 0.032 ≤ 0.3. Above 0.1 loses | month-avg: no rate ≤ 0.032 beats base beyond 0.01 | +| E15 | `e15_wis_sources` | replay, cold | where the WIS gain comes from | offsets a few % of interval width; REF-op gain from the lower half and the median shift | width-only variant's coverage not measured; split is not additive | +| E16 | `e16_sqrt_constant_lr` | replay, cold | constant rate on sqrt scale | sqrt 0.018 about +1 over rate constants at h0; at h1–h3 within 0.2 pooled, up to +0.7 in 2025-26. vs REF-op cold: behind in 2024-25 h0–h1, ahead elsewhere; month-avg coverage tie | 0.018 picked in-sample | +| E17 | `e17_late_decay` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | +| E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (see Known issues) | + +E06 (ILI+ burn-in) and E08 (old covid) were run on code or data later found +broken and were not rebuilt. + +## Known issues + +Things that could change the numbers above; none checked yet. + +- **Run cache ignores code.** `cal_run_cached()` keys on inputs and + arguments, not code. Some cached runs date from 2026-09-30 before the + exact-learning commit landed that evening. The E00 and E09 caches are + keyed by chunk code or string and must be cleared by hand. +- **E10 includes a third season.** Headline and month tables pool every + round to 2026-09-19, so Aug–Sep 2026 (season 2026-27) is in them. +- **`scales` may be ignored.** `rate` equals `count` to every digit in E03 + and E09 (adaptive), and E18 steps 4 and 5 are identical. Scale invariance + of the adaptive rate predicts this, but nothing checks that `scales` was + applied. +- **Month-view base column.** `_month_view.Rmd` builds `mv_abs_base` with + `slice(1)`, so in E07 the absolute table shows the ensemble's base for the + replay's rows. Plots are fine. +- **`late_decay` doc mismatch.** The roxygen in `calibrate.R` says the + offset is shrunk before it is played and learning continues; the code + shrinks the stored state. +- **Stale notebook text.** E00, E05, E10 Findings say "render of 30 + September" (numbers match the current render); E18 says 1 October. +- **Stale runbook.** `notes/calibration-runbook.md` has old paths + (`~/allHail/...`, `scripts/reports/...`) and doesn't mention the + experiment notebooks or `calibration_ws_replay.R`. + +## Review status + +| notebook | status | +|---|---| +| E00, E01 | verified | +| E02, E07, E09 | commented; changes made, not re-verified | +| E18 | changes made (reference rows), not reviewed | +| E03, E04, E05, E10, E11, E14, E15, E16, E17 | not reviewed | + +## Code, data and prod + +- `R/calibration/qt.R`: the tracker (`qt_track`, `qt_learning_rate`, + `qt_project`). Bit-exact against the authors' Python on the + `delphi-fixes` branch of `~/repos/delphi/multiQT`; oracle fixtures in + `tests/testthat/test-qt.R`. +- `R/calibration/calibrate.R`: `calibrate_hub_forecasts()` and + `calibrate_hub_forecasts_exact()`, plus metrics. Options beyond the + paper: `transform`, `scales`, `lr_slow` / `fast_decay` / + `burn_in_learns_slow` / `slow_init` (two-term offset and warm start), + `off_after`, `late_decay`, `lr_seasonal`. +- `R/calibration/hub_data.R`: hub readers (`hub_read_forecasts()` drops + spoiled submissions), `nhsn_read_truth()`. +- `R/calibration/views.R`: reference configs, cached runs, standard views. +- `scripts/calibration/calibration_ws_replay.R`: E12/E13. Scores in + `cache/calibration/ws_replay_scores_*.csv`. +- `reports/writeups/calibration/`: the pre-experiment notebooks (findings, + sweep, seasons and gallery, flu and covid). Nothing in code or pipelines + uses them; last rendered 2026-09-23, before exact learning truth. + Candidates for deletion. +- **Data.** Flu hub submissions 2023-10-14 … 2026-05-30, 53 locations, + h−1…h3. 2023-24 is burn-in. NHSN vintages from + `get_nhsn_data_archive()` start 2024-11-19. With `settle_days` 14 the h3 + offset is always 5 rounds stale. +- **Prod (2026-09-21).** Flu and covid prod write a secondary submission, + `CMU-TimeSeries-Calibrated`, via the targets `calibrated_ensemble_nhsn`, + `make_calibrated_submission_csv` and `local_calibrated_scores_nhsn`. Flu + runs REF-op with the 2023-24 burn-in; covid runs REF-op without burn-in + or warm start. The covid submission is mostly + `windowed_seasonal_extra_sources` at h1–h3 and `revision_aware` at h−1 + (weights in `pipelines/covid_geo_exclusions.csv`, edited weekly). + +## Open threads + +Checked against the code on 2026-10-02. Repo-wide items are in +`notes/ROADMAP.md`. + +| item | status | +|---|---| +| Uncertainty for WIS and coverage differences (bootstrap over rounds or locations) | open; needed before any ranking under ~2 points means anything | +| Fix the Known issues above and re-render | open | +| Report ramp and post-peak months separately | open; only the month view exists | +| sqrt constant 0.018 with a warm start | open | +| Shrink only the played offset after the peak, keep the stored state | open; `late_decay` shrinks the stored state | +| Covid with sqrt constant; covid at h−1/h0 only | open; "h0 only" can be read from existing columns | +| h−1 calibration | blocked on the revision-method backtests (ROADMAP 3) | +| HHS burn-in level (3–9% below NHSN) | open; E13 used it as-is | +| ILI+ burn-in on the fixed replay | open; replay rebuilt 2026-09-23, nothing re-run | +| `windowed_seasonal_extra_sources` retrospective | parked (ROADMAP) | +| Gallery review (do losses sit at turning points?) | stratified picker exists; review not done | +| Method ideas: cross-horizon gradients for staleness, data-driven phase gating, scale proxy | open; no code | +| Speed up exact runs (sort-based snapshot lookup, skip isoreg when ordered; checkpoint/resume in `qt_track()`) | open; no resumable state yet | +| Collaborator email | unknown | diff --git a/notes/calibration-review.md b/notes/calibration-review.md deleted file mode 100644 index c68ee66b..00000000 --- a/notes/calibration-review.md +++ /dev/null @@ -1,98 +0,0 @@ -# Calibration review - -Tracks which experiment notebooks Dmitry has read and verified, his -comments, and what was done about them. Notebooks render to -`reports/calibration_experiments/`; sources are in -`reports/writeups/calibration_experiments/`. - -Status: **verified** (read and agreed), **commented** (read, changes asked -for), **not reviewed**. - -## How the experiments build on each other - -1. **E00: learn from data as published.** Finalized data flatters the h−1 - and h0 gains, though not by much. Every experiment from E00 on learns - from each week's published data (the "exact" design), with no further - work needed. -2. **E07: one fixed forecaster is enough.** On identical rows, calibration - gains are similar on the submitted ensemble and on `windowed_seasonal`, - and the trackers rank the same way. So experiments can focus on the - clean `windowed_seasonal` replay, which is free of hand edits and - spoiled submissions. Caveat: the tracker ranking transfers, but a - date-based switch-off (`off_after`) does not; it depends on each - forecaster's spring bias (last on the ensemble in 2024-25, second on - `windowed_seasonal`). -3. Experiments before E07's re-run (E03–E05, E09–E11) used the submitted - ensemble; E01, E02 (new versions) and E12–E18 use the clean replay. -4. **Anything that acts mainly after the peak depends on the forecaster.** - The knee constant rate looks good after the 2024-25 peak in E09 because - the ensemble was still under-predicting that spring; on - `windowed_seasonal`, which over-predicts after the peak, the same - offsets cost WIS and coverage (E18). Same for switching off in March - (E07). - -## Month view (added 2026-10-01) - -Every notebook now has a "Month view: WIS, coverage, median error" section -(WIS reduction %, within-month L1 coverage bias, median absolute-error -reduction %, per month and horizon, states only; all three rows read -"positive = better than the uncalibrated base", with absolute monthly -coverage bias in a table under the plots) and a month-average -coverage column next to its season-pooled one. Season-pooled coverage -claims mostly don't survive: where one changed, the notebook's Findings end -with a "**Month view (2026-10-01):**" line. Verdicts: - -| notebook | coverage claim | verdict | -|---|---|---| -| E00 | exact ≈ final; vintage +0 weeks worse at h−1 | holds | -| E01 | 0.03 is the coverage elbow; carry beats reset | reversed: 0.01 lowest month by month; carry ≈ reset | -| E02 | switching off costs coverage | weakened: mostly an artifact; better at h3 | -| E02 | per-level eta, seasonal window coverage | reversed (small differences) | -| E03 | sqrt ≈ count on coverage | holds; sqrt better at h2–h3 | -| E04 | the leak nearly triples coverage bias | reversed: no difference month by month | -| E05 | warm + single best calibrated; warm start cuts bias a third | reversed / weakened: REF-op lowest; warm start −5 to −15% | -| E07 | coverage ranks the same on A and B | reversed: rankings differ by month | -| E09 | knee beats REF-paper on coverage | reversed: REF-op lowest; knee worse than base at h1–h3 | -| E10 | covid base already calibrated at h1–h3 | weakened | -| E11 | constant rate better coverage everywhere | reversed: worst of five, worse than base at h1–h3 | -| E14 | coverage keeps improving with lr | reversed: minimum near 0.01–0.032; ≥0.056 worse than base at h1–h3 | -| E15 | REF-op width changes buy coverage | holds, smaller | -| E16 | sqrt 0.018 ≈ rate 0.032 on coverage | reversed: sqrt better; rate worse than base in 2025-26 | -| E17 | decay / off-after give back coverage | weakened: small effect month by month | -| E18 | warm start, leak, constant rate move coverage | reversed: no step moves month-avg coverage by more than 0.008 | - -Differences under about 0.01 are within the month-level noise floor. - -## Notebooks - -| exp | forecaster | status | comments | done | -|---|---|---|---|---| -| E00 vintage backtest | ensemble | verified | Finalized data exaggerates h−1/h0 gains a little; use vintages, no more work. | — | -| E01 eta settings | clean replay | verified | Recap of the August sweep: carry slightly better, window 20 as good as larger, the multiplier has an elbow. Note: on this replay 0.03 is the coverage elbow and 0.01 the WIS-safe choice. Earlier: (1) Curve axis looked flipped vs E09. (2) x/o markers unlabeled. (3) "Rebuilds the original" is misleading: it re-asks the question on a new setup. | (1) E09 flipped to match; (2) legend added; (3) intro reworded with a what-changed table | -| E02 eta variants | clean replay | commented | Headline: switching off in Feb/Mar recovers WIS, loses coverage. (1) Coverage numbers hard to find in the WIS / bias cells. (2) Month plot: unclear whether positive % is good; the sign convention must be labeled explicitly in every notebook. | intro reworded; (1) tables split into WIS and coverage, plus a WIS vs coverage scatter; (2) every notebook now labels "WIS reduction % (positive = calibrated WIS lower than base)" and "L1 coverage bias (lower is better)" (all computations already used that sign). Month view settled the open question: the switch-off's coverage cost is mostly an artifact of season pooling (better at h3 month by month). | -| E03 scale | ensemble | not reviewed | | | -| E04 offset structure | ensemble | not reviewed | | | -| E05 warm start | ensemble | not reviewed | | | -| E07 across eras | both | commented | Needs concrete base WIS numbers, not "similar". Expand into the ensemble vs `windowed_seasonal` comparison: base WIS side by side, base bias by month, every operating-point candidate on both. | intro reworded; expanded with base WIS side by side (B/A 0.93–1.01 in 2024-25, 1.05–1.33 in 2025-26), base bias by month, and five candidates on both forecasters | -| E09 lr × delay | ensemble | commented | (1) WIS axis inverted relative to the other notebooks. (2) Month view: dashed base coverage line hard to see. (3) Why does the knee look so good in 2024-25? | (1) flipped; now "WIS reduction % (positive = calibrated WIS lower than base)"; (2) coverage row is now "bias reduction % vs base" in every notebook, absolute values in a table; (3) its post-peak gains come from the ensemble's spring 2024-25 under-prediction, and its ramp months are flat; on `windowed_seasonal` it is the weakest candidate (E18) | -| E10 covid | ensemble | not reviewed | | | -| E11 constant lr | ensemble | not reviewed | | | -| E14 lr by season | clean replay | not reviewed | | | -| E15 WIS sources | clean replay | not reviewed | | | -| E16 sqrt constant lr | clean replay | not reviewed | | | -| E17 late decay | clean replay | not reviewed | | | -| E18 REF-op bridge | clean replay | not reviewed | | added rate 0.032 (E14) and 0.056 (E09 knee) as reference rows; step 6 labelled "sqrt constant 0.018 (E16)" | - -E12 and E13 have no notebooks; they live in `notes/CALIBRATION.md` and -`scripts/calibration/calibration_ws_replay.R`. E06 and E08 were superseded -(by E13 and E10) and not rebuilt. - -## Queued (not started) - -- Headline split: report the ramp (up to the peak) and post-peak months - separately, instead of season-pooled or equally weighted month averages. -- sqrt constant 0.018 with a warm start: the one REF-op piece it lacks (E18). -- Possibly widen the month plot's clip on the coverage row (its ceiling is - +100%), if the +40% saturation hides differences. - -The summary for sharing is `notes/calibration-summary-2026-10-01.md`. diff --git a/notes/calibration-runbook.md b/notes/calibration-runbook.md index 2d35081f..888d83fa 100644 --- a/notes/calibration-runbook.md +++ b/notes/calibration-runbook.md @@ -173,7 +173,7 @@ It renders the settings sweep itself, so it takes ~5 min (32 full calibration ru ## 5. The gallery notebook Fixed operating point (`lr_mult = 0.03`, `lr_window = 20`, carry) plus a -ranked per-forecast panel gallery; see `notes/CALIBRATION.md` for the design. +ranked per-forecast panel gallery; see `notes/calibration-ledger.md` for the design. ```sh cd ~/allHail/delphi/exploration-tooling @@ -183,7 +183,7 @@ distrobox enter rocker -- Rscript -e \ As-of vintages come from `cache/calibration/nhsn_archive_flu.parquet` (a copy of an oracle-capture NHSN archive; provenance and staleness caveats in -`notes/CALIBRATION.md`). `n_panels` is a YAML param (default 150). +`notes/calibration-ledger.md`). `n_panels` is a YAML param (default 150). ## 6. The seasons notebook @@ -205,7 +205,7 @@ Output: `reports/calibration_qt_seasons.html`. Runs flu prod's `windowed_seasonal` forecaster over the ILI+ state history and caches hub-schema forecasts and truth for multi-season calibration burn-in -(design and caveats in `notes/CALIBRATION.md`, "ILI+ burn-in"). +(the `scales` option of `calibrate_hub_forecasts()` handles the scale mismatch). ```sh cd ~/allHail/delphi/exploration-tooling diff --git a/notes/calibration-summary-2026-10-01.md b/notes/calibration-summary-2026-10-01.md deleted file mode 100644 index 8337e2ba..00000000 --- a/notes/calibration-summary-2026-10-01.md +++ /dev/null @@ -1,186 +0,0 @@ -# Calibration: where the experiments stand (2026-10-01) - -A summary of E01, E02, E07 and E11–E17. Each notebook is in -`reports/calibration_experiments/` (`index.html` lists them all). - -## How these were run - -All of these are flu only, except E12 and E13, which cover covid as well. - -- **Forecaster.** Every experiment except E11 calibrates one fixed forecaster, - `windowed_seasonal`. It was replayed weekly from 2024-11-20 with none of - the hand edits we make to prod data (the "clean replay"). The submitted - ensemble changes weights and inputs by hand week to week, and some of its - rounds are known to be spoiled. A fixed forecaster removes both from the - comparison. E11 used the submitted ensemble, as E00–E10 did. -- **Learning truth.** The tracker learns from data as it was published each - week, and is re-run from scratch each week, as prod would run it. Scores - use finalized data. -- **Rounds and seasons.** The 55 hub rounds, 27 in 2024-25 and 28 in - 2025-26. Results are shown per season, because a setting can look good - pooled and still fail in one season. -- **Horizons.** h0–h3 only. h−1 is left to the revision-aware methods. -- **States only.** US is excluded from the headline numbers. WIS is in - counts, so US alone is about 45% of the all-locations total. The - all-locations numbers point the same way throughout. -- **Cold start.** The tracker starts from zero offsets in November 2024 - unless a burn-in is stated. - -"WIS change" is % against the uncalibrated forecast, positive is better. -"Coverage bias" is the mean |coverage − nominal| over quantile levels. - -## Read this first: coverage claims and the month view - -Coverage bias pooled over a whole season lets under-prediction on the ramp -cancel over-prediction after the peak. Scored within each month and then -averaged over months, the coverage gains from calibration mostly disappear: -calibrated configs that beat the base by 0.03 vs 0.10 season-pooled come -within 0-0.02 of the base month by month, and some are worse than the base. -A month holds only 1-5 rounds, so month-level bias has a noise floor above -zero; differences under about 0.01 between configs shouldn't be read as -real. - -So every coverage statement below that comes from season-pooled numbers is -weaker than it looks. WIS and median-error statements are unaffected. What -holds month by month: - -- REF-op (warm, leaky) has the lowest month-level coverage bias of the - hub-ensemble configs. -- Constant rates above about 0.03 per 100k are worse than the base at - h1-h3 month by month; sqrt 0.018 beats rate 0.032. -- Switching calibration off from March costs little coverage month by - month (and helps at h3), so its coverage cost was mostly an artifact. -- In March 2025-26 the median-error losses are about twice the WIS losses - (sqrt 0.018 at h2: −52% vs −28%). - -Each notebook now has a "Month view: WIS, coverage, median error" section -and, where a coverage claim changed, a "Month view (2026-10-01)" line at the -end of its Findings. - -## Findings - -**1. A constant learning rate on the sqrt scale is the best simple -tracker so far (E14, E16).** - -On the rate scale (per 100k), a constant rate of 0.018–0.032 improves WIS -at every horizon in both seasons (E14). E11's 0.1 is past that range: WIS -gets worse at h1–h3 in both seasons. The sqrt scale does better than the -rate scale (E16). Every sqrt rate from 0.001 to 0.018 improves WIS at -every horizon in both seasons, and 0.018 gains the most. - -| config | 2024-25 h0 | 2024-25 h2 | 2025-26 h0 | 2025-26 h2 | -|---|---|---|---|---| -| sqrt, constant 0.018 | +1.7 | +0.3 | +3.0 | +1.1 | -| rate, constant 0.032 | +1.0 | +0.3 | +1.9 | +0.8 | -| REF-op (current, cold) | +2.2 | 0.0 | +1.8 | +0.3 | - -REF-op gains more at h0–h1 in 2024-25 but loses at h3 there (−0.8). It also barely -moves 2024-25 coverage (bias 0.106–0.124, base 0.127–0.169). Neither REF-op -nor the adaptive sqrt tracker is good in both seasons at every horizon. - -**2. The calibration is a small upward shift, which is why the plots barely -change (E15).** - -- **The offsets are small.** The typical offset is 2–5% of the 90% - interval's width, and calibrated intervals are 1.00–1.04× the base width. -- **They point up.** The base under-predicts, and the tracker moves the - whole forecast up a little. -- **The gain is a median shift.** For REF-op, shifting every quantile by - the median offset alone gives more WIS than the full calibration. The - width and shape changes cost WIS and only buy coverage. -- **The gain is in the center and shoulders, not the tails.** The six tail - levels contribute at most 0.1 point. -- **Why a rate-scale constant step hurts:** a fixed step per 100k is large - for a forecast with a narrow interval, such as a small state or an - off-season week. For 1 forecast in 10 under E11, the offset exceeds - 0.26–0.55 of the 90% width. The sqrt scale shrinks this tail (E16). - -**3. Offsets go stale after the peak, and March is where it costs (E16, -E17, E02).** - -- **The March loss.** Every tracker loses WIS at h1–h3 in March of both - seasons. After the peak the base already over-predicts, and the offsets - are still positive. March is 4–8% of a season's WIS, so this costs about - a point over the season. -- **Season-level coverage hides it.** Under-prediction on the ramp and - over-prediction after the peak partly cancel in the season total. -- **Decaying the offsets toward zero from 1 March fixes March but loses - overall (E17).** The offsets carry into the next season, so the decay - also hands that season a near-zero start. -- **Switching calibration off from 1 March (or 15 February) is the one - variant with a large WIS gain on `windowed_seasonal` (E02, E17).** It - doesn't transfer to the ensemble: in 2024-25 the ensemble was still - under-predicting in spring, and the switch-off ranks last there (E07). It is positive at every - horizon in both seasons, but it gives up most of the coverage gain: - - **sqrt 0.018, pooled WIS:** +2.3/+1.2/+1.2/+1.3 switched off from - March, against +2.2/+0.8/+0.6/+0.6 left on; - - **sqrt 0.018, coverage bias:** 0.094–0.116 switched off, 0.059–0.078 - left on, and 0.105–0.127 for the base. - -**4. On the adaptive tracker, the multiplier is the only setting that -matters (E01, E02).** - -| setting | effect on pooled WIS | -|---|---| -| multiplier | 16–81 points | -| window | 1–2 points | -| carry vs reset | 0.3–1.7 points | - -- **On the sqrt scale, multipliers of 0.1 and 0.3 are too large.** -- **0.03** gains at h0 and loses a little at h1–h3. -- **0.01** never loses more than 0.2 points, but from a cold start it - barely fixes 2024-25 coverage. -- **Carry beats reset.** -- **Per-level eta** gives a small gain at h1–h3; the seasonal window gives - none. -- These agree with the original (pre-fix, finalized-truth) versions of - E01 and E02. - -**5. The gains are not specific to the ensemble (E07).** - -On identical rows, the trackers rank the same way on the submitted ensemble -and on `windowed_seasonal`. Base WIS (states, B/A) is 0.93–1.01 in 2024-25 -and 1.05–1.33 in 2025-26 (worst at h0); per state, `windowed_seasonal` is -better in 29 of 52 states in 2024-25 and 7 of 52 in 2025-26. Both have the -same monthly bias shape: under-prediction on the ramp, over-prediction -after the peak. The ensemble gains about a point more with -REF-op (pooled +3.9/+1.9/+1.5/+2.3 vs +2.8/+1.1/+0.8/+1.2). The two are -equally miscalibrated in 2024-25. In 2025-26 the ensemble is better -calibrated. The original E07's explanation for the gap, that -`windowed_seasonal` was better calibrated to begin with, does not hold at -h0–h3. - -**6. A 2023-24 burn-in helps the adaptive trackers on flu a little, and -doesn't help a constant rate (E13).** - -- **Burn-in data.** 2023-24 comes from HHS hospital data at its real - weekly versions. It runs about 3% below NHSN for flu and 6% for covid. -- **Flu.** A warm start adds 0.5–1.6 points for REF-op and E05's single - term. The warm configs under-cover the 50% interval (0.40–0.44). -- **Covid.** A warm start is wrong-signed: 2023-24 over-predicted, while - the live seasons under-predict at h0. -- **Constant rate.** A warm start gives no gain on flu, and is badly - harmful on covid (−18% at h3). - -## Where this leaves the operating point - -| candidate | for | against | -|---|---|---| -| sqrt, constant 0.018 | WIS-positive at every horizon in both seasons on `windowed_seasonal` (on the ensemble, −0.3/−0.1 at h2/h3 in 2025-26); small offsets; no learning-rate adaptation to tune | small gains at h2–h3 in 2024-25; loses in March | -| REF-op (current) | largest h0 gain in 2024-25; best with a flu burn-in | loses at h3 in 2024-25; barely improves 2024-25 coverage when cold | -| either, switched off from March | more WIS at h1–h3 | gives up most of the coverage gain | - -The gains are 1–3% of WIS and come from a small upward shift. Whether that's -worth running in prod depends on how much weight goes on coverage against -WIS. - -## Open questions - -- **A decay that keeps the next season warm.** Shrink only the offset - that's applied to the forecast after the peak, and keep the tracker's - learned state, so the next season doesn't start cold. -- **Covid.** Every result here is flu except E12 and E13. Covid under the - sqrt constant rate is untested. The covid results so far (E10, E12, E13) - point to calibrating h0 only. -- **Month-level coverage as a headline metric**, so cancellation within a - season stops flattering the season totals. diff --git a/pipelines/covid_hosp_prod.R b/pipelines/covid_hosp_prod.R index 955737fc..3c603939 100644 --- a/pipelines/covid_hosp_prod.R +++ b/pipelines/covid_hosp_prod.R @@ -587,7 +587,7 @@ joined_targets <- list2( combined_targets <- build_combined_targets(external_forecast_targets) # ============================== CALIBRATION TARGETS ============================ -# Same design as flu; see flu_hosp_prod.R and notes/CALIBRATION.md. No 2023-24 +# Same design as flu; see flu_hosp_prod.R and notes/calibration-ledger.md. No 2023-24 # burn-in for covid (hub started 2024-11-23), so slow_init is NULL. calibration_targets <- list( tar_target( diff --git a/pipelines/flu_hosp_prod.R b/pipelines/flu_hosp_prod.R index 965ea097..bf8aa570 100644 --- a/pipelines/flu_hosp_prod.R +++ b/pipelines/flu_hosp_prod.R @@ -587,7 +587,7 @@ combined_targets <- build_combined_targets(external_forecast_targets) # burn-in for free), appends the current week's ensemble_mix if not yet # submitted, and runs calibrate_hub_forecasts() over the full history. The # current round's value_cal is then written as the CMU-TimeSeries-Calibrated -# submission. Skipped in cache/evaluation mode. See notes/CALIBRATION.md for +# submission. Skipped in cache/evaluation mode. See notes/calibration-ledger.md for # the parameter choices. calibration_targets <- list( tar_target( diff --git a/reports/writeups/calibration/calibration_findings_flu.Rmd b/reports/writeups/calibration/calibration_findings_flu.Rmd index 2a09d417..2c25d947 100644 --- a/reports/writeups/calibration/calibration_findings_flu.Rmd +++ b/reports/writeups/calibration/calibration_findings_flu.Rmd @@ -27,7 +27,7 @@ ggplot2::theme_set(ggplot2::theme_bw()) ``` This notebook collects the main findings of the MultiQT calibration work -(`notes/CALIBRATION.md` has the full record and the method). In short: per +(`notes/calibration-ledger.md` has the full record and the method). In short: per (location, horizon) series, an additive offset over the 23 hub quantile levels is learned online from past coverage misses and added to the base forecast. The questions: @@ -376,7 +376,7 @@ fan_plot(fans, truth_rows, ylab = "ILI+ (%)") + ``` An earlier version of this replay was wrong at every horizon, and the ILI+ -results in `notes/CALIBRATION.md` dated before 2026-09-23 came from it (the +results in `notes/calibration-ledger.md` dated before 2026-09-23 came from it (the "bias flips sign on ILI+" reading, the ILI+ burn-in and setup C). The cause was epipredict's `extend_lags` latency adjustment. It pushes every lag back by the worst latency of any state, and some ILI+ state series end years diff --git a/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd b/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd index 59190418..3d089317 100644 --- a/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd +++ b/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd @@ -59,7 +59,7 @@ cal <- calibrate_hub_forecasts( ``` Per-forecast panel gallery for COVID hub submissions at the chosen operating point -(`calibration_qt_covid.Rmd` has the parameter sweeps; see `notes/CALIBRATION.md` +(`calibration_qt_covid.Rmd` has the parameter sweeps; see `notes/calibration-ledger.md` for project state). Parameters: diff --git a/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd b/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd index eb5883c7..8d4c7839 100644 --- a/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd +++ b/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd @@ -58,7 +58,7 @@ cal <- calibrate_hub_forecasts( ``` This document contains a gallery of forecasts that were adjusted the most (in mean absolute deviation, see below) (`calibration_qt_flu.Rmd` has the parameter-sweeps, this notebook gets more granular about forecast adjustments; see -`notes/CALIBRATION.md` for the project state). +`notes/calibration-ledger.md` for the project state). The parameters for the calibration method are fixed: - `lr_mult = `r params$lr_mult`` (the learning rate multiplier, which was found to be too slow at 0.01 and too fast at 0.1), diff --git a/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd b/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd index d637b158..107ed7f9 100644 --- a/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd +++ b/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd @@ -31,7 +31,7 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw()) ``` -This notebook takes the sqrt-space operating point from `notes/CALIBRATION.md` +This notebook takes the sqrt-space operating point from `notes/calibration-ledger.md` (`transform = "sqrt"`, `lr_mult = 0.03`, `lr_window = 20`, carry, burn-in 2023-24) and looks at four states across whole seasons rather than one panel at a time diff --git a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd index 2fcd0a08..e506aeed 100644 --- a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd +++ b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd @@ -32,8 +32,8 @@ Variants of the adaptive tracker, each changed alone from the reference. start of a ramp is sized by last year's ramp. Run with carry and with reset at the summer. -This asks the original E02's question (`calibration_qt_seasons_flu.Rmd` -and the "Method variants" table in `notes/CALIBRATION.md`) on the current +This asks the original E02's question (`calibration_qt_seasons_flu.Rmd`, +pre-fix) on the current setup. It is a new experiment, not a reproduction: four things changed at once, so it can't say which change matters. diff --git a/scripts/calibration/calibration_ili_backfill.R b/scripts/calibration/calibration_ili_backfill.R index 336a7a01..344b9c15 100644 --- a/scripts/calibration/calibration_ili_backfill.R +++ b/scripts/calibration/calibration_ili_backfill.R @@ -4,8 +4,8 @@ # that calibrate_hub_forecasts() consumes. ILI+ is the same augmentation source # the prod forecasters train on (see scripts/flu_hosp_prod.R `nhsn_prod_archive`), # so this is "what would our seasonal forecaster have said" for 2010-2024, on the -# ILI+ percent scale rather than NHSN counts. See notes/CALIBRATION.md, "ILI+ -# burn-in", for how the scale mismatch is handled downstream. +# ILI+ percent scale rather than NHSN counts. The `scales` option of +# calibrate_hub_forecasts() handles the scale mismatch downstream. # # ILI+ is faux-versioned (version = time_value), so every snapshot is finalized # data cut at the forecast date; there is no revision realism here. To mimic diff --git a/scripts/calibration/calibration_ws_experiments.R b/scripts/calibration/calibration_ws_experiments.R index 5268ece9..39f15b23 100644 --- a/scripts/calibration/calibration_ws_experiments.R +++ b/scripts/calibration/calibration_ws_experiments.R @@ -2,7 +2,7 @@ # behave the same on the ILI+ decade (2011-2023, percent scale, no revisions) # as on the NHSN seasons (2023-2026, counts, real vintages), and how does that # compare with calibrating the submitted ensemble? Prints markdown tables; -# results are recorded in notes/CALIBRATION.md. +# results are recorded in notes/calibration-ledger.md. # # Inputs (all cached in cache/calibration/): # scripts/calibration/calibration_ili_backfill.R -> windowed_seasonal on ILI+, 2010-2024 diff --git a/scripts/calibration/calibration_ws_replay.R b/scripts/calibration/calibration_ws_replay.R index 43a6eb49..a719c450 100644 --- a/scripts/calibration/calibration_ws_replay.R +++ b/scripts/calibration/calibration_ws_replay.R @@ -5,7 +5,7 @@ # Forecasts come straight from the evaluation stores through the harness # (ch_use() + ch_read_store()); truth and vintages from the same store's NHSN # archive. Rounds are the hub's CMU-TimeSeries submission rounds, so the round -# axis matches the hub-based experiments in notes/CALIBRATION.md. The E12 +# axis matches the hub-based experiments in notes/calibration-ledger.md. The E12 # configs use no burn-in season. # # With `burn_in`, warm-started configs also run with 2023-24 as a burn-in @@ -34,7 +34,7 @@ season_of <- function(d) { # E11's constant rate (0.1 admissions per 100k per step, rate scale) and the # covid prod config (REF-op without its warm start), both with no burn-in. With # `burn_in`, also E05's sqrt single term cold and warm, REF-op with its warm -# start, and E11's constant rate warm-started (CALIBRATION open thread 1). +# start, and E11's constant rate warm-started. ws_configs <- function(rate_scales, burn_in = BURN_IN) { no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) warm <- list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile") From 2930b0a29bc2569d805e7941f6900ff765f1ac8a Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Fri, 2 Oct 2026 01:51:11 -0700 Subject: [PATCH 077/107] docs(calibration): neutral notebook prose, archived index section, drop legacy notebooks, fold runbook into ledger; fix month-view base column and late_decay doc --- R/calibration/calibrate.R | 67 +- notes/calibration-ledger.md | 116 ++- notes/calibration-runbook.md | 231 ------ .../calibration/calibration_findings_flu.Rmd | 713 ------------------ .../calibration/calibration_qt_covid.Rmd | 496 ------------ .../calibration/calibration_qt_flu.Rmd | 586 -------------- .../calibration_qt_gallery_covid.Rmd | 245 ------ .../calibration_qt_gallery_flu.Rmd | 252 ------- .../calibration_qt_seasons_covid.Rmd | 354 --------- .../calibration_qt_seasons_flu.Rmd | 400 ---------- .../calibration_experiments/_month_view.Rmd | 23 +- .../e00_vintage_backtest.Rmd | 119 +-- .../e01_eta_settings.Rmd | 131 ++-- .../e02_eta_variants.Rmd | 149 ++-- .../calibration_experiments/e03_scale.Rmd | 90 ++- .../e04_offset_structure.Rmd | 89 ++- .../e05_warm_start.Rmd | 103 +-- .../e07_across_eras.Rmd | 188 ++--- .../calibration_experiments/e09_lr_delay.Rmd | 202 +++-- .../calibration_experiments/e10_covid.Rmd | 89 ++- .../e11_constant_lr.Rmd | 98 ++- .../e14_lr_by_season.Rmd | 116 ++- .../e15_wis_sources.Rmd | 115 ++- .../e16_sqrt_constant_lr.Rmd | 172 ++--- .../e17_late_decay.Rmd | 132 ++-- .../e18_ref_op_bridge.Rmd | 147 ++-- .../calibration_experiments/index.Rmd | 100 ++- scripts/calibration/calibration_ws_backfill.R | 126 ---- .../calibration/calibration_ws_experiments.R | 136 ---- tests/testthat/test-calibrate.R | 1 - 30 files changed, 1032 insertions(+), 4754 deletions(-) delete mode 100644 notes/calibration-runbook.md delete mode 100644 reports/writeups/calibration/calibration_findings_flu.Rmd delete mode 100644 reports/writeups/calibration/calibration_qt_covid.Rmd delete mode 100644 reports/writeups/calibration/calibration_qt_flu.Rmd delete mode 100644 reports/writeups/calibration/calibration_qt_gallery_covid.Rmd delete mode 100644 reports/writeups/calibration/calibration_qt_gallery_flu.Rmd delete mode 100644 reports/writeups/calibration/calibration_qt_seasons_covid.Rmd delete mode 100644 reports/writeups/calibration/calibration_qt_seasons_flu.Rmd delete mode 100644 scripts/calibration/calibration_ws_backfill.R delete mode 100644 scripts/calibration/calibration_ws_experiments.R diff --git a/R/calibration/calibrate.R b/R/calibration/calibrate.R index 48d7a2c9..39764639 100644 --- a/R/calibration/calibrate.R +++ b/R/calibration/calibrate.R @@ -97,10 +97,11 @@ hub_series_matrix <- function(series, round_date, n_levels) { #' unchanged into the next season. `"04-01"` switches calibration off for the #' spring tail, where the peak-tuned offsets are out of regime. #' @param late_decay `NULL`, or `list(from = "MM-DD", factor = )`. At each round -#' whose reference date falls on or after `from` (and before July), the fast -#' offset is multiplied by `factor` in `[0, 1]` before it is played; learning -#' continues as usual. The slow term is not decayed. Mutually exclusive with -#' `off_after`. +#' whose reference date falls on or after `from` (and before July), the stored +#' fast offset is multiplied by `factor` in `[0, 1]`, so the shrink persists: +#' later rounds, and the next season under `carry`, start from the shrunk +#' value. Learning continues as usual. The slow term is not decayed. Mutually +#' exclusive with `off_after`. #' @param lr_seasonal `NULL`, or `list(half_width_weeks = 5)`. Pools into the #' eta residual window the rounds of every earlier season that fall within #' `half_width_weeks` of the same point in the season (same calendar date @@ -474,64 +475,6 @@ hub_quantile_loss <- function(cal, drop_burn_in = TRUE, by = character(0)) { } -#' Rolling calibration error against rolling quantile loss, the trade-off curve. -#' -#' Both quantities are rolling averages over the last `window` rounds of one -#' series (or of a geo-pooled set of series): -#' calibration error = mean over levels of |rolling coverage at a - a| -#' quantile loss = mean over levels and rounds of the pinball loss -#' Plotting one against the other shows whether calibration is being bought at -#' the cost of sharpness, which is the thing a tracker can quietly get wrong. -#' -#' @param cal [calibrate_hub_forecasts()] output. -#' @param window number of rounds in the rolling window. -#' @param by grouping columns; `character(0)` pools all locations, which is the -#' default because per-series rolling coverage over 20 rounds is mostly noise. -#' @export -hub_rolling_tradeoff <- function(cal, window = 20L, by = character(0), drop_burn_in = TRUE) { - fc <- if (is.list(cal) && !is.data.frame(cal)) cal$forecasts else cal - # Rounds a location was not forecast at are all-NA in value_base/value_cal. - fc <- fc %>% filter(!is.na(.data$truth), !is.na(.data$value_base)) - if (drop_burn_in) fc <- fc %>% filter(!.data$is_burn_in) - pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) - - # Per (round, level) within each group: coverage indicator and pinball loss, - # pooled over whatever `by` does not name (locations, by default). - per_round <- fc %>% - group_by(across(all_of(c(by, "horizon", "round_index", "reference_date", "level")))) %>% - summarize( - cov_base = mean(.data$truth <= .data$value_base), - cov_cal = mean(.data$truth <= .data$value_cal), - loss_base = mean(pinball(.data$truth, .data$value_base, .data$level)), - loss_cal = mean(pinball(.data$truth, .data$value_cal, .data$level)), - .groups = "drop" - ) - - roll <- function(x) slider::slide_dbl(x, mean, .before = window - 1L, .complete = TRUE) - per_round %>% - group_by(across(all_of(c(by, "horizon", "level")))) %>% - arrange(.data$round_index, .by_group = TRUE) %>% - mutate( - roll_cov_base = roll(.data$cov_base), - roll_cov_cal = roll(.data$cov_cal), - roll_loss_base = roll(.data$loss_base), - roll_loss_cal = roll(.data$loss_cal) - ) %>% - ungroup() %>% - filter(!is.na(.data$roll_cov_base)) %>% - # Now collapse the level axis: calibration error is the mean over levels of - # |rolling coverage - nominal|. - group_by(across(all_of(c(by, "horizon", "round_index", "reference_date")))) %>% - summarize( - cal_error_base = mean(abs(.data$roll_cov_base - .data$level)), - cal_error_cal = mean(abs(.data$roll_cov_cal - .data$level)), - qloss_base = mean(.data$roll_loss_base), - qloss_cal = mean(.data$roll_loss_cal), - .groups = "drop" - ) -} - - #' Calibrate as prod does: re-run the tracker at every round on that round's #' vintage and keep only the forecast it plays at that round. #' diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 84e7dbd8..b0b5933a 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -3,9 +3,9 @@ Post-hoc online calibration of our hub forecasts with MultiQT (Ding, Gibbs & Tibshirani 2025). Each (location, horizon) series gets an additive offset over the 23 quantile levels, updated by online gradient descent on pinball loss -and projected to be monotone. Repro commands are in -`notes/calibration-runbook.md`; dropped hub rounds are in -`notes/spoiled-submissions.md`. +and projected to be monotone. Dropped hub rounds are in +`notes/spoiled-submissions.md`; how to run the tests and notebooks is under +"Running things". Every number below was re-read from the rendered notebook (or, for E12/E13, the score CSV) on 2026-10-02. Older notes, pre-fix tables and the @@ -131,12 +131,12 @@ gating), exact learning truth, spoiled submissions excluded. | E00 | `e00_vintage_backtest` | ensemble | learn from final, vintage or exact data? | exact ≈ final within 0.6 points except REF-op h−1 (2.2, 2024-25 only). Vintage at settle 7 loses the h−1 gain. Settle 7 vs 14: REF-paper +1 to +3 at every horizon; REF-op tie (flips by season) | | | E01 | `e01_eta_settings` | replay, cold | adaptive eta settings | mult ≥ 0.1 costs 7–81 at h1–h3. 0.01 beats 0.03 by 0.5–2.4 at h1–h3 in both seasons; 0.03 better at h0 in 2024-25 (0.8). Window and carry vs reset: ≤ 2.4, mostly < 1. Count is 2–5 worse than sqrt | month-avg coverage of 0.01 vs 0.03 within noise | | E02 | `e02_eta_variants` | replay, cold | `off_after`, per-level eta, seasonal window | off 1 Mar / 15 Feb +2 to +4 at h1–h3 over the 0.03 reference, both seasons, no month-avg coverage cost. Per-level +1. Seasonal window, burn-in, off 1 Apr: < 1 | dates in-sample; half the 2025-26 gain is December carry-over; "seasonal window" also changes `lr_window` 20 → 10 | -| E03 | `e03_scale` | ensemble | count vs sqrt vs rate | sqrt +1.4 to +3.9 over count pooled, mostly 2025-26. Coverage: neither better (sqrt worse per season pooled, better month-avg only in 2024-25) | rate identical to count in every cell (see Known issues) | +| E03 | `e03_scale` | ensemble | count vs sqrt vs rate | sqrt +1.4 to +3.9 over count pooled, mostly 2025-26. Coverage: neither better (sqrt worse per season pooled, better month-avg only in 2024-25) | rate identical to count in every cell (adaptive rate is scale invariant) | | E04 | `e04_offset_structure` | ensemble | leak, slow term | leak +2.5 to +5.6 at h1–h3 in 2025-26, −0.4 to −1.3 in 2024-25; month-avg coverage flips the same way (±0.015). Slow term without warm start: ≤ 0.5 | | | E05 | `e05_warm_start` | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | -| E07 | `e07_across_eras` | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | month-view absolute table shows A's base for B rows (bug) | +| E07 | `e07_across_eras` | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | | | E09 | `e09_lr_delay` | ensemble | constant rate × revision delay | knee 0.056: +3 to +5.5 over REF-paper at h1–h3; REF-op +1.2 to +2.4 over knee. Extra delay costs about 1 h−1 point per week. Above 0.1 loses steeply | month-avg: no constant rate beats base at h1–h3 | -| E10 | `e10_covid` | covid ensemble | REF-paper and REF-op on covid | h−1 +10 to +11. REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 at h1–h3 in 2025-26 | headline and month tables include Aug–Sep 2026 rounds (see Known issues) | +| E10 | `e10_covid` | covid ensemble | REF-paper and REF-op on covid | h−1 +10 to +11. REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 at h1–h3 in 2025-26 | headline and month tables run to 2026-09-19, including the 2026 summer wave | | E11 | `e11_constant_lr` | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | | E12 | none (`calibration_ws_replay.R`) | replay, cold, flu + covid | first look at the clean replay | flu: REF-op cold leads constant 0.1 by 0–2.5. Covid: REF-op cold h0 +1.3 states, losses at h1–h3 in 2025-26 | **all locations** (US is 47% of WIS); covid h0 gains are mostly US | | E13 | none (`calibration_ws_replay.R burn_in`) | replay, flu + covid | 2023-24 HHS burn-in | flu warm +1 to +2.6 in 2024-25, ≤ 0 in 2025-26. Covid: warm worse at h0–h1, better at h2–h3 (+1.3, +4.4); constant 0.1 warm −8 to −27 | HHS used as-is (3–9% below NHSN); coverage is season-pooled only | @@ -144,36 +144,31 @@ gating), exact learning truth, spoiled submissions excluded. | E15 | `e15_wis_sources` | replay, cold | where the WIS gain comes from | offsets a few % of interval width; REF-op gain from the lower half and the median shift | width-only variant's coverage not measured; split is not additive | | E16 | `e16_sqrt_constant_lr` | replay, cold | constant rate on sqrt scale | sqrt 0.018 about +1 over rate constants at h0; at h1–h3 within 0.2 pooled, up to +0.7 in 2025-26. vs REF-op cold: behind in 2024-25 h0–h1, ahead elsewhere; month-avg coverage tie | 0.018 picked in-sample | | E17 | `e17_late_decay` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | -| E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (see Known issues) | +| E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance) | E06 (ILI+ burn-in) and E08 (old covid) were run on code or data later found broken and were not rebuilt. ## Known issues -Things that could change the numbers above; none checked yet. - - **Run cache ignores code.** `cal_run_cached()` keys on inputs and arguments, not code. Some cached runs date from 2026-09-30 before the exact-learning commit landed that evening. The E00 and E09 caches are - keyed by chunk code or string and must be cleared by hand. -- **E10 includes a third season.** Headline and month tables pool every - round to 2026-09-19, so Aug–Sep 2026 (season 2026-27) is in them. -- **`scales` may be ignored.** `rate` equals `count` to every digit in E03 - and E09 (adaptive), and E18 steps 4 and 5 are identical. Scale invariance - of the adaptive rate predicts this, but nothing checks that `scales` was - applied. -- **Month-view base column.** `_month_view.Rmd` builds `mv_abs_base` with - `slice(1)`, so in E07 the absolute table shows the ensemble's base for the - replay's rows. Plots are fine. -- **`late_decay` doc mismatch.** The roxygen in `calibrate.R` says the - offset is shrunk before it is played and learning continues; the code - shrinks the stored state. -- **Stale notebook text.** E00, E05, E10 Findings say "render of 30 - September" (numbers match the current render); E18 says 1 October. -- **Stale runbook.** `notes/calibration-runbook.md` has old paths - (`~/allHail/...`, `scripts/reports/...`) and doesn't mention the - experiment notebooks or `calibration_ws_replay.R`. + keyed by chunk code or string. Plan: clear the whole cache and re-run + everything exact from scratch. + +Checked and resolved on 2026-10-02: + +- **`scales` is applied.** `rate` equals `count` to every digit with the + adaptive rate (E03, E09, E18 steps 4–5) because the adaptive rate is + scale invariant: all 53 locations match a population scale, adaptive + runs agree to 1e-11 with and without it, and with a constant rate the + calibrated quantiles differ by up to 300 admissions. +- **E10 pools through the 2026 summer on purpose**, to include covid's + summer wave; its headline and month tables run to 2026-09-19. +- **Month-view base column** in E07 (one base shown for configs on two + forecasters) and the **`late_decay` docstring** (now says it shrinks the + stored offset) were fixed. ## Review status @@ -184,12 +179,15 @@ Things that could change the numbers above; none checked yet. | E18 | changes made (reference rows), not reviewed | | E03, E04, E05, E10, E11, E14, E15, E16, E17 | not reviewed | +On 2026-10-02 every notebook's introduction and Findings were rewritten to +state setup and measured sizes only; the rewritten text has not been +reviewed. The index lists E00, E02, E04, E05 and E07 as archived. + ## Code, data and prod - `R/calibration/qt.R`: the tracker (`qt_track`, `qt_learning_rate`, - `qt_project`). Bit-exact against the authors' Python on the - `delphi-fixes` branch of `~/repos/delphi/multiQT`; oracle fixtures in - `tests/testthat/test-qt.R`. + `qt_project`). A port of the authors' Python, tested against it (see + "Running things"). - `R/calibration/calibrate.R`: `calibrate_hub_forecasts()` and `calibrate_hub_forecasts_exact()`, plus metrics. Options beyond the paper: `transform`, `scales`, `lr_slow` / `fast_decay` / @@ -200,10 +198,14 @@ Things that could change the numbers above; none checked yet. - `R/calibration/views.R`: reference configs, cached runs, standard views. - `scripts/calibration/calibration_ws_replay.R`: E12/E13. Scores in `cache/calibration/ws_replay_scores_*.csv`. -- `reports/writeups/calibration/`: the pre-experiment notebooks (findings, - sweep, seasons and gallery, flu and covid). Nothing in code or pipelines - uses them; last rendered 2026-09-23, before exact learning truth. - Candidates for deletion. +- `scripts/calibration/calibration_ili_backfill.R`: replays + `windowed_seasonal` over the ILI+ state history (2010–2024) into + hub-schema parquets in `cache/calibration/`, for the ILI+ burn-in thread. +- The pre-experiment notebooks (`reports/writeups/calibration/`) and the old + E07 scripts were deleted on 2026-10-02; they are in VCS history. Their + only analyses not in the current suite are the staleness lagged + correlation, the ILI+ base-bias table, per-round tracker internals (eta, + offsets before and after projection) and a per-level reliability plot. - **Data.** Flu hub submissions 2023-10-14 … 2026-05-30, 53 locations, h−1…h3. 2023-24 is burn-in. NHSN vintages from `get_nhsn_data_archive()` start 2024-11-19. With `settle_days` 14 the h3 @@ -216,6 +218,50 @@ Things that could change the numbers above; none checked yet. `windowed_seasonal_extra_sources` at h1–h3 and `revision_aware` at h−1 (weights in `pipelines/covid_geo_exclusions.csv`, edited weekly). +## Running things + +Commands run from the repo root (inside the rocker container, +`distrobox enter rocker -- `, if R isn't on the host). The hub +checkouts are siblings of this repo: `../FluSight-forecast-hub` (sparse: +CMU model-output only) and `../covid19-forecast-hub`. + +**Port tests.** `R/calibration/qt.R` ports the authors' Python +(`projectedQT`). The oracle is `../multiQT` on branch `delphi-fixes`, which +fixes several defects in the published code (see that branch's commit +message); the R port implements only the fixed behavior. + +```sh +# tracker tests: bit-exact oracle fixtures plus properties +Rscript -e 'testthat::test_file("tests/testthat/test-qt.R")' +# whole suite +Rscript -e 'testthat::test_dir("tests/testthat")' + +# regenerate the fixtures (made with lr_window = 50, which test-qt.R spells out) +(cd ../multiQT && uv run --with numpy --with scikit-learn --with matplotlib python make_r_fixtures.py) +cp ../multiQT/r_fixtures/*.csv tests/testthat/fixtures/qt/ + +# cross-check on real hub series; expect ALL MATCH within 1e-8 +Rscript scripts/calibration/calibration_export_series.R +(cd ../multiQT && uv run --with numpy --with scikit-learn --with matplotlib python check_r_port_real_series.py) +``` + +One porting trap: numpy broadcasts `Y - Yhat` along the last axis, while R +recycles a vector down columns. Getting it wrong silently corrupts every +learning rate; `test-qt.R` pins `eta` against a hand-built residual matrix. + +**Notebooks and scripts.** + +```sh +just calibration-experiments # all notebooks and the index +just calibration-experiments e03_scale # one notebook and the index +Rscript scripts/calibration/calibration_ws_replay.R 12 # E12 (12 workers) +Rscript scripts/calibration/calibration_ws_replay.R 12 burn_in # E13 +``` + +Tracker runs are cached under `cache/calibration/` by `cal_run_cached()`, +keyed on inputs and arguments but not code: clear the cache after changing +`R/calibration/`. E00 and E09 keep their own caches, cleared by hand. + ## Open threads Checked against the code on 2026-10-02. Repo-wide items are in @@ -224,7 +270,7 @@ Checked against the code on 2026-10-02. Repo-wide items are in | item | status | |---|---| | Uncertainty for WIS and coverage differences (bootstrap over rounds or locations) | open; needed before any ranking under ~2 points means anything | -| Fix the Known issues above and re-render | open | +| Clear the run cache and re-run everything exact | open | | Report ramp and post-peak months separately | open; only the month view exists | | sqrt constant 0.018 with a warm start | open | | Shrink only the played offset after the peak, keep the stored state | open; `late_decay` shrinks the stored state | diff --git a/notes/calibration-runbook.md b/notes/calibration-runbook.md deleted file mode 100644 index 888d83fa..00000000 --- a/notes/calibration-runbook.md +++ /dev/null @@ -1,231 +0,0 @@ -# Calibration runbook - -Every command needed to reproduce the online-calibration work from scratch. -Three repos are involved: - -| path | role | -|---|---| -| `~/allHail/delphi/exploration-tooling` | the R implementation, tests, driver, notebook | -| `~/allHail/delphi/multiQT` | the authors' code plus our upstreamed bugfixes (branch `delphi-fixes`) — the oracle for the R tests | -| `~/allHail/delphi/FluSight-forecast-hub` | flu submitted forecasts (sparse checkout: CMU model-output only) | -| `~/allHail/delphi/covid19-forecast-hub` | covid submitted forecasts + truth CSV | - -The multiQT clone must be on `delphi-fixes`. The as-published code had a -handful of defects (a learning-rate window that sliced the level axis instead -of time, an unpopulated first played column, a default `lr='adaptive++'` with -no implementation, a numpy read-only-view crash), all reported to the authors -and fixed in that branch's single commit — see its message for the details. -The R port implements the corrected semantics only; there are no -compatibility knobs for the as-published behavior. - -R is not on `$PATH`; it lives in the rocker container. Every R command below is -shown with the `distrobox` prefix. Drop it if you are already inside the -container. - ---- - -## 0. One-time setup - -```sh -# Python deps are supplied per-invocation by uv; nothing to install. -# The multiQT oracle must be on the bugfix branch: -cd ~/allHail/delphi/multiQT && git checkout delphi-fixes - -# Refresh the hub submissions (83 CMU-TimeSeries rounds, 2023-10-14 .. 2026-05-30): -cd ~/allHail/delphi/FluSight-forecast-hub && git pull --ff-only - -# Truth is the latest NHSN vintage (nhsn_read_truth()), fetched from the -# Delphi API; nothing to download by hand. -``` - -The hub checkout is sparse (`hub-config`, `model-output/CMU-TimeSeries`, -`model-output/CMU-climate_baseline`). Leave it that way — `target-data/time-series.csv` -alone is 84 MB, and we fetch the 570 KB truth file over HTTP instead. - ---- - -## 1. R implementation and its oracle tests - -```sh -cd ~/allHail/delphi/exploration-tooling - -# just the tracker's tests (bit-exact oracle fixtures + properties) -distrobox enter rocker -- Rscript -e \ - 'testthat::set_max_fails(Inf); testthat::test_file("tests/testthat/test-qt.R")' - -# the whole suite, to confirm nothing else moved (expect 183 pass / 0 fail / 9 skip) -distrobox enter rocker -- Rscript -e 'testthat::test_dir("tests/testthat")' -``` - -### Regenerating the oracle fixtures - -`tests/testthat/fixtures/qt/` is the output of `projectedQT` on the -`delphi-fixes` branch, generated with `lr_window = 50` — the same default the -R side uses, so `test-qt.R` calls `qt_track()` with its defaults (plus -`lr_window = 50` spelled out). If you change the fixture set, regenerate and -copy: - -```sh -cd ~/allHail/delphi/multiQT # on delphi-fixes -uv run --with numpy --with scikit-learn --with matplotlib \ - python make_r_fixtures.py -cp r_fixtures/*.csv ~/allHail/delphi/exploration-tooling/tests/testthat/fixtures/qt/ -``` - ---- - -## 2. Cross-check the R port against Python on **real** hub series - -The fixtures are synthetic. This runs the same comparison on real submitted -forecasts, real hub truth, and the real `horizon + 2` delay. - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript scripts/calibration_export_series.R -# optional: Rscript scripts/calibration_export_series.R - -cd ~/allHail/delphi/multiQT -uv run --with numpy --with scikit-learn --with matplotlib \ - python check_r_port_real_series.py -``` - -Expect `ALL MATCH`, max abs diff ~1e-12 (CSV round-trip noise) against a -tolerance of 1e-8. Only series complete across all 83 rounds are exported (52 of -265), because the reference Python has no notion of an inactive round. - ---- - -## 3. Run the calibration - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript -e ' -suppressPackageStartupMessages(source("R/load_all.R")) -fc <- hub_read_forecasts() -tr <- nhsn_read_truth("flu") -cal <- calibrate_hub_forecasts(fc, tr, - burn_in_seasons = "2023-2024", - season_policy = "carry", # or "reset" / "shrink" - lr_window = 50, # Inf is an expanding window - lr_args = list(mult = 0.1)) -print(as.data.frame(hub_coverage_summary(cal))) -print(as.data.frame(hub_quantile_loss(cal))) -saveRDS(cal, "cache/calibration/cal.rds") -' -``` - -Takes ~5 s for all 265 (location, horizon) series. Other entry points: - -- `hub_coverage(cal, by = "location")` — per-location coverage, for the fan plot -- `hub_rolling_tradeoff(cal, window = 20)` — rolling calibration error vs quantile loss -- `cal$series` — per-series learning-rate trajectory and reveal counts -- `cal$rounds` — the global round axis with season labels, burn-in flags, `hidden_scale` - -## 3b. COVID calibration - -The COVID hub checkout at `~/allHail/delphi/covid19-forecast-hub` carries all -model-output for CMU-TimeSeries (89 rounds, 2024-11-23 … present); truth is -the latest NHSN covid vintage. - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript -e ' -suppressPackageStartupMessages(source("R/load_all.R")) -fc <- hub_read_forecasts(hub_dir = "../covid19-forecast-hub", target = HUB_COVID_TARGET) -tr <- nhsn_read_truth("covid") -cal <- calibrate_hub_forecasts(fc, tr, - burn_in_seasons = character(0), # no off-season in covid data yet - season_policy = "carry", - transform = "sqrt", - lr_window = 20, - lr_args = list(mult = 0.03, floor = 1e-3)) -print(as.data.frame(hub_coverage_summary(cal))) -print(as.data.frame(hub_quantile_loss(cal))) -' -``` - -Takes ~5 s for all 265 (location, horizon) series. -Refresh the submissions before running: `cd ~/allHail/delphi/covid19-forecast-hub && git pull --ff-only`. - ---- - -## 4. The EDA notebook - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript -e \ - 'rmarkdown::render("scripts/reports/calibration_qt_flu.Rmd", output_file = "calibration_qt_flu.html")' -``` - -Output: `scripts/reports/calibration_qt_flu.html`. Parameters (`burn_in_seasons`, -`season_policy`, `lr_window`, `lr_mult`, `roll_window`) are in the YAML header; -override per-render with: - -```sh -distrobox enter rocker -- Rscript -e ' -rmarkdown::render("scripts/reports/calibration_qt_flu.Rmd", - output_file = "calibration_qt_mult003.html", - params = list(lr_mult = 0.03, season_policy = "reset"))' -``` - -It renders the settings sweep itself, so it takes ~5 min (32 full calibration runs). - -## 5. The gallery notebook - -Fixed operating point (`lr_mult = 0.03`, `lr_window = 20`, carry) plus a -ranked per-forecast panel gallery; see `notes/calibration-ledger.md` for the design. - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript -e \ - 'rmarkdown::render("scripts/reports/calibration_qt_gallery_flu.Rmd", output_file = "calibration_qt_gallery.html")' -``` - -As-of vintages come from `cache/calibration/nhsn_archive_flu.parquet` (a copy -of an oracle-capture NHSN archive; provenance and staleness caveats in -`notes/calibration-ledger.md`). `n_panels` is a YAML param (default 150). - -## 6. The seasons notebook - -Four states (best/worst by WIS and by median AE change, chosen from the -baseline run) × two live seasons × six method variants, plus a by-month -breakdown of where the baseline gains and loses. Chunks `load` and `variants` -are knitr-cached; clear `cache/calibration/knitr_cache_seasons/` after -changing anything in `R/calibration/`. - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript -e \ - 'rmarkdown::render("scripts/reports/calibration_qt_seasons_flu.Rmd", output_dir = here::here("reports"))' -``` - -Output: `reports/calibration_qt_seasons.html`. - -## 7. ILI+ pseudo-hub backfill - -Runs flu prod's `windowed_seasonal` forecaster over the ILI+ state history and -caches hub-schema forecasts and truth for multi-season calibration burn-in -(the `scales` option of `calibrate_hub_forecasts()` handles the scale mismatch). - -```sh -cd ~/allHail/delphi/exploration-tooling -distrobox enter rocker -- Rscript scripts/calibration_ili_backfill.R # ~30 min -distrobox enter rocker -- Rscript scripts/calibration_ili_backfill.R --refresh # rebuild -``` - -Output: `cache/calibration/ili_pseudo_hub_forecasts.parquet` and -`ili_pseudo_hub_truth.parquet`; read them with `ili_read_pseudo_hub()` after -sourcing the script. To combine with the hub tables, bind the rows, pass every -pre-2024-25 season as `burn_in_seasons`, and hand `calibrate_hub_forecasts()` -a `scales` table with one ILI+-era and one NHSN-era divisor per location. - ---- - -## Porting gotcha worth knowing about - -The one bug hit in the port itself, invisible by construction: numpy -broadcasts `Y - Yhat` along the **last** axis, R recycles a vector -**column-major down** the matrix. Getting it wrong transposes the outcome onto -the level axis and silently corrupts every learning rate while leaving the -algorithm's shape intact. `test-qt.R` pins `eta` against a hand-rolled residual -matrix to catch it — see "the adaptive lr is the 90th percentile of |Y - Yhat|". diff --git a/reports/writeups/calibration/calibration_findings_flu.Rmd b/reports/writeups/calibration/calibration_findings_flu.Rmd deleted file mode 100644 index 2c25d947..00000000 --- a/reports/writeups/calibration/calibration_findings_flu.Rmd +++ /dev/null @@ -1,713 +0,0 @@ ---- -title: Online calibration of flu forecasts — what we are seeing -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - toc: true - toc_float: true - toc_depth: 2 -params: - hub_dir: "../FluSight-forecast-hub" - rep_min_pop: 5000000 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code, NOT on functions in R/ -- after -# changing calibration code, clear cache/calibration/knitr_cache_findings_flu/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_findings_flu/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -This notebook collects the main findings of the MultiQT calibration work -(`notes/calibration-ledger.md` has the full record and the method). In short: per -(location, horizon) series, an additive offset over the 23 hub quantile levels -is learned online from past coverage misses and added to the base forecast. -The questions: - -1. **Which way is the base biased?** On NHSN, low, in every season, for both - `windowed_seasonal` and the submitted ensemble. On ILI+, `windowed_seasonal` - has little level bias, but shares the NHSN shortfall at the peak (section 1.2). -2. **Why do short and long horizons behave so differently?** An outcome is - only learned from once it has settled, `h + 2` rounds after the forecast, - so the h3 offset is always at least five rounds stale. -3. **What does a leak on the fast term buy?** It stops a stale offset from - persisting into the part of the season where it has the wrong sign. -4. **What does the sqrt transform buy?** Offsets that stay closer to - proportionate between trough and peak. - -All tracker runs use `lr_mult = 0.03`, `lr_window = 20`, `season_policy = -"carry"`, and 2023-24 as the burn-in season. Scores are on counts against the -latest NHSN vintage, over all hub locations (US included) unless stated. "WIS -change %" is positive when calibration improves on the base forecast. Burn-in -rounds are excluded from every score. - -```{r libs} -# Not cached: attaches packages and defines functions every render. -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -source(here::here("scripts/calibration/calibration_ili_backfill.R")) -source(here::here("scripts/calibration/calibration_ws_backfill.R")) -``` - -```{r load, cache=TRUE} -nhsn_arch <- get_nhsn_data_archive("flu") -nhsn_truth <- nhsn_read_truth("flu", nhsn_arch) -hub_fc <- hub_read_forecasts(hub_dir = here::here(params$hub_dir)) -ili <- ili_read_pseudo_hub() -ws <- ws_read_pseudo_hub() - -geo_map <- get_population_data() %>% - distinct(state_id, .keep_all = TRUE) %>% - transmute(geo_value = state_id, abbrev = toupper(state_id), location = state_code, population) -``` - -```{r prep, cache=TRUE, dependson="load"} -LIVE_SEASONS <- c("2024-2025", "2025-2026") - -# windowed_seasonal replay on the hub's submission rounds only, so seasons, -# gaps and round counts match the ensemble's. -ws_fc <- ws %>% - filter(reference_date %in% unique(hub_fc$reference_date)) %>% - select(-as_of_policy) -# Locations both bases cover, for the like-for-like comparison in section 5. -common_loc <- intersect(unique(ws_fc$location), unique(hub_fc$location)) - -ili_rounds <- hub_label_seasons(unique(ili$forecasts$reference_date)) -ili_fc <- ili$forecasts %>% - filter(reference_date %in% ili_rounds$round_date[!ili_rounds$season %in% c("2010-2011", "2020-2021", "2021-2022", "2023-2024")]) - -in_season <- function(d) as.integer(format(d, "%m")) %in% c(10:12, 1:5) -pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) -``` - -```{r runs, cache=TRUE, dependson="prep"} -run <- function(fc, truth, ...) { - args <- modifyList( - list( - burn_in_seasons = "2023-2024", transform = "sqrt", lr_window = 20, season_policy = "carry", - lr_args = list(mult = 0.03, floor = 1e-3), progress = FALSE - ), - list(...) - ) - suppressMessages(do.call(calibrate_hub_forecasts, c(list(fc, truth), args))) -} -operating <- list(slow_init = "burn_in_quantile", lr_slow = list(mult = 0.003), fast_decay = 0.1) - -# The submitted ensemble, all hub locations: the method variants. -ens <- list( - count = run(hub_fc, nhsn_truth, transform = "identity"), - count_leaky = run(hub_fc, nhsn_truth, transform = "identity", fast_decay = 0.1), - sqrt = run(hub_fc, nhsn_truth), - sqrt_leaky = run(hub_fc, nhsn_truth, fast_decay = 0.1), - warm_static = run(hub_fc, nhsn_truth, slow_init = "burn_in_quantile", lr_args = list(mult = 1e-6, floor = 0)), - warm_leaky = run(hub_fc, nhsn_truth, slow_init = "burn_in_quantile", fast_decay = 0.1), - operating = do.call(run, c(list(hub_fc, nhsn_truth), operating)) -) -# Ensemble (A) and windowed_seasonal (B) on their common locations. -ens_common <- hub_fc %>% filter(location %in% common_loc) -ws_common <- ws_fc %>% filter(location %in% common_loc) -ab <- list( - A_single = run(ens_common, nhsn_truth), - A_leaky = run(ens_common, nhsn_truth, fast_decay = 0.1), - A_operating = do.call(run, c(list(ens_common, nhsn_truth), operating)), - B_single = run(ws_common, nhsn_truth), - B_leaky = run(ws_common, nhsn_truth, fast_decay = 0.1), - B_operating = do.call(run, c(list(ws_common, nhsn_truth), operating)) -) - -variant_labels <- c( - count = "count space, single term (the paper)", - count_leaky = "count space, leaky fast (decay 0.1)", - sqrt = "sqrt, single term", - sqrt_leaky = "sqrt, leaky fast (decay 0.1)", - warm_static = "sqrt, warm start only (no tracking)", - warm_leaky = "sqrt, warm start + leaky fast", - operating = "sqrt, warm start + slow 0.003 + leaky fast (operating point)" -) -``` - -```{r score-helpers} -wis_table <- function(cals, labels = names(cals), by = character(0), seasons = NULL) { - purrr::imap(cals, function(cal, nm) { - left_join( - hub_quantile_loss(cal, by = by), - hub_coverage_summary(cal, by = by) %>% select(all_of(c(by, "horizon")), "cal_error_base", "cal_error_cal"), - by = c(by, "horizon") - ) %>% mutate(variant = nm) - }) %>% - bind_rows() %>% - { if (!is.null(seasons)) filter(., season %in% seasons) else . } %>% - mutate(variant = factor(variant, levels = names(cals), labels = labels)) -} -wide <- function(tbl, col, digits = 1, id = "variant") { - tbl %>% - select(all_of(id), horizon, value = all_of(col)) %>% - mutate(value = round(value, digits)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = value, names_prefix = "h") -} -median_rows <- function(fc, truth, base) { - rounds <- hub_label_seasons(unique(fc$reference_date)) %>% select(reference_date = round_date, season) - fc %>% - filter(level == 0.5) %>% - inner_join(truth, by = c("location", "target_end_date")) %>% - inner_join(rounds, by = "reference_date") %>% - mutate(base = base) -} -season_of <- function(d) { - y <- as.integer(format(d, "%Y")) - (as.integer(format(d, "%m")) < 8) - paste0(y, "-", y + 1) -} -fan_rows <- function(fc, loc, seasons, every = 3L) { - rounds <- hub_label_seasons(unique(fc$reference_date)) %>% - filter(season %in% seasons) %>% - group_by(season) %>% - filter((season_round - 1L) %% every == 0L) %>% - ungroup() %>% - select(reference_date = round_date, season) - fc %>% - filter(location == loc, level %in% c(0.1, 0.5, 0.9)) %>% - inner_join(rounds, by = "reference_date") %>% - mutate(level = paste0("q", level * 100)) %>% - select(season, reference_date, target_end_date, level, value) %>% - tidyr::pivot_wider(names_from = level, values_from = value) -} -fan_plot <- function(fans, truth_rows, vintages = NULL, ylab) { - p <- ggplot(fans, aes(target_end_date, group = reference_date)) - if (!is.null(vintages)) { - p <- p + geom_line(aes(time_value, value, group = reference_date), data = vintages, color = "orange3", linewidth = 0.4, alpha = 0.8) - } - p + - geom_line(aes(target_end_date, truth), data = truth_rows, inherit.aes = FALSE, color = "black") + - geom_ribbon(aes(ymin = q10, ymax = q90), fill = "steelblue", alpha = 0.25) + - geom_line(aes(y = q50), color = "steelblue4") + - facet_wrap(~season, scales = "free", ncol = 3) + - labs(x = NULL, y = ylab) -} -``` - -# 1. The base bias - -## 1.1 On NHSN, both bases forecast too low - -The tracker's largest gains come from correcting a persistent bias, so the -first question is whether the base has one. Two bases on NHSN: - -- **the submitted ensemble** (`CMU-TimeSeries`), read from the hub checkout; -- **`windowed_seasonal`**, flu prod's seasonal component, replayed by - `scripts/calibration/calibration_ws_backfill.R` with prod's archive, seed - and substitutions. 2024-25 and 2025-26 are honest as-of replays; 2023-24 - uses finalized data, since no NHSN vintages exist before 2024-11-19. - -```{r bias-data} -med <- bind_rows( - median_rows(ws_fc, nhsn_truth, "windowed_seasonal"), - median_rows(hub_fc, nhsn_truth, "submitted ensemble") -) -bias_by_season <- med %>% - group_by(base, season, horizon) %>% - summarize(above = 100 * mean(truth > value), .groups = "drop") -``` - -```{r bias-plot, fig.height=4} -ggplot(bias_by_season, aes(season, above, color = factor(horizon), group = horizon)) + - geom_hline(yintercept = 50, linetype = "dashed") + - geom_line() + - geom_point() + - facet_wrap(~base) + - scale_color_viridis_d(name = "horizon", end = 0.9) + - labs(x = NULL, y = "% of forecasts with truth above the median", title = "Above the dashed line: forecast too low") -``` - -Pooled over all three seasons: the share of forecasts with truth above the -median, and the summed median as a percentage error on summed truth (negative -means too low; this weights the peak weeks). - -```{r bias-table} -bias_pooled <- med %>% - group_by(base, horizon) %>% - summarize(above = 100 * mean(truth > value), level_err = 100 * (sum(value) / sum(truth) - 1), .groups = "drop") -knitr::kable( - left_join( - wide(bias_pooled, "above", 0, id = "base") %>% rename_with(~ paste(.x, "above %"), starts_with("h")), - wide(bias_pooled, "level_err", 0, id = "base") %>% rename_with(~ paste(.x, "level %"), starts_with("h")), - by = "base" - ) -) -``` - -Where in the season the bias sits: the same two numbers at h0 and h3, split -by how high truth is relative to the location's 90th-percentile in-season -level (trough on the left, peak on the right). - -```{r bias-bins} -loc_scale <- nhsn_truth %>% - filter(target_end_date >= as.Date("2023-10-01"), in_season(target_end_date)) %>% - group_by(location) %>% - summarize(scale = stats::quantile(truth, 0.9, names = FALSE), .groups = "drop") -bins <- med %>% - filter(horizon %in% c(0, 3)) %>% - inner_join(loc_scale, by = "location") %>% - mutate(level_bin = cut(truth / scale, c(0, 0.1, 0.25, 0.5, 0.75, 1, Inf), right = FALSE)) %>% - filter(!is.na(level_bin)) %>% - group_by(base, horizon, level_bin) %>% - summarize(above = round(100 * mean(truth > value)), level = round(100 * (sum(value) / sum(truth) - 1)), .groups = "drop") %>% - mutate(cell = sprintf("%d%% / %+d%%", above, level)) %>% - select(base, horizon, level_bin, cell) %>% - tidyr::pivot_wider(names_from = level_bin, values_from = cell) -knitr::kable(bins, caption = "truth above median % / level error %, by truth ÷ location's in-season 90th percentile") -``` - -- Both bases are low in all three NHSN seasons, and the bias grows with - horizon. At h0 it is a fairly even level shift (roughly 5–20% low across the - range). At h3 it concentrates at the peak: near the top of the season the - median is 60–70% low and truth is above it almost every time. That is the - base failing to anticipate how high the peak goes, not a constant offset. -- `windowed_seasonal` is the less biased of the two at h−1 and the more biased - of the two by h3. The ensemble's h−1 is a different model - (`climate_linear`) and is further off there. - -### A representative state - -The state whose `windowed_seasonal` above-median share (pooled over h0–h3 -and both live seasons) is closest to the pooled value, among states with -population over `r format(params$rep_min_pop, big.mark = ",", scientific = FALSE)`. -Every third round is drawn: the median path from h−1 to h3 with an 80% band, -over finalized truth (black). The orange lines are the NHSN vintage each drawn -round saw (its last eight weeks); where the end of an orange line sits below -the black line, the forecaster was working from data that was later revised -up. - -```{r rep-state} -pooled <- med %>% - filter(base == "windowed_seasonal", horizon >= 0, season %in% LIVE_SEASONS) %>% - summarize(p = mean(truth > value)) %>% - pull(p) -rep_scores <- med %>% - filter(base == "windowed_seasonal", horizon >= 0, season %in% LIVE_SEASONS) %>% - group_by(location) %>% - summarize(score = abs(mean(truth > value) - pooled), .groups = "drop") %>% - inner_join(geo_map, by = "location") %>% - filter(population > params$rep_min_pop) %>% - arrange(score) -rep_loc <- rep_scores$location[1] -rep_abbrev <- rep_scores$abbrev[1] -rep_geo <- rep_scores$geo_value[1] -``` - -```{r nhsn-fans, fig.height=4.5} -fans <- fan_rows(ws_fc, rep_loc, LIVE_SEASONS) -truth_rows <- nhsn_truth %>% - filter(location == rep_loc) %>% - mutate(season = season_of(target_end_date)) %>% - filter(season %in% LIVE_SEASONS, in_season(target_end_date)) -vintages <- purrr::map(unique(fans$reference_date), function(rd) { - as_of <- rd - 3L - if (as_of < min(nhsn_arch$DT$version)) return(NULL) - epix_as_of(nhsn_arch, as_of) %>% - as_tibble() %>% - filter(geo_value == rep_geo, time_value > rd - 7L * 9L) %>% - transmute(reference_date = rd, season = season_of(rd), time_value, value) -}) %>% bind_rows() -fan_plot(fans, truth_rows, vintages, ylab = "weekly flu admissions") + - ggtitle(sprintf("windowed_seasonal on NHSN, %s", rep_abbrev)) -``` - -## 1.2 On ILI+, the same forecaster has little level bias - -The same forecaster replayed on the ILI+ decade -(`scripts/calibration/calibration_ili_backfill.R`: finalized state ILI+, -snapshot cut one week before the forecast date, so hub h0 is one week past the -last observation). The same two numbers as the NHSN table: - -```{r ili-bias} -ili_med <- median_rows(ili_fc, ili$truth, "windowed_seasonal, ILI+ replay") -ili_pooled <- ili_med %>% - group_by(base, horizon) %>% - summarize(above = 100 * mean(truth > value), level_err = 100 * (sum(value) / sum(truth) - 1), .groups = "drop") -knitr::kable( - left_join( - wide(ili_pooled, "above", 0, id = "base") %>% rename_with(~ paste(.x, "above %"), starts_with("h")), - wide(ili_pooled, "level_err", 0, id = "base") %>% rename_with(~ paste(.x, "level %"), starts_with("h")), - by = "base" - ) -) -ili_scale <- ili$truth %>% - group_by(location) %>% - summarize(scale = stats::quantile(truth, 0.9, names = FALSE), .groups = "drop") -ili_bins <- ili_med %>% - filter(horizon %in% c(0, 3)) %>% - inner_join(ili_scale, by = "location") %>% - mutate(level_bin = cut(truth / scale, c(0, 0.1, 0.25, 0.5, 0.75, 1, Inf), right = FALSE)) %>% - filter(!is.na(level_bin)) %>% - group_by(horizon, level_bin) %>% - summarize(above = round(100 * mean(truth > value)), level = round(100 * (sum(value) / sum(truth) - 1)), .groups = "drop") %>% - mutate(cell = sprintf("%d%% / %+d%%", above, level)) %>% - select(horizon, level_bin, cell) %>% - tidyr::pivot_wider(names_from = level_bin, values_from = cell) -knitr::kable(ili_bins, caption = "ILI+: truth above median % / level error %, by truth ÷ location's 90th percentile") -``` - -- On ILI+ the median is about as often above truth as below (43–46% above), - and the summed level is within ±4% at every horizon. The persistent low bias - on NHSN does not show up here. -- The peak shortfall does. At h3, in the top bin, truth is above the median - almost nine times in ten, as on NHSN. At the trough the median is well - above truth. Across the season these roughly cancel on ILI+; on NHSN the peak - shortfall dominates. - -The replay's fans for the representative state: - -```{r ili-fans, fig.height=4.5} -ili_seasons <- c("2016-2017", "2017-2018", "2018-2019") -fans <- fan_rows(ili_fc, rep_loc, ili_seasons) -truth_rows <- ili$truth %>% - filter(location == rep_loc) %>% - mutate(season = season_of(target_end_date)) %>% - filter(season %in% ili_seasons, in_season(target_end_date)) -fan_plot(fans, truth_rows, ylab = "ILI+ (%)") + - ggtitle(sprintf("windowed_seasonal ILI+ replay, %s", rep_abbrev)) -``` - -An earlier version of this replay was wrong at every horizon, and the ILI+ -results in `notes/calibration-ledger.md` dated before 2026-09-23 came from it (the -"bias flips sign on ILI+" reading, the ILI+ burn-in and setup C). The cause -was epipredict's `extend_lags` latency adjustment. It pushes every lag back -by the worst latency of any state, and some ILI+ state series end years -before the forecast date (DC stops in 2015), so on 2018-01-03 every lag was -extended by 1001 days. The replay now leaves such states out of the latency -check. The ILI+ burn-in and setup C have not been re-run on the fixed replay. - -# 2. Staleness: why h−1/h0 and h2/h3 respond differently - -The tracker learns from an outcome only once it has settled (14 days after -the target week), so the outcome of an h-step forecast is revealed `h + 2` -rounds after it was issued, and the step at round `t` is driven by forecasts -issued at `t − h − 2` or earlier. The long horizons have the largest reveal -lag *and* are the hardest to forecast, so they also carry most of the base -error: - -```{r lag-table} -base_wis <- hub_quantile_loss(ens$sqrt) %>% - mutate( - `reveal lag (rounds)` = horizon + 2L, - `base WIS (ensemble)` = round(2 * loss_base, 1), - `relative to h0` = round(loss_base / loss_base[horizon == 0], 2) - ) %>% - select(horizon, `reveal lag (rounds)`, `base WIS (ensemble)`, `relative to h0`) -knitr::kable(base_wis) -``` - -## The offset echoes the past - -For the median of each series, define the *needed* shift as -`sqrt(truth) − sqrt(base median)` (what an oracle would have added, in the -tracker's working units) and the *played* shift as -`sqrt(calibrated median) − sqrt(base median)`. Below: the correlation between -the played shift at round `t` and the needed shift at round `t − k`, averaged -over locations, for the single-term sqrt tracker. - -```{r lag-corr, fig.height=4} -shift_rows <- function(cal) { - cal$forecasts %>% - filter(!is_burn_in, level == 0.5, !is.na(truth), !is.na(value_base)) %>% - mutate(needed = sqrt(truth) - sqrt(value_base), played = sqrt(value_cal) - sqrt(value_base)) %>% - select(location, horizon, round_index, reference_date, season, value_base, value_cal, needed, played) -} -ks <- 0:12 -lag_corr <- shift_rows(ens$sqrt) %>% - group_by(location, horizon) %>% - arrange(round_index, .by_group = TRUE) %>% - reframe(k = ks, r = purrr::map_dbl(ks, function(k) { - n <- length(played) - if (n - k < 10) return(NA_real_) - x <- played[(k + 1):n] - y <- needed[1:(n - k)] - if (sd(x) == 0 || sd(y) == 0) return(NA_real_) - stats::cor(x, y) - })) %>% - group_by(horizon, k) %>% - summarize(r = mean(r, na.rm = TRUE), .groups = "drop") -peaks <- lag_corr %>% group_by(horizon) %>% slice_max(r, n = 1) %>% ungroup() -ggplot(lag_corr, aes(k, r, color = factor(horizon))) + - geom_hline(yintercept = 0, color = "grey50") + - geom_line() + - geom_point(data = peaks, size = 3) + - geom_vline(aes(xintercept = horizon + 2, color = factor(horizon)), data = distinct(lag_corr, horizon), linetype = "dotted") + - scale_color_viridis_d(name = "horizon", end = 0.9) + - scale_x_continuous(breaks = ks) + - labs( - x = "k (rounds): played shift at t vs needed shift at t − k", - y = "mean correlation over locations", - title = "Dotted: reveal lag h + 2. Large dots: best-matching lag." - ) -knitr::kable( - lag_corr %>% - filter(k == 0) %>% - select(horizon, `r at k = 0` = r) %>% - left_join(peaks %>% select(horizon, `best k` = k, `r at best k` = r), by = "horizon") %>% - mutate(`reveal lag` = horizon + 2L, across(starts_with("r "), ~ round(.x, 2))) -) -``` - -- **The current needed shift and the played shift are negatively - correlated** at every horizon from h0 up, and more so at longer horizons: - on average the tracker is pushing the wrong way *right now*. -- **The played shift best matches the needed shift of `h + 4` to `h + 6` - rounds earlier** (h0–h3): the reveal lag plus two to four rounds for the - tracker to build up an offset. At h3 that is nine rounds, about two months. - h−1 has almost no structure: its lag is one round. - -Share of rounds where the calibrated median moved in the right direction -(same sign as the needed shift), among rounds where it moved the median by -more than 1%: - -```{r sign-table} -sign_right <- purrr::imap(ens[c("count", "sqrt", "sqrt_leaky", "warm_static", "operating")], function(cal, nm) { - shift_rows(cal) %>% - filter(abs(value_cal - value_base) > 0.01 * value_base) %>% - group_by(horizon) %>% - summarize(right = 100 * mean(sign(played) == sign(needed)), .groups = "drop") %>% - mutate(variant = variant_labels[[nm]]) -}) %>% bind_rows() -knitr::kable(wide(sign_right, "right", 0), caption = "% of moves in the right direction") -``` - -Because the base is low most of the time, "shift up" is right more often than -not, so every variant is above 50%. The static warm start (a constant -upward shift, no tracking) is right as often as any tracker at h0–h3: what -the trackers add is size and coverage, not timing. - -## One series through a season - -`r rep_abbrev` at h0 and h3 over both live seasons: the needed shift (black) -and the played shift for three variants, in sqrt units. - -```{r series-plot, fig.height=6} -series <- purrr::imap(ens[c("sqrt", "sqrt_leaky", "operating")], function(cal, nm) { - shift_rows(cal) %>% filter(location == rep_loc, horizon %in% c(0, 3)) %>% mutate(variant = variant_labels[[nm]]) -}) %>% bind_rows() -needed <- series %>% distinct(horizon, season, reference_date, needed) -ggplot(series, aes(reference_date)) + - geom_hline(yintercept = 0, color = "grey50") + - geom_line(aes(y = needed), data = needed, color = "black", linewidth = 0.8) + - geom_line(aes(y = played, color = variant)) + - facet_grid(paste0("h", horizon) ~ season, scales = "free") + - scale_color_brewer(palette = "Set1", name = NULL) + - labs(x = NULL, y = "shift of the median (sqrt units)") + - theme(legend.position = "bottom", legend.direction = "vertical") -``` - -The needed shift is dominated by the base missing the peak: large, fast and -changing sign within a few weeks, more so at h3. None of the trackers -follows it; at this learning rate they only build a small, smooth level. That -is by design (a faster tracker overshoots; see the sweep in -`calibration_qt_flu.Rmd`), and it is why the gain is mostly a level -correction plus coverage rather than a fix for the peak miss. - -## Score by horizon - -WIS change % over both live seasons, and calibration error (mean absolute -coverage miss over the 23 levels; base in the first row). - -```{r horizon-table} -tbl <- wis_table(ens, variant_labels[names(ens)]) -knitr::kable(wide(tbl, "pct_improvement"), caption = "WIS change % (positive is better)") -cal_err <- bind_rows( - tbl %>% filter(variant == variant_labels[["sqrt"]]) %>% mutate(variant = "base", cal_error_cal = cal_error_base), - tbl %>% mutate(variant = as.character(variant)) -) -knitr::kable(wide(cal_err, "cal_error_cal", 3), caption = "Calibration error") -``` - -- **h−1 and h0 gain under every tracker.** The lag is one to two rounds, - short enough that recent misses are still informative. At h−1 the - ensemble's base is `climate_linear`, a weaker model, so there is more to - correct. -- **h2 and h3 are where the method choices matter.** A tracker alone is - negative (single term) or roughly neutral (leaky) there; the warm start (a constant correction - learned from the burn-in season, which needs no timely feedback) is what - makes them positive. - -# 3. The leak: letting stale offsets decay - -With `fast_decay = 0.1`, the fast term is multiplied by 0.9 before each -step, so a correction that new outcomes do not refresh fades (half-life about -6.6 rounds). The effect by calendar month of the reference date: - -```{r by-month} -by_month <- function(cal) { - cal$forecasts %>% - filter(!is_burn_in, !is.na(truth), !is.na(value_base)) %>% - mutate(month = format(reference_date, "%b")) %>% - group_by(horizon, month) %>% - summarize( - base = sum(pinball(truth, value_base, level)), - cal = sum(pinball(truth, value_cal, level)), - .groups = "drop" - ) %>% - group_by(horizon) %>% - mutate(pct = 100 * (base - cal) / base, share = 100 * base / sum(base)) %>% - ungroup() -} -month_levels <- month.abb[c(10:12, 1:5)] -month_tbl <- purrr::imap(ens[c("sqrt", "sqrt_leaky", "warm_leaky", "operating")], function(cal, nm) { - by_month(cal) %>% mutate(variant = variant_labels[[nm]]) -}) %>% - bind_rows() %>% - filter(month %in% month_levels) %>% - mutate(month = factor(month, levels = month_levels)) - -ggplot(month_tbl %>% filter(horizon %in% c(0, 3)), aes(month, pct, color = variant, group = variant)) + - geom_hline(yintercept = 0, color = "grey50") + - geom_line() + - geom_point() + - facet_wrap(~ paste0("h", horizon)) + - scale_color_brewer(palette = "Set1", name = NULL) + - coord_cartesian(ylim = c(-60, 30)) + - labs(x = "month of reference date", y = "WIS change % (clipped at −60)") + - theme(legend.position = "bottom", legend.direction = "vertical") -knitr::kable( - month_tbl %>% - filter(horizon == 3) %>% - mutate(pct = round(pct, 1)) %>% - select(variant, month, pct) %>% - arrange(month) %>% - tidyr::pivot_wider(names_from = month, values_from = pct) %>% - bind_rows( - month_tbl %>% - filter(horizon == 3, variant == variant_labels[["sqrt"]]) %>% - transmute(variant = "share of base WIS %", month, pct = round(share, 1)) %>% - arrange(month) %>% - tidyr::pivot_wider(names_from = month, values_from = pct) - ), - caption = "h3: WIS change % by month, and each month's share of the base WIS" -) -``` - -- **The losses are in March–April and October, which carry little WIS.** In - March–April the offset built while chasing the winter peak is still in - place when the base has already come down; in October the offset carried - over from the previous spring has not been unwound yet (the reveal lag - means the first live outcomes arrive weeks into the season). -- **The leak alone cuts those losses by two thirds or more** (h3: March −17% - against −47% for the single-term tracker, October −14% against −48%, April - +9% against −36%), at the cost of coverage: - a correction that must be refreshed every round cannot hold a level through - a five-round reveal lag (calibration error in the table above: the leaky - variant sits between single-term and uncalibrated). -- **The warm start wins in December–January**, the months that carry most of - the base WIS (+4% at h3 against +1–2% without it), and so wins overall at - h1–h3 even though part of the spring loss comes back (March −28 to −33%). The warm start is a constant, - so it is too large once the season comes down; the leak cannot take it - away because it only decays the fast term. - -# 4. The sqrt transform - -In sqrt space the tracker sees `sqrt(truth)` and `sqrt(base quantiles)`, -learns and projects offsets there, and the played quantiles are squared back -(clamped at zero) and scored on counts. The coverage gradient is unchanged, -since a monotone map preserves which side of a quantile the truth falls on; -what changes is the size of a step. A fixed sqrt-space offset `d` on a base -median `b` moves it by `2 d sqrt(b) + d²` counts, so it grows with the square -root of the level instead of staying a fixed count. - -The plot shows the played median shift relative to the base median, split by -where the base median sits within the location's season (base median divided -by that location's 90th-percentile in-season truth). - -```{r relative-shift, fig.height=4} -rel <- purrr::imap(ens[c("count", "sqrt", "count_leaky", "sqrt_leaky")], function(cal, nm) { - cal$forecasts %>% - filter(!is_burn_in, level == 0.5, !is.na(value_base), value_base > 0) %>% - inner_join(loc_scale, by = "location") %>% - mutate( - level_bin = cut(value_base / scale, c(0, 0.05, 0.1, 0.2, 0.4, 0.7, Inf), right = FALSE), - rel_shift = abs(value_cal - value_base) / value_base, - variant = variant_labels[[nm]] - ) -}) %>% bind_rows() -rel_summary <- rel %>% - filter(horizon %in% c(0, 3)) %>% - group_by(variant, horizon, level_bin) %>% - summarize(med = 100 * median(rel_shift), .groups = "drop") -ggplot(rel_summary, aes(level_bin, med, color = variant, group = variant)) + - geom_line() + - geom_point() + - facet_wrap(~ paste0("h", horizon)) + - scale_y_log10() + - scale_color_brewer(palette = "Paired", name = NULL, direction = -1) + - labs( - x = "base median / location's in-season 90th percentile (trough → peak)", - y = "median |shift| / base median (%, log)" - ) + - theme(legend.position = "bottom", legend.direction = "vertical") -knitr::kable( - rel_summary %>% - group_by(variant, horizon) %>% - summarize(`trough ÷ peak` = round(first(med) / last(med), 1), .groups = "drop") %>% - tidyr::pivot_wider(names_from = horizon, values_from = `trough ÷ peak`, names_prefix = "h"), - caption = "Relative shift at the trough bin divided by the relative shift at the peak bin" -) -``` - -```{r transform-table} -tt <- wis_table(ens[c("count", "sqrt", "count_leaky", "sqrt_leaky")], variant_labels[c("count", "sqrt", "count_leaky", "sqrt_leaky")]) -knitr::kable(wide(tt, "pct_improvement"), caption = "WIS change %, both live seasons") -knitr::kable(wide(tt, "cal_error_cal", 3), caption = "Calibration error") -``` - -- **In count space the relative shift spans a much wider range** between - trough and peak than in sqrt space: an offset sized on peak residuals is - far too large for the trough. Sqrt narrows that range but does not remove - it, and it makes the relative shift at the peak somewhat larger. -- **Sqrt is better on WIS at every horizon**, with or without the leak. Its - calibration error is similar (slightly better at h1–h3 for the single-term - tracker, slightly worse with the leak). -- **The transform does not address staleness.** A stale sqrt-space offset is - still wrong-signed, just more modestly sized. The leak and the transform - are complementary: count space with a leak is comparable to sqrt without - one at h1–h3. - -# 5. The same tracker on two bases - -The ensemble (A) and `windowed_seasonal` (B) calibrated the same way, on the -locations both cover. `windowed_seasonal` is less miscalibrated to start with, -so there is less to gain. - -```{r ab-table} -ab_tbl <- wis_table(ab, names(ab), by = "season", seasons = LIVE_SEASONS) %>% - mutate(variant = as.character(variant)) -knitr::kable(wide(ab_tbl, "pct_improvement", id = c("variant", "season")) %>% arrange(season, variant), caption = "WIS change %") -ab_err <- bind_rows( - ab_tbl %>% filter(variant %in% c("A_single", "B_single")) %>% mutate(variant = paste0(substr(variant, 1, 1), "_base"), cal_error_cal = cal_error_base), - ab_tbl -) -knitr::kable(wide(ab_err, "cal_error_cal", 3, id = c("variant", "season")) %>% arrange(season, variant), caption = "Calibration error") -``` - -- The operating point helps both bases in both seasons at h0–h3, with small - gains on `windowed_seasonal` (under 4%). -- On `windowed_seasonal` in 2025-26 the single-term tracker has far better - calibration error than the operating point (0.009–0.014 against - 0.019–0.045) for slightly less WIS gain: the leak is the price of the - spring fix, paid in coverage. -- Whether the warm start transfers to a *different data source* (an ILI+ - burn-in before NHSN) has not been re-run on the fixed ILI+ replay. Within NHSN, the warm start comes from the same - base's burn-in season, and it helps both. - -# Summary - -| question | answer | -|---|---| -| Is there a stable bias to correct? | On NHSN yes: both bases forecast too low in every season, worse at longer horizons and, at h3, concentrated at the peak. On ILI+ the same forecaster is roughly level-unbiased, with the same peak shortfall. | -| Why do h2/h3 behave differently from h−1/h0? | The offset is at least `h + 2` rounds stale. Right now it is negatively correlated with the needed shift, and it best matches the needed shift of `h + 4` to `h + 6` rounds earlier. The long horizons are also the hardest and carry most of the base WIS. | -| What does the leak do? | Cuts the spring and October losses from stale offsets by two thirds or more, at the cost of coverage. | -| What does the warm start do? | Holds the persistent low bias as a constant: gains where the WIS mass is (December–January), which the trackers alone cannot at h2/h3. Its spring losses come back, since the leak does not touch it. | -| What does sqrt do? | Narrows the range of relative offset size between trough and peak, and is better on WIS than count space at every horizon. It does not fix staleness. | diff --git a/reports/writeups/calibration/calibration_qt_covid.Rmd b/reports/writeups/calibration/calibration_qt_covid.Rmd deleted file mode 100644 index a4b83c51..00000000 --- a/reports/writeups/calibration/calibration_qt_covid.Rmd +++ /dev/null @@ -1,496 +0,0 @@ ---- -title: Online quantile calibration (MultiQT) on submitted COVID hub forecasts -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - df_print: paged - toc: true - toc_float: true -params: - hub_dir: "../covid19-forecast-hub" - burn_in_seasons: "" - season_policy: "carry" - lr_window: 50 - lr_mult: 0.1 - roll_window: 20 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code + cache.extra, NOT on functions -# in R/ -- after changing calibration code, clear cache/calibration/knitr_cache_covid/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_covid/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -```{r libs} -# Not cached: attaches packages every render. -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -``` - -```{r load, cache=TRUE, cache.extra=list(params)} -forecasts <- hub_read_forecasts(hub_dir = here::here(params$hub_dir), target = HUB_COVID_TARGET) -truth <- nhsn_read_truth("covid") -rounds <- hub_label_seasons(unique(forecasts$reference_date)) - -burn_in <- if (nzchar(params$burn_in_seasons)) { - strsplit(params$burn_in_seasons, ",")[[1]] -} else { - character(0) -} - -cal <- calibrate_hub_forecasts( - forecasts, truth, - burn_in_seasons = burn_in, - season_policy = params$season_policy, - lr_window = params$lr_window, - lr_args = list(mult = params$lr_mult), - progress = FALSE -) -``` - -We calibrate the forecasts CMU-TimeSeries submitted to the COVID hub. https://github.com/cdcepi/covid19-forecast-hub -The hub ships one consistent truth series (`target-data/covid-hospital-admissions.csv`). -The method is MultiQT (Ding, Gibbs & Tibshirani 2025); our port is `R/calibration/qt.R`, verified against the authors' Python in `tests/testthat/test-qt.R`. - -COVID submissions run year-round with no meaningful off-season gap, so there is no burn-in season. - -Data facts: - -```{r data-shape} -forecasts %>% - left_join(select(rounds, reference_date = round_date, season), by = "reference_date") %>% - group_by(season) %>% - summarize( - rounds = n_distinct(reference_date), - locations = n_distinct(location), - horizons = paste(sort(unique(horizon)), collapse = ","), - cell_completeness = sprintf( - "%.0f%%", 100 * (n() / 23) / (n_distinct(reference_date) * 265) - ), - .groups = "drop" - ) -``` - -Notes: - -1. Waiting for revisions costs us `horizon + 2` rounds. `reference_date` and - `target_end_date` are both Saturdays with - `target_end_date = reference_date + 7 * horizon`, so waiting 14 days for NHSN - to settle puts the delay at exactly `h + 2` rounds. The offset applied at - horizon 3 is therefore always 5 rounds stale. -2. Unlike flu, COVID has no off-season: all rounds are treated as live. - Season policy (`carry`/`reset`/`shrink`) therefore has no effect on these results. -3. Per-series coverage cannot be measured at the extreme levels: the 1% and 99% - levels see ~0.3 expected exceedances per season per series. Every coverage - number below is pooled over the 53 locations. - -# Score results - -Coverage improves at some cost in sharpness. Below we show that the sharpness -cost can be reduced by changing the learning rate. - -```{r headline} -headline <- left_join( - hub_coverage_summary(cal), - select(hub_quantile_loss(cal), horizon, loss_base, loss_cal, loss_pct = pct_improvement), - by = "horizon" -) -headline %>% - transmute( - horizon, - n_obs, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) -``` - -Calibration error here is the mean over the 23 quantile levels of -`|geo-pooled empirical coverage at a - a|`. WIS is reported with the standard -factor-of-two convention (2 x the level-averaged pinball loss on this level -set), so it is directly comparable to hub scores; improvement percentages are -identical either way. - -# 1. What feeds the learning rate - -`eta_t = max(floor, mult * quantile_0.9(|Y - Yhat|))` over a window of recent -observed rounds, pooled across all 23 quantile levels -- one step size per -(series, round). The residual distribution below therefore determines the -learning rate, up to the multiplier. - -```{r resid-series, fig.height=7} -resid <- cal$forecasts %>% - filter(!is.na(truth), !is.na(value_base)) %>% - mutate(abs_resid = abs(truth - value_base)) %>% - group_by(location, horizon, reference_date, season, is_burn_in) %>% - summarize(q90 = quantile(abs_resid, 0.9), .groups = "drop") - -show_geos <- c("06", "36", "48", "50", "US") -resid %>% - filter(location %in% show_geos) %>% - mutate(location = recode(location, - "06" = "CA", "36" = "NY", "48" = "TX", "50" = "VT", "US" = "US" - )) %>% - ggplot(aes(reference_date, q90, colour = factor(horizon))) + - geom_line() + - facet_wrap(~location, scales = "free_y", ncol = 1) + - scale_y_log10() + - labs( - title = "90th percentile of |truth - base forecast|, per round", - subtitle = "The quantity the adaptive learning rate is a fraction of. Log scale.", - x = NULL, y = "q90 |residual| (log)", colour = "horizon" - ) -``` - -## Does the window length matter? - -`lr_window` sets how many of the most recent observed rounds feed the adaptive -learning rate; `Inf` is an expanding window over the full history. - -```{r lr-window, cache=TRUE, dependson="load"} -lr_by_window <- purrr::map(c(8, 20, 50, Inf), function(w) { - purrr::map(show_geos, function(loc) { - series <- forecasts %>% filter(location == loc, horizon == 2) - if (nrow(series) == 0) { - return(NULL) - } - Yhat <- hub_series_matrix(series, rounds$round_date, 23) - ted <- rounds$round_date + 14L - Y <- truth$truth[match(paste(loc, ted), paste(truth$location, truth$target_end_date))] - learnable <- !is.na(Y) & !apply(is.na(Yhat), 2, any) - delay <- qt_delay_from_dates(rounds$round_date, ted, 14L) - delay <- lapply(delay, function(i) i[learnable[i]]) - res <- qt_track(Y, Yhat, HUB_QUANTILE_LEVELS, - delay = delay, lr_window = w, nonneg = TRUE - ) - tibble( - location = loc, round_date = rounds$round_date, - lr = res$lr, window = as.character(w) - ) - }) %>% bind_rows() -}) %>% bind_rows() - -lr_by_window %>% - filter(lr > 0) %>% - mutate( - location = recode(location, - "06" = "CA", "36" = "NY", "48" = "TX", "50" = "VT", "US" = "US" - ), - window = factor(window, levels = c("8", "20", "50", "Inf")) - ) %>% - ggplot(aes(round_date, lr, colour = window)) + - geom_line() + - facet_wrap(~location, scales = "free_y", ncol = 1) + - scale_y_log10() + - labs( - title = "Learning-rate trajectory by window length (horizon 2)", - subtitle = "A short window tracks the season; an expanding one (Inf) ratchets up and stays.", - x = NULL, y = "eta (log)", colour = "lr_window" - ) -``` - -# 2. Actual vs nominal coverage, per location - -```{r coverage-plot, fig.height=6} -cov_by_geo <- hub_coverage(cal, by = "location") - -cov_long <- cov_by_geo %>% - select(location, horizon, level, coverage_base, coverage_cal) %>% - tidyr::pivot_longer( - c(coverage_base, coverage_cal), - names_to = "which", values_to = "coverage" - ) %>% - mutate(which = factor( - if_else(which == "coverage_base", "pre-calibration", "post-calibration"), - levels = c("pre-calibration", "post-calibration") - )) - -cov_long %>% - filter(horizon %in% c(0, 2)) %>% - ggplot(aes(level, coverage, group = location)) + - geom_abline(slope = 1, intercept = 0, linetype = "dashed", colour = "grey30") + - geom_line(alpha = 0.28, colour = "steelblue4") + - facet_grid(horizon ~ which, labeller = label_both) + - coord_equal(xlim = c(0, 1), ylim = c(0, 1)) + - labs( - title = "Empirical vs nominal coverage, one line per location", - subtitle = "Dashed line is perfect calibration. Each line is one of 53 locations, over all rounds.", - x = "nominal level", y = "empirical coverage" - ) -``` - -```{r coverage-pooled} -hub_coverage(cal) %>% - select(horizon, level, coverage_base, coverage_cal) %>% - tidyr::pivot_longer(c(coverage_base, coverage_cal), - names_to = "which", values_to = "coverage" - ) %>% - mutate(which = if_else(which == "coverage_base", "pre", "post")) %>% - ggplot(aes(level, coverage, colour = which)) + - geom_abline(slope = 1, intercept = 0, linetype = "dashed", colour = "grey30") + - geom_line() + - geom_point(size = 1) + - facet_wrap(~horizon, labeller = label_both, nrow = 1) + - coord_equal() + - labs( - title = "Geo-pooled coverage, pre and post calibration", - x = "nominal level", y = "empirical coverage", colour = NULL - ) -``` - -# 3. Calibration error against quantile loss - -Both axes are rolling averages over the last `r params$roll_window` rounds, pooled -over locations. Moving down is better calibrated; moving left is sharper. - -```{r tradeoff, fig.height=6} -tradeoff <- hub_rolling_tradeoff(cal, window = params$roll_window) - -tradeoff %>% - select(horizon, reference_date, cal_error_base, cal_error_cal, qloss_base, qloss_cal) %>% - tidyr::pivot_longer( - -c(horizon, reference_date), - names_to = c(".value", "which"), names_pattern = "(.*)_(base|cal)$" - ) %>% - mutate(which = if_else(which == "base", "pre-calibration", "post-calibration")) %>% - ggplot(aes(qloss, cal_error, colour = which)) + - geom_path(alpha = 0.7) + - geom_point(aes(size = as.numeric(reference_date)), alpha = 0.5, show.legend = FALSE) + - scale_size_continuous(range = c(0.4, 2.2)) + - facet_wrap(~horizon, scales = "free_x", labeller = label_both) + - labs( - title = "Rolling calibration error vs rolling quantile loss", - subtitle = "Point size increases with time. Down = better calibrated, left = sharper.", - x = "rolling mean pinball loss (proportional to WIS)", y = "rolling calibration error", colour = NULL - ) -``` - -```{r tradeoff-time, fig.height=5} -tradeoff %>% - select(horizon, reference_date, cal_error_base, cal_error_cal) %>% - tidyr::pivot_longer(-c(horizon, reference_date), - names_to = "which", values_to = "cal_error" - ) %>% - mutate(which = if_else(which == "cal_error_base", "pre", "post")) %>% - ggplot(aes(reference_date, cal_error, colour = which)) + - geom_line() + - facet_wrap(~horizon, labeller = label_both, nrow = 1) + - labs( - title = "Rolling calibration error over time", - subtitle = "How fast the tracker converges and what it looks like without an off-season gap.", - x = NULL, y = "rolling calibration error", colour = NULL - ) -``` - -# 4. Interval widths - -The WIS cost has to show up somewhere in the quantiles; central interval widths -are the direct measure. - -```{r interval-widths} -interval_defs <- tibble::tribble( - ~interval, ~lo, ~hi, - "20% (0.4-0.6)", 0.4, 0.6, - "60% (0.2-0.8)", 0.2, 0.8, - "80% (0.1-0.9)", 0.1, 0.9 -) - -width_rows <- cal$forecasts %>% - filter(!is_burn_in, !is.na(value_base), !is.na(value_cal)) %>% - select(reference_date, season, location, horizon, level, value_base, value_cal) - -widths <- purrr::pmap(interval_defs, function(interval, lo, hi) { - inner_join( - width_rows %>% filter(level == lo) %>% - select(-level) %>% rename(base_lo = value_base, cal_lo = value_cal), - width_rows %>% filter(level == hi) %>% - select(-level) %>% rename(base_hi = value_base, cal_hi = value_cal), - by = c("reference_date", "season", "location", "horizon") - ) %>% - transmute( - reference_date, season, location, horizon, - interval, - width_base = base_hi - base_lo, - width_cal = cal_hi - cal_lo - ) -}) %>% - bind_rows() %>% - mutate(interval = factor(interval, levels = interval_defs$interval)) -``` - -```{r width-time, fig.height=6} -widths %>% - group_by(interval, horizon, reference_date) %>% - summarize( - base = mean(width_base), calibrated = mean(width_cal), - .groups = "drop" - ) %>% - tidyr::pivot_longer(c(base, calibrated), names_to = "which", values_to = "width") %>% - ggplot(aes(reference_date, width, colour = which)) + - geom_line() + - facet_grid(interval ~ horizon, scales = "free_y", labeller = label_both) + - labs( - title = "Geo-pooled mean central interval width, base vs calibrated", - x = NULL, y = "mean interval width (admissions)", colour = NULL - ) -``` - -```{r width-change, fig.height=4} -widths %>% - group_by(interval, horizon) %>% - summarize( - pct_change = 100 * (mean(width_cal) / mean(width_base) - 1), - .groups = "drop" - ) %>% - ggplot(aes(horizon, pct_change, colour = interval)) + - geom_hline(yintercept = 0, linetype = "dashed", colour = "grey40") + - geom_line() + - geom_point(size = 2) + - labs( - title = "Change in mean interval width from calibration", - subtitle = "Positive means the calibrated intervals are wider than the base intervals.", - x = "horizon", y = "width change (%)", colour = "central interval" - ) -``` - -# 5. Sensitivity: window or multiplier? - -```{r sweep, cache=TRUE, dependson="load"} -sweep_grid <- tidyr::expand_grid( - season_policy = c("carry", "reset"), - lr_window = c(8, 20, 50, Inf), - lr_mult = c(0.3, 0.1, 0.03, 0.01) -) -sweep <- purrr::pmap(sweep_grid, function(season_policy, lr_window, lr_mult) { - s <- calibrate_hub_forecasts(forecasts, truth, - burn_in_seasons = burn_in, season_policy = season_policy, - lr_window = lr_window, lr_args = list(mult = lr_mult), progress = FALSE - ) - overall <- left_join( - hub_coverage_summary(s), - select(hub_quantile_loss(s), horizon, loss_pct = pct_improvement), - by = "horizon" - ) %>% - mutate(season = "all") - by_season <- left_join( - hub_coverage_summary(s, by = "season"), - select(hub_quantile_loss(s, by = "season"), - season, horizon, - loss_pct = pct_improvement - ), - by = c("season", "horizon") - ) - bind_rows(overall, by_season) %>% - mutate(season_policy = season_policy, lr_window = lr_window, lr_mult = lr_mult) -}) %>% bind_rows() - -dir.create(here::here("cache/calibration"), showWarnings = FALSE, recursive = TRUE) -saveRDS(sweep, here::here("cache/calibration/sweep_covid.rds")) -``` - -```{r sweep-cal-error, fig.height=5} -sweep %>% - filter(season == "all") %>% - ggplot(aes(lr_mult, cal_error_cal, colour = factor(lr_window))) + - geom_line() + - geom_point(size = 1.6) + - scale_x_log10() + - facet_grid(season_policy ~ horizon, labeller = label_both) + - labs( - title = "Calibration error by multiplier, one line per window", - x = "lr mult (log)", y = "calibration error (calibrated)", - colour = "lr_window" - ) -``` - -```{r sweep-wis, fig.height=5} -sweep %>% - filter(season == "all") %>% - ggplot(aes(lr_mult, loss_pct, colour = factor(lr_window))) + - geom_hline(yintercept = 0, linetype = "dashed", colour = "grey40") + - geom_line() + - geom_point(size = 1.6) + - scale_x_log10() + - facet_grid(season_policy ~ horizon, labeller = label_both) + - labs( - title = "WIS improvement by multiplier, one line per window", - subtitle = "Dashed line is WIS parity with the raw submission.", - x = "lr mult (log)", y = "WIS improvement (%)", - colour = "lr_window" - ) -``` - -```{r sweep-corner, fig.height=5} -sweep %>% - filter(season == "all", season_policy == "carry") %>% - ggplot(aes(cal_error_cal, loss_pct, - colour = factor(lr_mult), shape = factor(lr_window) - )) + - geom_hline(yintercept = 0, linetype = "dashed", colour = "grey40") + - geom_point(size = 3, alpha = 0.85) + - facet_wrap(~horizon, scales = "free_x", labeller = label_both, nrow = 1) + - labs( - title = "Calibration error vs WIS change (season_policy = carry)", - subtitle = "Upper-left is the target region: well calibrated and no worse on WIS.", - x = "calibration error (calibrated)", y = "WIS change (%)", - colour = "lr mult", shape = "lr_window" - ) -``` - -```{r sweep-table} -sweep %>% - filter(season == "all") %>% - transmute( - season_policy, lr_window, lr_mult, horizon, - cal_error_cal = round(cal_error_cal, 4), - wis_pct = round(loss_pct, 1) - ) %>% - arrange(horizon, season_policy, lr_window, lr_mult) -``` - -# 6. Offset trajectories - -```{r offsets, fig.height=6} -cal$forecasts %>% - filter(location %in% show_geos, horizon == 2, level %in% c(0.025, 0.5, 0.975)) %>% - mutate(location = recode(location, - "06" = "CA", "36" = "NY", "48" = "TX", "50" = "VT", "US" = "US" - )) %>% - ggplot(aes(reference_date, offset, colour = factor(level))) + - geom_hline(yintercept = 0, colour = "grey50") + - geom_line() + - facet_wrap(~location, scales = "free_y", ncol = 1) + - labs( - title = "Realised offset (played - base), horizon 2", - x = NULL, y = "offset (admissions)", colour = "level" - ) -``` - -```{r offset-scale} -cal$forecasts %>% - filter(!is_burn_in, !is.na(value_base), value_base > 0) %>% - group_by(horizon, level) %>% - summarize( - median_abs_offset = median(abs(offset)), - median_rel = median(abs(offset) / pmax(value_base, 1)), - .groups = "drop" - ) %>% - ggplot(aes(level, median_rel, colour = factor(horizon))) + - geom_line() + - scale_y_continuous(labels = scales::percent) + - labs( - title = "Median |offset| as a fraction of the base forecast", - x = "quantile level", y = "median |offset| / base", colour = "horizon" - ) -``` diff --git a/reports/writeups/calibration/calibration_qt_flu.Rmd b/reports/writeups/calibration/calibration_qt_flu.Rmd deleted file mode 100644 index 192865d4..00000000 --- a/reports/writeups/calibration/calibration_qt_flu.Rmd +++ /dev/null @@ -1,586 +0,0 @@ ---- -title: Online quantile calibration (MultiQT) on submitted FluSight forecasts -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - df_print: paged - toc: true - toc_float: true -params: - hub_dir: "../FluSight-forecast-hub" - burn_in_seasons: "2023-2024" - season_policy: "carry" - lr_window: 50 - lr_mult: 0.1 - roll_window: 20 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code + cache.extra, NOT on functions -# in R/ -- after changing calibration code, clear cache/calibration/knitr_cache_flu/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_flu/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -```{r libs} -# Not cached: attaches packages every render. -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -``` - -```{r load, cache=TRUE, cache.extra=list(params)} -forecasts <- hub_read_forecasts(hub_dir = here::here(params$hub_dir)) -truth <- nhsn_read_truth("flu") -rounds <- hub_label_seasons(unique(forecasts$reference_date)) - -burn_in <- if (nzchar(params$burn_in_seasons)) { - strsplit(params$burn_in_seasons, ",")[[1]] -} else { - character(0) -} - -cal <- calibrate_hub_forecasts( - forecasts, truth, - burn_in_seasons = burn_in, - season_policy = params$season_policy, - lr_window = params$lr_window, - lr_args = list(mult = params$lr_mult), - progress = FALSE -) -``` - -We calibrate the forecasts CMU-TimeSeries submitted to FluSight. https://github.com/cdcepi/FluSight-forecast-hub The hub -ships one consistent truth series (`target-data/target-hospital-admissions.csv`) -covering every season. The method is MultiQT (Ding, Gibbs & Tibshirani 2025); our -port is `R/calibration/qt.R`, verified against the authors' Python in -`tests/testthat/test-qt.R` (same input, same output). - -Data facts: - -```{r data-shape} -forecasts %>% - left_join(select(rounds, reference_date = round_date, season), by = "reference_date") %>% - group_by(season) %>% - summarize( - rounds = n_distinct(reference_date), - locations = n_distinct(location), - horizons = paste(sort(unique(horizon)), collapse = ","), - cell_completeness = sprintf( - "%.0f%%", 100 * (n() / 23) / (n_distinct(reference_date) * 265) - ), - .groups = "drop" - ) -``` - -Notes: - -1. Waiting for revisions costs us `horizon + 2` rounds. `reference_date` and - `target_end_date` are both Saturdays with - `target_end_date = reference_date + 7 * horizon`, so waiting 14 days for NHSN - to settle puts the delay at exactly `h + 2` rounds. The offset applied at - horizon 3 is therefore always 5 rounds stale. -2. 2023-24 is incomplete: horizon -1 exists in only 5 of 28 rounds and - locations drop in and out, so it is used as a burn-in season. Its residuals - warm up the learning rate, but no gradient step is taken and the offsets - stay at zero. -3. Per-series coverage cannot be measured at the extreme levels: ~27 rounds - per season minus delay means the 1% and 99% levels see ~0.3 expected - exceedances per series. Every coverage number below is pooled over the 53 - locations. - -# Score results - -Coverage improves at some cost in sharpness. Below we show that the sharpness -cost can be reduced by changing the learning rate. - -```{r headline} -headline <- left_join( - hub_coverage_summary(cal), - select(hub_quantile_loss(cal), horizon, loss_base, loss_cal, loss_pct = pct_improvement), - by = "horizon" -) -headline %>% - transmute( - horizon, - n_obs, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) -``` - -Calibration error here is the mean over the 23 quantile levels of -`|geo-pooled empirical coverage at a - a|`. WIS is reported with the standard -factor-of-two convention (2 x the level-averaged pinball loss on this level -set), so it is directly comparable to hub scores; improvement percentages are -identical either way. The two metrics move in opposite directions, and the WIS -cost grows with the horizon. This is consistent with the `h + 2` staleness: a -stale additive offset in count space does the most damage where the seasonal -ramp is steep relative to the offset. - -## By season - -```{r headline-by-season} -left_join( - hub_coverage_summary(cal, by = "season"), - select(hub_quantile_loss(cal, by = "season"), - season, horizon, loss_base, loss_cal, loss_pct = pct_improvement - ), - by = c("season", "horizon") -) %>% - transmute( - season, horizon, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) %>% - arrange(season, horizon) -``` - -The burn-in season (2023-24) is excluded throughout. In the per-season view, -season 2 is the tracker's first live season; season 3 shows the effect of a -second season of accumulated state (under `carry` across the off-season gap). - -# 1. What feeds the learning rate - -`eta_t = max(floor, mult * quantile_0.9(|Y - Yhat|))` over a window of recent -observed rounds, pooled across all 23 quantile levels -- one step size per -(series, round). The residual distribution below therefore determines the -learning rate, up to the multiplier. - -```{r resid-series, fig.height=7} -resid <- cal$forecasts %>% - filter(!is.na(truth), !is.na(value_base)) %>% - mutate(abs_resid = abs(truth - value_base)) %>% - group_by(location, horizon, reference_date, season, is_burn_in) %>% - summarize(q90 = quantile(abs_resid, 0.9), .groups = "drop") - -show_geos <- c("06", "36", "48", "50", "US") -resid %>% - filter(location %in% show_geos) %>% - mutate(location = recode(location, - "06" = "CA", "36" = "NY", "48" = "TX", "50" = "VT", "US" = "US" - )) %>% - ggplot(aes(reference_date, q90, colour = factor(horizon))) + - geom_line() + - facet_wrap(~location, scales = "free_y", ncol = 1) + - scale_y_log10() + - labs( - title = "90th percentile of |truth - base forecast|, per round", - subtitle = "The quantity the adaptive learning rate is a fraction of. Log scale.", - x = NULL, y = "q90 |residual| (log)", colour = "horizon" - ) -``` - -The residual scale swings by 1-2 orders of magnitude within a season, so an -additive offset learned near the peak is badly sized for the trough, and vice -versa. - -## Does the window length matter? - -`lr_window` sets how many of the most recent observed rounds feed the adaptive -learning rate; `Inf` is an expanding window over the full history. - -```{r lr-window, cache=TRUE, dependson="load"} -lr_by_window <- purrr::map(c(8, 20, 50, Inf), function(w) { - purrr::map(show_geos, function(loc) { - series <- forecasts %>% filter(location == loc, horizon == 2) - if (nrow(series) == 0) { - return(NULL) - } - Yhat <- hub_series_matrix(series, rounds$round_date, 23) - ted <- rounds$round_date + 14L - Y <- truth$truth[match(paste(loc, ted), paste(truth$location, truth$target_end_date))] - learnable <- !is.na(Y) & !apply(is.na(Yhat), 2, any) - delay <- qt_delay_from_dates(rounds$round_date, ted, 14L) - delay <- lapply(delay, function(i) i[learnable[i]]) - res <- qt_track(Y, Yhat, HUB_QUANTILE_LEVELS, - delay = delay, lr_window = w, nonneg = TRUE - ) - tibble( - location = loc, round_date = rounds$round_date, - lr = res$lr, window = as.character(w) - ) - }) %>% bind_rows() -}) %>% bind_rows() - -lr_by_window %>% - filter(lr > 0) %>% - mutate( - location = recode(location, - "06" = "CA", "36" = "NY", "48" = "TX", "50" = "VT", "US" = "US" - ), - window = factor(window, levels = c("8", "20", "50", "Inf")) - ) %>% - ggplot(aes(round_date, lr, colour = window)) + - geom_line() + - facet_wrap(~location, scales = "free_y", ncol = 1) + - scale_y_log10() + - labs( - title = "Learning-rate trajectory by window length (horizon 2)", - subtitle = "A short window tracks the season; an expanding one (Inf) ratchets up and stays.", - x = NULL, y = "eta (log)", colour = "lr_window" - ) -``` - -The expanding window is monotone by construction: once a peak-season residual -enters the pool it never leaves, so `eta` rises to peak scale and stays there -through the following low season. A short window decays back. On 83 rounds the -difference in outcome is small (below), but it would likely grow over more -seasons. - -# 2. Actual vs nominal coverage, per location - -```{r coverage-plot, fig.height=6} -cov_by_geo <- hub_coverage(cal, by = "location") - -cov_long <- cov_by_geo %>% - select(location, horizon, level, coverage_base, coverage_cal) %>% - tidyr::pivot_longer( - c(coverage_base, coverage_cal), - names_to = "which", values_to = "coverage" - ) %>% - mutate(which = factor( - if_else(which == "coverage_base", "pre-calibration", "post-calibration"), - levels = c("pre-calibration", "post-calibration") - )) - -cov_long %>% - filter(horizon %in% c(0, 2)) %>% - ggplot(aes(level, coverage, group = location)) + - geom_abline(slope = 1, intercept = 0, linetype = "dashed", colour = "grey30") + - geom_line(alpha = 0.28, colour = "steelblue4") + - facet_grid(horizon ~ which, labeller = label_both) + - coord_equal(xlim = c(0, 1), ylim = c(0, 1)) + - labs( - title = "Empirical vs nominal coverage, one line per location", - subtitle = "Dashed line is perfect calibration. Each line is one of 53 locations, over all non-burn-in rounds.", - x = "nominal level", y = "empirical coverage" - ) -``` - -```{r coverage-pooled} -hub_coverage(cal) %>% - select(horizon, level, coverage_base, coverage_cal) %>% - tidyr::pivot_longer(c(coverage_base, coverage_cal), - names_to = "which", values_to = "coverage" - ) %>% - mutate(which = if_else(which == "coverage_base", "pre", "post")) %>% - ggplot(aes(level, coverage, colour = which)) + - geom_abline(slope = 1, intercept = 0, linetype = "dashed", colour = "grey30") + - geom_line() + - geom_point(size = 1) + - facet_wrap(~horizon, labeller = label_both, nrow = 1) + - coord_equal() + - labs( - title = "Geo-pooled coverage, pre and post calibration", - x = "nominal level", y = "empirical coverage", colour = NULL - ) -``` - -The per-location spread is not visible in the pooled number: it shows whether -the pooled improvement is a per-series effect or an averaging artefact, and it -bears on whether the offsets themselves should be pooled across geos. - -# 3. Calibration error against quantile loss - -Both axes are rolling averages over the last `r params$roll_window` rounds, pooled -over locations. Moving down is better calibrated; moving left is sharper. A -trajectory that goes down and to the right means calibration is being bought -with sharpness. - -```{r tradeoff, fig.height=6} -tradeoff <- hub_rolling_tradeoff(cal, window = params$roll_window) - -tradeoff %>% - select(horizon, reference_date, cal_error_base, cal_error_cal, qloss_base, qloss_cal) %>% - tidyr::pivot_longer( - -c(horizon, reference_date), - names_to = c(".value", "which"), names_pattern = "(.*)_(base|cal)$" - ) %>% - mutate(which = if_else(which == "base", "pre-calibration", "post-calibration")) %>% - ggplot(aes(qloss, cal_error, colour = which)) + - geom_path(alpha = 0.7) + - geom_point(aes(size = as.numeric(reference_date)), alpha = 0.5, show.legend = FALSE) + - scale_size_continuous(range = c(0.4, 2.2)) + - facet_wrap(~horizon, scales = "free_x", labeller = label_both) + - labs( - title = "Rolling calibration error vs rolling quantile loss", - subtitle = "Point size increases with time. Down = better calibrated, left = sharper.", - x = "rolling mean pinball loss (proportional to WIS)", y = "rolling calibration error", colour = NULL - ) -``` - -```{r tradeoff-time, fig.height=5} -tradeoff %>% - select(horizon, reference_date, cal_error_base, cal_error_cal) %>% - tidyr::pivot_longer(-c(horizon, reference_date), - names_to = "which", values_to = "cal_error" - ) %>% - mutate(which = if_else(which == "cal_error_base", "pre", "post")) %>% - ggplot(aes(reference_date, cal_error, colour = which)) + - geom_line() + - facet_wrap(~horizon, labeller = label_both, nrow = 1) + - labs( - title = "Rolling calibration error over time", - subtitle = "How fast the tracker converges after the burn-in season, and what the off-season gap does to it.", - x = NULL, y = "rolling calibration error", colour = NULL - ) -``` - -# 4. Interval widths - -The WIS cost has to show up somewhere in the quantiles; central interval widths -are the direct measure. For three central intervals -- 20% (0.4 to 0.6), 60% -(0.2 to 0.8), 80% (0.1 to 0.9) -- we compare the calibrated width to the base -width, per (round, location, horizon). Widths are geo-pooled by mean, matching -the implicit weighting of the WIS aggregate (large-count series dominate both). - -```{r interval-widths} -interval_defs <- tibble::tribble( - ~interval, ~lo, ~hi, - "20% (0.4-0.6)", 0.4, 0.6, - "60% (0.2-0.8)", 0.2, 0.8, - "80% (0.1-0.9)", 0.1, 0.9 -) - -width_rows <- cal$forecasts %>% - filter(!is_burn_in, !is.na(value_base), !is.na(value_cal)) %>% - select(reference_date, season, location, horizon, level, value_base, value_cal) - -widths <- purrr::pmap(interval_defs, function(interval, lo, hi) { - inner_join( - width_rows %>% filter(level == lo) %>% - select(-level) %>% rename(base_lo = value_base, cal_lo = value_cal), - width_rows %>% filter(level == hi) %>% - select(-level) %>% rename(base_hi = value_base, cal_hi = value_cal), - by = c("reference_date", "season", "location", "horizon") - ) %>% - transmute( - reference_date, season, location, horizon, - interval, - width_base = base_hi - base_lo, - width_cal = cal_hi - cal_lo - ) -}) %>% - bind_rows() %>% - mutate(interval = factor(interval, levels = interval_defs$interval)) -``` - -```{r width-time, fig.height=6} -widths %>% - group_by(interval, horizon, reference_date) %>% - summarize( - base = mean(width_base), calibrated = mean(width_cal), - .groups = "drop" - ) %>% - tidyr::pivot_longer(c(base, calibrated), names_to = "which", values_to = "width") %>% - ggplot(aes(reference_date, width, colour = which)) + - geom_line() + - facet_grid(interval ~ horizon, scales = "free_y", labeller = label_both) + - labs( - title = "Geo-pooled mean central interval width, base vs calibrated", - subtitle = "Mean over the 53 locations, per round. The gap between the lines is the sharpness cost.", - x = NULL, y = "mean interval width (admissions)", colour = NULL - ) -``` - -```{r width-change, fig.height=4} -widths %>% - group_by(interval, horizon) %>% - summarize( - pct_change = 100 * (mean(width_cal) / mean(width_base) - 1), - .groups = "drop" - ) %>% - ggplot(aes(horizon, pct_change, colour = interval)) + - geom_hline(yintercept = 0, linetype = "dashed", colour = "grey40") + - geom_line() + - geom_point(size = 2) + - labs( - title = "Change in mean interval width from calibration", - subtitle = "Positive means the calibrated intervals are wider than the base intervals.", - x = "horizon", y = "width change (%)", colour = "central interval" - ) -``` - -Wider intervals where the base under-covers are the intended correction; the -question is whether the widening is concentrated in the intervals and horizons -where the coverage plots (section 2) show a deficit, or spread indiscriminately. - -# 5. Sensitivity: window or multiplier? - -The learning rate has two knobs: how far back the residual pool reaches -(`lr_window`) and what fraction of its 90th percentile becomes the step size -(`lr_mult`). The factorial sweep below shows which one matters: if the lines -for different windows sit on top of each other while the multiplier moves -everything, the multiplier is the tuning parameter and the window is a detail -(and vice versa). - -```{r sweep, cache=TRUE, dependson="load"} -sweep_grid <- tidyr::expand_grid( - season_policy = c("carry", "reset"), - lr_window = c(8, 20, 50, Inf), - lr_mult = c(0.3, 0.1, 0.03, 0.01) -) -sweep <- purrr::pmap(sweep_grid, function(season_policy, lr_window, lr_mult) { - s <- calibrate_hub_forecasts(forecasts, truth, - burn_in_seasons = burn_in, season_policy = season_policy, - lr_window = lr_window, lr_args = list(mult = lr_mult), progress = FALSE - ) - overall <- left_join( - hub_coverage_summary(s), - select(hub_quantile_loss(s), horizon, loss_pct = pct_improvement), - by = "horizon" - ) %>% - mutate(season = "all") - by_season <- left_join( - hub_coverage_summary(s, by = "season"), - select(hub_quantile_loss(s, by = "season"), - season, horizon, - loss_pct = pct_improvement - ), - by = c("season", "horizon") - ) - bind_rows(overall, by_season) %>% - mutate(season_policy = season_policy, lr_window = lr_window, lr_mult = lr_mult) -}) %>% bind_rows() - -# Keep the sweep around for analysis outside the notebook. -dir.create(here::here("cache/calibration"), showWarnings = FALSE, recursive = TRUE) -saveRDS(sweep, here::here("cache/calibration/sweep.rds")) -``` - -```{r sweep-cal-error, fig.height=5} -sweep %>% - filter(season == "all") %>% - ggplot(aes(lr_mult, cal_error_cal, colour = factor(lr_window))) + - geom_line() + - geom_point(size = 1.6) + - scale_x_log10() + - facet_grid(season_policy ~ horizon, labeller = label_both) + - labs( - title = "Calibration error by multiplier, one line per window", - subtitle = "Separated lines would indicate a window effect; sloped lines indicate a multiplier effect.", - x = "lr mult (log)", y = "calibration error (calibrated)", - colour = "lr_window" - ) -``` - -```{r sweep-wis, fig.height=5} -sweep %>% - filter(season == "all") %>% - ggplot(aes(lr_mult, loss_pct, colour = factor(lr_window))) + - geom_hline(yintercept = 0, linetype = "dashed", colour = "grey40") + - geom_line() + - geom_point(size = 1.6) + - scale_x_log10() + - facet_grid(season_policy ~ horizon, labeller = label_both) + - labs( - title = "WIS improvement by multiplier, one line per window", - subtitle = "Dashed line is WIS parity with the raw submission; above it calibration has no WIS cost.", - x = "lr mult (log)", y = "WIS improvement (%)", - colour = "lr_window" - ) -``` - -The target region is the combination of low calibration error and WIS at or -above parity. - -```{r sweep-corner, fig.height=5} -sweep %>% - filter(season == "all", season_policy == "carry") %>% - ggplot(aes(cal_error_cal, loss_pct, - colour = factor(lr_mult), shape = factor(lr_window) - )) + - geom_hline(yintercept = 0, linetype = "dashed", colour = "grey40") + - geom_point(size = 3, alpha = 0.85) + - facet_wrap(~horizon, scales = "free_x", labeller = label_both, nrow = 1) + - labs( - title = "Calibration error vs WIS change (season_policy = carry)", - subtitle = "Upper-left is the target region: well calibrated and no worse on WIS.", - x = "calibration error (calibrated)", y = "WIS change (%)", - colour = "lr mult", shape = "lr_window" - ) -``` - -If the expanding window's monotone behavior matters, its effect should grow -with history: season 2 starts with at most one season in the pool, season 3 -with two. The per-season view checks that. - -```{r sweep-by-season, fig.height=5} -sweep %>% - filter(season != "all", season_policy == "carry", horizon == 2) %>% - ggplot(aes(lr_mult, cal_error_cal, colour = factor(lr_window))) + - geom_line() + - geom_point(size = 1.6) + - scale_x_log10() + - facet_wrap(~season, labeller = label_both) + - labs( - title = "Calibration error by season (horizon 2, season_policy = carry)", - subtitle = "Does the window matter more once multiple seasons of residuals are available?", - x = "lr mult (log)", y = "calibration error (calibrated)", - colour = "lr_window" - ) -``` - -```{r sweep-table} -sweep %>% - filter(season == "all") %>% - transmute( - season_policy, lr_window, lr_mult, horizon, - cal_error_cal = round(cal_error_cal, 4), - wis_pct = round(loss_pct, 1) - ) %>% - arrange(horizon, season_policy, lr_window, lr_mult) -``` - -# 6. Offset trajectories - -```{r offsets, fig.height=6} -cal$forecasts %>% - filter(location %in% show_geos, horizon == 2, level %in% c(0.025, 0.5, 0.975)) %>% - mutate(location = recode(location, - "06" = "CA", "36" = "NY", "48" = "TX", "50" = "VT", "US" = "US" - )) %>% - ggplot(aes(reference_date, offset, colour = factor(level))) + - geom_hline(yintercept = 0, colour = "grey50") + - geom_line() + - facet_wrap(~location, scales = "free_y", ncol = 1) + - labs( - title = "Realised offset (played - base), horizon 2", - subtitle = "Zero through the burn-in season by construction. With season_policy='carry' the offset is held across the off-season gap.", - x = NULL, y = "offset (admissions)", colour = "level" - ) -``` - -```{r offset-scale} -cal$forecasts %>% - filter(!is_burn_in, !is.na(value_base), value_base > 0) %>% - group_by(horizon, level) %>% - summarize( - median_abs_offset = median(abs(offset)), - median_rel = median(abs(offset) / pmax(value_base, 1)), - .groups = "drop" - ) %>% - ggplot(aes(level, median_rel, colour = factor(horizon))) + - geom_line() + - scale_y_continuous(labels = scales::percent) + - labs( - title = "Median |offset| as a fraction of the base forecast", - subtitle = "How large a correction the tracker is applying, and how that scales with horizon.", - x = "quantile level", y = "median |offset| / base", colour = "horizon" - ) -``` diff --git a/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd b/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd deleted file mode 100644 index 3d089317..00000000 --- a/reports/writeups/calibration/calibration_qt_gallery_covid.Rmd +++ /dev/null @@ -1,245 +0,0 @@ ---- -title: MultiQT calibration gallery — per-forecast panels, COVID hub -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - df_print: paged - toc: true - toc_float: true - toc_depth: 2 -params: - hub_dir: "../covid19-forecast-hub" - burn_in_seasons: "" - season_policy: "carry" - lr_window: 20 - lr_mult: 0.03 - n_panels: 150 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code + cache.extra, NOT on functions -# in R/ -- after changing calibration code, clear cache/calibration/knitr_cache_gallery_covid/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_gallery_covid/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -```{r libs} -# Not cached: attaches packages every render. -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -``` - -```{r load, cache=TRUE, cache.extra=list(params)} -forecasts <- hub_read_forecasts(hub_dir = here::here(params$hub_dir), target = HUB_COVID_TARGET) -truth <- nhsn_read_truth("covid") -rounds <- hub_label_seasons(unique(forecasts$reference_date)) - -burn_in <- if (nzchar(params$burn_in_seasons)) { - strsplit(params$burn_in_seasons, ",")[[1]] -} else { - character(0) -} - -cal <- calibrate_hub_forecasts( - forecasts, truth, - burn_in_seasons = burn_in, - season_policy = params$season_policy, - lr_window = params$lr_window, - lr_args = list(mult = params$lr_mult), - transform = "sqrt", - progress = FALSE -) -``` - -Per-forecast panel gallery for COVID hub submissions at the chosen operating point -(`calibration_qt_covid.Rmd` has the parameter sweeps; see `notes/calibration-ledger.md` -for project state). - -Parameters: -- `lr_mult = `r params$lr_mult`` -- `lr_window = `r params$lr_window`` -- `season_policy = "`r params$season_policy`"` -- `transform = "sqrt"` - -# Summary scores - -```{r headline} -bind_rows( - left_join( - hub_coverage_summary(cal), - select(hub_quantile_loss(cal), horizon, loss_base, loss_cal, loss_pct = pct_improvement), - by = "horizon" - ) %>% mutate(season = "all"), - left_join( - hub_coverage_summary(cal, by = "season"), - select(hub_quantile_loss(cal, by = "season"), - season, horizon, loss_base, loss_cal, loss_pct = pct_improvement - ), - by = c("season", "horizon") - ) -) %>% - transmute( - season, horizon, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) %>% - arrange(season, horizon) -``` - -# Data vintages for the panels - -Each panel shows the NHSN data as of the Wednesday before the reference -date (`reference_date - 3`) alongside the finalized truth. - -```{r snapshots, cache=TRUE, dependson="load"} -archive <- get_nhsn_data_archive("covid")$DT %>% - mutate(geo_value = ifelse(geo_value == "usa", "us", geo_value)) %>% - as_epi_archive(compactify = TRUE) - -geo_map <- get_population_data() %>% - distinct(state_id, .keep_all = TRUE) %>% - transmute(state_id, abbrev = toupper(state_id), location = state_code) - -live_rounds <- rounds %>% filter(!season %in% burn_in) -min_version <- min(archive$DT$version) -max_version <- archive$versions_end - -snapshots <- purrr::map(live_rounds$round_date, function(rd) { - as_of <- rd - 3L - if (as_of < min_version || as_of > max_version) { - return(NULL) - } - epix_as_of(archive, as_of) %>% - as_tibble() %>% - inner_join(geo_map, by = c("geo_value" = "state_id")) %>% - transmute(reference_date = rd, location, time_value, value) -}) %>% bind_rows() -``` - -# Ranking - -Panels are ordered by mean absolute displacement between calibrated and base -quantiles, divided by the round's mean base median (floored at 20 admissions). - -```{r ranking} -pinball <- function(truth, value, level) { - ifelse(truth >= value, level * (truth - value), (1 - level) * (value - truth)) -} - -live <- cal$forecasts %>% - filter(!is_burn_in, !is.na(value_base), !is.na(value_cal)) - -escapes <- live %>% - filter(level %in% c(0.05, 0.95), !is.na(truth)) %>% - select(location, reference_date, horizon, level, value_base, truth) %>% - tidyr::pivot_wider(names_from = level, values_from = value_base, names_prefix = "q") %>% - group_by(location, reference_date) %>% - summarize(escape90 = any(truth < q0.05 | truth > q0.95), .groups = "drop") - -panel_stats <- live %>% - group_by(season, location, reference_date) %>% - summarize( - mad = mean(abs(value_cal - value_base)), - base_med = mean(value_base[level == 0.5]), - wis_base = 2 * mean(pinball(truth, value_base, level)[!is.na(truth)]), - wis_cal = 2 * mean(pinball(truth, value_cal, level)[!is.na(truth)]), - .groups = "drop" - ) %>% - mutate( - mad_rel = mad / pmax(base_med, 20), - wis_pct = if_else(wis_base > 0, 100 * (1 - wis_cal / wis_base), NA_real_) - ) %>% - left_join(escapes, by = c("location", "reference_date")) %>% - left_join(select(geo_map, location, abbrev), by = "location") %>% - arrange(desc(mad_rel)) - -panel_stats %>% - transmute( - season, abbrev, reference_date, - mad = round(mad, 1), mad_rel = round(mad_rel, 3), - wis_pct = round(wis_pct, 1), escape90 - ) -``` - -# Gallery: top `r params$n_panels` panels - -Black: NHSN data as of the forecast Wednesday (dashed vertical line). -Grey: finalized truth. Blue: submitted (base) forecast. Orange: calibrated. -Ribbons are the 50% and 90% central intervals; line is the median, across -horizons −1 to 3. - -```{r panel-fn} -fan_cols <- c(base = "#0072B2", calibrated = "#D55E00") -ribbon_levels <- c( - "0.05" = "lo90", "0.25" = "lo50", "0.5" = "med", - "0.75" = "hi50", "0.95" = "hi90" -) - -panel_plot <- function(loc, ref_date, caption) { - win_start <- ref_date - 84L - win_end <- ref_date + 28L - fin <- truth %>% - filter(location == loc, target_end_date >= win_start, target_end_date <= win_end) - snap <- snapshots %>% - filter(reference_date == ref_date, location == loc, time_value >= win_start) - fan <- cal$forecasts %>% - filter( - location == loc, reference_date == ref_date, - level %in% c(0.05, 0.25, 0.5, 0.75, 0.95) - ) %>% - mutate( - target_end_date = reference_date + 7L * horizon, - level = ribbon_levels[as.character(level)] - ) %>% - select(target_end_date, level, base = value_base, calibrated = value_cal) %>% - tidyr::pivot_longer(c(base, calibrated), names_to = "which") %>% - tidyr::pivot_wider(names_from = level, values_from = value) - - ggplot() + - geom_line(data = fin, aes(target_end_date, truth), colour = "grey65") + - geom_point(data = fin, aes(target_end_date, truth), colour = "grey65", size = 0.7) + - geom_ribbon( - data = fan, aes(target_end_date, ymin = lo90, ymax = hi90, fill = which), - alpha = 0.16 - ) + - geom_ribbon( - data = fan, aes(target_end_date, ymin = lo50, ymax = hi50, fill = which), - alpha = 0.3 - ) + - geom_line(data = fan, aes(target_end_date, med, colour = which)) + - geom_line(data = snap, aes(time_value, value), colour = "black") + - geom_point(data = snap, aes(time_value, value), colour = "black", size = 0.7) + - geom_vline(xintercept = ref_date - 3L, linetype = "dashed", colour = "grey40") + - scale_fill_manual(values = fan_cols, aesthetics = c("fill", "colour")) + - labs(x = NULL, y = "admissions", colour = NULL, fill = NULL, subtitle = caption) -} -``` - -```{r gallery, results='asis', fig.width=9, fig.height=4} -top <- head(panel_stats, params$n_panels) -for (i in seq_len(nrow(top))) { - p <- top[i, ] - cat(sprintf( - "\n\n### %d. %s — %s (%s)\n\n", - i, p$abbrev, format(p$reference_date), p$season - )) - caption <- sprintf( - "MAD %.1f admissions (base median %.1f) | WIS %s | truth escaped base 90%%: %s", - p$mad, p$base_med, - if (is.na(p$wis_pct)) "n/a" else sprintf("%+.1f%%", p$wis_pct), - if (isTRUE(p$escape90)) "yes" else "no" - ) - print(panel_plot(p$location, p$reference_date, caption)) -} -``` diff --git a/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd b/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd deleted file mode 100644 index 8d4c7839..00000000 --- a/reports/writeups/calibration/calibration_qt_gallery_flu.Rmd +++ /dev/null @@ -1,252 +0,0 @@ ---- -title: MultiQT calibration gallery — per-forecast panels at the chosen operating point -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - df_print: paged - toc: true - toc_float: true - toc_depth: 2 -params: - hub_dir: "../FluSight-forecast-hub" - burn_in_seasons: "2023-2024" - season_policy: "carry" - lr_window: 20 - lr_mult: 0.03 - n_panels: 150 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code + cache.extra, NOT on functions -# in R/ -- after changing calibration code, clear cache/calibration/knitr_cache_gallery_flu/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_gallery_flu/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -```{r libs} -# Not cached: attaches packages every render. -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -``` - -```{r load, cache=TRUE, cache.extra=list(params)} -forecasts <- hub_read_forecasts(hub_dir = here::here(params$hub_dir)) -truth <- nhsn_read_truth("flu") -rounds <- hub_label_seasons(unique(forecasts$reference_date)) - -burn_in <- if (nzchar(params$burn_in_seasons)) { - strsplit(params$burn_in_seasons, ",")[[1]] -} else { - character(0) -} - -cal <- calibrate_hub_forecasts( - forecasts, truth, - burn_in_seasons = burn_in, - season_policy = params$season_policy, - lr_window = params$lr_window, - lr_args = list(mult = params$lr_mult), - progress = FALSE -) -``` - -This document contains a gallery of forecasts that were adjusted the most (in mean absolute deviation, see below) (`calibration_qt_flu.Rmd` has the parameter-sweeps, this notebook gets more granular about forecast adjustments; see -`notes/calibration-ledger.md` for the project state). - -The parameters for the calibration method are fixed: -- `lr_mult = `r params$lr_mult`` (the learning rate multiplier, which was found to be too slow at 0.01 and too fast at 0.1), -- `lr_window = `r params$lr_window`` (a window of 8 weeks paid the cost across aheads, while 20, 50, and Inf were essentially indistinguishable and had consistent calibration improvements across aheads) -- `season_policy = "`r params$season_policy`"` (this parameter was either "carry" for carrying the learning rate over from the previous season or "reset" to reset it; we found that this did not influence things very much) - -# Summary scores at the parameter sweep peak settings - -Calibration error is the mean over the 23 quantile levels of -`|geo-pooled empirical coverage at a - a|`; WIS uses the standard -factor-of-two convention. The burn-in season (2023-24) is excluded, so testing is across (2024-25 and 2025-26). - -```{r headline} -bind_rows( - left_join( - hub_coverage_summary(cal), - select(hub_quantile_loss(cal), horizon, loss_base, loss_cal, loss_pct = pct_improvement), - by = "horizon" - ) %>% mutate(season = "all"), - left_join( - hub_coverage_summary(cal, by = "season"), - select(hub_quantile_loss(cal, by = "season"), - season, horizon, loss_base, loss_cal, loss_pct = pct_improvement - ), - by = c("season", "horizon") - ) -) %>% - transmute( - season, horizon, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) %>% - arrange(season, horizon) -``` - -# Data vintages for the panels - -Each panel shows the NHSN data as of the Wednesday before the reference -date (`reference_date - 3`, which should be what the forecaster saw at submission -time, unless there were delays) alongside the finalized truth. - -```{r snapshots, cache=TRUE, dependson="load"} -archive <- nanoparquet::read_parquet( - here::here("cache/calibration/nhsn_archive_flu.parquet") -) %>% - as_epi_archive(compactify = TRUE) - -geo_map <- get_population_data() %>% - distinct(state_id, .keep_all = TRUE) %>% - transmute(state_id, abbrev = toupper(state_id), location = state_code) - -live_rounds <- rounds %>% filter(!season %in% burn_in) -min_version <- min(archive$DT$version) - -snapshots <- purrr::map(live_rounds$round_date, function(rd) { - as_of <- rd - 3L - if (as_of < min_version) { - return(NULL) - } - epix_as_of(archive, as_of) %>% - as_tibble() %>% - inner_join(geo_map, by = c("geo_value" = "state_id")) %>% - transmute(reference_date = rd, location, time_value, value) -}) %>% bind_rows() -``` - -# Ranking - -Panels are ordered by the size of the correction: the mean absolute -displacement between calibrated and base quantiles (over the 5 horizons x 23 -levels of a round), divided by the round's mean base median (this way we rank small states -on relative shift). We floored the denominator at 20 admissions (with out this the entire top of the list is -end-of-season rounds where the base median is ~1). - -Even with that correction, season edges (May, October) dominate: the mid-season offsets are carried into the very low signal values at the end. - -```{r ranking} -pinball <- function(truth, value, level) { - ifelse(truth >= value, level * (truth - value), (1 - level) * (value - truth)) -} - -live <- cal$forecasts %>% - filter(!is_burn_in, !is.na(value_base), !is.na(value_cal)) - -escapes <- live %>% - filter(level %in% c(0.05, 0.95), !is.na(truth)) %>% - select(location, reference_date, horizon, level, value_base, truth) %>% - tidyr::pivot_wider(names_from = level, values_from = value_base, names_prefix = "q") %>% - group_by(location, reference_date) %>% - summarize(escape90 = any(truth < q0.05 | truth > q0.95), .groups = "drop") - -panel_stats <- live %>% - group_by(season, location, reference_date) %>% - summarize( - mad = mean(abs(value_cal - value_base)), - base_med = mean(value_base[level == 0.5]), - wis_base = 2 * mean(pinball(truth, value_base, level)[!is.na(truth)]), - wis_cal = 2 * mean(pinball(truth, value_cal, level)[!is.na(truth)]), - .groups = "drop" - ) %>% - mutate( - mad_rel = mad / pmax(base_med, 20), - wis_pct = if_else(wis_base > 0, 100 * (1 - wis_cal / wis_base), NA_real_) - ) %>% - left_join(escapes, by = c("location", "reference_date")) %>% - left_join(select(geo_map, location, abbrev), by = "location") %>% - arrange(desc(mad_rel)) - -panel_stats %>% - transmute( - season, abbrev, reference_date, - mad = round(mad, 1), mad_rel = round(mad_rel, 3), - wis_pct = round(wis_pct, 1), escape90 - ) -``` - -# Gallery: top `r params$n_panels` panels - -In the plots below, black is the NHSN data as of the forecast Wednesday -(dashed vertical line), while gray is the finalized series. Blue is the submitted -(base) forecast, orange the calibrated one; ribbons are the 50% and 90% -central intervals, the line is the median, across horizons -1 to 3. - -```{r panel-fn} -fan_cols <- c(base = "#0072B2", calibrated = "#D55E00") -ribbon_levels <- c( - "0.05" = "lo90", "0.25" = "lo50", "0.5" = "med", - "0.75" = "hi50", "0.95" = "hi90" -) - -panel_plot <- function(loc, ref_date, caption) { - win_start <- ref_date - 84L - win_end <- ref_date + 28L - fin <- truth %>% - filter(location == loc, target_end_date >= win_start, target_end_date <= win_end) - snap <- snapshots %>% - filter(reference_date == ref_date, location == loc, time_value >= win_start) - fan <- cal$forecasts %>% - filter( - location == loc, reference_date == ref_date, - level %in% c(0.05, 0.25, 0.5, 0.75, 0.95) - ) %>% - mutate( - target_end_date = reference_date + 7L * horizon, - level = ribbon_levels[as.character(level)] - ) %>% - select(target_end_date, level, base = value_base, calibrated = value_cal) %>% - tidyr::pivot_longer(c(base, calibrated), names_to = "which") %>% - tidyr::pivot_wider(names_from = level, values_from = value) - - ggplot() + - geom_line(data = fin, aes(target_end_date, truth), colour = "grey65") + - geom_point(data = fin, aes(target_end_date, truth), colour = "grey65", size = 0.7) + - geom_ribbon( - data = fan, aes(target_end_date, ymin = lo90, ymax = hi90, fill = which), - alpha = 0.16 - ) + - geom_ribbon( - data = fan, aes(target_end_date, ymin = lo50, ymax = hi50, fill = which), - alpha = 0.3 - ) + - geom_line(data = fan, aes(target_end_date, med, colour = which)) + - geom_line(data = snap, aes(time_value, value), colour = "black") + - geom_point(data = snap, aes(time_value, value), colour = "black", size = 0.7) + - geom_vline(xintercept = ref_date - 3L, linetype = "dashed", colour = "grey40") + - scale_fill_manual(values = fan_cols, aesthetics = c("fill", "colour")) + - labs(x = NULL, y = "admissions", colour = NULL, fill = NULL, subtitle = caption) -} -``` - -```{r gallery, results='asis', fig.width=9, fig.height=4} -top <- head(panel_stats, params$n_panels) -for (i in seq_len(nrow(top))) { - p <- top[i, ] - cat(sprintf( - "\n\n### %d. %s — %s (%s)\n\n", - i, p$abbrev, format(p$reference_date), p$season - )) - caption <- sprintf( - "MAD %.1f admissions (base median %.1f) | WIS %s | truth escaped base 90%%: %s", - p$mad, p$base_med, - if (is.na(p$wis_pct)) "n/a" else sprintf("%+.1f%%", p$wis_pct), - if (isTRUE(p$escape90)) "yes" else "no" - ) - print(panel_plot(p$location, p$reference_date, caption)) -} -``` diff --git a/reports/writeups/calibration/calibration_qt_seasons_covid.Rmd b/reports/writeups/calibration/calibration_qt_seasons_covid.Rmd deleted file mode 100644 index c0972d60..00000000 --- a/reports/writeups/calibration/calibration_qt_seasons_covid.Rmd +++ /dev/null @@ -1,354 +0,0 @@ ---- -title: MultiQT calibration — whole-season views of four states, COVID hub -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - df_print: paged - toc: true - toc_float: true - toc_depth: 2 -params: - hub_dir: "../covid19-forecast-hub" - burn_in_seasons: "" - season_policy: "carry" - lr_window: 20 - lr_mult: 0.03 - lr_floor: 0.001 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code + cache.extra, NOT on functions -# in R/ -- after changing calibration code, clear cache/calibration/knitr_cache_seasons_covid/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_seasons_covid/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -This notebook takes the sqrt-space operating point (`transform = "sqrt"`, -`lr_mult = 0.03`, `lr_window = 20`, carry, no burn-in) and looks at four -states across the available season of COVID hub submissions. -COVID runs year-round with no off-season gap, so season policy has no practical -effect; burn-in-dependent variants (conformal warm start) are not included. -The states are chosen by how much the baseline calibration changed them, pooled -over horizons −1…3. - -```{r libs} -# Not cached: attaches packages every render. -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -``` - -```{r load, cache=TRUE} -forecasts <- hub_read_forecasts(hub_dir = here::here(params$hub_dir), target = HUB_COVID_TARGET) -truth <- nhsn_read_truth("covid") -rounds <- hub_label_seasons(unique(forecasts$reference_date)) -burn_in <- if (nzchar(params$burn_in_seasons)) { - strsplit(params$burn_in_seasons, ",")[[1]] -} else { - character(0) -} - -geo_map <- get_population_data() %>% - distinct(state_id, .keep_all = TRUE) %>% - transmute(state_id, abbrev = toupper(state_id), location = state_code) - -archive <- get_nhsn_data_archive("covid")$DT %>% - mutate(geo_value = ifelse(geo_value == "usa", "us", geo_value)) %>% - as_epi_archive(compactify = TRUE) - -live_rounds <- rounds %>% filter(!season %in% burn_in) -min_version <- min(archive$DT$version) -max_version <- archive$versions_end -snapshots <- purrr::map(live_rounds$round_date, function(rd) { - as_of <- rd - 3L - if (as_of < min_version || as_of > max_version) { - return(NULL) - } - epix_as_of(archive, as_of) %>% - as_tibble() %>% - inner_join(geo_map, by = c("geo_value" = "state_id")) %>% - transmute(reference_date = rd, location, time_value, value) -}) %>% bind_rows() - -seasons <- unique(live_rounds$season) -``` - -```{r variants, cache=TRUE, dependson="load"} -base_args <- list( - burn_in_seasons = burn_in, season_policy = params$season_policy, - lr_window = params$lr_window, transform = "sqrt", - lr_args = list(mult = params$lr_mult, floor = params$lr_floor), - progress = FALSE -) -# Burn-in-dependent variants (warm_static, warm_fast, warm_slow_fast) are not -# included: COVID has no burn-in season so slow_init = "burn_in_quantile" has -# no data to warm from. off_feb is also excluded -- COVID has no spring tail. -variant_args <- list( - baseline = list(), - count_space = list(transform = "identity", lr_args = list(mult = params$lr_mult)), - slow_term = list(lr_slow = list(mult = 0.003)), - slow_fast = list(lr_slow = list(mult = 0.003), fast_decay = 0.1) -) -variant_titles <- c( - baseline = "Sqrt-space tracker (baseline)", - count_space = "Count-space tracker (the paper's method)", - slow_term = "Sqrt tracker + persistent slow term", - slow_fast = "Sqrt tracker + slow term + leaky fast term" -) - -cals <- purrr::map(variant_args, function(v) { - do.call(calibrate_hub_forecasts, c(list(forecasts, truth), modifyList(base_args, v))) -}) -``` - -```{r select-states, dependson="variants"} -pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) - -by_state <- cals$baseline$forecasts %>% - filter(!is_burn_in, !is.na(truth)) %>% - group_by(location) %>% - summarize( - wis_pct = 100 * (1 - sum(pinball(truth, value_cal, level)) / sum(pinball(truth, value_base, level))), - ae_pct = 100 * (1 - sum(abs(truth - value_cal)[level == 0.5]) / sum(abs(truth - value_base)[level == 0.5])), - .groups = "drop" - ) %>% - inner_join(select(geo_map, location, abbrev), by = "location") - -pick <- function(ranked, taken) ranked$location[!ranked$location %in% taken][1] -roles <- list( - `best WIS` = arrange(by_state, desc(wis_pct)), - `worst WIS` = arrange(by_state, wis_pct), - `best AE` = arrange(by_state, desc(ae_pct)), - `worst AE` = arrange(by_state, ae_pct) -) -taken <- character(0) -for (r in names(roles)) taken <- c(taken, pick(roles[[r]], taken)) -states <- tibble(location = taken, role = names(roles)) %>% - left_join(by_state, by = "location") %>% - mutate(label = sprintf("%s (%s: WIS %+.1f%%, AE %+.1f%%)", abbrev, role, wis_pct, ae_pct)) -``` - -# Which states, and why - -Per-state change from baseline calibration over all rounds, all horizons pooled. - -```{r state-table} -knitr::kable(states %>% transmute(state = abbrev, role, `WIS change %` = round(wis_pct, 1), `AE change %` = round(ae_pct, 1))) -``` - -# Summary across variants - -WIS % improvement over the base forecast (all rounds), per horizon. - -```{r variant-summary} -purrr::imap(cals, function(cal, nm) { - hub_quantile_loss(cal) %>% transmute(variant = variant_titles[[nm]], horizon, pct = round(pct_improvement, 1)) -}) %>% - bind_rows() %>% - tidyr::pivot_wider(names_from = horizon, values_from = pct, names_prefix = "h") -``` - -## Where in the year the baseline gains and loses - -Baseline variant, pooled over all locations, by calendar month of the reference -date. - -```{r by-month} -by_month <- cals$baseline$forecasts %>% - filter(!is_burn_in, !is.na(truth)) %>% - mutate(month = format(reference_date, "%b"), month_n = as.integer(format(reference_date, "%m"))) %>% - group_by(horizon, month, month_n) %>% - summarize( - base = sum(pinball(truth, value_base, level)), - cal = sum(pinball(truth, value_cal, level)), - shift = 100 * sum((value_cal - value_base)[level == 0.5]) / sum(truth[level == 0.5]), - .groups = "drop" - ) %>% - group_by(horizon) %>% - mutate(pct = 100 * (base - cal) / base, share = 100 * base / sum(base)) %>% - ungroup() %>% - mutate(month = factor(month, levels = unique(month[order(month_n)]))) - -show_wide <- function(col, digits = 1) { - by_month %>% - select(horizon, month, value = all_of(col)) %>% - mutate(value = round(value, digits)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = value, names_prefix = "h") %>% - arrange(month) -} -knitr::kable(show_wide("pct"), caption = "WIS change % (calibrated vs base)") -knitr::kable(show_wide("share"), caption = "Share of base WIS %") -knitr::kable(show_wide("shift"), caption = "Median shift % of truth") -``` - -```{r helpers} -headline_table <- function(cal) { - bind_rows( - left_join( - hub_coverage_summary(cal), - select(hub_quantile_loss(cal), horizon, loss_base, loss_cal, loss_pct = pct_improvement), - by = "horizon" - ) %>% mutate(season = "all"), - left_join( - hub_coverage_summary(cal, by = "season"), - select(hub_quantile_loss(cal, by = "season"), - season, horizon, loss_base, loss_cal, loss_pct = pct_improvement - ), - by = c("season", "horizon") - ) - ) %>% - transmute( - season, horizon, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) %>% - arrange(season, horizon) -} - -fan_cols <- c(base = "#0072B2", calibrated = "#D55E00") - -season_plot <- function(cal, loc, season) { - rr <- cal$rounds %>% filter(season == !!season, season_round %% 2 == 1) - ref_dates <- sort(rr$round_date) - - fin <- truth %>% - filter(location == loc, target_end_date >= min(ref_dates) - 21L, target_end_date <= max(ref_dates) + 28L) - - fan <- cal$forecasts %>% - filter(location == loc, reference_date %in% ref_dates, horizon %in% 0:3, level %in% c(0.1, 0.5, 0.9)) %>% - mutate( - target_end_date = reference_date + 7L * horizon, - lvl = c(`0.1` = "lo", `0.5` = "med", `0.9` = "hi")[as.character(level)] - ) %>% - select(reference_date, target_end_date, lvl, base = value_base, calibrated = value_cal) %>% - tidyr::pivot_longer(c(base, calibrated), names_to = "which") %>% - tidyr::pivot_wider(names_from = lvl, values_from = value) - - vint_windows <- tibble(ref = ref_dates) %>% - mutate( - win_start = dplyr::lag(ref, default = ref[1] - 14L) - 3L, - win_end = ref - 3L - ) - vint <- purrr::pmap(vint_windows, function(ref, win_start, win_end) { - snapshots %>% - filter(reference_date == ref, location == loc, time_value >= win_start, time_value <= win_end) - }) %>% bind_rows() - - main <- ggplot() + - geom_line(data = fin, aes(target_end_date, truth), colour = "grey65") + - geom_point(data = fin, aes(target_end_date, truth), colour = "grey65", size = 0.7) + - geom_ribbon( - data = filter(fan, which == "base"), - aes(target_end_date, ymin = lo, ymax = hi, fill = which, group = reference_date), - alpha = 0.2 - ) + - geom_line( - data = filter(fan, which == "calibrated"), - aes(target_end_date, lo, colour = which, group = reference_date), linewidth = 0.3 - ) + - geom_line( - data = filter(fan, which == "calibrated"), - aes(target_end_date, hi, colour = which, group = reference_date), linewidth = 0.3 - ) + - geom_line( - data = fan, - aes(target_end_date, med, colour = which, group = interaction(reference_date, which)) - ) + - geom_line(data = vint, aes(time_value, value), colour = "black") + - geom_point(data = vint, aes(time_value, value), colour = "black", size = 0.6) + - geom_vline(xintercept = ref_dates - 3L, linetype = "dashed", colour = "grey40", linewidth = 0.3) + - scale_fill_manual(values = fan_cols, aesthetics = c("fill", "colour")) + - labs(x = NULL, y = "admissions", colour = NULL, fill = NULL) + - theme(legend.position = "top") - - eta <- cal$series %>% - filter(location == loc, season == !!season, horizon %in% 0:3) %>% - mutate(lr = if_else(lr > 0, lr, NA_real_)) %>% - ggplot(aes(round_date, lr, colour = factor(horizon))) + - geom_line(na.rm = TRUE) + - geom_point(size = 0.6, na.rm = TRUE) + - labs(x = NULL, y = "eta", colour = "horizon") + - theme(legend.position = "top") - - title <- paste0(geo_map$abbrev[match(loc, geo_map$location)], " — ", season) - cowplot::plot_grid( - cowplot::ggdraw() + cowplot::draw_label(title, fontface = "bold"), - cowplot::plot_grid(main, eta, ncol = 1, rel_heights = c(3, 1), align = "v", axis = "lr"), - ncol = 1, rel_heights = c(0.08, 1) - ) -} - -oh_table <- function(cal, loc, h, season) { - cal$forecasts %>% - filter(location == loc, horizon == h, season == !!season, level %in% c(0.05, 0.25, 0.5, 0.75, 0.95)) %>% - mutate(level = paste0("q", level)) %>% - select(reference_date, level, lr_level, truth, hidden, offset, value_base, value_cal) %>% - tidyr::pivot_wider( - names_from = level, - values_from = c(hidden, offset, value_base, value_cal, lr_level), - names_glue = "{level}_{.value}" - ) %>% - mutate( - eta = round(q0.5_lr_level, 3), - across(c(truth, matches("hidden|offset|value_base|value_cal")), ~ round(.x)) - ) %>% - select( - reference_date, eta, truth, - matches("^q.*_hidden$"), matches("^q.*_offset$"), - matches("^q.*_value_base$"), matches("^q.*_value_cal$") - ) %>% - arrange(reference_date) -} - -internals_block <- function(cal, season) { - for (i in seq_len(nrow(states))) { - loc <- states$location[i] - cat(sprintf("\n\n
%s internals (h=2)\n\n", states$label[i])) - print(knitr::kable(oh_table(cal, loc, 2L, season))) - cat("\n\n
\n\n") - } -} -``` - -```{r sections, results='asis'} -for (nm in names(cals)) { - cal <- cals[[nm]] - cat(sprintf("\n\n# %s\n\n", variant_titles[[nm]])) - - desc <- switch(nm, - baseline = "The single-term tracker run on sqrt(counts). lr_mult = 0.03, lr_window = 20, carry, isotonic projection.", - count_space = "The paper's method as originally run: additive offsets in count space, same multiplier and window.", - slow_term = "Sqrt tracker plus a slow persistent offset term (mult 0.003, expanding window). No fast decay.", - slow_fast = "Sqrt tracker plus slow persistent term (mult 0.003) and a 10% per-round leak on the fast term." - ) - cat(desc, "\n\n") - - cat("## Headline\n\n") - print(knitr::kable(headline_table(cal))) - - for (season in seasons) { - cat(sprintf("\n\n## %s\n\n", season)) - for (i in seq_len(nrow(states))) { - loc <- states$location[i] - cat(sprintf("\n\n### %s — %s\n\n", states$label[i], season)) - print(season_plot(cal, loc, season)) - } - } - - cat("\n\n## Internals\n\n") - for (season in seasons) { - cat(sprintf("\n\n### %s\n\n", season)) - internals_block(cal, season) - } -} -``` diff --git a/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd b/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd deleted file mode 100644 index 107ed7f9..00000000 --- a/reports/writeups/calibration/calibration_qt_seasons_flu.Rmd +++ /dev/null @@ -1,400 +0,0 @@ ---- -title: MultiQT calibration — whole-season views of four states, across method variants -date: "`r format(Sys.time(), '%d %B %Y')`" -output: - html_document: - code_folding: hide - df_print: paged - toc: true - toc_float: true - toc_depth: 2 -params: - hub_dir: "../FluSight-forecast-hub" - burn_in_seasons: "2023-2024" - season_policy: "carry" - lr_window: 20 - lr_mult: 0.03 - lr_floor: 0.001 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -# cache = TRUE chunks are keyed on chunk code + cache.extra, NOT on functions -# in R/ -- after changing calibration code, clear cache/calibration/knitr_cache_seasons_flu/. -knitr::opts_chunk$set( - fig.align = "center", fig.width = 10, fig.height = 6, - message = FALSE, warning = FALSE, cache = FALSE, - cache.path = here::here("cache/calibration/knitr_cache_seasons_flu/") -) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw()) -``` - -This notebook takes the sqrt-space operating point from `notes/calibration-ledger.md` -(`transform = "sqrt"`, `lr_mult = 0.03`, `lr_window = 20`, carry, burn-in -2023-24) and looks at four -states across whole seasons rather than one panel at a time -(`calibration_qt_gallery_flu.Rmd` has the per-forecast gallery view). The states -are chosen by how much the baseline calibration changed them over the two live -seasons, pooled over horizons −1…3: the best and worst by WIS change, and the -best and worst by absolute error of the median. If one state is best (or -worst) on both, the next-ranked distinct state is taken so four panels are -shown. Method variants compared: the sqrt single-term tracker (baseline), the -paper's count-space tracker, a static conformal warm start from the burn-in -season with no tracking, the warm start plus a leaky fast tracker, the warm -start plus slow and leaky fast terms (the current operating point), and -switching off after February 15 as the earlier reference. - -```{r libs} -# Not cached: attaches packages every render. -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -``` - -```{r load, cache=TRUE} -forecasts <- hub_read_forecasts(hub_dir = here::here(params$hub_dir)) -truth <- nhsn_read_truth("flu") -rounds <- hub_label_seasons(unique(forecasts$reference_date)) -burn_in <- strsplit(params$burn_in_seasons, ",")[[1]] - -geo_map <- get_population_data() %>% - distinct(state_id, .keep_all = TRUE) %>% - transmute(state_id, abbrev = toupper(state_id), location = state_code) - -archive <- nanoparquet::read_parquet( - here::here("cache/calibration/nhsn_archive_flu.parquet") -) %>% - as_epi_archive(compactify = TRUE) - -live_rounds <- rounds %>% filter(!season %in% burn_in) -min_version <- min(archive$DT$version) -max_version <- archive$versions_end -snapshots <- purrr::map(live_rounds$round_date, function(rd) { - as_of <- rd - 3L - if (as_of < min_version || as_of > max_version) { - return(NULL) - } - epix_as_of(archive, as_of) %>% - as_tibble() %>% - inner_join(geo_map, by = c("geo_value" = "state_id")) %>% - transmute(reference_date = rd, location, time_value, value) -}) %>% bind_rows() - -seasons <- c("2024-2025", "2025-2026") -``` - -```{r variants, cache=TRUE, dependson="load"} -base_args <- list( - burn_in_seasons = burn_in, season_policy = params$season_policy, - lr_window = params$lr_window, transform = "sqrt", - lr_args = list(mult = params$lr_mult, floor = params$lr_floor), - progress = FALSE -) -variant_args <- list( - baseline = list(), - count_space = list(transform = "identity", lr_args = list(mult = params$lr_mult)), - warm_static = list(slow_init = "burn_in_quantile", lr_args = list(mult = 1e-6, floor = 0)), - warm_fast = list(slow_init = "burn_in_quantile", fast_decay = 0.1), - warm_slow_fast = list(slow_init = "burn_in_quantile", lr_slow = list(mult = 0.003), fast_decay = 0.1), - off_feb = list(off_after = "02-15") -) -variant_titles <- c( - baseline = "Sqrt-space tracker (baseline)", - count_space = "Count-space tracker (the paper's method)", - warm_static = "Conformal warm start only, no tracking", - warm_fast = "Warm start + leaky fast tracker", - warm_slow_fast = "Warm start + slow term + leaky fast tracker (operating point)", - off_feb = "Calibration off after February 15" -) - -cals <- purrr::map(variant_args, function(v) { - do.call(calibrate_hub_forecasts, c(list(forecasts, truth), modifyList(base_args, v))) -}) -``` - -```{r select-states, dependson="variants"} -pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) - -by_state <- cals$baseline$forecasts %>% - filter(!is_burn_in, !is.na(truth)) %>% - group_by(location) %>% - summarize( - wis_pct = 100 * (1 - sum(pinball(truth, value_cal, level)) / sum(pinball(truth, value_base, level))), - ae_pct = 100 * (1 - sum(abs(truth - value_cal)[level == 0.5]) / sum(abs(truth - value_base)[level == 0.5])), - .groups = "drop" - ) %>% - inner_join(select(geo_map, location, abbrev), by = "location") - -# Best/worst by WIS then AE, skipping a state already taken so four are shown. -pick <- function(ranked, taken) ranked$location[!ranked$location %in% taken][1] -roles <- list( - `best WIS` = arrange(by_state, desc(wis_pct)), - `worst WIS` = arrange(by_state, wis_pct), - `best AE` = arrange(by_state, desc(ae_pct)), - `worst AE` = arrange(by_state, ae_pct) -) -taken <- character(0) -for (r in names(roles)) taken <- c(taken, pick(roles[[r]], taken)) -states <- tibble(location = taken, role = names(roles)) %>% - left_join(by_state, by = "location") %>% - mutate(label = sprintf("%s (%s: WIS %+.1f%%, AE %+.1f%%)", abbrev, role, wis_pct, ae_pct)) -``` - -# Which states, and why - -Per-state change from baseline calibration over both live seasons, all -horizons pooled (positive is an improvement). The four panels below are the -extremes of these two columns. - -```{r state-table} -knitr::kable(states %>% transmute(state = abbrev, role, `WIS change %` = round(wis_pct, 1), `AE change %` = round(ae_pct, 1))) -``` - -# Summary across variants - -WIS % improvement over the base forecast (all seasons, burn-in excluded), per -horizon. - -```{r variant-summary} -purrr::imap(cals, function(cal, nm) { - hub_quantile_loss(cal) %>% transmute(variant = variant_titles[[nm]], horizon, pct = round(pct_improvement, 1)) -}) %>% - bind_rows() %>% - tidyr::pivot_wider(names_from = horizon, values_from = pct, names_prefix = "h") -``` - -## Where in the season the baseline gains and loses - -Baseline variant, pooled over all locations and both live seasons, by calendar -month of the reference date. `WIS change %` is per month and horizon; `share of -base WIS %` is how much of the season's total base WIS that month carries, so a -large percentage loss in April costs little in absolute terms while a small -gain in December or January is worth a lot. `median shift %` is the calibrated -median minus the base median, summed over the month, as a percentage of summed -truth: the direction and size of the correction the tracker was applying. - -```{r by-month} -by_month <- cals$baseline$forecasts %>% - filter(!is_burn_in, !is.na(truth)) %>% - mutate(month = format(reference_date, "%b"), month_n = as.integer(format(reference_date, "%m"))) %>% - group_by(horizon, month, month_n) %>% - summarize( - base = sum(pinball(truth, value_base, level)), - cal = sum(pinball(truth, value_cal, level)), - shift = 100 * sum((value_cal - value_base)[level == 0.5]) / sum(truth[level == 0.5]), - .groups = "drop" - ) %>% - group_by(horizon) %>% - mutate(pct = 100 * (base - cal) / base, share = 100 * base / sum(base)) %>% - ungroup() %>% - mutate(month = factor(month, levels = unique(month[order((month_n - 7) %% 12)]))) - -show_wide <- function(col, digits = 1) { - by_month %>% - select(horizon, month, value = all_of(col)) %>% - mutate(value = round(value, digits)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = value, names_prefix = "h") %>% - arrange(month) -} -knitr::kable(show_wide("pct"), caption = "WIS change % (calibrated vs base)") -knitr::kable(show_wide("share"), caption = "Share of base WIS %") -knitr::kable(show_wide("shift"), caption = "Median shift % of truth") -``` - -```{r helpers} -headline_table <- function(cal) { - bind_rows( - left_join( - hub_coverage_summary(cal), - select(hub_quantile_loss(cal), horizon, loss_base, loss_cal, loss_pct = pct_improvement), - by = "horizon" - ) %>% mutate(season = "all"), - left_join( - hub_coverage_summary(cal, by = "season"), - select(hub_quantile_loss(cal, by = "season"), - season, horizon, loss_base, loss_cal, loss_pct = pct_improvement - ), - by = c("season", "horizon") - ) - ) %>% - transmute( - season, horizon, - `cal err (base)` = round(cal_error_base, 4), - `cal err (calibrated)` = round(cal_error_cal, 4), - `cal err improvement %` = round(pct_improvement, 1), - `WIS (base)` = round(2 * loss_base, 1), - `WIS (calibrated)` = round(2 * loss_cal, 1), - `WIS improvement %` = round(loss_pct, 1) - ) %>% - arrange(season, horizon) -} - -fan_cols <- c(base = "#0072B2", calibrated = "#D55E00") - -# One plot per (state, season): whole-season fan (odd season_rounds only, h 0..3; -# h -1 dropped -- with h -1 in, 5 horizons x ~13 rounds x 2 fans was unreadable) -# plus vintage patchwork strips underneath, plus an eta strip. -season_plot <- function(cal, loc, season) { - rr <- cal$rounds %>% filter(season == !!season, season_round %% 2 == 1) - ref_dates <- sort(rr$round_date) - - fin <- truth %>% - filter(location == loc, target_end_date >= min(ref_dates) - 21L, target_end_date <= max(ref_dates) + 28L) - - fan <- cal$forecasts %>% - filter(location == loc, reference_date %in% ref_dates, horizon %in% 0:3, level %in% c(0.1, 0.5, 0.9)) %>% - mutate( - target_end_date = reference_date + 7L * horizon, - lvl = c(`0.1` = "lo", `0.5` = "med", `0.9` = "hi")[as.character(level)] - ) %>% - select(reference_date, target_end_date, lvl, base = value_base, calibrated = value_cal) %>% - tidyr::pivot_longer(c(base, calibrated), names_to = "which") %>% - tidyr::pivot_wider(names_from = lvl, values_from = value) - - # Vintage patchwork: each plotted round's as-of snapshot paints only the - # fortnight since the previous plotted round (like backtesting.Rmd's vignette). - vint_windows <- tibble(ref = ref_dates) %>% - mutate( - win_start = dplyr::lag(ref, default = ref[1] - 14L) - 3L, - win_end = ref - 3L - ) - vint <- purrr::pmap(vint_windows, function(ref, win_start, win_end) { - snapshots %>% - filter(reference_date == ref, location == loc, time_value >= win_start, time_value <= win_end) - }) %>% bind_rows() - - main <- ggplot() + - geom_line(data = fin, aes(target_end_date, truth), colour = "grey65") + - geom_point(data = fin, aes(target_end_date, truth), colour = "grey65", size = 0.7) + - geom_ribbon( - data = filter(fan, which == "base"), - aes(target_end_date, ymin = lo, ymax = hi, fill = which, group = reference_date), - alpha = 0.2 - ) + - geom_line( - data = filter(fan, which == "calibrated"), - aes(target_end_date, lo, colour = which, group = reference_date), linewidth = 0.3 - ) + - geom_line( - data = filter(fan, which == "calibrated"), - aes(target_end_date, hi, colour = which, group = reference_date), linewidth = 0.3 - ) + - geom_line( - data = fan, - aes(target_end_date, med, colour = which, group = interaction(reference_date, which)) - ) + - geom_line(data = vint, aes(time_value, value), colour = "black") + - geom_point(data = vint, aes(time_value, value), colour = "black", size = 0.6) + - geom_vline(xintercept = ref_dates - 3L, linetype = "dashed", colour = "grey40", linewidth = 0.3) + - scale_fill_manual(values = fan_cols, aesthetics = c("fill", "colour")) + - labs(x = NULL, y = "admissions", colour = NULL, fill = NULL) + - theme(legend.position = "top") - - eta <- cal$series %>% - filter(location == loc, season == !!season, horizon %in% 0:3) %>% - # lr is 0 at rounds with no reveal (gaps); blank those rather than plot dips. - mutate(lr = if_else(lr > 0, lr, NA_real_)) %>% - ggplot(aes(round_date, lr, colour = factor(horizon))) + - geom_line(na.rm = TRUE) + - geom_point(size = 0.6, na.rm = TRUE) + - labs(x = NULL, y = "eta", colour = "horizon") + - theme(legend.position = "top") - - title <- paste0(geo_map$abbrev[match(loc, geo_map$location)], " — ", season) - cowplot::plot_grid( - cowplot::ggdraw() + cowplot::draw_label(title, fontface = "bold"), - cowplot::plot_grid(main, eta, ncol = 1, rel_heights = c(3, 1), align = "v", axis = "lr"), - ncol = 1, rel_heights = c(0.08, 1) - ) -} - -# oh_table.R's internals table, generalized to any location/horizon/season. -oh_table <- function(cal, loc, h, season) { - cal$forecasts %>% - filter(location == loc, horizon == h, season == !!season, level %in% c(0.05, 0.25, 0.5, 0.75, 0.95)) %>% - mutate(level = paste0("q", level)) %>% - select(reference_date, level, lr_level, truth, hidden, offset, value_base, value_cal) %>% - tidyr::pivot_wider( - names_from = level, - values_from = c(hidden, offset, value_base, value_cal, lr_level), - names_glue = "{level}_{.value}" - ) %>% - mutate( - eta = round(q0.5_lr_level, 3), - across(c(truth, matches("hidden|offset|value_base|value_cal")), ~ round(.x)) - ) %>% - select( - reference_date, eta, truth, - matches("^q.*_hidden$"), matches("^q.*_offset$"), - matches("^q.*_value_base$"), matches("^q.*_value_cal$") - ) %>% - arrange(reference_date) -} - -internals_block <- function(cal, season) { - for (i in seq_len(nrow(states))) { - loc <- states$location[i] - cat(sprintf("\n\n
%s internals (h=2)\n\n", states$label[i])) - print(knitr::kable(oh_table(cal, loc, 2L, season))) - cat("\n\n
\n\n") - } -} -``` - -The whole-season fans plot horizons 0–3 only (dropping −1) and only every -other round. Blue ribbon and line: base 80% band and median; orange lines: -calibrated 10%, 50%, 90% quantiles. Black: the NHSN vintage each forecast saw, -painted over the fortnight before its reference date (dashed); grey: finalized -truth. The strip below is eta per horizon (blank where a round revealed no -outcome). Read it as "where do the medians diverge, and does the orange band -contain the grey truth better than the blue", not round-by-round. - -```{r sections, results='asis'} -for (nm in names(cals)) { - cal <- cals[[nm]] - cat(sprintf("\n\n# %s\n\n", variant_titles[[nm]])) - - desc <- switch(nm, - baseline = "The single-term tracker run on sqrt(counts): offsets and eta live in sqrt - units, so a correction learned at the peak is smaller in counts when applied in the - trough. lr_mult = 0.03, lr_window = 20, carry, isotonic projection; offsets update - every round including the spring tail. The eta strip is in sqrt units.", - count_space = "The paper's method as originally run: additive offsets in count space, same - multiplier and window. Kept for reference; the eta strip is in counts.", - warm_static = "No online tracking at all (eta is effectively zero). The offset per (horizon, - level) is the conformal offset implied by the 2023-24 burn-in season pooled over - locations -- the level-quantile of truth minus the base quantile, in sqrt units -- and - it is applied unchanged to both live seasons. Shows how much of the gain is the - persistent low bias alone.", - warm_fast = "The conformal warm start plus the sqrt-space tracker with a 10% per-round leak - on its offset: a correction that is not refreshed by new outcomes decays back toward - the warm-start level instead of persisting through the reveal lag.", - warm_slow_fast = "The current operating point: warm start, a slow term that keeps updating - the persistent level with a tiny step (mult 0.003, expanding window, never decays, - carries across seasons), and the leaky fast tracker on top. The internals table's - hidden column is slow + fast.", - off_feb = "The earlier reference: rounds from February 15 through June play the base - forecast unchanged and take no gradient step; the offset learned through the peak is - frozen and carried into the next season." - ) - cat(desc, "\n\n") - - cat("## Headline\n\n") - print(knitr::kable(headline_table(cal))) - - for (season in seasons) { - cat(sprintf("\n\n## %s\n\n", season)) - for (i in seq_len(nrow(states))) { - loc <- states$location[i] - cat(sprintf("\n\n### %s — %s\n\n", states$label[i], season)) - print(season_plot(cal, loc, season)) - } - } - - cat("\n\n## Internals\n\n") - for (season in seasons) { - cat(sprintf("\n\n### %s\n\n", season)) - internals_block(cal, season) - } -} -``` diff --git a/reports/writeups/calibration_experiments/_month_view.Rmd b/reports/writeups/calibration_experiments/_month_view.Rmd index 107f29b1..4fbe0a30 100644 --- a/reports/writeups/calibration_experiments/_month_view.Rmd +++ b/reports/writeups/calibration_experiments/_month_view.Rmd @@ -48,14 +48,23 @@ if (!isFALSE(get0("mv_show_table"))) { ```{r mview-abs, results="asis"} # Absolute within-month coverage bias behind the plot's reduction %, at h0 and h2. mv_abs <- mv_plot_month %>% filter(horizon %in% c(0, 2)) -mv_abs_base <- mv_abs %>% distinct(season, month, horizon, cov_base) %>% group_by(season, month, horizon) %>% slice(1) %>% ungroup() +# Configs on different base forecasters (E07) have different bases, so each cell then carries its own base. +mv_one_base <- mv_abs %>% distinct(season, month, horizon, cov_base) %>% count(season, month, horizon) %>% pull(n) %>% max() == 1L +mv_abs_tbl <- mv_abs %>% + transmute(season, month, h = paste0("h", horizon), config, + v = if (mv_one_base) sprintf("%.3f", cov_cal) else sprintf("%.3f (base %.3f)", cov_cal, cov_base)) %>% + tidyr::pivot_wider(names_from = config, values_from = v) +if (mv_one_base) { + mv_abs_base <- mv_abs %>% distinct(season, month, h = paste0("h", horizon), base = sprintf("%.3f", cov_base)) + mv_abs_tbl <- mv_abs_tbl %>% left_join(mv_abs_base, by = c("season", "month", "h")) +} cat("\n\n## Monthly L1 coverage bias, absolute (h0 and h2)\n\n") print(knitr::kable( - mv_abs %>% - transmute(season, month, h = paste0("h", horizon), config, v = sprintf("%.3f", cov_cal)) %>% - tidyr::pivot_wider(names_from = config, values_from = v) %>% - left_join(mv_abs_base %>% transmute(season, month, h = paste0("h", horizon), base = sprintf("%.3f", cov_base)), by = c("season", "month", "h")) %>% - arrange(h, season, month), - caption = "Within-month L1 coverage bias (lower is better), calibrated configs and the uncalibrated base." + mv_abs_tbl %>% arrange(h, season, month), + caption = if (mv_one_base) { + "Within-month L1 coverage bias (lower is better), calibrated configs and the uncalibrated base." + } else { + "Within-month L1 coverage bias (lower is better) of each config, with its own uncalibrated base in parentheses." + } )) ``` diff --git a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd index 230034f1..5ea3fe92 100644 --- a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd +++ b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd @@ -25,54 +25,35 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw()) ``` -Every result before this notebook trained the tracker on finalized NHSN -values. Only the *timing* of each reveal was realistic (when the tracker -learns from an outcome). This notebook varies one thing, the data the tracker -learns from, at both reference configs (REF-paper and REF-op; see the index): - -- **final**: the latest vintage, as in all earlier results. -- **exact**: what prod does. Prod re-runs the tracker from the first round - each week on that week's data, so at round `t` every revealed outcome has - its value as published by `t`. We replay that by re-running the tracker - once per round. -- **vintage**: one run in which each outcome keeps the value it had at the - round that revealed it. This was meant as a cheap stand-in for exact, but - it is also a deployment design in its own right: a stateful tracker that - steps once per outcome and never revisits it. - -A round with reference date `d` sees data published by `d - 3` (the Wednesday -submission). Vintages start 2024-11-19; earlier as-of dates use that first -vintage. Scores are always against finalized truth, over both live seasons, -all hub locations, with burn-in rounds excluded. "WIS reduction %" is -100 × (base WIS − calibrated WIS) / base WIS, so positive = calibrated WIS -lower than base. "L1 coverage bias" is lower-is-better. - -Questions: (1) how much did learning from finalized values flatter the -earlier results, and (2) is the cheap mode close enough to exact to use in -sweeps? - -# Findings - -Numbers below are from the render of 30 September 2026. - -- **Finalized truth flattered the earlier results only a little.** At REF-op - with one week of extra delay, exact vs final WIS reduction is +5.4 vs +7.6 at - h−1, within 0.5 points at h0–h3, and the h−1 coverage bias is 0.074 vs - 0.067. At REF-paper, exact is within 0.6 points of final at every horizon: - slightly better at h2/h3, slightly worse at h1. Earlier conclusions stand; - the h−1 gain was overstated by about 2 points. -- **The cheap mode is not a usable stand-in.** It drifts from exact as the - season goes on (figure below). With no extra delay it learns once from the - first report, which under-reports and is never corrected: REF-op h−1 falls - to 0.0 (exact +6.1) and coverage bias rises to 0.121 (exact 0.076). Every - later experiment uses exact. -- **Under prod's re-run design, a shorter delay is better.** Exact with no - extra delay beats or ties one week of extra delay at every horizon, at both - references (REF-op +6.1/+5.1/+2.6/+1.8/+2.2 vs +5.4/+4.9/+2.4/+1.7/+2.2), - with nearly the same coverage bias. Learning early from a first report costs - little when next week's run replaces it with the revised value. A stateful - tracker (the vintage mode) shows the opposite. So the value of an extra - revision delay depends on the deployment design, and E09 reports both. +Question: how do results change when the tracker learns from NHSN data as +published, rather than from finalized values? + +Setup: + +- Disease: flu. Base forecaster: the submitted hub ensemble (spoiled + submissions dropped). +- References: REF-paper (count scale, one offset per level) and REF-op (sqrt + scale, leak 0.1, slow term with a burn-in-quantile warm start); see the index. +- Burn-in: 2023-24, used only to warm up the learning rate. Offsets start at + zero at the first 2024-25 round (REF-op's slow term starts from its warm + start). +- Seasons scored: 2024-25 and 2025-26, burn-in rounds excluded. Horizons + h−1 to h3. Headline tables: all hub locations including US; month view: + states only. +- Scores are always against finalized truth. + +Configs are `ref/settle_days/learn`: + +| axis | values | +|---|---| +| `ref` | `paper`, `op` | +| `settle_days` | `14` (one week of extra delay, the reference) or `7` (none) | +| `learn` | `final`: latest vintage. `exact`: re-run from the first round at every round on the data published by then, as prod does. `vintage`: one run where each outcome keeps the value it had at the round that revealed it (a stateful tracker that never revisits a step) | + +A round with reference date `d` sees data published by `d - 3`. Vintages +start 2024-11-19; earlier as-of dates use that first vintage. "WIS reduction +%" is 100 × (base WIS − calibrated WIS) / base WIS, so positive = calibrated +WIS lower than base. "L1 coverage bias" is lower-is-better. ```{r libs, cache=FALSE} suppressPackageStartupMessages(source(here::here("R/load_all.R"))) @@ -147,10 +128,10 @@ knitr::kable( ) ``` -# How far the cheap mode drifts from exact +# Vintage vs exact: difference in the calibrated median -Median over locations of |cheap − exact| in the played median, relative to -the base median, by month: +Median over locations of |vintage − exact| in the calibrated median, +relative to the base median, by month, one panel per `ref/settle_days`: ```{r drift, fig.height=6} drift <- purrr::map(unique(paste(grid$ref, grid$settle_days, sep = "/")), function(cfg) { @@ -170,13 +151,45 @@ ggplot(drift, aes(month, 100 * rel, colour = factor(horizon), group = horizon)) geom_line() + geom_point(size = 0.8) + facet_wrap(~config) + - labs(x = NULL, y = "median |cheap − exact| / base median (%)", colour = "horizon") + labs(x = NULL, y = "median |vintage − exact| / base median (%)", colour = "horizon") ``` ```{r month-view-cfgs} -# The runs the Findings compare: final vs exact at one week of delay, and the cheap (vintage) mode vs exact with none. +# Runs drawn in the month view. mv_plot_cfgs <- c("op/14/final", "op/14/exact", "op/7/vintage", "op/7/exact", "paper/14/final", "paper/14/exact") ``` ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` + +# Findings + +WIS reduction % is positive = calibrated WIS lower than base; coverage bias is +lower-is-better. A WIS gap under 1 point or a month-avg coverage gap under +0.01 is read as no clear difference. + +- **final vs exact.** REF-op at settle 14: h−1 +7.6 (final) vs +5.4 (exact), + a 2.2-point gap that comes from 2024-25 (11.2 vs 7.7; 2025-26 2.4 vs 2.1). + At h0–h3 the gap is 0.5 points or less. REF-paper: under 1 point at every + horizon (0.3–0.6). Month-avg coverage bias: no clear difference (REF-op + 0.106 vs 0.107 at h−1; REF-paper 0.122 vs 0.115). Season-pooled bias is + higher under exact (REF-paper h−1 0.044 vs 0.030), but pooling lets ramp + and post-peak errors cancel. +- **vintage vs exact.** At settle 7 the vintage mode loses most of the h−1 + gain: REF-op 0.0 vs +6.1, REF-paper −1.1 vs +4.5. Its month-avg coverage + bias at h−1 is worse than the base (REF-op 0.136, REF-paper 0.149, base + 0.134; exact 0.107 and 0.110). At settle 14 the gaps are smaller (REF-op h−1 + +4.9 vs +5.4). At REF-paper h1–h3 the vintage mode has higher WIS reduction + than exact (settle 14: −0.9/−1.6/−4.2 vs −2.0/−2.7/−5.3). +- **settle 7 vs 14 under exact.** REF-paper: settle 7 is 1.3–2.8 points + higher at every horizon, in both seasons. REF-op: no clear difference + pooled (0.0–0.7); in 2025-26 settle 14 is 0.5–0.6 points higher at h1–h3. + Month-avg coverage: no clear difference at either reference. +- **settle 7 vs 14 under vintage.** settle 14 is higher at h−1/h0 (REF-op + +4.9/+4.5 vs 0.0/+3.0). At REF-paper h1–h3 the order is the other way + (−0.1/−0.4/−2.3 at settle 7 vs −0.9/−1.6/−4.2 at settle 14). +- **Month-avg coverage at h2–h3.** Every REF-paper config is at or above the + base (h3 0.160–0.170 vs 0.151); every REF-op config is below it (h3 + 0.135–0.137). +- Offsets start from zero in November 2024 (the burn-in only warms the + learning rate), so early 2024-25 rounds of the REF-paper runs equal the base. diff --git a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd index 1970e2d2..140a5cbf 100644 --- a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd +++ b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd @@ -19,84 +19,35 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` -The paper's tracker sets its learning rate from recent errors: +Question: how do the adaptive learning-rate settings change WIS and +coverage? The paper's tracker sets `eta = mult × 90th percentile of |outcome − forecast|` over the last -`lr_window` observed rounds. Three settings: the multiplier (`lr_mult`), the -window (`lr_window`), and what happens to the offsets over the summer -(`season_policy`: carry them into the next season, or reset them to zero). -**Which settings should we use, and which of the three matters?** - -This asks the original E01's question (`calibration_qt_flu.Rmd`) on the -current setup. It is a new experiment, not a reproduction: four things -changed at once, so it can't say which change matters. - -| | original E01 | this notebook | -|---|---|---| -| forecaster | submitted ensemble | clean `windowed_seasonal` replay | -| scale | count | sqrt | -| learning truth | finalized | as published each week | -| tracker code | before the 2026-09-17 fixes | after | - -**Setup.** As E14–E17: flu, the clean `windowed_seasonal` replay (all 53 -locations, the 55 hub rounds, h0–h3), learning from data as published each -week and re-run from scratch each week as prod does, one week of extra -revision delay, no burn-in season. The tracker is the plain single-term one -with the adaptive learning rate (REF-paper), **on the square-root scale** -with an eta floor of 0.001: that is the scale prod's tracker (REF-op) runs -on, and E03 found it better than counts at every horizon. One count-scale -run at the original operating point (mult 0.03, window 20, carry) links to -the old sweep. - -Grid: `lr_mult` {0.3, 0.1, 0.03, 0.01} × `lr_window` {8, 20, Inf} × -`season_policy` {carry, reset}, 24 runs. The old sweep also had window 50, -which it found indistinguishable from 20 and Inf; it is dropped here. With -no burn-in there is one summer to cross (2025 into 2025-26), so the season -policy only changes 2025-26. - -"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so -**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is -better) is the mean over the 23 levels of |empirical coverage − level|. Each season is scored on its own rounds. -Results lead with states only (US excluded). - -# Findings - -Numbers are from the render of 1 October 2026, states only, WIS reduction % -(positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), h0/h1/h2/h3, both seasons pooled -unless stated. All-locations numbers point the same way. - -- **The multiplier is the setting that matters.** Changing it moves WIS by - 16–81 points (pooled, averaged over the other settings); changing the - window moves it by 1–2 points and the season policy by 0.3–1.7. -- **On the sqrt scale, 0.1 and 0.3 are far too large.** 0.1 loses WIS at - every horizon (−1.3/−6.8/−11.3/−16.7 at window 20, carry) and 0.3 loses - 14–77%. **0.03** (REF-paper's value) gains at h0 and loses a little at - h1–h3 (+1.9/−0.2/−1.1/−1.7), with coverage bias 0.032–0.045 against the - base's 0.105–0.127. **0.01** is the only multiplier that never loses more - than 0.2 (+1.7/+0.7/+0.5/+0.5), but from a cold start it barely moves - 2024-25's coverage (0.107–0.129 vs base 0.127–0.169). It catches up in - 2025-26 (0.015–0.041). -- **Window: keep 20.** It changes nothing in 2024-25 (under 0.2 points). In - 2025-26, Inf is up to a point worse than 20 at h1–h3 for mult ≤ 0.03; 8 - and 20 are within half a point. -- **Season policy: carry.** Reset only acts at the 2025 summer, and there it - restarts the tracker from zero: at mult 0.03, window 20, 2025-26 goes from - +2.2/−0.6/−0.7/−0.2 (carry) to +0.7/−0.9/−1.3/−1.6 (reset), and coverage - bias from 0.008–0.016 to 0.039–0.050. -- **Count scale** at the same point loses more (−0.4/−2.5/−4.9/−7.0), as E03 - found. -- **Against the old E01** (count scale, ensemble, finalized truth, pre-fix): - the same answer. The multiplier is the knob, the window is immaterial, 0.1 - overpays and 0.01 is near WIS parity but slow. One change: carry vs reset - was "minor" before; here carry is clearly better in 2025-26. -- **Month view (2026-10-01):** scored within each month, the coverage - ranking changes. Month-avg L1 coverage bias (both seasons, h0/h1/h2/h3, - window 20, carry): base 0.112/0.127/0.153/0.177, mult 0.01 - 0.093/0.114/0.142/0.168, 0.03 0.095/0.120/0.150/0.176, 0.1 - 0.097/0.130/0.161/0.190. So 0.01, not 0.03, has the lowest month-level - bias, and at h2–h3 0.03 is level with the base: most of its season-pooled - coverage gain is ramp and post-peak errors cancelling. Carry vs reset - flips slightly in 2025-26: reset 0.076/0.103/0.122/0.138, carry - 0.078/0.105/0.130/0.143. +`lr_window` observed rounds; the settings are the multiplier (`lr_mult`), the +window (`lr_window`), and the summer policy (`season_policy`: carry offsets +into the next season, or reset them to zero). + +This re-asks the original E01 (a pre-fix, count-scale sweep on the submitted +ensemble with finalized truth, since deleted) on the current setup. + +Setup: + +- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (all 53 + locations, the 55 hub rounds; 27 in 2024-25, 28 in 2025-26). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. +- Cold start: no burn-in season, offsets start at zero in November 2024. With + no burn-in there is one summer to cross (2025), so the season policy only + changes 2025-26. +- Tracker: REF-paper (single term, adaptive rate) on the square-root scale, + learning-rate floor 0.001. +- Grid: `lr_mult` {0.3, 0.1, 0.03, 0.01} × `lr_window` {8, 20, Inf} × + `season_policy` {carry, reset}, 24 runs, plus one count-scale run at mult + 0.03, window 20, carry. +- Tables lead with states only (US excluded). Each season is scored on its + own rounds. + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) @@ -310,3 +261,29 @@ mv_plot_cfgs <- c( ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` + +# Findings + +There are no uncertainty estimates; a WIS gap under 1 point or a month-avg +coverage gap under 0.01 is read as no clear difference. The cold start leaves +Nov–Dec 2024 (about 38% of 2024-25's base WIS at h2) equal to the base for +every config. States only, window 20, carry unless stated. + +- **Large multipliers:** 0.1 is −1.3/−6.8/−11.3/−16.7 pooled at h0–h3 and + 0.3 loses 14–77. Across multipliers WIS moves 16–81 points; across windows + 0.8–2.4; across season policies 0.3–1.7. +- **0.01 vs 0.03:** 0.01 is ahead at h1–h3 in both seasons (by 0.5–2.4); + 0.03 is ahead at h0 in 2024-25 (by 0.8). Pooled: 0.01 + +1.7/+0.7/+0.5/+0.5, 0.03 +1.9/−0.2/−1.1/−1.7. +- **Month-avg coverage (both seasons, h0…h3):** base 0.112/0.127/0.153/0.177; + 0.01 0.093/0.114/0.142/0.168; 0.03 0.095/0.120/0.150/0.176; 0.1 + 0.097/0.130/0.161/0.190. 0.01 vs 0.03: no clear difference. 0.1 is worse + than the base at h1–h3. Season-pooled, 0.03 and 0.1 look much better than + the base (0.017–0.045), but pooling lets ramp and post-peak errors cancel. +- **Window:** no strong result (under 1 point in most cells; up to 2.4 at h3, + 2025-26, driven by mult 0.3). +- **Carry vs reset (2025-26 only):** at mult ≤ 0.03 carry is 0.3–1.9 points + ahead on WIS; at 0.1 and 0.3 the order is mixed. Month-avg coverage: no + clear difference (reset lower by 0.002–0.008). +- **Count scale** (mult 0.03): −0.4/−2.5/−4.9/−7.0 pooled, 2–5 points below + sqrt at h1–h3 in both seasons. diff --git a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd index e506aeed..a3891186 100644 --- a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd +++ b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd @@ -19,101 +19,39 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` -Variants of the adaptive tracker, each changed alone from the reference. -**Does any of them beat the reference?** +Question: how do three variants of the adaptive tracker, each changed alone +from a reference, change WIS and coverage? - **Switch off in spring** (`off_after`): from a calendar date until July, play the base forecast and stop learning; the offsets carry unchanged into the next season. Dates: 1 April, 1 March, 15 February. -- **Per-level learning rate** (`lr_args$per_level`): each of the 23 - quantile levels gets its own eta, instead of one eta pooled over levels. +- **Per-level learning rate** (`lr_args$per_level`): each of the 23 quantile + levels gets its own eta, instead of one eta pooled over levels. - **Seasonal window** (`lr_seasonal`): eta is sized from the last 10 rounds - plus the rounds within 5 weeks of the same point in earlier seasons, so the - start of a ramp is sized by last year's ramp. Run with carry and with - reset at the summer. - -This asks the original E02's question (`calibration_qt_seasons_flu.Rmd`, -pre-fix) on the current -setup. It is a new experiment, not a reproduction: four things changed at -once, so it can't say which change matters. - -| | original E02 | this notebook | -|---|---|---| -| forecaster | submitted ensemble | clean `windowed_seasonal` replay | -| scale | count | sqrt | -| learning truth | finalized | as published each week | -| tracker code | before the 2026-09-17 fixes | after | - -The original also -had a geo-pooled eta (one eta shared across locations, population-weighted). -That option never beat the baseline and was removed from the code on -2026-09-17, so it is not run here. - -**Setup.** As E01: flu, the clean `windowed_seasonal` replay (all 53 -locations, the 55 hub rounds, h0–h3), learning from data as published each -week and re-run from scratch each week, one week of extra revision delay. -The reference is the sqrt single-term tracker with the adaptive learning -rate (mult 0.03, floor 0.001, window 20, carry), no burn-in: E01's center -point. The seasonal window needs an earlier season to look back on, so it -runs with the 2023-24 burn-in (the HHS-stitched backfill from E13), which -only warms eta; the reference is re-run with the same burn-in as a bridge. - -"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so -**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is -better) is the mean over the 23 levels of |empirical coverage − level|. Results lead with states only. - -# Findings - -Numbers are from the render of 1 October 2026, states only, WIS reduction % -(positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), h0/h1/h2/h3, both seasons pooled -unless stated. All-locations numbers point the same way. - -- **Switching off in spring is the only variant with a large WIS gain.** - The reference is +1.9/−0.2/−1.1/−1.7. `off_after` 1 March gives - +2.8/+1.7/+2.0/+2.5 and 15 February +2.7/+2.1/+2.5/+3.2, positive at every - horizon in both seasons. 1 April only halves the h2–h3 loss - (+1.6/−0.2/−0.7/−0.8). -- **It costs coverage.** Coverage bias rises from 0.032–0.045 to - 0.080–0.091 (1 March) and 0.086–0.100 (15 February), against the base's - 0.105–0.127. From the switch-off date the base forecast is played, and the - over-prediction after the peak no longer offsets the under-prediction on - the ramp (E16). -- **Where the gain comes from.** It removes the reference's March–April - losses (2024-25 h2: −31% in March, −41% in April). The offsets frozen in - February also carry into autumn 2025 and help November–December (h2 +14% - vs +5–6%) but hurt October (−60%, on under 1% of the season's WIS). -- **Same as E17.** E17 found `off_after = "03-01"` raised WIS and gave back - most of the coverage gain for constant rates (sqrt 0.018: - +2.3/+1.2/+1.2/+1.3, coverage bias 0.094–0.116). The adaptive tracker - gains more from it because it loses more after the peak. -- **Per-level eta: a small gain at h1–h3** (+2.1/+0.5/0.0/−0.3), mostly from - smaller March–April losses, with slightly higher coverage bias - (0.043–0.055). -- **Seasonal window: no gain.** Against the reference with the same burn-in - (+2.0/0.0/−0.6/−1.0), carry gives +2.0/−0.1/−0.8/−1.3 with lower coverage - bias at h2–h3 (0.025–0.032 vs 0.040–0.044); reset gives - +1.4/−0.2/−0.9/−1.6. The burn-in alone (warming eta on 2023-24) is worth - +0.5–0.7 at h2–h3 over the cold reference. -- **Against the old E02** (count scale, ensemble, finalized truth, pre-fix): - the same ranking. Switching off on 1 March or 15 February removes the WIS - cost (15 February best at h1–h3), per-level eta is a modest gain and the - seasonal window does not help. Two changes: reset now trails carry for the - seasonal window (it used to recover some ground), and the coverage cost of - switching off, not reported before, is large. -- **Month view (2026-10-01):** the coverage cost of switching off is - mostly an artifact of pooling the season. Month-avg L1 coverage bias - (h0/h1/h2/h3): reference 0.095/0.120/0.150/0.176, off 1 March - 0.104/0.124/0.153/0.170, off 15 February 0.106/0.126/0.153/0.169, base - 0.112/0.127/0.153/0.177. The reference itself gains only 0.001–0.017 over - the base by month, so switching off gives up at most 0.01 and is better at - h3. In March 2025-26 switching off lowers the monthly bias (h2: 0.094 vs - 0.196 for the reference). Per-level eta is slightly better than the - reference by month (0.092/0.117/0.147/0.173), not worse, and the seasonal - window with carry is worse than the burn-in reference at h2–h3 - (0.156/0.183 vs 0.149/0.174). -- **Month view (2026-10-01), median error:** in March the reference's median loses more - than its WIS (2025-26 h0–h3: absolute error −34/−54/−86/−109%, WIS - −22/−38/−60/−81%); switching off removes it. + plus the rounds within 5 weeks of the same point in earlier seasons. Run + with carry and with reset at the summer. This also shortens the window + from 20 to 10, so it changes two things at once. + +This re-asks the original E02 (a pre-fix, count-scale comparison on the +submitted ensemble with finalized truth, since deleted) on the current setup. + +Setup: + +- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (all 53 + locations, the 55 hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. +- Reference: E01's center point, the sqrt single-term tracker with the + adaptive rate (mult 0.03, floor 0.001, window 20, carry), cold start (no + burn-in). +- The seasonal window needs an earlier season to look back on, so it runs + with the 2023-24 burn-in (the HHS-stitched backfill from E13), which only + warms eta. The reference is re-run with the same burn-in + (`reference + burn-in`) for comparison. +- Tables lead with states only (US excluded). + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) @@ -298,3 +236,34 @@ mv_cols <- CFG_COLS ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` + +# Findings + +There are no uncertainty estimates; a WIS gap under 1 point or a month-avg +coverage gap under 0.01 is read as no clear difference. The cold start leaves +Nov–Dec 2024 equal to the base for the cold configs. States only, pooled over +both seasons unless stated. + +- **Switching off from 1 March or 15 February:** +2.8/+1.7/+2.0/+2.5 and + +2.7/+2.1/+2.5/+3.2 at h0–h3, against the reference's + +1.9/−0.2/−1.1/−1.7, ahead at every horizon in both seasons. The two dates + differ by 0.1–0.7 (no clear difference). Both dates were chosen with these + seasons in view. +- **Where that gain sits:** in 2024-25 it is the March–April months, where + the reference loses (h2: −31% in March, −41% in April). In 2025-26 about + half of the h2 gain is December, where offsets frozen in February 2025 are + still in play (+13.6% vs +6.3%), partly offset by January and October. +- **Switching off from 1 April:** +1.6/−0.2/−0.7/−0.8, within 1 point of the + reference at h0–h2. +- **Per-level eta:** +2.1/+0.5/−0.0/−0.3, about 1 point above the reference + at h1–h3 in both seasons. +- **Seasonal window and burn-in:** against `reference + burn-in` + (+2.0/−0.0/−0.6/−1.0), the seasonal window is +2.0/−0.1/−0.8/−1.3 with carry + and +1.4/−0.2/−0.9/−1.6 with reset. The burn-in alone is +0.5 to +0.7 at + h2–h3 over the cold reference, in 2024-25 only. No strong result. +- **Month-avg coverage (h0…h3):** base 0.112/0.127/0.153/0.177; reference + 0.095/0.120/0.150/0.176; off 1 March 0.104/0.124/0.153/0.170; off 15 + February 0.106/0.126/0.153/0.169; per-level 0.092/0.117/0.147/0.173. No + clear difference between any variant and the reference. Season-pooled the + switch-offs look much worse than the reference (0.080–0.100 vs + 0.032–0.045), but pooling lets ramp and post-peak errors cancel. diff --git a/reports/writeups/calibration_experiments/e03_scale.Rmd b/reports/writeups/calibration_experiments/e03_scale.Rmd index 5445c3b6..4a3ed2d4 100644 --- a/reports/writeups/calibration_experiments/e03_scale.Rmd +++ b/reports/writeups/calibration_experiments/e03_scale.Rmd @@ -23,50 +23,31 @@ ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplo ```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} ``` -One axis: the scale the tracker works on. Everything else is REF-paper (one -offset per level, adaptive learning rate of 0.03 × the 90th percentile of the -last 20 rounds' absolute errors, offsets carried across seasons, 2023-24 -burn-in, one week of extra delay), learning from vintages the way prod does -(re-run each week; E00). +Question: does the scale the tracker works on (count, sqrt, rate) change +WIS and coverage? -- **count**: REF-paper itself; additive offsets in admissions. -- **sqrt**: offsets on the square root of admissions, mapped back. -- **rate**: offsets in admissions per 100k, so one location's step size is - comparable to another's. +Setup: -The adaptive learning rate has a floor (0.1 by default) in the tracker's -working units, so the sqrt and rate variants use a floor of 0.001; that floor -change goes with the scale. +- Disease: flu. Base forecaster: the submitted hub ensemble (spoiled + submissions dropped). +- Reference: REF-paper. One offset per level; adaptive learning rate of 0.03 + × the 90th percentile of the last 20 rounds' absolute errors; offsets + carried across seasons; one week of extra delay (`settle_days = 14`). +- Burn-in: 2023-24, used only to warm up the learning rate. Offsets start at + zero at the first 2024-25 round (cold start). +- Learning truth: exact (re-run at every round on the data published by then, + as prod does; E00). Scores against finalized truth. +- Seasons scored: 2024-25 and 2025-26. Horizons h−1 to h3. Headline tables: + all hub locations including US; month view: states only. -# Findings - -Numbers are from the render of 30 September 2026, pooled over both live -seasons and all locations. WIS reduction % is 100 × (base − calibrated) / base, -so positive = calibrated WIS lower than base; the absolute-error change of the -median is defined the same way. L1 coverage bias: lower is better. +| config | change from REF-paper | +|---|---| +| `count` | none (additive offsets in admissions) | +| `sqrt` | offsets on the square root of admissions, mapped back; learning-rate floor 0.001 | +| `rate` | offsets in admissions per 100k; learning-rate floor 0.001 | -- **sqrt beats count at every horizon.** WIS +5.1/+4.0/+0.7/−1.3/−1.4 vs - +3.2/+1.9/−2.0/−2.7/−5.3 at h−1…h3, with similar coverage bias - (0.049–0.026 vs 0.044–0.032). The median improves too: absolute error - +9.5/+7.3/+2.4/−0.6/+1.2 vs +5.0/+4.7/+0.2/−1.3/−4.9. -- **The gain is in the spring.** At h3, count loses 122% of April's and 63% of - May's base WIS; sqrt loses 20% and gains 18%. An offset learned at the peak - is sized in admissions, so in count space it is far too large for the tail. -- **sqrt is worse at the start of the season and in March.** At h3 it loses - 35% in October and 42% in March, against 27% and 26% for count. These - months carry little of the base WIS (0.3% and 5.4%), so the spring gain - dominates. -- **rate equals count with the adaptive learning rate.** The adaptive rate is - a quantile of the location's own residuals, so dividing a location by its - population rescales steps and offsets together. The rate scale matters only - with a constant learning rate (E09). -- **2025-26 is harder for both.** sqrt loses 1.8–4.6% at h1–h3 there (count - 6.9–7.9%), against gains in 2024-25. -- **Month view (2026-10-01):** scored within each month (states only), - sqrt's coverage edge is clearer than "similar": month-avg L1 coverage bias - 0.108/0.102/0.117/0.130/0.150 at h−1…h3 vs count - 0.115/0.102/0.120/0.143/0.168, base 0.134/0.116/0.123/0.137/0.151. Count - is worse than the base at h2–h3 month by month. +The learning-rate floor (0.1 by default) is in the tracker's working units, +so the sqrt and rate configs use 0.001. ```{r runs} cals <- list( @@ -79,3 +60,32 @@ focus <- "sqrt" ```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} ``` + +# Findings + +WIS reduction % is positive = calibrated WIS lower than base; coverage bias is +lower-is-better. A WIS gap under 1 point or a month-avg coverage gap under +0.01 is read as no clear difference. + +- **sqrt vs count, WIS.** sqrt is 1.4 to 3.9 points higher pooled at h−1…h3 + (+5.1/+4.0/+0.7/−1.3/−1.4 vs +3.2/+1.9/−2.0/−2.7/−5.3). In 2025-26 the gap + is 1.5–6.8 points at every horizon. In 2024-25 it is 2.2 at h−1 and 3.5 at + h3, and under 1 point at h0–h2. +- **sqrt vs count, h3 by month.** The largest gaps are in months with little + base WIS: April −20.0 vs −122.2 and May +17.7 vs −63.4 (1.3% and 1.2% of + base WIS); sqrt is lower in October (−35.1 vs −26.9) and March (−42.1 vs + −26.0). Weighted by base WIS share, April–May and November–January each + account for about half of the 3.9-point h3 gap; March takes back about 0.9. +- **sqrt vs count, coverage.** Month-avg bias (states): sqrt is 0.013 and + 0.018 lower at h2 and h3, from 2024-25 only (h3 0.181 vs 0.223); in 2025-26 + there is no clear difference (h3 0.122 vs 0.120). At h3 neither is below the + base month-avg (sqrt 0.150, count 0.168, base 0.151). Season-pooled per + season, sqrt is higher than count (2024-25 by 0.02–0.05); pooled over both + seasons the two look alike, because pooling lets errors cancel across + seasons as well as within one. +- **rate vs count.** Identical in every table. With the adaptive learning rate, + dividing a location by its population rescales steps and offsets together. + A direct check (2026-10-02) confirmed `scales` is applied: with a constant + learning rate the two differ by up to 300 admissions. +- Offsets start from zero in November 2024, so 2024-25 includes several + rounds where every config equals the base. diff --git a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd index bcd96b32..38ef2362 100644 --- a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd +++ b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd @@ -23,49 +23,31 @@ ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplo ```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} ``` -One axis: the structure of the offset. Everything else is REF-paper on the sqrt scale (the -E03 sqrt variant), learning from vintages the way prod does (E00). +Question: does adding a leak or a second, slow offset term to the tracker +change WIS and coverage? -- **single**: the paper's tracker, one offset per level. -- **leak 0.1**: the offset shrinks by 10% each round, so - a correction that new outcomes do not refresh fades (half-life about 6.6 - rounds). -- **slow 0.003**: a second, persistent offset term with a small learning rate - (0.003 × the error percentile, against 0.03 for the main term) added to the paper's one. -- **slow 0.003 + leak 0.1**: both. This is REF-op without its warm start - (E05). +Setup: -# Findings - -Numbers are from the render of 30 September 2026, pooled over both live -seasons and all locations. WIS reduction % is 100 × (base − calibrated) / base, -so positive = calibrated WIS lower than base; the absolute-error change of the -median is defined the same way. L1 coverage bias: lower is better. +- Disease: flu. Base forecaster: the submitted hub ensemble (spoiled + submissions dropped). +- Reference: REF-paper on the sqrt scale (E03's `sqrt`): adaptive learning + rate of 0.03 × the 90th percentile of the last 20 rounds' absolute errors, + floor 0.001; offsets carried across seasons; one week of extra delay + (`settle_days = 14`). +- Burn-in: 2023-24, used only to warm up the learning rate. Offsets start at + zero at the first 2024-25 round (cold start; no warm start for the slow + term). +- Learning truth: exact (re-run at every round on the data published by then, + as prod does; E00). Scores against finalized truth. +- Seasons scored: 2024-25 and 2025-26. Horizons h−1 to h3. Headline tables: + all hub locations including US; month view: states only. -- **The leak buys WIS at h1–h3 with coverage.** `leak 0.1` moves WIS from - +0.7/−1.3/−1.4 to +1.6/+0.8/+0.6 at h1–h3, but coverage bias nearly - triples (0.026–0.028 → 0.075–0.081), close to the base's 0.095–0.102. - A correction that fades between refreshes cannot hold coverage through a - 3–5 round reveal lag. -- **The leak is what repairs the tail and the second season.** At h3 it cuts - the October loss from 35% to 12% and March from 42% to 14%, and turns April - from −20% to +10%. In 2025-26, h1–h3 go from −1.8/−4.6/−3.5 to - +1.5/+1.0/+0.9. -- **A slow term alone does nothing.** `slow 0.003` is within 0.2 points of - `single` everywhere, and `slow 0.003 + leak 0.1` is within 0.5 points of - `leak 0.1` (slightly better coverage, 0.068–0.084). At this learning rate - the slow term never accumulates enough to matter without a warm start - (E05). -- The median follows WIS: the leak improves absolute error at h1–h3 - (+3.4/+2.8/+3.4 vs +2.4/−0.6/+1.2) and costs 1.1 points at h−1 and 0.3 at - h0. -- **Month view (2026-10-01):** scored within each month (states only), the - leak costs no coverage. Month-avg L1 coverage bias at h−1…h3: single - 0.108/0.102/0.117/0.130/0.150, leak 0.1 0.115/0.102/0.115/0.132/0.143, - base 0.134/0.116/0.123/0.137/0.151. The near-tripling of the - season-pooled bias is the leak no longer letting ramp and post-peak errors - cancel; month by month the two are level at h0–h2, the leak is better at - h3 and slightly worse at h−1. +| config | change from the sqrt reference | +|---|---| +| `single` | none (one offset per level) | +| `leak 0.1` | the offset shrinks by 10% each round (half-life about 6.6 rounds) | +| `slow 0.003` | adds a second offset term with learning-rate multiplier 0.003 (vs 0.03 for the main term), no leak | +| `slow 0.003 + leak 0.1` | both; this is REF-op without its warm start (E05) | ```{r runs} sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) @@ -80,3 +62,30 @@ focus <- "leak 0.1" ```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} ``` + +# Findings + +WIS reduction % is positive = calibrated WIS lower than base; coverage bias is +lower-is-better. A WIS gap under 1 point or a month-avg coverage gap under +0.01 is read as no clear difference. + +- **leak 0.1 vs single, WIS: the order flips by season.** Pooled, the leak is + 0.9–2.1 points higher at h1–h3 (+1.6/+0.8/+0.6 vs +0.7/−1.3/−1.4) and 0.3 + lower at h−1/h0. In 2025-26 it is 3.3–5.6 points higher at h1–h3. In + 2024-25 it is 0.4–1.3 points lower at h−1…h2 and 0.4 higher at h3. +- **leak 0.1 vs single, h3 by month.** Most of the pooled 2.0-point h3 gap + comes from March (−14.2 vs −42.1, 5.4% of base WIS) and February (+0.3 vs + −3.8). The leak is lower in December (+0.8 vs +1.7) and May (+5.7 vs +17.7). +- **leak 0.1 vs single, coverage: the order flips by season.** Month-avg bias + (states) over both seasons: no clear difference (0.115/0.102/0.115/0.132/0.143 + vs 0.108/0.102/0.117/0.130/0.150; base 0.134/0.116/0.123/0.137/0.151). Per + season, the leak is 0.014–0.017 higher in 2024-25 at h−1…h2 and 0.010–0.018 + lower in 2025-26 at h0…h3. Season-pooled bias roughly triples with the leak + (0.026–0.028 → 0.075–0.081 at h1–h3), but pooling lets errors cancel, and + `single`'s two-season value at h1 (0.026) is below either season's (0.089, + 0.038). +- **No strong result for the slow term.** `slow 0.003` vs `single`: at most + 0.2 points pooled, at most 0.5 per season. `slow 0.003 + leak 0.1` vs + `leak 0.1`: 0.1–0.5 points; month-avg coverage within 0.004. +- Offsets start from zero in November 2024, so early 2024-25 rounds of every + config equal the base. diff --git a/reports/writeups/calibration_experiments/e05_warm_start.Rmd b/reports/writeups/calibration_experiments/e05_warm_start.Rmd index f938b461..13e45841 100644 --- a/reports/writeups/calibration_experiments/e05_warm_start.Rmd +++ b/reports/writeups/calibration_experiments/e05_warm_start.Rmd @@ -23,54 +23,37 @@ ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplo ```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} ``` -One axis: whether the persistent offset starts from the burn-in season's -conformal offset (per horizon and level, the level-quantile of the burn-in -season's errors pooled over locations, on the sqrt scale) or from zero. Everything else is REF-paper on the sqrt scale, learning from vintages -the way prod does (E00). Each warm-started run is paired with its cold twin -from E04: +Question: does starting the persistent offset from the burn-in season's +errors, rather than from zero, change WIS and coverage for each tracker? -- **warm only**: the warm start, then no tracking (learning rate ≈ 0). -- **single** / **warm + single**: the paper's tracker. -- **leak 0.1** / **warm + leak 0.1**. -- **slow + leak** / **warm + slow + leak**: the second is REF-op, the - operating point in prod. +**Setup.** -# Findings +- Flu, the submitted hub ensemble (`CMU-TimeSeries`), horizons h−1 to h3. +- 2023-24 is the burn-in season; 2024-25 and 2025-26 are scored. Burn-in + outcomes never move the offsets, so a cold run plays the base forecast + until its first live outcomes arrive. +- Square-root scale, REF-paper otherwise (adaptive learning rate, mult + 0.03, floor 0.001, window 20, offsets carried across seasons). +- Learning truth: NHSN data as published each week, re-run from scratch + each week (prod's design, E00). Scoring: finalized truth. +- Headline tables cover all hub locations; the month view and its tables + cover states only. +- The warm start sets the slow offset, per horizon and level, to the + level-quantile of the burn-in season's errors pooled over locations, on + the sqrt scale. -Numbers are from the render of 30 September 2026, pooled over both live -seasons and all locations. WIS reduction % is 100 × (base − calibrated) / base, -so positive = calibrated WIS lower than base; the absolute-error change of the -median is defined the same way. L1 coverage bias: lower is better. +| config | warm start | tracking after the start | +|---|---|---| +| `warm only` | yes | none (learning rate ≈ 0) | +| `single` | no | REF-paper tracker | +| `warm + single` | yes | REF-paper tracker | +| `leak 0.1` | no | fast offset shrinks 10% per round | +| `warm + leak 0.1` | yes | as `leak 0.1` | +| `slow + leak` | no | leak 0.1 plus a slow offset (mult 0.003) | +| `warm + slow + leak (REF-op)` | yes | as `slow + leak`; prod's config | -- **The warm start helps every tracker, and never hurts.** Each warm run is - at least as good as its cold twin on WIS at every horizon, and better on - coverage: `single` +5.1/+4.0/+0.7/−1.3/−1.4 → `warm + single` - +5.1/+4.7/+1.3/−0.2/+0.3, coverage bias 0.049–0.026 → 0.046–0.014. -- **`warm + single` is the best-calibrated config so far.** Its coverage bias - is 0.013–0.020 at h0–h3, a fifth of the base's, with WIS within 0.2 points - of the base or better at every horizon. REF-op (`warm + slow + leak`) has - the best WIS (+5.4/+4.9/+2.4/+1.7/+2.2) at about three times the coverage - bias (0.074–0.041). These two mark the ends of the WIS/coverage trade-off. -- **The warm start's gain is in the first live season.** In 2024-25 it adds - 1.7–3.2 points at h2/h3 to every tracker; in 2025-26 warm and cold runs are - within 0.5 points, since by then the tracker has learned the level itself. - `warm + single` still loses 1.8–4.3% at h1–h3 in 2025-26; only the leaky - variants gain there. -- **The warm start alone** (no tracking) gains 0.5–2.1% WIS and cuts - coverage bias by about a third at h0–h3 (12% at h−1): the persistent low - bias is real and cheap to correct. -- **Where REF-op loses** is unchanged from the finalized-truth runs: at h3, - October (−30%) and March (−29%), 6% of the base WIS between them; - December–January (71% of it) gain about 5%. -- **Month view (2026-10-01):** scored within each month (states only), - `warm + single` is not the best-calibrated config. Month-avg L1 coverage - bias at h−1…h3: `warm + single` 0.107/0.100/0.116/0.128/0.146, REF-op - 0.107/0.091/0.111/0.125/0.136, base 0.134/0.116/0.123/0.137/0.151. REF-op - is lowest at h0–h3, so the two do not mark opposite ends of a coverage - trade-off; `warm + single`'s low season-pooled bias is largely ramp and - post-peak errors cancelling (in March 2025-26 it loses 71/83% of WIS at - h2/h3 with bias 0.25/0.27 vs the base's 0.15/0.18). The warm start alone - cuts month-avg bias by 5–15% at h0–h3, not a third. +WIS reduction % is `100 × (WIS_base − WIS_cal) / WIS_base`, so positive = +calibrated WIS lower than base. L1 coverage bias: lower is better. ```{r runs} sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) @@ -91,3 +74,35 @@ mv_plot_cfgs <- c("warm only", "single", "warm + single", "slow + leak", "warm + ```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} ``` + +# Findings + +WIS reduction % (positive = calibrated WIS lower than base) differences are +in points. L1 coverage bias (lower is better) leads with the month-avg, +states only. + +- **Warm start, WIS: depends on the season.** In 2024-25 it adds 1.7–3.2 + points at h2/h3 and 0.1–1.2 at h−1…h1 to every tracker. In 2025-26 warm + and cold differ by −0.4 to +0.4 at every horizon: no clear difference. + Part of the 2024-25 gain is against cold runs that play the base + forecast for the first live rounds. +- **Warm start, coverage.** In 2024-25 month-avg bias is 0.007–0.012 lower + for `warm + single` than `single`, and 0.011–0.017 lower for REF-op than + `slow + leak`, at h0–h3. In 2025-26 the pairs are within 0.008 in either + direction (`warm + single` is 0.002–0.004 higher than `single`): no clear + difference. Season-pooled bias makes `warm + single` look lowest + (0.013–0.020 at h0–h3, all locations), but pooling lets ramp + under-prediction cancel post-peak over-prediction. +- **Leak vs plain tracker.** In 2025-26 at h1–h3, `single` is + −1.8/−4.6/−3.5 and `leak 0.1` +1.5/+1.0/+0.9, 3.3–5.6 points apart. + In 2024-25 `single` is ahead by 1.3 points at h−1 and by under 1 point + elsewhere. Month-avg bias, both seasons: within 0.007 at every horizon. +- **Slow term (REF-op vs `warm + leak 0.1`).** No strong result: WIS within + 0.4 points at every horizon (`warm + leak 0.1` ahead at h2/h3), + month-avg bias within 0.003. +- **Warm start alone.** Pooled WIS +0.5 to +2.1 (2024-25 +0.7 to +3.5, + 2025-26 +0.1 to +1.3). Month-avg bias 0.007–0.017 below the base at + h−1…h3. +- **By month, h3, both seasons.** Every config loses in October (−12 to + −49%) and March (−11 to −51%), months holding 0.3% and 5.4% of the base + WIS; December–January (71% of it) gain 1–6%. diff --git a/reports/writeups/calibration_experiments/e07_across_eras.Rmd b/reports/writeups/calibration_experiments/e07_across_eras.Rmd index 5fa19a85..c3ed9bf9 100644 --- a/reports/writeups/calibration_experiments/e07_across_eras.Rmd +++ b/reports/writeups/calibration_experiments/e07_across_eras.Rmd @@ -20,136 +20,46 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` -The hub-era results (E00–E05, E09) calibrate our submitted FluSight -forecasts: an ensemble whose members changed between seasons. **Are the -gains an artifact of that ensemble, or does the tracker help a single, -fixed forecaster the same way?** The fixed forecaster is flu prod's -`windowed_seasonal`. - -This asks the NHSN half of the original E07's question -(`scripts/calibration/calibration_ws_experiments.R`) on the current setup. -It is a new experiment, not a reproduction: the original learned from -finalized data and replayed `windowed_seasonal` with prod's data -substitutions; here both learn from data as published each week, and B is -the clean replay. The original's third setup, C (the ILI+ decade), was -invalid and is not repeated. +Question: does calibration change WIS and coverage the same way for the +submitted ensemble as for one fixed forecaster? **Setup.** Two forecasters, everything else shared: -- **A**: the submitted ensemble, read from the hub as in E00 and E03–E05. +- **A**: the submitted flu ensemble, read from the hub as in E00 and + E03–E05. - **B**: the clean `windowed_seasonal` replay (E12–E17), with the 2023-24 HHS backfill (E13) as its burn-in season. -Both are cut to the same (round, location, horizon) rows: the hub's -submission rounds, all 53 locations, h0–h3 (the ensemble's h−1 is a -different model, so it is left out), and the submissions excluded from the -hub reads are dropped from B too. Both learn from and are scored on the -same NHSN data, as published each week and re-run from scratch each week, -one week of extra revision delay, with 2023-24 as a burn-in. - -The trackers are the original's three, all on the square-root scale with -the adaptive learning rate (mult 0.03, floor 0.001, window 20, carry): - -- **single**: the paper's tracker (REF-paper at sqrt). -- **leaky**: single with the offset shrinking 10% per round (`fast_decay`). -- **operating**: REF-op, prod's tracker: leaky plus a slow term and a warm - start from the burn-in season's errors. - -Section 3 also runs the operating-point candidates (E14, E16, E17) on both -forecasters. +| | | +|---|---| +| rows | identical (round, location, horizon, level) rows for A and B: hub submission rounds, submissions excluded from the hub reads dropped from both | +| seasons | 2023-24 burn-in (27 rounds), 2024-25 (27) and 2025-26 (28) scored | +| locations | all 53; results lead with states only | +| horizons | h0–h3 (the ensemble's h−1 is a different model) | +| learning truth | NHSN as published each week, re-run from scratch each week, one week of extra revision delay | +| scoring truth | the store's NHSN truth | + +Trackers (section 2), all sqrt scale, adaptive learning rate (mult 0.03, +floor 0.001, window 20, carry): + +- **single**: REF-paper at sqrt. +- **leaky**: single with the fast offset shrinking 10% per round. +- **operating**: REF-op (leaky plus a slow offset and a warm start from the + burn-in season's errors). + +Candidates (section 3): REF-op warm (= operating) and cold; constant rate +0.018 on the sqrt of the per-100k rate (E16); constant 0.032 per 100k on +the rate scale (E14); sqrt 0.018 switched off from 1 March +(`off_after = "03-01"`, E17). The cold candidates see the live seasons +only and start from zero offsets. + +This rebuilds the original E07 (a script since deleted), +which learned from finalized data, used the replay with prod's data +substitutions and included h−1. "WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is -better) is the mean over the 23 levels of |empirical coverage − level|. Results lead with states only. - -# Findings - -Numbers are from the render of 1 October 2026, states only, WIS reduction % -(positive = calibrated WIS lower than base) / L1 coverage bias (lower is -better), h0/h1/h2/h3. All-locations numbers -point the same way. - -- **Base WIS: equal in 2024-25, B worse in 2025-26.** B/A base WIS is - 0.93/0.99/1.01/1.01 in 2024-25 and 1.33/1.22/1.11/1.05 in 2025-26 - (1.06/1.07/1.05/1.02 pooled). Per state, pooled over h0–h3, the median - ratio is 0.99 in 2024-25 (B better in 29 of 52 states) and 1.12 in - 2025-26 (B better in 7 of 52). So in 2025-26 the ensemble beats B almost - everywhere, by about 10%, most at h0–h1. -- **The two bases are equally miscalibrated in 2024-25** (coverage bias - 0.13–0.17, 68–77% of outcomes above the median for both). In 2025-26 the - ensemble is better calibrated (0.046–0.078 vs 0.081–0.090). -- **By month, both bases have the same shape.** Both under-predict on the - ramp (December: 96–97% of outcomes above the median in 2024-25, 86–94% in - 2025-26, coverage bias 0.24–0.34) and over-predict after the peak (March - 2025-26: 32% above for A, 38% for B). B's ramp bias is larger in 2025-26 - (November–December coverage bias 0.19/0.31 vs 0.14/0.24). The one place - they differ is spring 2024-25: A still under-predicts in March–April (59% - and 75% above the median), while B is close to calibrated (45% and 56%). -- **The operating-point candidates rank mostly the same on both, with one - exception.** Pooled over both seasons, h0/h1/h2/h3 WIS reduction %: - - | candidate | A ensemble | B `windowed_seasonal` | - |---|---|---| - | REF-op warm | +3.9/+1.9/+1.5/+2.3 | +2.8/+1.1/+0.8/+1.2 | - | REF-op cold | +2.8/+0.7/−0.3/−0.7 | +2.0/+0.6/+0.1/−0.2 | - | sqrt 0.018 | +2.4/+0.6/+0.1/0.0 | +2.2/+0.8/+0.6/+0.6 | - | rate 0.032 | +1.5/+0.7/+0.5/+0.2 | +1.3/+0.7/+0.5/+0.4 | - | sqrt 0.018, off 1 Mar | +1.7/+0.6/+0.4/+0.6 | +2.3/+1.2/+1.2/+1.3 | - - Ordered by the mean over h0–h3, sqrt 0.018 > rate 0.032 > REF-op cold on - both. REF-op warm is first on A (+2.4 mean) and tied first with - off-1-March on B (both +1.5). Coverage bias ranks identically on both, - pooled: REF-op warm best (A 0.035–0.047, B 0.037–0.061), then rate 0.032, - sqrt 0.018, REF-op cold, and off-1-March worst (A 0.084–0.092, B - 0.094–0.116). -- **The exception is the March switch-off, and it follows the base's spring - bias.** In 2024-25, off-1-March is last of the five on A (+0.6/+0.2/0.0/+0.1) - but second on B (+1.6/+0.8/+0.7/+0.5): A was still under-predicting that - spring, so the offsets it switches off were still helping. In 2025-26, - when both over-predict in March, it is first on both. So whether to - switch off in spring depends on the forecaster, and B's result for it - should not be carried over to the ensemble without checking. -- **sqrt 0.018 is WIS-positive at every horizon in both seasons only on B.** - On A it is −0.3/−0.1 at h2/h3 in 2025-26, where the ensemble was already - close to calibrated. -- **Calibration does the same thing to both.** The trackers rank the same - way on each: operating gains the most WIS, single gives the best coverage - and loses at h2–h3. Pooled over both seasons: - - | tracker | A ensemble | B `windowed_seasonal` | - |---|---|---| - | single | +2.3/0.0/−1.3/−1.8 | +2.0/0.0/−0.6/−1.0 | - | leaky | +2.2/+0.8/0.0/−0.3 | +1.7/+0.6/+0.4/+0.2 | - | operating | +3.9/+1.9/+1.5/+2.3 | +2.8/+1.1/+0.8/+1.2 | - -- **The ensemble gains about 1 point more with REF-op**, mostly at h2–h3 in - 2024-25 (+2.8/+4.3 vs +1.4/+1.7) and h0 in 2025-26 (+4.3 vs +2.2). The - single tracker loses more on the ensemble in 2025-26 (h2 −3.1 vs −0.7), - where the ensemble was already close to calibrated. -- **Answer: the gains are not an artifact of the ensemble.** On the same - rows and data, one fixed forecaster gains the same way, a little less. - B's base is about 5% worse on WIS pooled and its bias has the same - monthly shape, so experiments on B transfer to the ensemble for the - choice of tracker. They do not transfer for date-based switches, which - depend on where each base is biased in spring. -- **Against the old E07** (finalized truth, the replay with prod's data - substitutions, 50 locations, h−1 included): the verdict holds. The old - explanation for B's smaller gains does not: at h0–h3 the gap is about a - point, not up to 11 (that was h−1, a different model), and B is not less - miscalibrated than the ensemble (equal in 2024-25, worse in 2025-26). - B single still has much the best 2025-26 coverage (0.010–0.013). -- **Month view (2026-10-01):** the coverage rankings do not survive - scoring within each month. Month-avg L1 coverage bias, both seasons, mean - over h0–h3: on A, REF-op warm 0.115 < REF-op cold 0.120 < sqrt 0.018 - 0.124 < off 1 Mar 0.126 < rate 0.032 0.133 (base 0.132); on B, sqrt 0.018 - 0.130 < REF-op cold 0.133 < REF-op warm 0.135 < off 1 Mar 0.136 < rate - 0.032 0.142 (base 0.142). So off-1-March is not the worst, rate 0.032 is - (no better than the base), and the order differs between A and B. Among - the trackers, operating has the lowest month-avg bias on A - (0.090/0.110/0.124/0.138 vs single 0.101/0.117/0.130/0.150) and the three - tie on B; B single's 2025-26 lead disappears (0.078/0.105/0.128/0.142 vs - operating 0.078/0.111/0.128/0.145). The base comparison holds: in - 2025-26 A's month-avg bias is 0.089–0.117 against B's 0.097–0.146. +better) is the mean over the 23 levels of |empirical coverage − level|. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) @@ -522,3 +432,39 @@ mv_linetype <- "forecaster" ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` + +# Findings + +States only. WIS reduction % (positive = calibrated WIS lower than base) +differences are in points; L1 coverage bias is lower-is-better and leads +with the month-avg. + +- **Bases.** B/A base WIS is 0.93–1.01 in 2024-25 and 1.05–1.33 in 2025-26 + (largest at h0); per state B is lower in 29 of 52 states, then 7 of 52. + Month-avg base bias over h0–h3 is 0.116–0.151 for A and 0.112–0.177 for + B (both seasons), 0.089–0.117 vs 0.097–0.146 in 2025-26. Both bases have + over 86% of outcomes above the median in December and under 40% in March + 2025-26. They differ in November 2024-25 (51% vs 73% above), March–April + 2024-25 (59/75% vs 45/56%) and February 2025-26 (60% vs 71%). +- **Trackers.** On both, pooled WIS at h1–h3 orders operating > leaky > + single. Operating gains 0.7–1.1 points more on A than on B (A + +3.9/+1.9/+1.5/+2.3, B +2.8/+1.1/+0.8/+1.2), mostly 2024-25 h2/h3 + (+2.8/+4.3 vs +1.4/+1.7) and 2025-26 h0 (+4.3 vs +2.2). On B, leaky and + single are under 1 point apart except at h3 (1.2). +- **Candidates, WIS: the order flips between seasons.** Mean over h0–h3, + REF-op warm leads in 2024-25 on both (A +3.44, B +2.07; next +0.79 and + +0.90). In 2025-26 the 1 March switch-off leads on both (A +1.96, B + +2.37) and REF-op warm is 4th on A, 5th on B (+0.69, +0.60). Pooled over + both seasons the other four candidates are within 0.2 points of each + other on A (+0.64 to +0.83) and within 0.9 on B (+0.63 to +1.48). +- **1 March switch-off in 2024-25.** No strong result: 0.47 points below + sqrt 0.018 on A and 0.24 above it on B. The rates (0.018, 0.032) and the + 1 March date were picked in E14/E16/E17 on these same seasons, on B. +- **Coverage, month-avg.** No strong result among candidates: over h0–h3, + A ranges 0.115–0.126 for four candidates (base 0.132) and B 0.130–0.136 + (base 0.142). Rate 0.032 is no better than the base on either (A 0.133, + B 0.142), and in 2025-26 it is above the base at every horizon (A by + 0.016–0.024, B by 0.007–0.017). Season-pooled, rate 0.032 has the lowest + 2025-26 bias on B (0.010–0.014), because pooling lets ramp and post-peak + errors cancel. Season-pooled coverage ranks agree between A and B over + both seasons (Spearman 0.97) but not in 2025-26 (0.31). diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd index 584d9f0b..179664ff 100644 --- a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -25,123 +25,54 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` -The question: for the plain quantile tracker applied on the **rate scale** -(admissions per 100k) with a **constant** learning rate, how do coverage and -WIS trade off across learning rates, and how does that change with the -**extra delay** before the tracker learns from an outcome? - -**The tracker.** For each location, horizon and quantile level, the tracker -keeps an additive offset that is added to the submitted quantile. When an -outcome is learned, each offset moves by `lr × (1{y > q} − (1 − τ))`: up by -`lr × τ` if the outcome landed above the played quantile `q`, down by -`lr × (1 − τ)` if below. `lr` is the learning rate (the step size). - -- **Constant learning rate** (the sweep in this notebook): `lr` is a fixed - number, the same for every location and round, and does not depend on the - forecast errors. On the rate scale it is in admissions per 100k, so one - step moves an offset by at most `lr` per 100k. The curves below trace this - one number from 0.001 to about 3.2. -- **Adaptive learning rate** (the paper's default, used by both reference - configs): `lr` is recomputed every round as 0.03 × the 90th percentile of - the base forecast's absolute errors over the location's last 20 observed - rounds. The paper uses 0.1 and 50 rounds; an earlier sweep found 0.03 gives - similar coverage at a much smaller WIS cost, and that the window length - (8, 20, 50 or all history) barely matters. The window has no role with a - constant learning rate, so it is not varied here; the reference marks use - 20. - -**Extra delay.** The first NHSN report of the week ending on Saturday `X` is -published on Wednesday `X + 4`, in time for the round with reference date -`X + 7`. Learning from it at that round is zero extra delay. Each further -week of waiting is one week of extra delay, giving later revisions time to -come in. The reference configs wait one extra week; the sweep covers 0–3. - -**What the tracker learns from.** At each round the tracker learns from the -NHSN data as published by then (a "vintage"), not from finalized values, -because finalized values are not available in real time (see E00). There are -two ways to deploy that, and E00 found they respond to the delay in opposite -directions, so both are shown: - -- **re-run** (what prod does): each week the tracker is run again from the - first round over the whole history, on that week's vintage. An outcome - first learned from its early report is learned from its revised value in - every later week's run. -- **stateful**: the tracker runs once, stepping forward a round at a time, - and learns each outcome once, from the value published at the round it - becomes available. It never sees later revisions of that outcome. - -This is a separate choice from what happens to the offsets between seasons. -In both designs the offsets carry over the summer off-season rather than -resetting to zero. - -**Other settings.** Everything else follows the REF-paper reference config: -one offset per level, carried across seasons, and a 2023-24 burn-in season -whose outcomes take no steps (the adaptive rate uses it to warm up; the -constant rate needs no warm-up). Scores are on counts against finalized -truth, over the two live seasons (2024-25 and 2025-26), all 53 hub -locations. "WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so -positive = calibrated WIS lower than base, as in the other notebooks. "L1 -coverage bias" (lower is better) is the mean over the 23 quantile levels of -|empirical coverage − level|. +Question: for the plain quantile tracker on the rate scale (admissions per +100k) with a constant learning rate, how do WIS and coverage change with +the learning rate and with the extra delay before an outcome is learned? + +**Setup.** + +| | | +|---|---| +| disease, base | flu, the submitted hub ensemble (`CMU-TimeSeries`) | +| seasons | 2023-24 burn-in (its outcomes take no steps; offsets start at zero); 2024-25 and 2025-26 scored | +| locations, horizons | all 53 hub locations, h−1 to h3; the month view is states only | +| scoring truth | finalized NHSN | +| tracker | REF-paper (one offset per level, carried across seasons) on the rate scale | +| learning rate | constant, 0.001 to about 3.2 per 100k per step (15 values, a quarter-decade apart) | +| extra delay | 0, 1, 2 or 3 weeks (`settle_days` 7/14/21/28) | +| learning truth | two designs, below | + +**The tracker.** For each location, horizon and quantile level, the offset +moves by `lr × (1{y > q} − (1 − τ))` when an outcome is learned. With a +constant `lr` the step does not depend on the forecast errors. The +reference configs use the adaptive rate instead: 0.03 × the 90th +percentile of the location's absolute base errors over its last 20 +observed rounds. + +**Extra delay.** The first NHSN report of the week ending Saturday `X` is +published Wednesday `X + 4`, in time for the round with reference date +`X + 7`. Learning from it at that round is zero extra delay; each further +week of waiting is one week of extra delay. The reference configs use one +week. + +**Learning truth designs.** + +- **re-run** (prod): each week the tracker is re-run from the first round + on that week's vintage, so an outcome is learned from its latest value. +- **stateful**: the tracker runs once and learns each outcome from the + value published at the round it becomes available. **Reference configs** (marked on the plots): -- **REF-paper**: the paper's tracker on counts, adaptive learning rate as - above. -- **REF-op**: the configuration running in prod. REF-paper on the - square-root scale, plus a 10% per-round shrink of the offset toward zero, a - second slow-moving offset, and a starting offset taken from the burn-in - season's errors (E03–E05). +- **REF-paper**: the paper's tracker on counts, adaptive learning rate. +- **rate, adaptive**: REF-paper on the rate scale (floor 0.001 per 100k). +- **REF-op**: prod's config: REF-paper on the sqrt scale, plus a 10% + per-round leak, a slow offset and a warm start from the burn-in season. -# Findings - -Numbers are from the render of 30 September 2026, pooled over locations. -WIS reduction %: positive = calibrated WIS lower than base. L1 coverage bias: -lower is better. -"Knee" below means a constant learning rate of 0.056 admissions per 100k per -step. - -- **The trade-off is L-shaped.** Raising the constant learning rate cuts the - coverage bias from about 0.10 to 0.03 at almost no WIS cost, up to a knee - near 0.05–0.1. Past the knee WIS worsens steeply and coverage barely - improves. At the reference delay, h−1 has its best WIS at 0.1 (at 0.32 - with no extra delay). The per-location lines spread widely: past 0.3 many - locations lose 20% of WIS or more. -- **On the rate scale, the knee matches or beats the paper's adaptive - tracker at every horizon.** At the reference delay (+1 week, re-run - design), the knee gives WIS reduction +3.2/+2.5/+1.1/+0.5/+0.2% and coverage - bias 0.040/0.031/0.026/0.027/0.027 at h−1…h3. REF-paper gives - +3.2/+1.9/−2.0/−2.7/−5.3% and 0.044/0.030/0.032/0.034/0.032: the same WIS - at h−1, worse WIS from h0 on, and worse coverage at h−1 and h1–h3. -- **The rate scale only matters with a constant learning rate.** "rate, - adaptive" matches REF-paper exactly. The adaptive rate is a quantile of the - location's own residuals, so dividing a location by its population rescales - the steps and the offsets together. -- **REF-op makes a different trade.** Its WIS gain (+5.4/+4.9/+2.4/+1.7/+2.2%) - is larger than any constant rate reaches at h0–h3, but at about twice the - coverage bias (0.074/0.054/0.045/0.046/0.041). The leak and the warm start - buy WIS; the plain tracker buys coverage. -- **Extra delay, re-run design (prod): shorter is better, but it matters only - at h−1/h0.** At a learning rate of 0.1, h−1 WIS reduction is +5.3/+3.6/+1.0% - at +0/+1/+2 weeks. At h1–h3 the curves nearly coincide, and coverage bias - barely moves with delay. A value learned early is replaced by its revision - in the next week's run, so waiting only delays the signal. -- **Extra delay, stateful design: one week is needed.** At +0 weeks the - tracker learns once from the first report, which under-reports, so the - offsets are pushed down: at the knee, h−1 coverage bias is 0.142, worse than - the uncalibrated 0.111, and h−1 WIS is 0.9% worse than the base. +1 week - recovers most of it (0.066); +2 and +3 weeks change little at h1–h3 and - lose WIS at h−1. -- **Month view (2026-10-01):** scored within each month (states only, +1 - week, re-run), the knee does not beat REF-paper on coverage and REF-op has - the lowest bias, not twice it. Month-avg L1 coverage bias at h−1…h3: knee - 0.122/0.117/0.129/0.142/0.164, 0.1 0.118/0.114/0.127/0.147/0.172, - REF-paper 0.115/0.102/0.120/0.143/0.168, REF-op - 0.107/0.091/0.111/0.125/0.136, base 0.134/0.116/0.123/0.137/0.151. At - h1–h3 the knee and 0.1 are worse than the base month by month, so the - L-shaped drop in season-pooled bias is ramp and post-peak errors - cancelling. The delay results hold: stateful +0 weeks is worse than the - base at h−1 (0.151 vs 0.134) and +1 week recovers (0.118). +"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so positive = +calibrated WIS lower than base. "L1 coverage bias" (lower is better) is the +mean over the 23 quantile levels of |empirical coverage − level|. "Knee" +below is the constant rate 0.056. ```{r libs, cache=FALSE} suppressPackageStartupMessages(source(here::here("R/load_all.R"))) @@ -259,12 +190,12 @@ best_const <- sweep_m %>% group_by(horizon) %>% slice_max(wis_pct, n = 1, with_ties = FALSE) %>% ungroup() %>% - mutate(ref = "rate, constant lr (best WIS per horizon)") + mutate(ref = "rate, constant lr (highest WIS per horizon)") bridge <- bind_rows( ref_m %>% filter(learn == "exact", settle_days == 14L), best_const ) %>% - mutate(ref = factor(ref, levels = c("REF-paper (count, adaptive)", "rate, adaptive", "rate, constant lr (best WIS per horizon)", "REF-op"))) + mutate(ref = factor(ref, levels = c("REF-paper (count, adaptive)", "rate, adaptive", "rate, constant lr (highest WIS per horizon)", "REF-op"))) base_row <- bridge %>% filter(ref == "REF-paper (count, adaptive)") %>% mutate(ref = "base", wis_pct = 0, bias_cal = bias_base) tbl <- bind_rows(base_row, bridge) %>% transmute(config = ref, horizon, cell = sprintf("%+.1f%% / %.3f", wis_pct, bias_cal)) %>% @@ -311,8 +242,7 @@ triangles are the reference configs at the same delay. Per-location coverage comes from about 50 forecasts per horizon, so a location's L1 bias has a noise floor of a few points even for a perfectly -calibrated forecaster. Read the shape of the pooled line; the grey lines show -spread. +calibrated forecaster. The grey lines show the spread across locations. ```{r curve-fn} ACCENT <- "#1f5fa8" @@ -422,3 +352,41 @@ mv_plot_cfgs <- c( ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` + +# Findings + +WIS reduction % (positive = calibrated WIS lower than base) differences +are in points. L1 coverage bias is lower-is-better; month-avg is states +only, season-pooled is all locations. Unless stated, +1 week, re-run. + +- **Constant rate.** WIS rises with the rate up to about 0.056–0.1 (knee: + +3.2/+2.5/+1.1/+0.5/+0.2 at h−1…h3) and falls fast past 0.3 (at 1.0: + −12 to −23 at h0–h3). The rate with the highest WIS per horizon is + 0.1/0.056/0.056/0.018/0.018, picked on the scored seasons; at h2, + 0.018–0.056 are within 0.1 point. +- **Knee vs REF-paper.** WIS: knee ahead by 3.1–5.5 points at h1–h3, 0.6 at + h0, tied at h−1. Coverage: no clear difference season-pooled (within + 0.007); month-avg REF-paper is lower at h−1…h1 by 0.007/0.015/0.009 and + the knee at h2/h3 by 0.001/0.004. +- **Month-avg coverage vs base.** Against the base + (0.134/0.116/0.123/0.137/0.151 at h−1…h3), the knee differs by + −0.012/+0.001/+0.006/+0.005/+0.013 and 0.1 by + −0.016/−0.002/+0.004/+0.010/+0.021: no lower than the base at h1–h3. REF-op is 0.012–0.027 below the base at every horizon. REF-op + differs from the sweep in scale, leak, slow offset and warm start, so + the comparison does not isolate any one of them. Season-pooled, the + knee's bias (0.026–0.040) is about half REF-op's (0.041–0.074), but + pooling lets ramp and post-peak errors cancel. +- **Extra delay, re-run.** At the knee, h−1 WIS is +4.1/+3.2/+2.1/+1.0 at + +0/+1/+2/+3 weeks and h0 +2.9/+2.5/+2.0/+1.5; at h1–h3 the delays are + within 1 point. Month-avg bias at the knee moves by under 0.01 between + +0 and +2 weeks. +- **Extra delay, stateful.** At +0 weeks the knee makes h−1 worse than the + base: WIS −0.9, season-pooled bias 0.142 vs 0.111, month-avg 0.151 vs + 0.134. At +1 week h−1 is +3.7, 0.066 pooled, 0.118 month-avg. At h0–h3, + +0 weeks has WIS within 0.6 points of +1 week. +- **Known issues.** `rate, adaptive` matches REF-paper to every printed + digit because the adaptive rate is scale invariant (`scales` is applied; + checked 2026-10-02). Runs are cached + by name, not by inputs or code (clear `cache/calibration/experiments/e09/` + after changes). In October 2025 (one round) the knee and 0.1 carry spring + offsets forward: h0 month bias 0.232/0.218 vs the base's 0.053. diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd index 14a3a913..46e86cc6 100644 --- a/reports/writeups/calibration_experiments/e10_covid.Rmd +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -24,57 +24,28 @@ ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplo ```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} ``` -One axis: the disease. Both reference configs run on the covid hub's `CMU-TimeSeries` -submissions, learning from NHSN covid vintages the way prod does (E00), and -scored against finalized covid NHSN. +Question: how do the two reference configs do on covid? -Covid submissions start 2024-11-23, so there is no burn-in season. As in the -covid prod pipeline, both configs run with no burn-in season, -and REF-op runs without its warm start, leaving sqrt + slow term 0.003 + -leak 0.1. The tracker therefore starts from zero offsets at -the first round. +Setup: -Covid is submitted year-round, so there is no off-season gap between -seasons. For the per-season views below, seasons are August–July years. +- Disease: covid. Base forecaster: the submitted `CMU-TimeSeries` ensemble. +- Learning truth: NHSN covid vintages, re-run at every round as prod does + (E00). Scores against finalized covid NHSN. +- No burn-in season (covid submissions start 2024-11-23); both trackers start + from zero offsets. REF-op runs without its warm start, as in covid prod. +- Rounds 2024-11-30 to 2026-09-19. The 2024-11-23 round is excluded (made + from a bad NHSN release corrected the next day). +- Covid is submitted year-round; per-season views use August–July seasons. +- Horizons h−1 to h3. Headline tables: all hub locations; month view: states + only. -# Findings - -Numbers are from the render of 30 September 2026, pooled over all rounds -(2024-11-30 to 2026-09-19) and locations. WIS reduction % is 100 × (base − -calibrated) / base, so positive = calibrated WIS lower than base. L1 coverage -bias: lower is better. The first covid round (2024-11-23) is excluded: it was made from a -bad NHSN release that was corrected the next day. +| config | change from REF-paper | +|---|---| +| `REF-paper` | none (counts, adaptive learning rate) | +| `REF-op (no warm start)` | sqrt scale, learning-rate floor, slow term 0.003, leak 0.1; no warm start | -- **The covid base is already calibrated at h1–h3,** so there is little to - fix there. Its coverage bias is 0.098/0.067/0.046/0.036/0.028 at h−1…h3, - against about 0.1 at every horizon for flu. -- **h−1 is a clear win.** WIS +10.2% (REF-op) and +11.4% (REF-paper), - absolute error of the median +20% for both. REF-paper's h−1 coverage bias - is 0.012, the lowest of any config in these notebooks. Over these rounds - the submitted h−1 came entirely from a climatology-plus-linear-trend - ensemble (the autoregressive models were left out of negative horizons), - and h−1 is where the base is least calibrated. -- **At h1–h3, REF-op is neutral to slightly harmful and REF-paper is - harmful.** REF-op: WIS 0.0/−0.6/−1.3%, coverage bias slightly worse than - base at h2/h3 (0.039/0.037 vs 0.036/0.028). REF-paper: WIS - −4.5/−7.3/−10.3%, coverage bias worse than base from h2 on (0.044/0.049). - An offset learned where the base is calibrated is noise. -- **The losses are in 2025-26 and around September.** REF-op gains - +2.4/+2.8/+1.9% at h1–h3 in 2024-25 and loses 3.0–4.3% in 2025-26. By month - at h3, REF-op loses 16% in September and 12% in October (20% of the base - WIS between them), around the late-summer wave. REF-paper also loses 42–49% - in April and May. -- **Implication for prod:** for covid, calibrating h−1 (and perhaps h0) and - passing h1–h3 through would keep the gain and drop the losses. The covid prod - pipeline calibrates every horizon today. -- **Month view (2026-10-01):** the covid base is calibrated only - season-pooled. Scored within each month (states only, August–July - seasons 2024-25 and 2025-26), its L1 coverage bias is - 0.148/0.102/0.125/0.143/0.157 at h−1…h3, about the flu base's level - (0.134/0.116/0.123/0.137/0.151). Neither config improves on it beyond - h−1/h0: REF-op - 0.123/0.096/0.126/0.147/0.161, REF-paper 0.111/0.094/0.126/0.155/0.171. - The h−1 gain and the h2–h3 coverage cost hold. +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. ```{r runs} no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) @@ -96,3 +67,27 @@ focus <- "REF-op (no warm start)" ```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} ``` + +# Findings + +There are no uncertainty estimates; a WIS gap under 1 point or a month-avg +coverage gap under 0.01 is read as no clear difference. The pooled headline, +median-error and by-month tables run to 2026-09-19 to include the 2026 summer +wave; those Aug–Sep 2026 rounds are in no per-season table. + +- **h−1:** both configs improve WIS by about 10–11% in each season + (REF-paper 11.6/10.5, REF-op 8.9/10.8) and the absolute error of the median + by about 20%. Which config is ahead flips between seasons. +- **REF-paper at h1–h3:** −4.5/−7.3/−10.3 pooled, almost all from 2025-26 + (−10.2/−14.4/−19.3). In 2024-25 it is within 0.3 points of the base. +- **REF-op at h1–h3:** 0.0/−0.6/−1.3 pooled; +2.4/+2.8/+1.9 in 2024-25 and + −3.0/−4.0/−4.3 in 2025-26. No strong result: the sign depends on the + season. +- **Month-avg coverage (states, h−1…h3):** base 0.148/0.102/0.125/0.143/0.157; + REF-op 0.123/0.096/0.126/0.147/0.161 (lower at h−1 only); REF-paper + 0.111/0.094/0.126/0.155/0.171 (about 0.01 worse at h2/h3). Season-pooled, + the base looks calibrated at h1–h3, but pooling lets the two seasons' + errors cancel: within each season its bias is 0.072–0.078 (2024-25) and + 0.029–0.051 (2025-26). +- **By month at h3:** REF-op −16.5% in September and −12.0% in October + (together 20% of base WIS); REF-paper −42% to −49% in April–May (about 6%). diff --git a/reports/writeups/calibration_experiments/e11_constant_lr.Rmd b/reports/writeups/calibration_experiments/e11_constant_lr.Rmd index 502d3809..61151585 100644 --- a/reports/writeups/calibration_experiments/e11_constant_lr.Rmd +++ b/reports/writeups/calibration_experiments/e11_constant_lr.Rmd @@ -23,61 +23,32 @@ ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplo ```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} ``` -One axis: the learning rate. The constant rate is the corner of E09's -trade-off curve, re-run design, one week of extra delay: 0.1 admissions per -100k per step on the rate scale. Below that rate, coverage bias falls with -little change in WIS. Above it, coverage stops improving and WIS gets worse -at every horizon. It is compared with the adaptive learning rate (0.03 × the -90th percentile of the last 20 rounds' absolute errors) in the configs from -E03 and E05. Everything else is REF-paper, learning from vintages the way -prod does (re-run each week; E00). +Question: does a constant learning rate on the rate scale do better than the +adaptive learning rate? -- **REF-paper**: counts, adaptive. -- **sqrt, adaptive**: the E03 winner. -- **warm + sqrt, adaptive**: the E05 `warm + single`, the best-calibrated - adaptive config. -- **rate, constant 0.1**: the E09 corner. -- **REF-op**: prod, for orientation; it differs in several settings. +Setup: -# Findings +- Disease: flu. Base forecaster: the submitted hub ensemble (spoiled + submissions dropped). +- Burn-in: 2023-24, used only to warm up the adaptive learning rate. Offsets + start at zero at the first 2024-25 round unless a config has a warm start. + The constant-rate config has no learning rate to warm up, so it is + effectively cold. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. Scores against finalized truth. +- Seasons scored: 2024-25 and 2025-26. Horizons h−1 to h3. Headline tables: + all hub locations; month view: states only. -Numbers are from the render of 1 October 2026, pooled over both live -seasons and all locations. WIS reduction % is 100 × (base − calibrated) / base, -so positive = calibrated WIS lower than base; the absolute-error change of the -median is defined the same way. L1 coverage bias: lower is better. The constant run matches E09's table at 0.1, +1 week, -re-run. +| config | change from REF-paper (counts, adaptive 0.03 × p90 of the last 20 errors) | +|---|---| +| `REF-paper` | none | +| `sqrt, adaptive` | sqrt scale, learning-rate floor 0.001 (E03 `sqrt`) | +| `warm + sqrt, adaptive` | as above, plus a warm start from the burn-in season (E05 `warm + single`) | +| `rate, constant 0.1` | rate scale (per 100k), constant learning rate 0.1 (E09, +1 week, re-run) | +| `REF-op` | the prod config; differs in several settings | -- **Against the cold adaptive trackers, the constant rate trades WIS at - h−1/h0 for better coverage everywhere.** WIS +3.6/+2.4/+0.7/+0.3/−0.5 at - h−1…h3, against +5.1/+4.0/+0.7/−1.3/−1.4 for `sqrt, adaptive`; coverage - bias 0.032/0.022/0.018/0.017/0.018 against 0.049/0.032/0.026/0.028/0.026. - It beats REF-paper on both at every horizon. -- **The warm-started adaptive tracker still beats it at h0–h1.** - `warm + sqrt, adaptive` has better WIS at h−1–h1 (+5.1/+4.7/+1.3) and - better coverage at h0–h3 (0.013–0.020). The constant rate keeps an edge only - in h−1 coverage (0.032 vs 0.046) and is within 0.8 points on WIS at h2/h3. -- **It is the most stable tracker across seasons and months.** In 2025-26 it - loses 1.8–2.0% at h1–h3, against up to 4.6% for the adaptive sqrt trackers - (REF-op is the only config that gains there). By month at h3 it has the - smallest March loss of the plain trackers (16% vs 42–51% for sqrt), but is - nearly neutral in December–January (about +1%, 71% of the base WIS), where - the warm start and REF-op gain 4–6%. Its h−1 is the weakest in 2025-26 - (−1.6%). -- **The median gains less near the present.** Absolute error improves - +6.3/+4.9% at h−1/h0, against about +10/+8% for the sqrt trackers; it is - better than them at h2 (+1.8%). -- **Untested:** a warm start on the constant-rate tracker. The warm start - is the largest single gain for the adaptive tracker (E05), and the constant - rate keeps coverage without a leak. -- **Month view (2026-10-01):** scored within each month (states only), the - constant rate has the worst coverage, not the best. Month-avg L1 coverage - bias at h−1…h3: constant 0.1 0.118/0.114/0.127/0.147/0.172, `sqrt, - adaptive` 0.108/0.102/0.117/0.130/0.150, REF-paper - 0.115/0.102/0.120/0.143/0.168, `warm + sqrt, adaptive` - 0.107/0.100/0.116/0.128/0.146, REF-op 0.107/0.091/0.111/0.125/0.136, base - 0.134/0.116/0.123/0.137/0.151. It is highest of the five at every horizon - and worse than the base at h1–h3; its h−1 edge over the warm tracker - disappears. +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. ```{r runs} sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) @@ -93,3 +64,28 @@ focus <- "rate, constant 0.1" ```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} ``` + +# Findings + +There are no uncertainty estimates; a WIS gap under 1 point or a month-avg +coverage gap under 0.01 is read as no clear difference. + +- **Constant 0.1 vs `sqrt, adaptive`:** WIS +3.6/+2.4/+0.7/+0.3/−0.5 vs + +5.1/+4.0/+0.7/−1.3/−1.4 at h−1…h3 pooled. The adaptive tracker is ahead + at h−1/h0 (1.5/1.6 points), the constant at h2/h3 (1.6/0.9). In 2025-26 the + gaps are larger in both directions (adaptive +4.6 at h0, constant +2.8 at + h2). The order depends on horizon. +- **Constant 0.1 vs `warm + sqrt, adaptive`:** the warm tracker is ahead in + every 2024-25 cell (by 0.8–2.7). In 2025-26 the constant is ahead at h2/h3 + (by 2.5/1.9). This compares a cold config with a warm one. +- **REF-op:** pooled WIS +5.4/+4.9/+2.4/+1.7/+2.2, the highest at every + horizon, by up to about 2 points over `warm + sqrt, adaptive`. Its 2025-26 + gains at h1–h3 are under 1.5 points. +- **Month-avg coverage (states, h−1…h3):** base 0.134/0.116/0.123/0.137/0.151; + constant 0.118/0.114/0.127/0.147/0.172; `sqrt, adaptive` + 0.108/0.102/0.117/0.130/0.150; `warm + sqrt, adaptive` + 0.107/0.100/0.116/0.128/0.146; REF-op 0.107/0.091/0.111/0.125/0.136. The + adaptive configs are within 0.01 of each other at most horizons. About half + of the constant's h0 gap comes from October 2025, a single sparse month + (0.3% of base WIS) with spring offsets carried over. Season-pooled the + constant has the lowest bias (0.017–0.032), but pooling lets errors cancel. diff --git a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd index f6ba4a4d..12a2fd64 100644 --- a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd +++ b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd @@ -19,73 +19,29 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` -E09 traced how coverage and WIS trade off as the constant learning rate of -the plain tracker changes, pooled over both live seasons. This notebook -splits those curves by season. **Is there a -range of learning rates that is good in both 2024-25 and 2025-26?** - -**Setup.** The clean `windowed_seasonal` replay for flu (E12: substitutions -off, all 53 locations, the 55 hub rounds, h0–h3; 27 rounds in 2024-25 and -28 in 2025-26), learning from data as published each week and re-run from -scratch each week as prod does, one week of extra revision delay (E09's -best re-run setting), no burn-in season. One axis: the constant learning -rate of the plain tracker on the rate scale (admissions per 100k per step), -12 values from 0.0032 to 1.8, a quarter decade apart. This is E11's -config with its learning rate swept; 0.1 is E11 itself. - -Reference points, run on the same data: - -- **REF-op cold**: prod's tracker without its warm start. -- **sqrt, adaptive**: the plain tracker on the square-root scale with the - adaptive learning rate (E03's winner; E13's "E05 single cold"). - -"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so -**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is -better) is the mean over the 23 levels of |empirical coverage − level|. Each season is -scored on its own rounds; the tracker still runs over both seasons, so -2025-26 starts from the offsets 2024-25 left behind. Results lead with states -only (US excluded), since US alone is about 45% of the all-locations WIS. - -# Findings - -Numbers are from the render of 1 October 2026, states only, WIS reduction % -(positive = calibrated WIS lower than base) / L1 coverage bias (lower is better). All-locations numbers point the -same way. - -- **Yes: 0.018–0.032 per 100k is good in both seasons.** At those rates WIS - improves at every horizon in both seasons (2024-25 +0.2 to +1.0, 2025-26 - +0.8 to +1.9) and coverage bias falls in both. At 0.056 WIS is still - non-negative everywhere (2024-25 h3 −0.0). From 0.1 up WIS gets worse at - h1–h3 in both seasons, and past about 0.18 it falls fast. -- **The seasons disagree about coverage, not WIS.** In 2025-26 coverage - bias reaches its floor (about 0.01–0.015) by 0.032. In 2024-25 it keeps - falling until about 0.3, and 0.032 leaves it at 0.10–0.12 (base - 0.13–0.17). The extra coverage 2024-25 gets from a larger rate costs WIS - in both seasons. Part of the gap is the cold start: with no burn-in, the - tracker starts at zero in November 2024, so 2024-25 is also the season it - is still learning in. -- **The knee is lower than E09's.** E09 (submitted ensemble, 2023-24 - burn-in) put the knee at 0.05–0.1. Here the WIS-best rate is 0.018–0.056 - at every horizon and season, and E11's 0.1 is already past it: −0.5/−0.9 - at h2/h3 in 2024-25 and −1.0/−1.2 in 2025-26. -- **The references.** REF-op cold has the most WIS gain at h0 (+2.2/+1.8) - but keeps 2024-25's coverage bias near the base (0.106–0.124). The adaptive - sqrt tracker reaches a 2025-26 coverage bias of 0.008–0.016 with +2.2 at - h0, but loses WIS at h2/h3 in 2024-25 (−1.4/−2.6). Neither is good in both - seasons at every horizon; a constant rate of 0.018–0.032 is, at a smaller - gain. -- **The 50% interval.** At 0.018–0.032 the 50% interval covers 0.47–0.51 - at h0/h2, close to nominal; 0.1 and above under-cover it in 2024-25 - (0.39–0.45 at h2). -- **Month view (2026-10-01):** scored within each month, coverage bias - does not keep falling with the rate. Month-avg L1 coverage bias (both - seasons, h0/h1/h2/h3) is lowest near 0.01 (0.099/0.115/0.144/0.170; base - 0.112/0.127/0.153/0.177) and is above the base at h1–h3 from 0.056 - (0.112/0.135/0.164/0.191) up (0.32: 0.113/0.145/0.178/0.207). In 2024-25 - it bottoms at 0.032 at h1–h3 (0.131/0.164/0.196), not 0.3; in 2025-26 at - 0.0056–0.01, and 0.032 is already worse than the base there - (0.104/0.123/0.147/0.161 vs 0.097/0.112/0.130/0.146). Above about 0.03, - the season-pooled coverage gain is ramp and post-peak errors cancelling. +Question: how do WIS and coverage change with a constant learning rate, +season by season? + +Setup: + +- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (E12: + substitutions off, all 53 locations, the 55 hub rounds; 27 in 2024-25 and + 28 in 2025-26). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. +- Cold start: no burn-in season, offsets start at zero in November 2024. The + tracker runs over both seasons, so 2025-26 starts from the offsets 2024-25 + left behind; each season is scored on its own rounds. +- Swept: the constant learning rate of the plain tracker on the rate scale + (admissions per 100k per step), 12 values from 0.0032 to 1.8, a quarter + decade apart. 0.1 is E11's config, though E11 used the submitted ensemble. +- References on the same data: `REF-op cold` (prod's tracker without its warm + start) and `sqrt, adaptive` (E13's "E05 single cold"). +- Tables lead with states only (US excluded; it is about 45% of the + all-locations WIS). + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) @@ -268,3 +224,29 @@ mv_plot_cfgs <- c("rate 0.018", "rate 0.032", "rate 0.056", "rate 0.1", "REF-op ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` + +# Findings + +There are no uncertainty estimates; a WIS gap under 1 point or a month-avg +coverage gap under 0.01 is read as no clear difference. The cold start leaves +Nov–Dec 2024 equal to the base for every config, so 2024-25 gains are +diluted. The rate range was looked at on these same seasons. + +- **WIS up to 0.056:** every rate from 0.0032 to 0.056 is at or above the + base in both seasons (at most +1.2 in 2024-25, +1.9 in 2025-26), and the + rates in this range are within 1.2 points of each other (0.018 vs 0.032: + within 0.3). No strong result for a best rate. +- **WIS above 0.056:** from 0.1 WIS falls (2024-25 h2/h3 −0.5/−0.9; 2025-26 + h1–h3 −0.8 to −1.2); by 0.18 it reaches −3 to −4 at h3. +- **Month-avg coverage (states, both seasons, h0…h3):** base + 0.112/0.127/0.153/0.177; 0.01 0.099/0.115/0.144/0.170; 0.032 about equal to + the base; 0.056 and above 0.008–0.026 worse than the base at h1–h3. In + 2025-26 at h0, 0.032 is above the base only because of October 2025. + Season-pooled bias falls steadily with the rate (2025-26 reaches about 0.01 + by 0.032), but pooling lets ramp and post-peak errors cancel. +- **References:** at h0, `REF-op cold` and `sqrt, adaptive` gain +1.8 to + +2.2 in each season. At h2/h3 in 2024-25 they lose (REF-op −0.0/−0.8, + sqrt −1.4/−2.6). Against the constant rates 0.018–0.032 the gaps are under + 1 point in most other cells. +- **50% interval:** at 0.018–0.032 it covers 0.47–0.51 at h0/h2; from 0.1 up + it under-covers in 2024-25 (0.39–0.45 at h2). diff --git a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd index 572cf804..3255914d 100644 --- a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd +++ b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd @@ -19,23 +19,26 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` -In the fan and vintage plots the calibrated intervals look almost the same -as the base ones, yet WIS improves. This notebook asks where that WIS -change comes from. - -**Setup.** The clean `windowed_seasonal` replay for flu (E12: substitutions -off, all 53 locations, the 55 hub rounds, h0–h3), learning from data as -published each week and re-run from scratch each week as prod does, no -burn-in season. Two calibrated configs, both from E12: - -- **REF-op cold**: prod's tracker without its warm start. -- **E11 constant 0.1**: the plain tracker on the rate scale (per 100k) - with a constant learning rate of 0.1. - -Scores are against finalized truth over 2024-25 and 2025-26. "WIS reduction %" -is `100 × (WIS_base − WIS_cal) / WIS_base`: **positive = calibrated WIS lower than base**. -Results lead with states only (US excluded), since US alone is about 45% of -the all-locations WIS. +Question: how does the WIS change split across quantile levels, how large +are the offsets next to the base forecast's spread, and how much of the +change comes from moving the median versus changing widths? + +Setup: + +- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (E12: + substitutions off, all 53 locations, the 55 hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. Cold start, no burn-in season. +- Scores against finalized truth over 2024-25 and 2025-26, pooled; states + only first (US excluded), then all locations. +- Configs, both from E12: + - **REF-op cold**: prod's tracker (sqrt, adaptive rate, slow term, leak) + without its warm start. + - **E11 constant 0.1**: the plain tracker on the rate scale (per 100k), + constant learning rate 0.1. + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. **Three ways to cut the WIS reduction.** @@ -58,48 +61,8 @@ the all-locations WIS. median sits on the base median. The calibrated widths and shape are kept, the base centre is kept. - If the two parts did not interact, their WIS reductions would add up to the - full change. Both parts are floored at zero, as the calibrated forecast - is. - -# Findings - -Numbers are from the render of 1 October 2026, states only, h0/h1/h2/h3, -WIS reduction % (positive = calibrated WIS lower than base). All-locations numbers point the same way. - -- **The offsets are small, and that is why the fans look unchanged.** The - typical (median) offset is 2–5% of the base forecast's 90% interval width - at every level and horizon, or 7–19% of its 50% interval width. The median - calibrated 90% width is 1.00–1.04 times the base width. Gains of 1–2% of - WIS need only shifts this size, which are hard to see on a fan plot. -- **The offsets nearly always point up.** The median offset is positive at - every level and horizon, and fewer than one forecast in ten is moved down - by more than about 3% of its 90% width. The tracker moves the whole - forecast up a little, which matches a base that under-predicts (E12). -- **The gain is in the centre and shoulders, not the tails.** The six tail - levels (0.01–0.05 and 0.95–0.99) add at most 0.1 point at any horizon. - REF-op's +2.0 at h0 is +1.0 from the centre (0.35–0.65) and +0.9 from the - shoulders (0.1–0.3, 0.7–0.9). -- **The lower half gains; the upper half loses beyond h0.** For REF-op the - levels below the median give +1.4/+0.7/+0.6/+0.6 and those above give - +0.5/−0.1/−0.5/−0.8. Raising the lower quantiles helps when the outcome - comes in above them. At h2–h3 the offsets grow with the level, so the upper - quantiles move up further and stretch the upper side past where outcomes - land. E11 shows the same pattern with a weaker lower-half gain - (+1.0/+0.4/+0.1/0.0) and a larger upper-half loss (−0.3 to −0.9). -- **For REF-op, shifting the median is the whole gain.** Moving every base - quantile by the median offset alone gives +2.3/+0.8/+0.5/+0.4, more than - the full calibration (+2.0/+0.6/+0.1/−0.2). Changing only the widths and - shape, around the base median, costs −0.6/−0.7/−0.9/−1.1. The two parts do - not add up: the full change is 0.3–0.6 points better than their sum. So - REF-op's width changes buy coverage at a WIS cost, and its WIS gain is a - small upward shift. -- **For E11 the median shift helps only at h0** (+0.9, then - −0.4/−1.2/−1.7), and the width change costs −0.2 to −0.8. Its typical - offset is about REF-op's size, but it has a long tail: in one forecast in - ten the offset exceeds 0.26–0.55 of the 90% width (REF-op 0.08–0.20). A - fixed step per 100k is large for a forecast whose interval is narrow, as in - small states or off-season weeks. + The interaction is the full change minus the two parts. Both parts are + floored at zero, as the calibrated forecast is. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) @@ -190,9 +153,7 @@ knitr::kable(group_tbl(GEOS[["all"]]), caption = "All locations: WIS reduction % ``` Change in pinball loss at each level relative to that level's own base -loss (not a share of WIS), states only. Tail levels carry little of the -WIS, so a large relative change there can still be a small piece of the -total. +loss (not a share of WIS), states only. ```{r level-pct} knitr::kable( @@ -333,3 +294,33 @@ mv <- cal_month_view(cals, seasons = LIVE) ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` + +# Findings + +There are no uncertainty estimates; a WIS gap under 1 point or a month-avg +coverage gap under 0.01 is read as no clear difference. Both configs are cold +and equal the base through December 2024 (about a third of 2024-25's base +WIS), so the pooled gains are diluted. There is no per-season split here. +States only, both seasons pooled, h0/h1/h2/h3. + +- **WIS:** REF-op +2.03/+0.59/+0.09/−0.17, E11 +0.73/−0.21/−0.70/−1.03. + REF-op is 1.3 points ahead at h0; at h1–h3 the gap is 0.8–0.9 (no clear + difference). +- **By level:** the six tail levels (0.01–0.05, 0.95–0.99) add at most 0.1 + point. REF-op's levels below the median give +1.36/+0.72/+0.59/+0.58 and + those above +0.54/−0.14/−0.48/−0.76. E11: below +1.00/+0.39/+0.12/−0.02, + above −0.31/−0.55/−0.73/−0.91. +- **Offset size:** median |offset| is 2–5% of the base 90% width at levels + 0.25–0.95, and 0.4–2.1% at the 0.05 level. The median calibrated 90% width + is 1.00–1.04 times the base. The 90th percentile of |offset| ÷ 90% width + (levels 0.25–0.95) is 0.07–0.20 for REF-op and 0.24–0.55 for E11. +- **Median shift vs width change:** for REF-op, median shift only gives + +2.29/+0.80/+0.47/+0.39 (0.2–0.6 above the full calibration) and width + change only −0.59 to −1.14. The interaction is +0.33 to +0.57 (E11: +0.43 + to +0.86), so the split is not additive. The coverage of the width-only + variant is not measured. +- **Month-avg coverage:** base 0.112/0.127/0.153/0.177; REF-op + 0.098/0.119/0.146/0.167 (0.007–0.014 lower); E11 0.116/0.143/0.174/0.203 + (0.004–0.026 higher than the base). Season-pooled, E11 looks lowest + (0.020–0.025, REF-op 0.077–0.085, base 0.105–0.127), but pooling lets ramp + and post-peak errors cancel. diff --git a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd index 0084fb80..45def696 100644 --- a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd +++ b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd @@ -19,97 +19,33 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` -E14 found a constant learning rate of 0.018–0.032 per 100k good in both -seasons. E15 found that E11's rate-scale tracker (0.1 per 100k) loses WIS -through a tail of offsets that are large next to narrow intervals: a fixed -step per 100k is big for a small state or an off-season week. Two questions: - -1. **Does the same tracker on the square-root scale do better?** On the - square root of the rate, a fixed step moves the forecast less where the - rate is low. -2. **Do the offsets go stale late in the season?** Offsets learned on the - ramp up point upward; after the peak they may linger and push the - forecast above outcomes that are now falling. - -**Setup.** As E14: the clean `windowed_seasonal` replay for flu (all 53 -locations, the 55 hub rounds, h0–h3), learning from data as published each -week and re-run from scratch each week as prod does, one week of extra -revision delay, no burn-in season. The tracker is the plain one with a -constant learning rate. The new axis is the scale: **sqrt** runs it on the -square root of admissions per 100k, so the learning rate is in -sqrt(per-100k) units and the grid is not comparable number for number with -the rate scale's. The sqrt grid is 11 values from 0.001 to 0.32, a quarter -decade apart, picked after a first look so the knee sits inside it. The rate -curve is E14's runs. - -Reference points: **REF-op cold** (prod's tracker without its warm start) -and the rate-scale rates **0.018** and **0.032** (E14's best range). - -"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so -**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is -better) is the mean over the 23 levels of |empirical coverage − level|. Each season is scored on its own rounds; the -tracker runs over both, so 2025-26 starts from the offsets 2024-25 left -behind. Results lead with states only (US excluded). - -# Findings - -Numbers are from the render of 1 October 2026, states only, WIS reduction % -(positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), h0/h1/h2/h3. All-locations numbers -point the same way. - -- **The sqrt scale works, with a knee at 0.01–0.018.** Every sqrt rate from - 0.001 to 0.018 improves WIS at every horizon in both seasons. At 0.032 h3 - of 2024-25 turns negative (−0.3), and from 0.056 up WIS falls fast. The - best rate by E14's criterion (highest mean gain, no loss anywhere) is - **0.018**. -- **sqrt 0.018 vs rate 0.032 (E14's best range).** 2024-25: - +1.7/+0.6/+0.3/+0.1 vs +1.0/+0.5/+0.3/+0.2. 2025-26: +3.0/+1.0/+1.1/+1.5 - vs +1.9/+1.0/+0.8/+0.8. Coverage bias is about the same: 0.099–0.137 vs - 0.098–0.123 in 2024-25, 0.021–0.024 vs 0.010–0.014 in 2025-26. So the sqrt - scale buys more WIS at h0 in both seasons and at every horizon in - 2025-26, for no coverage gain. It does not help h2–h3 in 2024-25. -- **Against REF-op cold**, sqrt 0.018 is ahead in 2025-26 at every horizon - and avoids REF-op's 2024-25 h3 loss (−0.8). REF-op has more WIS at h0–h1 - in 2024-25 (+2.2/+0.9) but keeps 2024-25 coverage bias near the base. -- **The tail shrinks, but not everywhere.** The 90th percentile of - |offset| ÷ base 90% width, over all levels, is 0.092–0.111 for sqrt 0.018 - vs 0.124–0.153 for rate 0.032 (rate 0.1: 0.24–0.34). At the low levels it - is much smaller (0.05 level: 0.008–0.025 vs 0.055–0.064). At the 0.95 level - it is larger (0.115–0.180 vs 0.096–0.152), because a fixed step on the - square root stretches the upper quantiles more where counts are high. - sqrt 0.01 has the smallest tail (0.056–0.071). -- **Yes, offsets go stale after the peak, and March is where it costs.** All - four trackers lose WIS in March of both seasons at h1–h3: rate 0.018 - −1.2 / −10.6 at h2 (2024-25 / 2025-26), rate 0.032 −2.2 / −18.7, sqrt - 0.018 −7.2 / −27.9, REF-op cold −19.8 / −24.3. In March the base - already over-predicts (58% and 67% of outcomes below its median at h2) - and the calibrated median sits higher still (62–77%). March is 4–8% of a - season's base WIS at h2, so a −20% March costs about 1 point of season - WIS. -- **April and May carry little WIS (1–3% of the season) and go both ways.** - In 2024-25 the base under-predicts in May and every tracker gains - (+6 to +18%). In 2025-26 rate 0.032 loses at h3 in May (−14%, 79% of - outcomes below the median). -- **Rate-scale offsets do not come down after the peak.** The mean offset - at the median stays at 0.07–0.2 per 100k from the peak through May, while - the base median falls from 10–12 to under 1 per 100k. On the sqrt scale and - for REF-op the offset falls with the level, but too slowly to avoid March. -- **Season-level coverage hides month-level errors.** In December the base's - monthly coverage bias is 0.25–0.38 and no tracker moves it; in March the - trackers make it worse. Part of the season-level coverage gain comes from - under-prediction on the ramp and over-prediction after the peak cancelling - out. -- **Month view (2026-10-01):** scored within each month, sqrt 0.018 is - better calibrated than rate 0.032, not equal. Month-avg L1 coverage bias in - 2025-26 (h0/h1/h2/h3): sqrt 0.018 0.075/0.098/0.119/0.138, rate 0.032 - 0.104/0.123/0.147/0.161, base 0.097/0.112/0.130/0.146; both seasons - 0.094/0.116/0.143/0.169 vs 0.108/0.127/0.155/0.177 (base - 0.112/0.127/0.153/0.177). Rate 0.032's low 2025-26 season-pooled bias is - cancellation; by month it is worse than the base. Larger sqrt rates rise - above the base at h1–h3 too (0.1: 0.095/0.129/0.157/0.182). -- **Month view (2026-10-01), median error:** the March losses are about twice as large - on the median as on WIS (h2, 2025-26: sqrt 0.018 −52% vs −28%, REF-op cold - −52% vs −24%). +Question: how does a constant-rate tracker on the square-root scale compare +with the same tracker on the rate scale and with REF-op cold, across learning +rates and by month? + +Setup: + +- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (all 53 + locations, the 55 hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. Cold start, no burn-in season. +- The tracker runs over both seasons, so 2025-26 starts from the offsets + 2024-25 left behind; each season is scored on its own rounds. Results lead + with states only (US excluded). +- Configs: + - `sqrt `: the plain tracker with a constant learning rate on the + square root of admissions per 100k; 11 rates from 0.001 to 0.32, a + quarter decade apart, in sqrt(per-100k) units (not comparable number for + number with the rate scale). The grid was set after a first look at + results. + - `rate `: E14's rate-scale runs; 0.018 and 0.032 are shown as + references. + - `REF-op cold`: prod's tracker without its warm start. +- A sqrt rate is selected below by E14's rule (highest mean gain with no loss + in any season × horizon). The selection is in-sample. + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) @@ -165,8 +101,8 @@ BEST_SQRT <- best_tbl %>% filter(worst > 0) %>% slice_max(mean, n = 1) %>% pull( One curve per season and scale, tracing the learning rate from smallest (near the base, the open circle) to largest. Solid lines are the rate scale (E14), dashed lines the sqrt scale. Labels mark a few rates on each curve. -Triangles are REF-op cold. The y axis is clipped at −10%. The best sqrt -rate by the rule below is **`r BEST_SQRT`**. +Triangles are REF-op cold. The y axis is clipped at −10%. The sqrt rate +selected by the rule below is **`r BEST_SQRT`**. ```{r curve-fn} LABELLED <- list(rate = c(0.01, 0.032, 0.1, 0.32), sqrt = c(0.0032, 0.01, 0.032, 0.1)) @@ -237,7 +173,7 @@ ggplot(long, aes(lr, value, colour = season)) + ## Tables WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better) per config (rows) and season × horizon -(columns). The sqrt sweep, then the rate-scale rates around E14's best range +(columns). The sqrt sweep, then the rate-scale rates 0.018 and 0.032 (E14) and REF-op cold. ```{r tables} @@ -283,11 +219,11 @@ knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) knitr::kable(season_table(GEOS[["all"]])) ``` -### Picking the best sqrt rate +### Selecting a sqrt rate For each sqrt rate, the worst and the mean WIS reduction % (positive = calibrated WIS lower than base) over the two seasons -and h0–h3 (states only). The best rate is the one with the highest mean -among those whose worst case is still a gain (E14's criterion). +and h0–h3 (states only). The selected rate has the highest mean among those +whose worst case is still a gain (E14's rule). ```{r best} knitr::kable(best_tbl %>% mutate(across(c(worst, mean), \(x) round(x, 2)))) @@ -310,7 +246,7 @@ bind_rows( knitr::kable() ``` -# 2. Does the offset tail shrink? +# 2. Offset tail E15's statistic: for each forecast and level, |offset| ÷ the base 90% interval width (q95 − q05), and its 90th percentile over forecasts (location @@ -358,10 +294,9 @@ knitr::kable( # 3. Month by month Five forecasts per state-round are compared: the base, the rate-scale rates -0.018 and 0.032, the best sqrt rate and REF-op cold. Months are calendar +0.018 and 0.032, the selected sqrt rate and REF-op cold. Months are calendar months of the reference date. States only, h0–h3. Each cell holds about -150–260 state forecasts, so single months are noisy, but the March losses -below repeat across seasons and horizons. +150–260 state forecasts, so single months are noisy. ```{r month-data} MONTH_COLS <- setNames(c("#009E73", "#0072B2", "#CC79A7", "#D55E00"), names(pick)[c(2, 3, 1, 5)]) @@ -455,3 +390,38 @@ mv_cols <- MONTH_COLS ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` + +# Findings + +There are no uncertainty estimates; a WIS gap under 1 point or a month-avg +coverage gap under 0.01 is read as no clear difference. The cold start leaves +Nov–Dec 2024 equal to the base for every config. States only, h0/h1/h2/h3. + +- **Sweep:** sqrt 0.001–0.018 are at or above the base in every season × + horizon, often by +0.0 to +0.3. From 0.032 some cells turn negative + (2024-25 h3 −0.3); from 0.056 the losses grow. The rule selects 0.018 + (mean +1.15, worst +0.06); 0.01 is close (mean +1.02, worst +0.10). +- **sqrt 0.018 vs rate 0.032:** pooled +2.2/+0.8/+0.6/+0.6 vs + +1.3/+0.7/+0.5/+0.4. About 1 point apart at h0 (2024-25 +0.7, 2025-26 + +1.1); within 0.2 at h1–h3 pooled (no clear difference). At h3 sqrt is + ahead in 2025-26 (+0.7) and rate in 2024-25 (+0.1). +- **sqrt 0.018 vs REF-op cold:** the order flips by season. In 2024-25 + REF-op is ahead at h0–h1 (+2.2/+0.9 vs +1.7/+0.6) and behind at h2–h3 + (−0.0/−0.8 vs +0.3/+0.1); in 2025-26 sqrt is ahead at every horizon. + Pooled gaps are 0.2–0.8. +- **Month-avg coverage (both seasons):** base 0.112/0.127/0.153/0.177; sqrt + 0.018 0.094/0.116/0.143/0.169; REF-op 0.098/0.119/0.146/0.167; rate 0.018 + 0.102/0.120/0.147/0.172; rate 0.032 0.108/0.127/0.155/0.177. sqrt vs REF-op + and vs rate 0.018: no clear difference. sqrt vs rate 0.032: 0.008–0.014 + lower, all from 2025-26 (0.023–0.029 there; in 2024-25 rate 0.032 is + 0.003–0.008 lower). Rate 0.032's low season-pooled 2025-26 bias + (0.010–0.014) is cancellation; month-avg it is above the base. +- **Offset tail:** 90th percentile of |offset| ÷ base 90% width, all levels: + sqrt 0.018 0.092–0.111, rate 0.018 0.078–0.096, rate 0.032 0.124–0.153, + REF-op 0.074–0.114. sqrt is smaller than rate 0.032 at the 0.05 level and + larger at 0.95. +- **March at h2** (within the month, 2024-25 / 2025-26): rate 0.018 + −1.2/−10.6, rate 0.032 −2.2/−18.7, sqrt 0.018 −7.2/−27.9, REF-op + −19.8/−24.3. March is 7.5% and 4.2% of the base WIS, so these cost 0.1–1.5 + season points. In March the base already over-predicts (58% and 67% of + outcomes below its median at h2). diff --git a/reports/writeups/calibration_experiments/e17_late_decay.Rmd b/reports/writeups/calibration_experiments/e17_late_decay.Rmd index 7c623b5d..f1bd0908 100644 --- a/reports/writeups/calibration_experiments/e17_late_decay.Rmd +++ b/reports/writeups/calibration_experiments/e17_late_decay.Rmd @@ -19,79 +19,32 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` -E16 found that the constant-rate trackers lose WIS in March of both seasons: -the offsets learned on the ramp up still point up after the peak, while -outcomes fall below the base median. This notebook asks whether **shrinking -the offset toward zero from a calendar date onward** removes those losses -without giving up the gains elsewhere. - -**Existing options.** None does exactly this: - -- `fast_decay` shrinks the offset every round of the year, so it also - weakens the ramp-up correction. -- `season_policy` (`carry`, `reset`, `shrink`) acts only at the season - boundary in the autumn, after the spring has been scored. -- `off_after` is the abrupt version: from a calendar date to July it plays - the base forecast and stops learning. It is run here as a comparison. - -So this notebook uses a new option, `late_decay = list(from = "MM-DD", -factor = f)`: from that date until July, the offset is multiplied by `f` -at each round before it is played. Learning continues, so new outcomes can -still push the offset. - -**Setup.** As E14/E16: flu, the clean `windowed_seasonal` replay, h0–h3, -learning from data as published each week and re-run from scratch each week, -one week of extra revision delay, no burn-in, states only. Three constant -rates: rate scale 0.018 and 0.032 per 100k (E14) and sqrt scale -0.018 (E16's best). For each, five settings: no decay, `late_decay` from -1 March with factor 0.85, 0.7 or 0.5 per round, and `off_after = "03-01"`. -The tracker carries offsets across seasons, so a decay in spring 2025 also -changes where 2025-26 starts. - -"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so **positive -= calibrated WIS lower than base**; "L1 coverage bias" (lower is better) is -the mean over the 23 levels of |empirical coverage − level|. - -# Findings - -Numbers are from the render of 1 October 2026, states only, WIS reduction % -(positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), h0/h1/h2/h3. - -- **The decay removes the March loss.** For sqrt 0.018 at h2, March 2025-26 - goes from −27.9% to −10.7 / −6.3 / −2.9% (factor 0.85 / 0.7 / 0.5), and - the calibrated median comes back toward the base's (69–73% of outcomes - below it, base 67%). The rate-scale trackers behave the same way. -- **But it costs more than it saves.** It also removes the April–May gains, - and because the offsets carry across seasons, it hands 2025-26 an offset of - almost zero: the next season starts cold again. 2025-26 at h0 drops from - +3.0 to +1.7–2.1 (sqrt 0.018) and its coverage bias triples (0.021–0.024 - to 0.066–0.078). Pooled over both seasons, every decay setting is worse - than no decay on coverage and no better on WIS for all three rates. -- **`off_after = "03-01"` does better on WIS.** It plays the base from 1 - March and freezes the offsets, which then carry into the next season - unchanged. Pooled WIS for sqrt 0.018 goes from +2.2/+0.8/+0.6/+0.6 to - +2.3/+1.2/+1.2/+1.3, and both seasons gain at h1–h3 (2025-26: - +3.4/+1.6/+1.9/+2.6). Rate 0.032 gains at h2–h3 too (+0.8/+0.9 pooled vs - +0.5/+0.4). The cost is the October 2025 start (about −10% at h2/h3 on a - month with under 1% of the WIS). -- **Any spring switch-off gives back most of the coverage gain.** With - `off_after`, pooled coverage bias for sqrt 0.018 is 0.094–0.116 (no decay - 0.059–0.078, base 0.105–0.127). Spring rounds then play the base, and - the over-prediction after the peak no longer cancels the under-prediction - on the ramp (E16). -- **Answer.** The offsets do go stale after the peak, but shrinking them - toward zero by the calendar is not a free fix. The gradual decay as built - here loses because it also wipes the offset the next season starts from. - `off_after` in March is the better WIS choice and the worse coverage choice. -- **Month view (2026-10-01):** scored within each month, the coverage - costs nearly vanish. Month-avg L1 coverage bias, both seasons, - h0/h1/h2/h3, sqrt 0.018: no decay 0.094/0.116/0.143/0.169, decay 0.85 - 0.103/0.123/0.149/0.173, off after 1 Mar 0.104/0.121/0.150/0.170, base - 0.112/0.127/0.153/0.177. No decay is still lowest, but by 0.001–0.010, - not the season-pooled 0.03–0.04. In 2025-26 the decays raise it by - 0.002–0.015 (decay 0.85: 0.084/0.108/0.127/0.140 vs - 0.075/0.098/0.119/0.138), not threefold. For rate 0.032, off after 1 Mar - ties no decay (0.112/0.128/0.156/0.173 vs 0.108/0.127/0.155/0.177). +Question: does shrinking the offset from 1 March, or switching calibration +off from 1 March, change WIS and coverage compared with the same tracker +without it? + +Setup: + +- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (all 53 + locations, the 55 hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. Cold start, no burn-in season. States only. +- Three constant-rate trackers: rate 0.018 and rate 0.032 per 100k (E14), + and sqrt 0.018 (E16). Each crossed with five settings: + - **no decay**. + - **`late_decay`** with factor 0.85, 0.7 or 0.5: from 1 March to 30 June + the stored offset is multiplied by the factor at each round before it is + played. Learning continues, and because offsets carry across seasons, + the shrink carries into the next season. + - **`off_after = "03-01"`**: from 1 March to 30 June the base is played, + those rounds produce no updates, and the offset carries unchanged into + the next season. +- The 1 March date was chosen after E16, on these same seasons. + (`fast_decay` and `season_policy` are other ways to shrink offsets; they + are not run here.) + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) @@ -245,3 +198,36 @@ mv_cols <- setNames(DECAY_COLS, mv_plot_cfgs) ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` + +# Findings + +There are no uncertainty estimates; a WIS gap under 1 point or a month-avg +coverage gap under 0.01 is read as no clear difference. The cold start leaves +Nov–Dec 2024 equal to the base for every config. States only, h0/h1/h2/h3. + +- **Decays vs no decay, pooled WIS:** sqrt 0.018 loses 0.4–0.5 at h0 and is + within 0.2 at h1–h3; rate 0.018 loses 0.5–0.7 at h0 and 0.2–0.3 at h1–h3; + rate 0.032 loses 0.2–0.5 at h0 and at most 0.2 at h1–h3. No strong result. + For sqrt the order flips by season: in 2024-25 the decays are 0.1–0.3 + ahead at h1–h3; in 2025-26 they are 0.9–1.3 behind at h0. +- **`off_after` vs no decay, pooled WIS:** sqrt 0.018 +2.3/+1.2/+1.2/+1.3 vs + +2.2/+0.8/+0.6/+0.6 (0.1–0.7 apart); rate 0.032 −0.1 to +0.5; rate 0.018 + −0.5 to 0. No pooled gap reaches 1 point. The sqrt gain is mostly 2025-26 + (+3.4/+1.6/+1.9/+2.6 vs +3.0/+1.0/+1.1/+1.5). In 2024-25 the rate-scale + trackers are lower at h0 with the switch-off. The date was tuned + in-sample. +- **March and spring at h2:** the decays shrink the March loss (sqrt + 2025-26: −27.9 to −10.7/−6.3/−2.9; `off_after` −0.0), and also remove the + April–May gains (sqrt May 2025-26: +17.3 to +3.3/−0.0/−0.1; `off_after` + +0.0). +- **Month-avg coverage (both seasons):** sqrt 0.018 no decay + 0.094/0.116/0.143/0.169, decay 0.85 0.103/0.123/0.149/0.173, `off_after` + 0.104/0.121/0.150/0.170, base 0.112/0.127/0.153/0.177: gaps ≤ 0.010, no + clear difference. For rate 0.032, decay 0.85 is 0.005–0.007 lower than no + decay overall and 0.014–0.024 lower in 2025-26. Season-pooled, no decay is + 0.032–0.043 lower for sqrt, but pooling lets ramp and post-peak errors + cancel. +- **Carry-over:** `off_after` carries the pre-March offset into the next + ramp. In 2025-26 that ramp under-predicted too (Nov 2025 h0 month bias + 0.065 for `off_after` vs 0.096–0.103 for the decays), which a season that + ramps differently would not repeat. diff --git a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd index dff2f897..d477a305 100644 --- a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd +++ b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd @@ -19,17 +19,22 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` -REF-op (prod's tracker) and sqrt constant 0.018 (E16's best) differ in -several settings at once. This notebook walks from one to the other, -changing one setting per step, to see which piece of REF-op does the work. +Question: REF-op (prod's tracker) and sqrt constant 0.018 (E16) differ in +several settings. How do WIS and coverage change as one setting at a time is +changed from one to the other? -**Setup.** As E13/E16: the clean `windowed_seasonal` replay for flu (53 -locations, the 55 hub rounds, h0–h3), learning from data as published each -week and re-run from scratch each week as prod does, one week of extra -revision delay. Scores are over the two live seasons; results are states -only (US excluded). +Setup: -**The steps.** Each row changes one setting from the row above. +- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (53 + locations, the 55 hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. +- Steps 0–3 run with the 2023-24 season present (the HHS-stitched backfill + from E13); steps 4–6 and the references are cold. +- Scores over the two live seasons, each on its own rounds; states only (US + excluded). + +Each row changes one setting from the row above: | step | change | config after the change | |---|---|---| @@ -39,88 +44,24 @@ only (US excluded). | 3 no leak | `fast_decay = 0` | sqrt, adaptive, mult 0.03, window 20, carry, with the 2023-24 season present | | 4 no burn-in season | drop the 2023-24 rounds | as 3, cold: the eta pool starts empty | | 5 per-100k units | `scales` = population / 100k | as 4, on the square root of admissions per 100k | -| 6 sqrt constant 0.018 (E16) | `lr = 0.018` | E16's best: sqrt per 100k, constant 0.018 | - -**Why steps 4 and 5.** The 2023-24 burn-in season does two things. With -`slow_init` it sets the slow term's starting value (the warm start). Without -it, its residuals still fill the adaptive learning rate's window, so eta is -sized from the first live round instead of growing from an empty pool; the -offsets themselves stay at zero, since burn-in outcomes never step. So the -burn-in season stays present through step 3, and dropping it is step 4: step -1 isolates the warm start, step 4 the eta warm-up (E13 follow-up 2). E16's -constant rate runs on the square root of the per-100k rate. The adaptive -rate is almost unchanged by that rescaling (it scales with the residuals; -only the 1e-3 floor does not), but it is still a change, so it is step 5, -and step 6 changes only the learning rate. +| 6 sqrt constant 0.018 (E16) | `lr = 0.018` | sqrt per 100k, constant 0.018 | -Steps 0, 3, 4 and 6 are the cached runs of E13 (REF-op warm), E02 -("reference + burn-in"), E01/E02 ("reference") and E16 (sqrt 0.018). Step 3 -is written as E02 writes it (REF-paper with sqrt and the 1e-3 floor), which -equals REF-op with `slow_init`, `lr_slow` and the leak removed. +The 2023-24 season does two things: with `slow_init` it sets the slow term's +starting value (step 1 removes this), and its residuals fill the adaptive +learning rate's window, so eta is sized from the first live round (step 4 +removes this). Burn-in outcomes never step the offsets. Step 5 changes only +units, so that step 6 changes only the learning rate. -"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so -**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is -better) is the mean over the 23 levels of |empirical coverage − level|. -"Month-avg" coverage bias is computed within each calendar month and then -averaged, so errors on the ramp and after the peak cannot cancel. - -# Findings +Two rate-scale constant rates are shown as references outside the chain: +rate 0.032 (E14) and rate 0.056 (E09's knee). -Numbers are from the render of 1 October 2026, states only, both seasons -pooled unless named, h0/h1/h2/h3. WIS is WIS reduction % (positive = -calibrated WIS lower than base); coverage is L1 coverage bias (lower is -better). +Steps 0, 3, 4 and 6 are the cached runs of E13 (REF-op warm), E02 +(`reference + burn-in`), E01/E02 (`reference`) and E16 (sqrt 0.018), so they +are not independent checks of those notebooks. -- **Endpoints.** REF-op +2.8/+1.1/+0.9/+1.1, sqrt constant 0.018 - +2.2/+0.8/+0.6/+0.6. REF-op is ahead overall because of 2024-25 - (+3.2/+1.9/+1.6/+1.6 vs +1.7/+0.6/+0.3/+0.1); sqrt 0.018 is ahead in - 2025-26 (+3.0/+1.0/+1.1/+1.5 vs +2.2/−0.1/−0.0/+0.2). Month-avg coverage - bias is about the same: 0.095/0.121/0.150/0.175 vs 0.094/0.116/0.143/0.169. -- **WIS: the learning rate moves it most at h1–h3, the warm start at h0.** - Switching the adaptive rate to constant 0.018 (step 6) adds - +0.3/+1.0/+1.7/+2.3. Inside REF-op, three pieces each add about a point: - the warm start (step 1 costs −0.9/−0.5/−0.6/−1.0, nearly all of it in - 2024-25: h3 goes from +1.6 to −0.4), the leak (step 3 costs - −0.6/−1.0/−1.2 at h1–h3 and gains +0.4 at h0) and, smaller, the burn-in - season's eta warm-up (step 4 costs −0.1/−0.2/−0.5/−0.7, again in - 2024-25). Without the warm start, the slow term does nothing (step 2: - within ±0.2). -- **Warm start vs eta warm-up (E13 follow-up 2).** Both matter only in - 2024-25, the first live season. The warm start is worth about twice as - much WIS as the eta warm-up and shows from the first rounds (the month - view gains at h2–h3 in Nov–Feb 2024-25). The eta warm-up is about half a - point at h2–h3 and nothing in 2025-26. -- **Month-level coverage: no step moves it beyond noise.** Every step - changes month-avg coverage bias by 0.008 or less, under the ~0.01 noise - floor. The largest are dropping the warm start at h3 (−0.008) and the - constant rate at h1–h3 (−0.005/−0.007/−0.007). -- **Season-pooled coverage moves a lot, and it is cancellation.** Dropping - the warm start worsens pooled bias by +0.024/+0.032/+0.044/+0.052, - dropping the leak improves it by −0.040/−0.042/−0.051/−0.060, and the - constant rate worsens it by +0.014/+0.023/+0.034/+0.047. None of these - shows up month by month, so they come from ramp and post-peak errors - cancelling differently, not from better-calibrated months. -- **Where the WIS steps act: March and April.** The month view shows the - WIS steps mostly as spring losses at h2–h3. In 2024-25 at h2, March costs - about −13% with the leak and no slow term (step 2), −23% without the leak - (step 3) and −31% without the burn-in season (steps 4–5); constant 0.018 - costs about −7%. April is worse for the adaptive trackers without the leak - (down to the −40% clip). In 2025-26 every step loses 27–40% in March at - h1–h2, constant 0.018 slightly less. -- **Per-100k units change nothing (step 5).** The adaptive rate scales with - the residuals, so on the square-root scale only the 1e-3 floor sees the - units. Calibrated values differ by at most 1.6 admissions and every score - matches step 4 to the reported digits. -- **Rate-scale constant rates (references, outside the chain).** Rate - 0.032 (E14's best) and 0.056 (E09's knee) give +1.3/+0.7/+0.5/+0.4 and - +1.3/+0.5/+0.3/0.0, below sqrt 0.018 (+2.2/+0.8/+0.6/+0.6) and REF-op - (+2.8/+1.1/+0.9/+1.1). Month-avg coverage bias is about the base's - (0.108–0.177 and 0.112–0.191 vs base 0.112–0.177; 0.056 is worse at - h2–h3). Both sit at zero through the ramp. On the ensemble (E09) the knee - looked good after the 2024-25 peak because the ensemble was still - under-predicting that spring (E07); `windowed_seasonal` over-predicts - after the peak, so here the same upward offsets cost WIS and coverage in - March–April. +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. "Month-avg" coverage +bias is computed within each calendar month and then averaged. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) @@ -264,3 +205,35 @@ mv_cols <- STEP_COLS ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` + +# Findings + +There are no uncertainty estimates; a WIS gap under 1 point or a month-avg +coverage gap under 0.01 is read as no clear difference. The cold steps equal +the base through Nov–Dec 2024; only the warm steps act there. States only, +h0/h1/h2/h3, both seasons pooled unless stated. + +- **Endpoints:** REF-op +2.8/+1.1/+0.9/+1.1, sqrt constant 0.018 + +2.2/+0.8/+0.6/+0.6 (0.3–0.6 apart, no clear difference). The order flips + by season: REF-op is ahead in 2024-25 (+3.2/+1.9/+1.6/+1.6 vs + +1.7/+0.6/+0.3/+0.1), sqrt 0.018 in 2025-26 (+3.0/+1.0/+1.1/+1.5 vs + +2.2/−0.1/−0.0/+0.2). +- **Warm start (step 1):** −0.9/−0.5/−0.6/−1.0 pooled when removed. In + 2024-25 it is worth 0.9–2.0 points; in 2025-26 removing it gains 0.2–0.7 + at h1–h3. +- **Slow term without warm start (step 2):** within ±0.2. No strong result. +- **Leak (step 3):** removing it costs 0.6–1.2 at h1–h3 and gains 0.4 at h0. +- **Burn-in season's eta warm-up (step 4):** −0.1 to −0.7, in 2024-25 only. + No strong result. +- **Per-100k units (step 5):** identical to step 4 in every reported digit. + The adaptive rate is scale invariant when the 1e-3 floor doesn't bind; a + direct check (2026-10-02) confirmed `scales` is applied. +- **Constant 0.018 vs cold adaptive (step 6):** +0.3/+1.0/+1.7/+2.3, ahead at + h2–h3 in both seasons. +- **Month-avg coverage:** no step changes it by more than 0.008. REF-op + 0.095/0.121/0.150/0.175, sqrt 0.018 0.094/0.116/0.143/0.169, base + 0.112/0.127/0.153/0.177. Season-pooled bias moves by up to 0.06 per step, + but pooling lets ramp and post-peak errors cancel. +- **References:** rate 0.032 +1.3/+0.7/+0.5/+0.4 and rate 0.056 + +1.3/+0.5/+0.3/+0.0 pooled. Month-avg coverage: rate 0.032 equals the base; + rate 0.056 is 0.008–0.014 above it. diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index 7afd2eb0..fcf372d8 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -16,39 +16,52 @@ hub submissions. The step adjusts each submitted quantile forecast by an additive offset that an online quantile tracker (MultiQT; Ding, Gibbs & Tibshirani, 2025) learns from past coverage misses. Each notebook changes one setting of that tracker and compares the result with the uncalibrated -submission. +forecast. -# Notebooks +The summaries below give sizes, not verdicts. There are no uncertainty +estimates, so a WIS gap under 1 point or a month-avg coverage gap under 0.01 +is read as no clear difference. -```{r notebooks, results="asis", echo=FALSE} -src <- list.files(here::here("reports/writeups/calibration_experiments"), pattern = "^e[0-9]+_.*[.]Rmd$", full.names = TRUE) +```{r rows, include=FALSE} out_dir <- here::here("reports/calibration_experiments") -for (f in src) { - html <- sub("[.]Rmd$", ".html", basename(f)) - title <- rmarkdown::yaml_front_matter(f)$title - rendered <- file.exists(file.path(out_dir, html)) - stamp <- if (rendered) format(file.mtime(file.path(out_dir, html)), "%Y-%m-%d %H:%M %Z", tz = "America/Los_Angeles") else "not rendered" - cat(sprintf("- %s (%s)\n", if (rendered) sprintf("[%s](%s)", title, html) else title, stamp)) +link <- function(id, file) { + html <- paste0(file, ".html") + if (!file.exists(file.path(out_dir, html))) return(paste(id, "(not rendered)")) + stamp <- format(file.mtime(file.path(out_dir, html)), "%Y-%m-%d", tz = "America/Los_Angeles") + sprintf("[%s](%s) (%s)", id, html, stamp) } ``` -| | question | short answer | -|---|---|---| -| E00 | Should the tracker learn from finalized data or from data as published at the time? | As published, re-run each week as prod does. Finalized data overstated the earlier gains only a little (about 2 points at h−1). | -| E01 | Which adaptive learning-rate settings (multiplier, window, carry or reset over the summer)? | The multiplier is what matters; the window barely does, and carrying offsets beats resetting. On the square-root scale 0.03 gains at h0 and loses a little at h1–h3; 0.01 never loses but corrects coverage slowly from a cold start. | -| E02 | Do a spring switch-off date, a per-level learning rate or a seasonal window improve on the adaptive tracker? | Switching off from 1 March or 15 February gains 2–5 points of WIS over the reference at h1–h3, at the cost of most of the coverage gain. A per-level rate gains a little; a seasonal window does not help. | -| E03 | Which scale should the offsets live on: counts, square root, or rate per 100k? | Square root, at every horizon, mostly by avoiding spring losses. Rate equals counts with the adaptive learning rate. | -| E04 | Should the offset shrink between updates, or have a second slow component? | Shrinking buys WIS at h1–h3 but nearly triples coverage error. A slow component alone does nothing. | -| E05 | Should the offset start from the burn-in season's errors instead of zero? | Yes; it never hurts. With the plain tracker it gives the best-calibrated config so far. | -| E07 | Are the gains an artifact of calibrating the submitted ensemble? | No. On the same rows and data, the single `windowed_seasonal` forecaster gains the same way, about a point less with prod's tracker. | -| E09 | With a constant learning rate on the rate scale, how do coverage and WIS trade off, and how much revision delay is best? | An L-shaped trade-off whose knee matches or beats the paper's adaptive rate at every horizon. Under prod's weekly re-run, less delay is better. | -| E10 | Do the flu results carry over to covid? | Only at h−1 (+10% WIS). The covid base is already calibrated at h1–h3, where calibrating is neutral or harmful. | -| E11 | How does E09's best constant learning rate compare with the adaptive trackers? | Better coverage than the cold adaptive trackers at less WIS gain near the present, and the most stable across seasons. The warm-started adaptive tracker still wins at h0–h1. | -| E14 | On the clean `windowed_seasonal` replay, is there a constant learning rate that is good in both seasons? | Yes, 0.018–0.032 per 100k: better WIS at every horizon in both seasons and lower coverage bias. 2024-25 needs a larger rate for good coverage, which costs WIS. | -| E15 | Why does WIS improve when the calibrated intervals barely move? | The offsets are small (2–5% of the 90% interval width) and point up. The gain comes from shifting the forecast up, mostly at the central levels below the median; width changes cost WIS. | -| E16 | Does a constant learning rate work better on the square-root scale, and do the offsets go stale late in the season? | sqrt 0.018 improves WIS at every horizon in both seasons, more than the rate scale at h0 and in 2025-26, with similar coverage. Every tracker loses WIS in March, after the peak, when the base already over-predicts and the offsets still push up. | -| E17 | Does shrinking the offset from 1 March fix the March losses? | It fixes March but wipes the offset the next season starts from, so it is worse overall. Switching calibration off from 1 March (`off_after`) gains WIS at h1–h3 but gives back most of the coverage gain. | -| E18 | Which part of prod's tracker (REF-op) does the work, stepping one setting at a time to E16's sqrt constant 0.018? | The warm start from the 2023-24 season adds 0.5–1 point of WIS (mostly in 2024-25) and the leak about a point at h1–h3 (both seasons); the burn-in season's learning-rate warm-up adds about half a point at h2–h3, the slow term nothing on its own. Switching to the constant rate adds 1–2 points at h1–h3. No step changes month-level coverage beyond noise. | +# Notebooks + +| | base | question | what the numbers show | +|---|---|---|---| +| `r link("E01", "e01_eta_settings")` | replay, cold | How do the adaptive learning-rate settings (multiplier, window, carry or reset) change WIS and coverage? | Multipliers 0.1 and 0.3 lose 7–77 points at h1–h3. 0.01 is 0.5–2.4 points above 0.03 at h1–h3; month-avg coverage shows no clear difference. Count scale is 2–5 points below sqrt. Window and carry vs reset: no strong result. | +| `r link("E03", "e03_scale")` | ensemble | Which scale should the offsets live on: counts, square root, or rate per 100k? | sqrt is 1.4–3.9 points above count pooled, mostly from 2025-26. Coverage: no strong result. Rate is identical to count in every cell. | +| `r link("E09", "e09_lr_delay")` | ensemble | With a constant learning rate on the rate scale, how do WIS and coverage change with the rate and with extra learning delay? | WIS peaks near 0.056–0.1 per 100k and falls fast past 0.3. Month-avg coverage is no lower than the base at h1–h3 for any constant rate. Each week of extra delay costs about 1 point at h−1. Stateful learning with no extra delay makes h−1 worse than the base. | +| `r link("E10", "e10_covid")` | covid ensemble | How do the two reference configs do on covid? | h−1 improves about 10–11% in both seasons. REF-paper loses 10–19% at h1–h3 in 2025-26. REF-op at h1–h3 is −4 to +3, sign depending on the season: no strong result. | +| `r link("E11", "e11_constant_lr")` | ensemble | Does a constant learning rate do better than the adaptive rate? | Constant 0.1 vs adaptive sqrt: the order flips by horizon (1–2 points either way). REF-op has the highest pooled WIS at every horizon, by up to 2 points. On month-avg coverage the constant rate is 0.003–0.022 worse. | +| `r link("E14", "e14_lr_by_season")` | replay, cold | How do WIS and coverage change with a constant learning rate, season by season? | Every rate from 0.0032 to 0.056 is at or above the base in both seasons, within 1.2 points of each other: no strong result for a best rate. From 0.1 WIS falls. Month-avg coverage for rates ≤ 0.032 stays within about 0.01 of the base. | +| `r link("E15", "e15_wis_sources")` | replay, cold | Where does the WIS change come from? | Offsets are 2–5% of the 90% interval width. REF-op's gain (+2.0 at h0, about 0 by h3) comes from raising the lower quantiles; a median shift alone matches it within 0.6 points. | +| `r link("E16", "e16_sqrt_constant_lr")` | replay, cold | Constant learning rate on the square-root scale vs the rate scale | sqrt 0.018 is about 1 point above rate 0.032 at h0 and within 0.2 at h1–h3 pooled. Against REF-op cold the order flips by season. Every tracker loses in March (up to −28% within the month at h2). | +| `r link("E17", "e17_late_decay")` | replay, cold | Does shrinking or switching off the offset from 1 March change WIS and coverage? | No strong result: pooled WIS gaps ≤ 0.7 points and month-avg coverage gaps ≤ 0.010. The switch-off's sqrt gain comes mostly from 2025-26; the date was chosen in-sample. | +| `r link("E18", "e18_ref_op_bridge")` | replay | Stepping from REF-op to sqrt constant 0.018 one setting at a time | REF-op vs sqrt 0.018: no strong result (0.3–0.6 points pooled, order flips by season). The warm start helps in 2024-25 only. Constant 0.018 is 1.7–2.3 points above cold adaptive at h2–h3. No step moves month-avg coverage by more than 0.008. | + +E12 and E13 (the clean replay without and with a 2023-24 burn-in) have no +notebook; they are `scripts/calibration/calibration_ws_replay.R`. + +# Archived + +Earlier experiments, kept for reference. Their questions are settled or +covered by the notebooks above. + +| | base | question | what the numbers show | +|---|---|---|---| +| `r link("E00", "e00_vintage_backtest")` | ensemble | Should the tracker learn from finalized data or from data as published? | Exact and final differ by 0.6 points or less, except REF-op h−1 (2.2, 2024-25). Learning once from the first report loses most of the h−1 gain. All other notebooks use the exact (prod) design. | +| `r link("E02", "e02_eta_variants")` | replay, cold | Spring switch-off, per-level learning rate, seasonal window | Switching off from 1 March or 15 February is 2–4 points above the reference at h1–h3 (dates in-sample; in 2025-26 about half the gain is December carry-over). Per-level about +1. Month-avg coverage: no clear difference. | +| `r link("E04", "e04_offset_structure")` | ensemble | Leak and slow term | The leak is +3.3 to +5.6 at h1–h3 in 2025-26 and −0.4 to −1.3 in 2024-25. The slow term without a warm start: no strong result. | +| `r link("E05", "e05_warm_start")` | ensemble | Warm start from the burn-in season | +1.7 to +3.2 at h2/h3 in 2024-25 for every tracker, within ±0.4 in 2025-26. | +| `r link("E07", "e07_across_eras")` | both | Does calibration act the same on the ensemble and the clean replay? | Same direction on both; REF-op gains 0.7–1.1 points more on the ensemble. The candidate order flips between seasons on both. | # How to read the results @@ -57,14 +70,24 @@ for (f in src) { later have not been reported yet. - **WIS reduction %** is `100 × (WIS_base − WIS_cal) / WIS_base`, the calibrated forecast's weighted interval score against the uncalibrated - submission's, pooled over locations. Positive = calibrated WIS lower than - base. Every notebook uses this sign. -- **L1 coverage bias** (lower is better) is the mean over the 23 quantile levels of - |empirical coverage − level|. 0 is perfect; the uncalibrated flu - submission sits near 0.1. -- **Scoring.** Every score is against finalized NHSN data, over the live - seasons (2024-25 and 2025-26 for flu), all hub locations. A burn-in season - (2023-24 for flu) is used only to warm up the tracker and is not scored. + forecast's, pooled over locations. Positive = calibrated WIS lower than + base. +- **L1 coverage bias** (lower is better) is the mean over the 23 quantile + levels of |empirical coverage − level|. **Month-avg** computes it within + each calendar month and averages; season-pooled lets under-prediction on + the ramp cancel over-prediction after the peak, and usually looks much + better. Month-level noise is about 0.01. +- **Two bases.** "Ensemble" notebooks calibrate our submitted forecasts + (h−1 to h3, headline tables over all hub locations). "Replay" notebooks + calibrate a clean weekly replay of `windowed_seasonal` with no hand edits + (h0–h3, states only). +- **Scoring.** Every score is against finalized NHSN data over the live + seasons (2024-25 and 2025-26). A burn-in season (2023-24 for flu) only warms + up the learning rate and is not scored. +- **Cold start.** Without a warm start, offsets start at zero, and every + config equals the base through Nov–Dec 2024, so 2024-25 gains are diluted. +- **In-sample choices.** The constant rates and switch-off dates in later + notebooks were picked on the same two seasons they are scored on. - **Excluded submissions.** A few submissions were broken by pipeline bugs or bad data and are left out, both from what the tracker learns and from the scores. Flu: h−1 of the 2025-11-22 round (every quantile was 0) and @@ -79,14 +102,11 @@ for (f in src) { - **Re-run vs stateful.** Prod re-runs the tracker from the first round each week on that week's data, so early reports are replaced by their revisions. A stateful tracker instead learns each outcome once, from the first value - it sees. All notebooks use the re-run design; E09 also shows the stateful - one. + it sees. All notebooks use the re-run design; E00 and E09 also show the + stateful one. ## Reference configs -Each experiment starts from one of two configurations and changes one -setting. - - **REF-paper**: the paper's tracker on counts. One offset per quantile level, updated with an adaptive learning rate (0.03 × the 90th percentile of the last 20 rounds' absolute errors), carried across seasons, with one week of diff --git a/scripts/calibration/calibration_ws_backfill.R b/scripts/calibration/calibration_ws_backfill.R deleted file mode 100644 index a081d9f7..00000000 --- a/scripts/calibration/calibration_ws_backfill.R +++ /dev/null @@ -1,126 +0,0 @@ -# Pseudo-hub history for calibration: replay flu prod's `windowed_seasonal` -# forecaster (scaled_pop_seasonal, seasonal_method = "window", trained on the -# nhsn + ILI+ + flusurv prod archive) over the NHSN seasons and write the result -# in the FluSight hub schema that calibrate_hub_forecasts() consumes. Together -# with scripts/calibration/calibration_ili_backfill.R this gives one forecaster's history -# from 2011 to today, so calibration can be studied without the submitted -# ensemble's changing composition in the way. -# -# The replay is the prod cell: nhsn_prod_archive from the flu_hosp_evaluation -# store, make_forecast_snapshot() + run_forecaster() with the grid row's spec -# values (copied from scripts/flu_hosp_prod.R; keep in sync), prod's seed and -# data substitutions. NHSN vintages begin 2024-11-19, so 2024-25 and 2025-26 are -# honest as-of replays; 2023-24 is replayed with as_of_policy = "cheating" -# (finalized data cut at the forecast date) and is marked as such in the output. -# -# Usage, from the repo root (~10 min cold, cached in cache/calibration/): -# distrobox enter rocker -- Rscript scripts/calibration/calibration_ws_backfill.R -# Then in R: ws <- ws_read_pseudo_hub() # forecasts; truth from get_nhsn_data_archive("flu") - -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) - -WS_CACHE_DIR <- here::here("cache/calibration") -WS_FORECASTS_FILE <- file.path(WS_CACHE_DIR, "ws_pseudo_hub_forecasts.parquet") -WS_HORIZONS <- -1:3 -WS_ID <- "windowed_seasonal" -WS_STORE <- "flu_hosp_evaluation" -WS_SUBSTITUTIONS <- here::here("pipelines/flu_data_substitutions.csv") -# scripts/flu_hosp_prod.R: windowed_seasonal grid row, modeling params only. -WS_PARAMS <- list( - outcome = "value", - trainer = epipredict::quantile_reg(), - seasonal_method = "window", - pop_scaling = FALSE, - lags = list(c(0, 7)), - keys_to_ignore = list(list(c("source"), c("flusurv", "ILI+"))) -) - -#' Wednesday forecast dates with the generation-date delays prod's evaluation -#' mode uses, plus the vintage-free 2023-24 season under the cheating policy. -ws_schedule <- function(versions_end) { - delays <- c("2024-11-20" = "2024-11-21", "2024-12-25" = "2024-12-26", "2025-01-01" = "2025-01-02", "2025-12-24" = "2025-12-29") - through <- as.Date(versions_end) - through <- through - ((as.integer(format(through, "%u")) - 3) %% 7) - live <- tibble(forecast_date = seq.Date(as.Date("2024-11-20"), through, by = 7L), as_of_policy = "asof") %>% - mutate(generation_date = forecast_date) - hit <- match(as.character(live$forecast_date), names(delays)) - live$generation_date[!is.na(hit)] <- as.Date(delays[hit[!is.na(hit)]]) - cheat <- tibble( - forecast_date = seq.Date(as.Date("2023-10-11"), as.Date("2024-05-29"), by = 7L), - as_of_policy = "cheating" - ) %>% mutate(generation_date = forecast_date) - bind_rows(cheat, live) -} - -ws_geo_map <- function() { - gm <- get_population_data() %>% - distinct(state_id, .keep_all = TRUE) %>% - transmute(geo_value = state_id, location = state_code) - if (!"us" %in% gm$geo_value) gm <- bind_rows(gm, tibble(geo_value = "us", location = "US")) - gm -} - -ws_forecast_one <- function(archive, fd, gd, policy, geo_map) { - snapshot <- make_forecast_snapshot( - archive, forecast_date = fd, generation_date = gd, as_of_policy = policy, - substitutions = if (policy == "asof") WS_SUBSTITUTIONS else NULL - ) - purrr::map(WS_HORIZONS, function(h) { - set.seed(targets::tar_seed_create(paste(WS_ID, "nhsn", as.character(fd), h, sep = "/"))) - run_forecaster( - snapshot = snapshot, forecaster = scaled_pop_seasonal, aheads = h * 7L, - params = WS_PARAMS, id = WS_ID, target_date_shift = 3L, sort_quantiles = TRUE - ) %>% - mutate(horizon = h) - }) %>% - bind_rows() %>% - inner_join(geo_map, by = "geo_value") %>% - transmute( - reference_date = fd + 3L, - horizon, - target_end_date, - location, - level_index = hub_level_index(quantile), - level = quantile, - value, - as_of_policy = policy - ) -} - -ws_backfill <- function(refresh = FALSE) { - if (file.exists(WS_FORECASTS_FILE) && !refresh) { - cli::cli_alert_success("Pseudo-hub cache present at {.path {WS_FORECASTS_FILE}}; pass refresh = TRUE to rebuild.") - return(invisible(ws_read_pseudo_hub())) - } - withr::local_envvar(TAR_PROJECT = WS_STORE) - archive <- targets::tar_read(nhsn_prod_archive) - schedule <- ws_schedule(archive$versions_end) - geo_map <- ws_geo_map() - cli::cli_alert_info("Backfilling {nrow(schedule)} forecast dates x {length(WS_HORIZONS)} horizons of {WS_ID}") - failures <- list() - forecasts <- purrr::pmap(schedule, .progress = "windowed_seasonal backfill", function(forecast_date, as_of_policy, generation_date) { - tryCatch(ws_forecast_one(archive, forecast_date, generation_date, as_of_policy, geo_map), error = function(e) { - failures[[as.character(forecast_date)]] <<- conditionMessage(e) - NULL - }) - }) %>% - bind_rows() - if (length(failures) > 0) { - cli::cli_warn("{length(failures)} date{?s} failed: {.val {names(failures)}}") - print(failures) - } - dir.create(WS_CACHE_DIR, showWarnings = FALSE, recursive = TRUE) - nanoparquet::write_parquet(forecasts, WS_FORECASTS_FILE) - cli::cli_alert_success("{nrow(forecasts)} forecast rows over {n_distinct(forecasts$reference_date)} rounds -> {.path {WS_FORECASTS_FILE}}") - invisible(forecasts) -} - -#' Read the cached pseudo-hub forecasts in hub_read_forecasts() shape (plus `as_of_policy`). -ws_read_pseudo_hub <- function() { - nanoparquet::read_parquet(WS_FORECASTS_FILE) %>% - mutate(across(c(reference_date, target_end_date), as.Date), horizon = as.integer(horizon)) -} - -if (sys.nframe() == 0L) { - ws_backfill(refresh = "--refresh" %in% commandArgs(trailingOnly = TRUE)) -} diff --git a/scripts/calibration/calibration_ws_experiments.R b/scripts/calibration/calibration_ws_experiments.R deleted file mode 100644 index 39f15b23..00000000 --- a/scripts/calibration/calibration_ws_experiments.R +++ /dev/null @@ -1,136 +0,0 @@ -# One forecaster across eras: does online calibration of `windowed_seasonal` -# behave the same on the ILI+ decade (2011-2023, percent scale, no revisions) -# as on the NHSN seasons (2023-2026, counts, real vintages), and how does that -# compare with calibrating the submitted ensemble? Prints markdown tables; -# results are recorded in notes/calibration-ledger.md. -# -# Inputs (all cached in cache/calibration/): -# scripts/calibration/calibration_ili_backfill.R -> windowed_seasonal on ILI+, 2010-2024 -# scripts/calibration/calibration_ws_backfill.R -> windowed_seasonal on NHSN, 2023-2026 -# hub_read_forecasts()/nhsn_read_truth() -> the submitted ensemble and NHSN truth -# -# Usage: distrobox enter rocker -- Rscript scripts/calibration/calibration_ws_experiments.R - -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -source(here::here("scripts/calibration/calibration_ili_backfill.R")) -source(here::here("scripts/calibration/calibration_ws_backfill.R")) - -ili <- ili_read_pseudo_hub() -ws <- ws_read_pseudo_hub() -hub_fc <- hub_read_forecasts() -hub_truth <- nhsn_read_truth("flu") -LIVE_SEASONS <- c("2024-2025", "2025-2026") -ERA_SPLIT <- as.Date("2023-08-01") # ILI+ history before, NHSN after - -# ILI+ seasons kept: drop 2010-11 (too little training data for h3), the two -# pandemic non-seasons, and 2023-24 (the NHSN era owns those rounds). -ili_rounds <- hub_label_seasons(unique(ili$forecasts$reference_date)) -ili_keep <- ili_rounds %>% filter(!season %in% c("2010-2011", "2020-2021", "2021-2022", "2023-2024")) -ili_fc <- ili$forecasts %>% filter(reference_date %in% ili_keep$round_date) - -# Locations present in every source, so eras are comparable per location. -common_loc <- Reduce(intersect, list(unique(ili_fc$location), unique(ws$location), unique(hub_fc$location))) -ili_fc <- ili_fc %>% filter(location %in% common_loc) -# The replay ran every Wednesday, summer included; keep only the hub's -# submission rounds so seasons, gaps and round counts match the ensemble's. -ws_fc <- ws %>% - filter(location %in% common_loc, reference_date %in% unique(hub_fc$reference_date)) %>% - select(-as_of_policy) -hub_fc <- hub_fc %>% filter(location %in% common_loc) - -combined_fc <- bind_rows(ili_fc, ws_fc) -combined_truth <- bind_rows( - ili$truth %>% filter(target_end_date < ERA_SPLIT), - hub_truth %>% filter(target_end_date >= ERA_SPLIT) -) %>% filter(location %in% common_loc) - -# Whitening across eras: divide each location's rounds by the 90th percentile of -# its in-season (Oct-May) truth for that era, so offsets and eta live in -# "fraction of a typical season peak" units in both eras. -in_season <- function(d) as.integer(format(d, "%m")) %in% c(10:12, 1:5) -era_scale <- function(truth, from, to) { - truth %>% - filter(target_end_date >= from, target_end_date < to, in_season(target_end_date)) %>% - group_by(location) %>% - summarize(scale = stats::quantile(truth, 0.9, names = FALSE), .groups = "drop") -} -scales <- bind_rows( - era_scale(ili$truth, as.Date("2011-08-01"), ERA_SPLIT) %>% mutate(from = as.Date("2000-01-01")), - era_scale(hub_truth, as.Date("2023-10-01"), as.Date("2024-06-01")) %>% mutate(from = ERA_SPLIT) -) %>% filter(location %in% common_loc, scale > 0) - -variants <- function(floor) list( - single = list(), - leaky = list(fast_decay = 0.1), - operating = list(slow_init = "burn_in_quantile", lr_slow = list(mult = 0.003), fast_decay = 0.1) -) -run_all <- function(fc, truth, burn_in, floor, ...) { - purrr::map(variants(floor), function(v) { - do.call(calibrate_hub_forecasts, c( - list(fc, truth, burn_in_seasons = burn_in, transform = "sqrt", lr_window = 20, - lr_args = list(mult = 0.03, floor = floor), progress = FALSE), - v, list(...) - )) - }) -} - -cli::cli_h1("A. submitted ensemble, NHSN only, burn-in 2023-24") -A <- run_all(hub_fc, hub_truth, "2023-2024", 1e-3) -cli::cli_h1("B. windowed_seasonal, NHSN only, burn-in 2023-24 (finalized-data replay)") -B <- run_all(ws_fc, hub_truth, "2023-2024", 1e-3) -cli::cli_h1("C. windowed_seasonal, ILI+ decade then NHSN, burn-in 2011-12, whitened") -C <- run_all(combined_fc, combined_truth, "2011-2012", 1e-4, scales = scales) - -season_table <- function(cals, seasons = LIVE_SEASONS) { - purrr::imap(cals, function(cal, nm) { - left_join( - hub_quantile_loss(cal, by = "season") %>% select(season, horizon, wis_pct = pct_improvement), - hub_coverage_summary(cal, by = "season") %>% select(season, horizon, cal_err_base = cal_error_base, cal_err_cal = cal_error_cal), - by = c("season", "horizon") - ) %>% mutate(variant = nm) - }) %>% - bind_rows() %>% - filter(season %in% seasons) -} -wide <- function(tbl, col, digits = 1) { - tbl %>% select(variant, season, horizon, value = all_of(col)) %>% - mutate(value = round(value, digits)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = value, names_prefix = "h") %>% - arrange(season, variant) -} -report <- function(label, cals) { - tbl <- season_table(cals) - cat("\n\n### ", label, "\n\nWIS change % vs base (positive is better)\n\n", sep = "") - print(knitr::kable(wide(tbl, "wis_pct"))) - cat("\nCalibration error, calibrated (base in first row)\n\n") - base <- tbl %>% filter(variant == "single") %>% transmute(variant = "base", season, horizon, cal_err_cal = cal_err_base) - print(knitr::kable(wide(bind_rows(base, tbl), "cal_err_cal", 3))) -} -report("A. ensemble, NHSN only", A) -report("B. windowed_seasonal, NHSN only", B) -report("C. windowed_seasonal, ILI+ decade + NHSN, whitened", C) -cat("\n\n### C, the 2023-24 NHSN season (tracked live here; burn-in in A and B)\n\n") -print(knitr::kable(wide(season_table(C, seasons = "2023-2024"), "wis_pct"))) - -cat("\n\n### Base bias: fraction of truth above the base median, live seasons\n\n") -bias <- bind_rows( - A$single$forecasts %>% mutate(base = "ensemble"), - B$single$forecasts %>% mutate(base = "windowed_seasonal") -) %>% - filter(level == 0.5, !is.na(truth), !is.na(value_base), season %in% c("2023-2024", LIVE_SEASONS)) %>% - group_by(base, season, horizon) %>% - summarize(p = round(mean(truth > value_base), 2), .groups = "drop") %>% - tidyr::pivot_wider(names_from = horizon, values_from = p, names_prefix = "h") -print(knitr::kable(bias)) - -cat("\n\n### C, ILI+ seasons: WIS change % (single / operating)\n\n") -print(knitr::kable(wide(season_table(C, seasons = unique(ili_keep$season)) %>% filter(variant != "leaky"), "wis_pct"))) - -cat("\n\n### Base WIS per season and horizon (mean pinball x 2), ensemble vs windowed_seasonal\n\n") -bw <- bind_rows( - hub_quantile_loss(A$single, by = "season") %>% mutate(base = "ensemble"), - hub_quantile_loss(B$single, by = "season") %>% mutate(base = "windowed_seasonal") -) %>% filter(season %in% LIVE_SEASONS) %>% - transmute(base, season, horizon, wis = round(2 * loss_base, 1)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = wis, names_prefix = "h") -print(knitr::kable(bw)) diff --git a/tests/testthat/test-calibrate.R b/tests/testthat/test-calibrate.R index 91f42ec8..2186af3d 100644 --- a/tests/testthat/test-calibrate.R +++ b/tests/testthat/test-calibrate.R @@ -87,7 +87,6 @@ test_that("metrics ignore rounds a location was not forecast at", { expect_true(anyNA(cal$forecasts$value_base)) expect_false(anyNA(hub_quantile_loss(cal)$loss_cal)) expect_false(anyNA(hub_coverage(cal)$coverage_cal)) - expect_false(anyNA(hub_rolling_tradeoff(cal, window = 5L)$qloss_cal)) }) # Vintages where each week is first reported at half its value on the From fa57205ce46b6ef552ccbb4a2f89d18c3ee149b0 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:28:08 -0700 Subject: [PATCH 078/107] =?UTF-8?q?refactor(calibration):=20consolidate=20?= =?UTF-8?q?experiment=20notebooks=20into=20six=20section-based=20notebooks?= =?UTF-8?q?;=20add=20sqrt=200.018=20warm,=20per-state=20win=20counts=20(E1?= =?UTF-8?q?8)=20and=20finalists=20on=20flu/covid=20=C3=97=20ensemble/repla?= =?UTF-8?q?y=20(E10)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Justfile | 2 +- R/calibration/views.R | 143 +++++++++++ notes/calibration-ledger.md | 48 ++-- ...eta_settings.Rmd => _e01_eta_settings.Rmd} | 0 ...eta_variants.Rmd => _e02_eta_variants.Rmd} | 0 .../{e03_scale.Rmd => _e03_scale.Rmd} | 0 ...tructure.Rmd => _e04_offset_structure.Rmd} | 0 ...e05_warm_start.Rmd => _e05_warm_start.Rmd} | 0 ...7_across_eras.Rmd => _e07_across_eras.Rmd} | 173 +------------ .../_e07b_candidates.Rmd | 237 ++++++++++++++++++ .../{e09_lr_delay.Rmd => _e09_lr_delay.Rmd} | 0 .../_e10_flu_covid.Rmd | 228 +++++++++++++++++ ...1_constant_lr.Rmd => _e11_constant_lr.Rmd} | 0 ...lr_by_season.Rmd => _e14_lr_by_season.Rmd} | 0 ...5_wis_sources.Rmd => _e15_wis_sources.Rmd} | 0 ...stant_lr.Rmd => _e16_sqrt_constant_lr.Rmd} | 0 ...e17_late_decay.Rmd => _e17_late_decay.Rmd} | 2 +- ...f_op_bridge.Rmd => _e18_ref_op_bridge.Rmd} | 48 +++- .../calibration_experiments/_month_view.Rmd | 6 +- .../_standard_views.Rmd | 4 +- .../calibration_experiments/e10_covid.Rmd | 93 ------- .../calibration_experiments/finalists.Rmd | 36 +++ .../calibration_experiments/flu_covid.Rmd | 32 +++ .../calibration_experiments/index.Rmd | 72 ++++-- .../calibration_experiments/late_season.Rmd | 32 +++ .../calibration_experiments/learning_rate.Rmd | 40 +++ ...intage_backtest.Rmd => learning_truth.Rmd} | 2 +- .../tracker_structure.Rmd | 40 +++ scripts/calibration/render_experiments.R | 4 +- 29 files changed, 922 insertions(+), 320 deletions(-) rename reports/writeups/calibration_experiments/{e01_eta_settings.Rmd => _e01_eta_settings.Rmd} (100%) rename reports/writeups/calibration_experiments/{e02_eta_variants.Rmd => _e02_eta_variants.Rmd} (100%) rename reports/writeups/calibration_experiments/{e03_scale.Rmd => _e03_scale.Rmd} (100%) rename reports/writeups/calibration_experiments/{e04_offset_structure.Rmd => _e04_offset_structure.Rmd} (100%) rename reports/writeups/calibration_experiments/{e05_warm_start.Rmd => _e05_warm_start.Rmd} (100%) rename reports/writeups/calibration_experiments/{e07_across_eras.Rmd => _e07_across_eras.Rmd} (60%) create mode 100644 reports/writeups/calibration_experiments/_e07b_candidates.Rmd rename reports/writeups/calibration_experiments/{e09_lr_delay.Rmd => _e09_lr_delay.Rmd} (100%) create mode 100644 reports/writeups/calibration_experiments/_e10_flu_covid.Rmd rename reports/writeups/calibration_experiments/{e11_constant_lr.Rmd => _e11_constant_lr.Rmd} (100%) rename reports/writeups/calibration_experiments/{e14_lr_by_season.Rmd => _e14_lr_by_season.Rmd} (100%) rename reports/writeups/calibration_experiments/{e15_wis_sources.Rmd => _e15_wis_sources.Rmd} (100%) rename reports/writeups/calibration_experiments/{e16_sqrt_constant_lr.Rmd => _e16_sqrt_constant_lr.Rmd} (100%) rename reports/writeups/calibration_experiments/{e17_late_decay.Rmd => _e17_late_decay.Rmd} (99%) rename reports/writeups/calibration_experiments/{e18_ref_op_bridge.Rmd => _e18_ref_op_bridge.Rmd} (82%) delete mode 100644 reports/writeups/calibration_experiments/e10_covid.Rmd create mode 100644 reports/writeups/calibration_experiments/finalists.Rmd create mode 100644 reports/writeups/calibration_experiments/flu_covid.Rmd create mode 100644 reports/writeups/calibration_experiments/late_season.Rmd create mode 100644 reports/writeups/calibration_experiments/learning_rate.Rmd rename reports/writeups/calibration_experiments/{e00_vintage_backtest.Rmd => learning_truth.Rmd} (99%) create mode 100644 reports/writeups/calibration_experiments/tracker_structure.Rmd diff --git a/Justfile b/Justfile index 8e0a0b93..d3b91644 100644 --- a/Justfile +++ b/Justfile @@ -1,6 +1,6 @@ # R isn't installed on the host; run recipes inside the distrobox: `distrobox enter rocker -- just `. rscript := env_var_or_default("RSCRIPT", "Rscript") -# Render calibration experiment notebooks (all, or the named ones, e.g. e00_vintage_backtest) and the index. +# Render calibration experiment notebooks (all, or the named ones, e.g. finalists) and the index. calibration-experiments *notebooks: {{rscript}} scripts/calibration/render_experiments.R {{notebooks}} diff --git a/R/calibration/views.R b/R/calibration/views.R index 3de1f4fb..3005cf52 100644 --- a/R/calibration/views.R +++ b/R/calibration/views.R @@ -481,3 +481,146 @@ cal_month_plot <- function(month_tbl, season, colours = NULL, linetype = NULL, c axis.text.x = ggplot2::element_text(angle = 45, hjust = 1) ) } + + +CAL_NOTEBOOK_DIR <- "reports/writeups/calibration_experiments" + +#' Markdown heading marks for a heading written by `cat()` inside a section: +#' level `n`, pushed down by the section's shift. +#' @export +cal_h <- function(n) strrep("#", n + getOption("cal.hshift", 0L)) + +#' Knit a section notebook (`_eNN_*.Rmd`) into a combined notebook. +#' +#' The section's title becomes a heading with id `#eNN`, its own headings +#' (and those of the children it includes) move down `shift` levels, and it +#' runs in its own environment with `params` from its YAML, overridden by +#' `params`. Call from a chunk with `results = "asis"`. See +#' notes/calibration-ledger.md, "Notebooks". +#' @export +cal_section <- function(file, params = list(), shift = 1L, envir = parent.frame()) { + path <- here::here(CAL_NOTEBOOK_DIR, file) + yaml <- rmarkdown::yaml_front_matter(path) + id <- sub("^_", "", tools::file_path_sans_ext(file)) + lines <- cal_shift_headings(cal_expand_children(readLines(path)), shift) + env <- new.env(parent = envir) + env$params <- utils::modifyList(yaml$params %||% list(), params) + old <- options(cal.hshift = shift, knitr.duplicate.label = "allow") + old_chunk <- knitr::opts_chunk$get() + on.exit({ + options(old) + knitr::opts_chunk$restore(old_chunk) + }) + out <- knitr::knit_child( + text = lines, envir = env, quiet = TRUE, + options = list(fig.path = paste0(knitr::opts_chunk$get("fig.path"), id, "-")) + ) + cat("\n\n", strrep("#", shift), " ", yaml$title, " {#", sub("_.*", "", id), "}\n\n", out, "\n\n", sep = "") +} + +# Drop the YAML header and the root.dir setting (the combined notebook sets +# it), and inline `child=` chunks, recursively, so their headings can be +# shifted with the rest. +cal_expand_children <- function(lines) { + if (length(lines) > 0 && lines[[1]] == "---") { + end <- which(lines == "---")[2] + lines <- lines[-seq_len(end)] + } + lines <- lines[!grepl("^knitr::opts_knit\\$set\\(root.dir", lines)] + out <- character(0) + i <- 1L + while (i <= length(lines)) { + m <- regmatches(lines[[i]], regexec("^```\\{r child=(.*)\\}\\s*$", lines[[i]]))[[1]] + if (length(m) == 2) { + out <- c(out, cal_expand_children(readLines(eval(parse(text = m[[2]]))))) + i <- i + 2L + } else { + out <- c(out, lines[[i]]) + i <- i + 1L + } + } + out +} + +# Add `shift` levels to every markdown heading outside code chunks. +cal_shift_headings <- function(lines, shift) { + in_code <- FALSE + for (i in seq_along(lines)) { + if (grepl("^```", lines[[i]])) { + in_code <- !in_code + } else if (!in_code && grepl("^#+ ", lines[[i]])) { + lines[[i]] <- paste0(strrep("#", shift), lines[[i]]) + } + } + lines +} + +#' Per-state WIS (summed pinball) of the base and each run, per (season, +#' location, horizon), live seasons, states only. Seasons run August to July. +#' @export +cal_state_wis <- function(cals, seasons = c("2024-2025", "2025-2026")) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + purrr::imap(cals, function(cal, nm) { + cal$forecasts %>% + mutate(season = cal_season_of(.data$reference_date)) %>% + filter(.data$location != "US", .data$season %in% seasons, !is.na(.data$truth), !is.na(.data$value_base)) %>% + group_by(across(all_of(c("season", "location", "horizon")))) %>% + summarize( + wis_base = sum(pinball(.data$truth, .data$value_base, .data$level)), + wis_cal = sum(pinball(.data$truth, .data$value_cal, .data$level)), .groups = "drop" + ) %>% + mutate(config = nm) + }) %>% bind_rows() +} + +#' For each pair `c(a, b)` of runs in `state_wis`: per (season, horizon), the +#' number of states where `a` has the lower WIS, and the median per-state gap +#' in WIS reduction points (positive = `a` better). A table for `knitr::kable()`. +#' @export +cal_pair_wins <- function(state_wis, pairs) { + labels <- purrr::map_chr(pairs, \(pr) paste(pr[1], "vs", pr[2])) + purrr::map2(pairs, labels, function(pr, lab) { + state_wis %>% + filter(.data$config %in% pr) %>% + select("season", "location", "horizon", "config", "wis_cal", "wis_base") %>% + tidyr::pivot_wider(names_from = "config", values_from = "wis_cal") %>% + transmute( + season = .data$season, horizon = .data$horizon, pair = lab, + a_wins = .data[[pr[1]]] < .data[[pr[2]]], + gap = 100 * (.data[[pr[2]]] - .data[[pr[1]]]) / .data$wis_base + ) + }) %>% + bind_rows() %>% + mutate(pair = factor(.data$pair, levels = labels)) %>% + mutate(h = paste0("h", .data$horizon)) %>% + group_by(across(all_of(c("pair", "season", "h")))) %>% + summarize(cell = sprintf("%d of %d (median %+.1f)", sum(.data$a_wins), n(), stats::median(.data$gap)), .groups = "drop") %>% + tidyr::pivot_wider(names_from = "h", values_from = "cell") %>% + arrange(.data$pair, .data$season) +} + +CAL_PAIR_WINS_CAPTION <- "States where the first config of the pair has the lower WIS, of the states scored, and the median per-state gap in WIS reduction points (positive = first config better)" + +# August-July season label, as `season_of()` in the replay scripts. +cal_season_of <- function(d) { + y <- as.integer(format(d, "%Y")) - (as.integer(format(d, "%m")) < 8) + paste0(y, "-", y + 1) +} + +#' Knit the month view (`_month_view.Rmd`) in the calling environment under +#' its own heading, `level` levels below the section's top. For notebooks that +#' show more than one month view; uses `mv` and the optional `mv_*` settings +#' from `envir`. Call from a chunk with `results = "asis"`. +#' @export +cal_month_section <- function(title, level = 2L, envir = parent.frame()) { + lines <- readLines(here::here(CAL_NOTEBOOK_DIR, "_month_view.Rmd")) + lines[grep("^# Month view", lines)] <- paste("#", title) + shift <- getOption("cal.hshift", 0L) + level - 1L + old <- options(cal.hshift = shift) + on.exit(options(old)) + slug <- gsub("[^a-z0-9]+", "-", tolower(title)) + cat(knitr::knit_child( + text = cal_shift_headings(lines, shift), envir = envir, quiet = TRUE, + options = list(fig.path = paste0(knitr::opts_chunk$get("fig.path"), slug, "-")) + )) +} diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index b0b5933a..f6a6c06d 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -110,9 +110,19 @@ seasons, not more single-run comparisons. Both are `cal_ref_args()` in `R/calibration/views.R`; notebooks run them via `cal_run()`, which also picks the learning truth (`"exact"`: re-run from -scratch each round on that round's data, as prod does). Notebooks live in -`reports/writeups/calibration_experiments/` and render to -`reports/calibration_experiments/` with `just calibration-experiments`. +scratch each round on that round's data, as prod does). + +**Notebooks.** Six notebooks in `reports/writeups/calibration_experiments/` +(`finalists`, `flu_covid`, `learning_rate`, `tracker_structure`, +`late_season`, `learning_truth`), rendered to `reports/calibration_experiments/` +with `just calibration-experiments [name]`. Each experiment is a section +file `_eNN_*.Rmd`; a notebook knits its sections with `cal_section()` +(`R/calibration/views.R`), which gives each its own environment, YAML +`params` and figure prefix, and shifts its headings down one level (headings +written with `cat()` use `cal_h()` so they shift too). The index links each +experiment as `.html#eNN`. `learning_truth` is E00 rendered on its +own, so that its knitr chunk cache (keyed on chunk options, including the +figure path) still hits. Standard views (`_standard_views.Rmd`, `_month_view.Rmd`): V-head (WIS and coverage by horizon × season), V-month (by month of reference date, with @@ -128,23 +138,23 @@ gating), exact learning truth, spoiled submissions excluded. | id | notebook | base | question | what the numbers show | caveats | |---|---|---|---|---|---| -| E00 | `e00_vintage_backtest` | ensemble | learn from final, vintage or exact data? | exact ≈ final within 0.6 points except REF-op h−1 (2.2, 2024-25 only). Vintage at settle 7 loses the h−1 gain. Settle 7 vs 14: REF-paper +1 to +3 at every horizon; REF-op tie (flips by season) | | -| E01 | `e01_eta_settings` | replay, cold | adaptive eta settings | mult ≥ 0.1 costs 7–81 at h1–h3. 0.01 beats 0.03 by 0.5–2.4 at h1–h3 in both seasons; 0.03 better at h0 in 2024-25 (0.8). Window and carry vs reset: ≤ 2.4, mostly < 1. Count is 2–5 worse than sqrt | month-avg coverage of 0.01 vs 0.03 within noise | -| E02 | `e02_eta_variants` | replay, cold | `off_after`, per-level eta, seasonal window | off 1 Mar / 15 Feb +2 to +4 at h1–h3 over the 0.03 reference, both seasons, no month-avg coverage cost. Per-level +1. Seasonal window, burn-in, off 1 Apr: < 1 | dates in-sample; half the 2025-26 gain is December carry-over; "seasonal window" also changes `lr_window` 20 → 10 | -| E03 | `e03_scale` | ensemble | count vs sqrt vs rate | sqrt +1.4 to +3.9 over count pooled, mostly 2025-26. Coverage: neither better (sqrt worse per season pooled, better month-avg only in 2024-25) | rate identical to count in every cell (adaptive rate is scale invariant) | -| E04 | `e04_offset_structure` | ensemble | leak, slow term | leak +2.5 to +5.6 at h1–h3 in 2025-26, −0.4 to −1.3 in 2024-25; month-avg coverage flips the same way (±0.015). Slow term without warm start: ≤ 0.5 | | -| E05 | `e05_warm_start` | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | -| E07 | `e07_across_eras` | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | | -| E09 | `e09_lr_delay` | ensemble | constant rate × revision delay | knee 0.056: +3 to +5.5 over REF-paper at h1–h3; REF-op +1.2 to +2.4 over knee. Extra delay costs about 1 h−1 point per week. Above 0.1 loses steeply | month-avg: no constant rate beats base at h1–h3 | -| E10 | `e10_covid` | covid ensemble | REF-paper and REF-op on covid | h−1 +10 to +11. REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 at h1–h3 in 2025-26 | headline and month tables run to 2026-09-19, including the 2026 summer wave | -| E11 | `e11_constant_lr` | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | +| E00 | `learning_truth` | ensemble | learn from final, vintage or exact data? | exact ≈ final within 0.6 points except REF-op h−1 (2.2, 2024-25 only). Vintage at settle 7 loses the h−1 gain. Settle 7 vs 14: REF-paper +1 to +3 at every horizon; REF-op tie (flips by season) | | +| E01 | `learning_rate` | replay, cold | adaptive eta settings | mult ≥ 0.1 costs 7–81 at h1–h3. 0.01 beats 0.03 by 0.5–2.4 at h1–h3 in both seasons; 0.03 better at h0 in 2024-25 (0.8). Window and carry vs reset: ≤ 2.4, mostly < 1. Count is 2–5 worse than sqrt | month-avg coverage of 0.01 vs 0.03 within noise | +| E02 | `late_season` | replay, cold | `off_after`, per-level eta, seasonal window | off 1 Mar / 15 Feb +2 to +4 at h1–h3 over the 0.03 reference, both seasons, no month-avg coverage cost. Per-level +1. Seasonal window, burn-in, off 1 Apr: < 1 | dates in-sample; half the 2025-26 gain is December carry-over; "seasonal window" also changes `lr_window` 20 → 10 | +| E03 | `tracker_structure` | ensemble | count vs sqrt vs rate | sqrt +1.4 to +3.9 over count pooled, mostly 2025-26. Coverage: neither better (sqrt worse per season pooled, better month-avg only in 2024-25) | rate identical to count in every cell (adaptive rate is scale invariant) | +| E04 | `tracker_structure` | ensemble | leak, slow term | leak +2.5 to +5.6 at h1–h3 in 2025-26, −0.4 to −1.3 in 2024-25; month-avg coverage flips the same way (±0.015). Slow term without warm start: ≤ 0.5 | | +| E05 | `tracker_structure` | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | +| E07 | `flu_covid` (E07), `finalists` (E07b, the candidates) | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | | +| E09 | `learning_rate` | ensemble | constant rate × revision delay | knee 0.056: +3 to +5.5 over REF-paper at h1–h3; REF-op +1.2 to +2.4 over knee. Extra delay costs about 1 h−1 point per week. Above 0.1 loses steeply | month-avg: no constant rate beats base at h1–h3 | +| E10 | `flu_covid` | covid ensemble | REF-paper and REF-op on covid | h−1 +10 to +11. REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 at h1–h3 in 2025-26 | headline and month tables run to 2026-09-19, including the 2026 summer wave | +| E11 | `tracker_structure` | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | | E12 | none (`calibration_ws_replay.R`) | replay, cold, flu + covid | first look at the clean replay | flu: REF-op cold leads constant 0.1 by 0–2.5. Covid: REF-op cold h0 +1.3 states, losses at h1–h3 in 2025-26 | **all locations** (US is 47% of WIS); covid h0 gains are mostly US | | E13 | none (`calibration_ws_replay.R burn_in`) | replay, flu + covid | 2023-24 HHS burn-in | flu warm +1 to +2.6 in 2024-25, ≤ 0 in 2025-26. Covid: warm worse at h0–h1, better at h2–h3 (+1.3, +4.4); constant 0.1 warm −8 to −27 | HHS used as-is (3–9% below NHSN); coverage is season-pooled only | -| E14 | `e14_lr_by_season` | replay, cold | constant rate grid by season | every rate 0.0032–0.056 is ≥ 0 in both seasons and within 1.2 of each other; 0.018 vs 0.032 ≤ 0.3. Above 0.1 loses | month-avg: no rate ≤ 0.032 beats base beyond 0.01 | -| E15 | `e15_wis_sources` | replay, cold | where the WIS gain comes from | offsets a few % of interval width; REF-op gain from the lower half and the median shift | width-only variant's coverage not measured; split is not additive | -| E16 | `e16_sqrt_constant_lr` | replay, cold | constant rate on sqrt scale | sqrt 0.018 about +1 over rate constants at h0; at h1–h3 within 0.2 pooled, up to +0.7 in 2025-26. vs REF-op cold: behind in 2024-25 h0–h1, ahead elsewhere; month-avg coverage tie | 0.018 picked in-sample | -| E17 | `e17_late_decay` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | -| E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance) | +| E14 | `learning_rate` | replay, cold | constant rate grid by season | every rate 0.0032–0.056 is ≥ 0 in both seasons and within 1.2 of each other; 0.018 vs 0.032 ≤ 0.3. Above 0.1 loses | month-avg: no rate ≤ 0.032 beats base beyond 0.01 | +| E15 | `finalists` | replay, cold | where the WIS gain comes from | offsets a few % of interval width; REF-op gain from the lower half and the median shift | width-only variant's coverage not measured; split is not additive | +| E16 | `learning_rate` | replay, cold | constant rate on sqrt scale | sqrt 0.018 about +1 over rate constants at h0; at h1–h3 within 0.2 pooled, up to +0.7 in 2025-26. vs REF-op cold: behind in 2024-25 h0–h1, ahead elsewhere; month-avg coverage tie | 0.018 picked in-sample | +| E17 | `late_season` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | +| E18 | `finalists` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance) | E06 (ILI+ burn-in) and E08 (old covid) were run on code or data later found broken and were not rebuilt. @@ -253,7 +263,7 @@ learning rate; `test-qt.R` pins `eta` against a hand-built residual matrix. ```sh just calibration-experiments # all notebooks and the index -just calibration-experiments e03_scale # one notebook and the index +just calibration-experiments finalists # one notebook and the index Rscript scripts/calibration/calibration_ws_replay.R 12 # E12 (12 workers) Rscript scripts/calibration/calibration_ws_replay.R 12 burn_in # E13 ``` diff --git a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd b/reports/writeups/calibration_experiments/_e01_eta_settings.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/e01_eta_settings.Rmd rename to reports/writeups/calibration_experiments/_e01_eta_settings.Rmd diff --git a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/_e02_eta_variants.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/e02_eta_variants.Rmd rename to reports/writeups/calibration_experiments/_e02_eta_variants.Rmd diff --git a/reports/writeups/calibration_experiments/e03_scale.Rmd b/reports/writeups/calibration_experiments/_e03_scale.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/e03_scale.Rmd rename to reports/writeups/calibration_experiments/_e03_scale.Rmd diff --git a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd b/reports/writeups/calibration_experiments/_e04_offset_structure.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/e04_offset_structure.Rmd rename to reports/writeups/calibration_experiments/_e04_offset_structure.Rmd diff --git a/reports/writeups/calibration_experiments/e05_warm_start.Rmd b/reports/writeups/calibration_experiments/_e05_warm_start.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/e05_warm_start.Rmd rename to reports/writeups/calibration_experiments/_e05_warm_start.Rmd diff --git a/reports/writeups/calibration_experiments/e07_across_eras.Rmd b/reports/writeups/calibration_experiments/_e07_across_eras.Rmd similarity index 60% rename from reports/writeups/calibration_experiments/e07_across_eras.Rmd rename to reports/writeups/calibration_experiments/_e07_across_eras.Rmd index c3ed9bf9..a7747685 100644 --- a/reports/writeups/calibration_experiments/e07_across_eras.Rmd +++ b/reports/writeups/calibration_experiments/_e07_across_eras.Rmd @@ -47,12 +47,6 @@ floor 0.001, window 20, carry): - **operating**: REF-op (leaky plus a slow offset and a warm start from the burn-in season's errors). -Candidates (section 3): REF-op warm (= operating) and cold; constant rate -0.018 on the sqrt of the per-100k rate (E16); constant 0.032 per 100k on -the rate scale (E14); sqrt 0.018 switched off from 1 March -(`off_after = "03-01"`, E17). The cold candidates see the live seasons -only and start from zero offsets. - This rebuilds the original E07 (a script since deleted), which learned from finalized data, used the replay with prod's data substitutions and included h−1. @@ -84,26 +78,8 @@ grid <- tidyr::expand_grid(forecaster = names(FCS), tracker = names(TRACKERS)) cals <- purrr::pmap(grid, function(forecaster, tracker) { do.call(cal_run_cached, c(list(FCS[[forecaster]], wi$truth), TRACKERS[[tracker]], list(learn = "exact", vintages = wi$vintages, workers = params$workers))) }) -# Operating-point candidates (E14, E16, E17). Cold ones see only the live seasons, so they start from zero offsets. -const_cfg <- ws_configs(wi$rate_scales, burn_in = FALSE)[["E11 constant 0.1"]] -sqrt_018 <- utils::modifyList(const_cfg, list(lr = 0.018, transform = "sqrt")) -CANDS <- list( - `REF-op warm` = list(cfg = list(ref = "op"), warm = TRUE), - `REF-op cold` = list(cfg = list(ref = "op", burn_in_seasons = character(0), slow_init = NULL), warm = FALSE), - `sqrt 0.018` = list(cfg = sqrt_018, warm = FALSE), - `rate 0.032` = list(cfg = utils::modifyList(const_cfg, list(lr = 0.032)), warm = FALSE), - `sqrt 0.018, off 1 Mar` = list(cfg = c(sqrt_018, list(off_after = "03-01")), warm = FALSE) -) -FCS_LIVE <- purrr::map(FCS, \(fc) filter(fc, season_of(reference_date) %in% LIVE)) -cgrid <- tidyr::expand_grid(forecaster = names(FCS), cand = names(CANDS)) -ccals <- purrr::pmap(cgrid, function(forecaster, cand) { - fc <- if (CANDS[[cand]]$warm) FCS[[forecaster]] else FCS_LIVE[[forecaster]] - do.call(cal_run_cached, c(list(fc, wi$truth), CANDS[[cand]]$cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) -}) ``` -Rows per forecaster after the cut, by season: - ```{r rows} purrr::imap(FCS, \(fc, nm) fc %>% mutate(season = season_of(reference_date)) %>% count(season, name = nm)) %>% purrr::reduce(full_join, by = "season") %>% @@ -290,7 +266,7 @@ season_table <- function(g) { knitr::kable(season_table(GEOS[["states"]])) ``` -L1 coverage bias, season-pooled against month-average (see "Month view" below): +L1 coverage bias, season-pooled against month-average (the month view is in E07b): ```{r table-states-month} mv_trk <- cal_month_view(setNames(cals, paste(grid$forecaster, grid$tracker, sep = ", ")), seasons = LIVE) @@ -303,136 +279,6 @@ knitr::kable(cal_month_cov_table(mv_trk), caption = CAL_MONTH_COV_CAPTION) knitr::kable(season_table(GEOS[["all"]])) ``` -# 3. Operating-point candidates on both - -The candidates from E14, E16 and E17, each run on both forecasters on the -same rows: REF-op warm (as in section 2) and cold, sqrt-scale constant -0.018 (E16), rate-scale constant 0.032 per 100k (E14), and sqrt 0.018 -switched off from 1 March (`off_after = "03-01"`, E17). The cold ones learn -from the live seasons only. - -```{r cand-metrics} -cm <- purrr::map2(ccals, seq_len(nrow(cgrid)), \(cal, i) score(cal) %>% mutate(cgrid[i, ])) %>% - bind_rows() %>% - mutate( - forecaster = factor(forecaster, levels = names(FCS)), cand = factor(cand, levels = names(CANDS)), - geos = factor(geos, levels = names(GEOS), labels = GEOS), - season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), - h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) - ) -``` - -WIS reduction % (positive = calibrated WIS lower than base) per candidate, states only. - -```{r cand-plot, fig.height=6.5} -ggplot(cm %>% filter(geos == GEOS[["states"]]), aes(cand, wis_pct, fill = forecaster)) + - geom_hline(yintercept = 0, colour = "grey50") + - geom_col(position = position_dodge(width = 0.85), width = 0.8) + - facet_grid(season ~ h) + - scale_fill_manual(values = FC_COLS, name = NULL) + - labs(x = NULL, y = "WIS reduction % (positive = calibrated WIS lower than base)") + - theme(legend.position = "bottom", axis.text.x = element_text(angle = 30, hjust = 1)) -``` - -L1 coverage bias (lower is better), calibrated (filled) against the base (open), states only. - -```{r cand-cov, fig.height=6.5} -ggplot(cm %>% filter(geos == GEOS[["states"]]), aes(cand, cal_err_cal, colour = forecaster)) + - geom_point(aes(y = cal_err_base), shape = 1, size = 2.4, position = position_dodge(width = 0.6)) + - geom_point(size = 2.4, position = position_dodge(width = 0.6)) + - facet_grid(season ~ h) + - scale_colour_manual(values = FC_COLS, name = NULL) + - expand_limits(y = 0) + - labs(x = NULL, y = "L1 coverage bias (lower is better; open: base)") + - theme(legend.position = "bottom", axis.text.x = element_text(angle = 30, hjust = 1)) -``` - -## Rankings on A and B - -Candidates ordered by mean WIS reduction % (positive = calibrated WIS lower than base) over h0–h3, separately on each -forecaster, states only. `rank A` / `rank B` are the positions (1 is the -largest WIS reduction; `cov rank` 1 is the lowest L1 coverage bias); Spearman ρ compares the two orders per season, over all -(candidate, horizon) cells. - -```{r cand-rank} -rk <- cm %>% - filter(geos == GEOS[["states"]]) %>% - group_by(forecaster, season, cand) %>% - summarize(wis = mean(wis_pct), cov = mean(cal_err_cal), .groups = "drop") %>% - group_by(forecaster, season) %>% - mutate(rank = rank(-wis), cov_rank = rank(cov)) %>% - ungroup() %>% - mutate(fc = ifelse(forecaster == names(FCS)[1], "A", "B")) %>% - select(season, cand, fc, wis, rank, cov, cov_rank) %>% - tidyr::pivot_wider(names_from = fc, values_from = c(wis, rank, cov, cov_rank)) %>% - arrange(season, rank_A) -rk %>% - transmute(season, cand, `WIS reduction % A` = wis_A, `rank A` = rank_A, `WIS reduction % B` = wis_B, `rank B` = rank_B, - `cov bias A` = cov_A, `cov rank A` = cov_rank_A, `cov bias B` = cov_B, `cov rank B` = cov_rank_B) %>% - knitr::kable(digits = c(0, 0, 2, 0, 2, 0, 3, 0, 3, 0), caption = "Mean over h0–h3, states only. WIS reduction %: positive = calibrated WIS lower than base. Coverage bias is L1 coverage bias (lower is better)") -cm %>% - filter(geos == GEOS[["states"]]) %>% - select(season, cand, h, forecaster, wis_pct, cal_err_cal) %>% - tidyr::pivot_wider(names_from = forecaster, values_from = c(wis_pct, cal_err_cal)) %>% - group_by(season) %>% - summarize( - `ρ WIS cells` = cor(.data[[paste0("wis_pct_", names(FCS)[1])]], .data[[paste0("wis_pct_", names(FCS)[2])]], method = "spearman"), - `ρ coverage cells` = cor(.data[[paste0("cal_err_cal_", names(FCS)[1])]], .data[[paste0("cal_err_cal_", names(FCS)[2])]], method = "spearman") - ) %>% - knitr::kable(digits = 2, caption = "Spearman correlation of A vs B over (candidate, horizon) cells") -``` - -## Tables - -WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better). - -```{r cand-tables} -cand_table <- function(g) { - bind_rows( - cm %>% filter(cand == names(CANDS)[1]) %>% mutate(cand = "base", wis_pct = 0, cal_err_cal = cal_err_base), - cm - ) %>% - filter(geos == g) %>% - mutate(config = factor(paste(forecaster, cand, sep = ", "), levels = paste(rep(names(FCS), each = length(CANDS) + 1), c("base", names(CANDS)), sep = ", "))) %>% - transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(config) -} -``` - -### States only - -```{r cand-table-states} -knitr::kable(cand_table(GEOS[["states"]])) -``` - -L1 coverage bias, season-pooled against month-average (see "Month view" below): - -```{r cand-table-states-month} -mv <- cal_month_view(setNames(ccals, paste(cgrid$forecaster, cgrid$cand, sep = ", ")), seasons = LIVE) -mv_show_table <- FALSE -knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) -``` - -### All locations - -```{r cand-table-all} -knitr::kable(cand_table(GEOS[["all"]])) -``` - -```{r month-view-cfgs} -# Three candidates on both forecasters: colour is the candidate, linetype the forecaster (also for the grey base lines). -MV_CANDS <- c(`REF-op warm` = "black", `sqrt 0.018` = "#0072B2", `sqrt 0.018, off 1 Mar` = "#D55E00") -mv_plot_cfgs <- paste(rep(names(FCS), each = length(MV_CANDS)), names(MV_CANDS), sep = ", ") -mv_cols <- MV_CANDS -mv$month <- mv$month %>% mutate(forecaster = sub(",.*", "", config), cand = factor(sub("^[^,]*, ", "", config), levels = names(CANDS))) -mv_colour_by <- "cand" -mv_linetype <- "forecaster" -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` - # Findings States only. WIS reduction % (positive = calibrated WIS lower than base) @@ -451,20 +297,3 @@ with the month-avg. +3.9/+1.9/+1.5/+2.3, B +2.8/+1.1/+0.8/+1.2), mostly 2024-25 h2/h3 (+2.8/+4.3 vs +1.4/+1.7) and 2025-26 h0 (+4.3 vs +2.2). On B, leaky and single are under 1 point apart except at h3 (1.2). -- **Candidates, WIS: the order flips between seasons.** Mean over h0–h3, - REF-op warm leads in 2024-25 on both (A +3.44, B +2.07; next +0.79 and - +0.90). In 2025-26 the 1 March switch-off leads on both (A +1.96, B - +2.37) and REF-op warm is 4th on A, 5th on B (+0.69, +0.60). Pooled over - both seasons the other four candidates are within 0.2 points of each - other on A (+0.64 to +0.83) and within 0.9 on B (+0.63 to +1.48). -- **1 March switch-off in 2024-25.** No strong result: 0.47 points below - sqrt 0.018 on A and 0.24 above it on B. The rates (0.018, 0.032) and the - 1 March date were picked in E14/E16/E17 on these same seasons, on B. -- **Coverage, month-avg.** No strong result among candidates: over h0–h3, - A ranges 0.115–0.126 for four candidates (base 0.132) and B 0.130–0.136 - (base 0.142). Rate 0.032 is no better than the base on either (A 0.133, - B 0.142), and in 2025-26 it is above the base at every horizon (A by - 0.016–0.024, B by 0.007–0.017). Season-pooled, rate 0.032 has the lowest - 2025-26 bias on B (0.010–0.014), because pooling lets ramp and post-peak - errors cancel. Season-pooled coverage ranks agree between A and B over - both seasons (Spearman 0.97) but not in 2025-26 (0.31). diff --git a/reports/writeups/calibration_experiments/_e07b_candidates.Rmd b/reports/writeups/calibration_experiments/_e07b_candidates.Rmd new file mode 100644 index 00000000..34919c43 --- /dev/null +++ b/reports/writeups/calibration_experiments/_e07b_candidates.Rmd @@ -0,0 +1,237 @@ +--- +title: "E07b: operating-point candidates on the ensemble and the replay" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + hub_dir: "../FluSight-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: do the operating-point candidates rank the same way on the +submitted ensemble as on one fixed forecaster? + +**Setup.** As in E07 (the sections above): A is the submitted flu ensemble, +B the clean `windowed_seasonal` replay with the 2023-24 HHS backfill as its +burn-in season, on identical (round, location, horizon, level) rows, h0–h3, +exact learning truth with one week of extra delay. + +Candidates: REF-op warm and cold; constant rate 0.018 on the sqrt of the +per-100k rate (E16); constant 0.032 per 100k on the rate scale (E14); sqrt +0.018 switched off from 1 March (`off_after = "03-01"`, E17). The cold +candidates see the live seasons only and start from zero offsets. + +"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so +**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is +better) is the mean over the 23 levels of |empirical coverage − level|. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs("flu") +hub_fc <- suppressMessages(hub_read_forecasts(hub_dir = here::here(params$hub_dir), target = HUB_FLU_TARGET)) %>% + filter(horizon %in% 0:3) +ws_fc <- bind_rows(ws_burn_in_forecasts("flu"), wi$fc) +# Keep only (round, location, horizon, level) rows both forecasters have. +key <- c("reference_date", "location", "horizon", "level") +common <- inner_join(distinct(hub_fc, across(all_of(key))), distinct(ws_fc, across(all_of(key))), by = key) +FCS <- list( + `A ensemble` = semi_join(hub_fc, common, by = key), + `B windowed_seasonal` = semi_join(ws_fc, common, by = key) +) +# Cold candidates see only the live seasons, so they start from zero offsets. +const_cfg <- ws_configs(wi$rate_scales, burn_in = FALSE)[["E11 constant 0.1"]] +sqrt_018 <- utils::modifyList(const_cfg, list(lr = 0.018, transform = "sqrt")) +CANDS <- list( + `REF-op warm` = list(cfg = list(ref = "op"), warm = TRUE), + `REF-op cold` = list(cfg = list(ref = "op", burn_in_seasons = character(0), slow_init = NULL), warm = FALSE), + `sqrt 0.018` = list(cfg = sqrt_018, warm = FALSE), + `rate 0.032` = list(cfg = utils::modifyList(const_cfg, list(lr = 0.032)), warm = FALSE), + `sqrt 0.018, off 1 Mar` = list(cfg = c(sqrt_018, list(off_after = "03-01")), warm = FALSE) +) +FCS_LIVE <- purrr::map(FCS, \(fc) filter(fc, season_of(reference_date) %in% LIVE)) +cgrid <- tidyr::expand_grid(forecaster = names(FCS), cand = names(CANDS)) +ccals <- purrr::pmap(cgrid, function(forecaster, cand) { + fc <- if (CANDS[[cand]]$warm) FCS[[forecaster]] else FCS_LIVE[[forecaster]] + do.call(cal_run_cached, c(list(fc, wi$truth), CANDS[[cand]]$cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) +}) +``` + +```{r metrics} +GEOS <- c(states = "states only (US excluded)", all = "all locations") +SEASONS <- c(LIVE, "both") +score <- function(cal) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) + purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% + bind_rows() %>% + select(geos, season, horizon, which, wis, cal_err, cov50, cov90, above_median) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err, cov50, cov90)) %>% + mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) +} +FC_COLS <- c(`A ensemble` = "#0072B2", `B windowed_seasonal` = "#D55E00") +``` + +# Candidates on both + +The candidates from E14, E16 and E17, each run on both forecasters on the +same rows: REF-op warm (as in section 2) and cold, sqrt-scale constant +0.018 (E16), rate-scale constant 0.032 per 100k (E14), and sqrt 0.018 +switched off from 1 March (`off_after = "03-01"`, E17). The cold ones learn +from the live seasons only. + +```{r cand-metrics} +cm <- purrr::map2(ccals, seq_len(nrow(cgrid)), \(cal, i) score(cal) %>% mutate(cgrid[i, ])) %>% + bind_rows() %>% + mutate( + forecaster = factor(forecaster, levels = names(FCS)), cand = factor(cand, levels = names(CANDS)), + geos = factor(geos, levels = names(GEOS), labels = GEOS), + season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +``` + +WIS reduction % (positive = calibrated WIS lower than base) per candidate, states only. + +```{r cand-plot, fig.height=6.5} +ggplot(cm %>% filter(geos == GEOS[["states"]]), aes(cand, wis_pct, fill = forecaster)) + + geom_hline(yintercept = 0, colour = "grey50") + + geom_col(position = position_dodge(width = 0.85), width = 0.8) + + facet_grid(season ~ h) + + scale_fill_manual(values = FC_COLS, name = NULL) + + labs(x = NULL, y = "WIS reduction % (positive = calibrated WIS lower than base)") + + theme(legend.position = "bottom", axis.text.x = element_text(angle = 30, hjust = 1)) +``` + +L1 coverage bias (lower is better), calibrated (filled) against the base (open), states only. + +```{r cand-cov, fig.height=6.5} +ggplot(cm %>% filter(geos == GEOS[["states"]]), aes(cand, cal_err_cal, colour = forecaster)) + + geom_point(aes(y = cal_err_base), shape = 1, size = 2.4, position = position_dodge(width = 0.6)) + + geom_point(size = 2.4, position = position_dodge(width = 0.6)) + + facet_grid(season ~ h) + + scale_colour_manual(values = FC_COLS, name = NULL) + + expand_limits(y = 0) + + labs(x = NULL, y = "L1 coverage bias (lower is better; open: base)") + + theme(legend.position = "bottom", axis.text.x = element_text(angle = 30, hjust = 1)) +``` + +## Rankings on A and B + +Candidates ordered by mean WIS reduction % (positive = calibrated WIS lower than base) over h0–h3, separately on each +forecaster, states only. `rank A` / `rank B` are the positions (1 is the +largest WIS reduction; `cov rank` 1 is the lowest L1 coverage bias); Spearman ρ compares the two orders per season, over all +(candidate, horizon) cells. + +```{r cand-rank} +rk <- cm %>% + filter(geos == GEOS[["states"]]) %>% + group_by(forecaster, season, cand) %>% + summarize(wis = mean(wis_pct), cov = mean(cal_err_cal), .groups = "drop") %>% + group_by(forecaster, season) %>% + mutate(rank = rank(-wis), cov_rank = rank(cov)) %>% + ungroup() %>% + mutate(fc = ifelse(forecaster == names(FCS)[1], "A", "B")) %>% + select(season, cand, fc, wis, rank, cov, cov_rank) %>% + tidyr::pivot_wider(names_from = fc, values_from = c(wis, rank, cov, cov_rank)) %>% + arrange(season, rank_A) +rk %>% + transmute(season, cand, `WIS reduction % A` = wis_A, `rank A` = rank_A, `WIS reduction % B` = wis_B, `rank B` = rank_B, + `cov bias A` = cov_A, `cov rank A` = cov_rank_A, `cov bias B` = cov_B, `cov rank B` = cov_rank_B) %>% + knitr::kable(digits = c(0, 0, 2, 0, 2, 0, 3, 0, 3, 0), caption = "Mean over h0–h3, states only. WIS reduction %: positive = calibrated WIS lower than base. Coverage bias is L1 coverage bias (lower is better)") +cm %>% + filter(geos == GEOS[["states"]]) %>% + select(season, cand, h, forecaster, wis_pct, cal_err_cal) %>% + tidyr::pivot_wider(names_from = forecaster, values_from = c(wis_pct, cal_err_cal)) %>% + group_by(season) %>% + summarize( + `ρ WIS cells` = cor(.data[[paste0("wis_pct_", names(FCS)[1])]], .data[[paste0("wis_pct_", names(FCS)[2])]], method = "spearman"), + `ρ coverage cells` = cor(.data[[paste0("cal_err_cal_", names(FCS)[1])]], .data[[paste0("cal_err_cal_", names(FCS)[2])]], method = "spearman") + ) %>% + knitr::kable(digits = 2, caption = "Spearman correlation of A vs B over (candidate, horizon) cells") +``` + +## Tables + +WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better). + +```{r cand-tables} +cand_table <- function(g) { + bind_rows( + cm %>% filter(cand == names(CANDS)[1]) %>% mutate(cand = "base", wis_pct = 0, cal_err_cal = cal_err_base), + cm + ) %>% + filter(geos == g) %>% + mutate(config = factor(paste(forecaster, cand, sep = ", "), levels = paste(rep(names(FCS), each = length(CANDS) + 1), c("base", names(CANDS)), sep = ", "))) %>% + transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(config) +} +``` + +### States only + +```{r cand-table-states} +knitr::kable(cand_table(GEOS[["states"]])) +``` + +L1 coverage bias, season-pooled against month-average (see "Month view" below): + +```{r cand-table-states-month} +mv <- cal_month_view(setNames(ccals, paste(cgrid$forecaster, cgrid$cand, sep = ", ")), seasons = LIVE) +mv_show_table <- FALSE +knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) +``` + +### All locations + +```{r cand-table-all} +knitr::kable(cand_table(GEOS[["all"]])) +``` + +```{r month-view-cfgs} +# Three candidates on both forecasters: colour is the candidate, linetype the forecaster (also for the grey base lines). +MV_CANDS <- c(`REF-op warm` = "black", `sqrt 0.018` = "#0072B2", `sqrt 0.018, off 1 Mar` = "#D55E00") +mv_plot_cfgs <- paste(rep(names(FCS), each = length(MV_CANDS)), names(MV_CANDS), sep = ", ") +mv_cols <- MV_CANDS +mv$month <- mv$month %>% mutate(forecaster = sub(",.*", "", config), cand = factor(sub("^[^,]*, ", "", config), levels = names(CANDS))) +mv_colour_by <- "cand" +mv_linetype <- "forecaster" +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` + +# Findings + +States only. WIS reduction % (positive = calibrated WIS lower than base) +differences are in points; L1 coverage bias is lower-is-better and leads +with the month-avg. + +- **Candidates, WIS: the order flips between seasons.** Mean over h0–h3, + REF-op warm leads in 2024-25 on both (A +3.44, B +2.07; next +0.79 and + +0.90). In 2025-26 the 1 March switch-off leads on both (A +1.96, B + +2.37) and REF-op warm is 4th on A, 5th on B (+0.69, +0.60). Pooled over + both seasons the other four candidates are within 0.2 points of each + other on A (+0.64 to +0.83) and within 0.9 on B (+0.63 to +1.48). +- **1 March switch-off in 2024-25.** No strong result: 0.47 points below + sqrt 0.018 on A and 0.24 above it on B. The rates (0.018, 0.032) and the + 1 March date were picked in E14/E16/E17 on these same seasons, on B. +- **Coverage, month-avg.** No strong result among candidates: over h0–h3, + A ranges 0.115–0.126 for four candidates (base 0.132) and B 0.130–0.136 + (base 0.142). Rate 0.032 is no better than the base on either (A 0.133, + B 0.142), and in 2025-26 it is above the base at every horizon (A by + 0.016–0.024, B by 0.007–0.017). Season-pooled, rate 0.032 has the lowest + 2025-26 bias on B (0.010–0.014), because pooling lets ramp and post-peak + errors cancel. Season-pooled coverage ranks agree between A and B over + both seasons (Spearman 0.97) but not in 2025-26 (0.31). diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/_e09_lr_delay.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/e09_lr_delay.Rmd rename to reports/writeups/calibration_experiments/_e09_lr_delay.Rmd diff --git a/reports/writeups/calibration_experiments/_e10_flu_covid.Rmd b/reports/writeups/calibration_experiments/_e10_flu_covid.Rmd new file mode 100644 index 00000000..54edf6d6 --- /dev/null +++ b/reports/writeups/calibration_experiments/_e10_flu_covid.Rmd @@ -0,0 +1,228 @@ +--- +title: "E10: the finalists on flu and covid" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + disease: "covid" + hub_dir: "../covid19-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} +``` + +Question: how do the two finalists, REF-op and sqrt constant 0.018, do on +flu and covid, on the submitted ensemble and on the clean replay? + +Part 1 runs them on both diseases and both bases, h0–h3, on identical rows. +Part 2 is the covid ensemble at h−1–h3 with REF-paper for reference, and the +standard views. + +# Part 1: both diseases, both bases + +Setup: + +- Bases: the submitted `CMU-TimeSeries` ensemble (hub) and the clean + `windowed_seasonal` replay, on the (round, location, horizon, level) rows + both have, h0–h3 (the ensemble's h−1 is a different model). +- Learning truth: exact (re-run every round on the store's NHSN vintages), + one week of extra delay. Scores against the store's finalized NHSN, states + only, the two live seasons (August–July). +- Configs: REF-op warm (flu prod) and cold (covid prod); sqrt constant 0.018 + per 100k (E16), cold and warm (`slow_init = "burn_in_quantile"`). Warm + runs see a 2023-24 burn-in season: for flu the rows both bases have (hub + submissions and the HHS-stitched backfill, as E07); for covid only the + replay's backfill, since covid submissions start 2024-11-23. So the covid + ensemble has no warm runs. Cold runs see the live seasons only. + +```{r fin-runs} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +key <- c("reference_date", "location", "horizon", "level") +fin_inputs <- function(d) { + wi <- ws_inputs(d) + # Right after ws_inputs(): both read the harness globals that ch_use() sets. + burn <- ws_burn_in_forecasts(d) + hub <- suppressMessages(hub_read_forecasts( + hub_dir = here::here(HUB_DIRS[[d]]), target = c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET)[[d]] + )) %>% filter(horizon %in% 0:3) + ws <- bind_rows(burn, wi$fc) + common <- inner_join(distinct(hub, across(all_of(key))), distinct(ws, across(all_of(key))), by = key) + fcs <- list(ensemble = semi_join(hub, common, by = key), replay = semi_join(ws, common, by = key)) + warm <- fcs + if (!any(season_of(fcs$ensemble$reference_date) == "2023-2024")) { + warm <- list(replay = bind_rows(burn, fcs$replay)) + } + list(wi = wi, cold = purrr::map(fcs, \(fc) filter(fc, season_of(reference_date) %in% LIVE)), warm = warm) +} +fin_cands <- function(rate_scales) { + const <- ws_configs(rate_scales, burn_in = FALSE)[["E11 constant 0.1"]] + sqrt_018 <- utils::modifyList(const, list(lr = 0.018, transform = "sqrt")) + list( + `REF-op warm` = list(cfg = list(ref = "op"), warm = TRUE), + `REF-op cold` = list(cfg = list(ref = "op", burn_in_seasons = character(0), slow_init = NULL), warm = FALSE), + `sqrt 0.018` = list(cfg = sqrt_018, warm = FALSE), + `sqrt 0.018 warm` = list(cfg = utils::modifyList(sqrt_018, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")), warm = TRUE) + ) +} +FIN <- purrr::map(c(flu = "flu", covid = "covid"), function(d) { + inp <- fin_inputs(d) + cands <- fin_cands(inp$wi$rate_scales) + grid <- tidyr::expand_grid(base = c("ensemble", "replay"), cand = names(cands)) %>% + filter(!cands[cand] %>% purrr::map_lgl("warm") | base %in% names(inp$warm)) + cals <- purrr::pmap(grid, function(base, cand) { + cc <- cands[[cand]] + fc <- if (cc$warm) inp$warm[[base]] else inp$cold[[base]] + do.call(cal_run_cached, c(list(fc, inp$wi$truth), cc$cfg, list(learn = "exact", vintages = inp$wi$vintages, workers = params$workers))) + }) + list(cals = setNames(cals, paste(grid$base, grid$cand, sep = ", ")), grid = grid, cands = names(cands)) +}) +fin_score <- function(cal) { + ws_scores(cal$forecasts %>% mutate(season = season_of(reference_date)) %>% filter(location != "US")) %>% + select(season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% + mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) +} +FIN_M <- purrr::map(FIN, function(f) { + purrr::map2(f$cals, seq_len(nrow(f$grid)), \(cal, i) fin_score(cal) %>% mutate(f$grid[i, ])) %>% + bind_rows() %>% + mutate( + base = factor(base, levels = c("ensemble", "replay")), cand = factor(cand, levels = f$cands), + season = factor(season, levels = c(LIVE, "both"), labels = c(LIVE, "both seasons")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +}) +BASE_COLS <- c(ensemble = "#0072B2", replay = "#D55E00") +fin_table <- function(m) { + bind_rows( + m %>% distinct(base, season, h, .keep_all = TRUE) %>% mutate(cand = "base", cell = sprintf("WIS %.1f", wis_base)), + m %>% mutate(cell = sprintf("%+.1f", wis_pct)) + ) %>% + mutate(config = paste(base, cand, sep = ", ")) %>% + arrange(base, cand != "base", cand) %>% + transmute(config = factor(config, levels = unique(config)), col = paste(season, h), cell) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) +} +fin_plot <- function(m) { + ggplot(m, aes(cand, wis_pct, fill = base)) + + geom_hline(yintercept = 0, colour = "grey50") + + geom_col(position = position_dodge(width = 0.85, preserve = "single"), width = 0.8) + + facet_grid(season ~ h) + + scale_fill_manual(values = BASE_COLS, name = NULL) + + labs(x = NULL, y = "WIS reduction % (positive = calibrated WIS lower than base)") + + theme(legend.position = "bottom", axis.text.x = element_text(angle = 30, hjust = 1)) +} +``` + +```{r fin-sections, results="asis", fig.height=6.5} +for (d in names(FIN)) { + f <- FIN[[d]] + m <- FIN_M[[d]] + cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") + cat("\n\n", cal_h(3), " WIS\n\n", sep = "") + cat("WIS reduction % (positive = calibrated WIS lower than base), states only; the base rows give the base WIS.\n\n") + print(knitr::kable(fin_table(m))) + print(fin_plot(m)) + cat("\n\n", cal_h(3), " Coverage\n\n", sep = "") + print(knitr::kable(cal_month_cov_table(cal_month_view(f$cals, seasons = LIVE)), caption = CAL_MONTH_COV_CAPTION)) + cat("\n\n", cal_h(3), " State by state\n\n", sep = "") + cat("States share the national wave, so the counts are not independent tests.\n\n") + op <- if (d == "flu") "REF-op warm" else "REF-op cold" + pairs <- unlist(purrr::map(c("ensemble", "replay"), function(b) { + nm <- function(x) paste(b, x, sep = ", ") + ps <- list(c(nm(op), nm("sqrt 0.018")), c(nm("sqrt 0.018 warm"), nm("sqrt 0.018"))) + purrr::keep(ps, \(p) all(p %in% names(f$cals))) + }), recursive = FALSE) + print(knitr::kable(cal_pair_wins(cal_state_wis(f$cals), pairs), caption = CAL_PAIR_WINS_CAPTION)) +} +``` + +```{r fin-month, results="asis"} +for (d in names(FIN)) { + for (b in c("ensemble", "replay")) { + mv <- cal_month_view(FIN[[d]]$cals[startsWith(names(FIN[[d]]$cals), paste0(b, ", "))], seasons = LIVE) + mv_show_table <- FALSE + cal_month_section(paste("Month view:", d, b), level = 2L) + } +} +``` + +# Part 2: covid ensemble, h−1–h3 + +- Disease: covid. Base forecaster: the submitted `CMU-TimeSeries` ensemble. +- Learning truth: NHSN covid vintages, re-run at every round as prod does + (E00). Scores against finalized covid NHSN. +- No burn-in season (covid submissions start 2024-11-23); both trackers start + from zero offsets. REF-op runs without its warm start, as in covid prod. +- Rounds 2024-11-30 to 2026-09-19. The 2024-11-23 round is excluded (made + from a bad NHSN release corrected the next day). +- Covid is submitted year-round; per-season views use August–July seasons. +- Horizons h−1 to h3. Headline tables: all hub locations; month view: states + only. + +| config | change from REF-paper | +|---|---| +| `REF-paper` | none (counts, adaptive learning rate) | +| `REF-op (no warm start)` | sqrt scale, learning-rate floor, slow term 0.003, leak 0.1; no warm start | +| `sqrt 0.018` | sqrt of the per-100k rate, constant learning rate 0.018 (E16), single term | + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. + +```{r runs} +no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) +cals <- list( + `REF-paper` = do.call(run, c(list("paper"), no_burn_in)), + `REF-op (no warm start)` = do.call(run, c(list("op"), no_burn_in)), + `sqrt 0.018` = do.call(run, c(list("paper", transform = "sqrt", lr = 0.018, scales = rate_scales), no_burn_in)) +) +season_of <- function(d) { + y <- as.integer(format(d, "%Y")) - (as.integer(format(d, "%m")) < 8) + paste0(y, "-", y + 1) +} +cals <- purrr::map(cals, function(cal) { + cal$forecasts <- cal$forecasts %>% mutate(season = season_of(reference_date)) + cal +}) +LIVE <- c("2024-2025", "2025-2026") +focus <- "REF-op (no warm start)" +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} +``` + +# Findings, part 2 + +There are no uncertainty estimates; a WIS gap under 1 point or a month-avg +coverage gap under 0.01 is read as no clear difference. The pooled headline, +median-error and by-month tables run to 2026-09-19 to include the 2026 summer +wave; those Aug–Sep 2026 rounds are in no per-season table. + +- **h−1:** both configs improve WIS by about 10–11% in each season + (REF-paper 11.6/10.5, REF-op 8.9/10.8) and the absolute error of the median + by about 20%. Which config is ahead flips between seasons. +- **REF-paper at h1–h3:** −4.5/−7.3/−10.3 pooled, almost all from 2025-26 + (−10.2/−14.4/−19.3). In 2024-25 it is within 0.3 points of the base. +- **REF-op at h1–h3:** 0.0/−0.6/−1.3 pooled; +2.4/+2.8/+1.9 in 2024-25 and + −3.0/−4.0/−4.3 in 2025-26. No strong result: the sign depends on the + season. +- **Month-avg coverage (states, h−1…h3):** base 0.148/0.102/0.125/0.143/0.157; + REF-op 0.123/0.096/0.126/0.147/0.161 (lower at h−1 only); REF-paper + 0.111/0.094/0.126/0.155/0.171 (about 0.01 worse at h2/h3). Season-pooled, + the base looks calibrated at h1–h3, but pooling lets the two seasons' + errors cancel: within each season its bias is 0.072–0.078 (2024-25) and + 0.029–0.051 (2025-26). +- **By month at h3:** REF-op −16.5% in September and −12.0% in October + (together 20% of base WIS); REF-paper −42% to −49% in April–May (about 6%). diff --git a/reports/writeups/calibration_experiments/e11_constant_lr.Rmd b/reports/writeups/calibration_experiments/_e11_constant_lr.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/e11_constant_lr.Rmd rename to reports/writeups/calibration_experiments/_e11_constant_lr.Rmd diff --git a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd b/reports/writeups/calibration_experiments/_e14_lr_by_season.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/e14_lr_by_season.Rmd rename to reports/writeups/calibration_experiments/_e14_lr_by_season.Rmd diff --git a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd b/reports/writeups/calibration_experiments/_e15_wis_sources.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/e15_wis_sources.Rmd rename to reports/writeups/calibration_experiments/_e15_wis_sources.Rmd diff --git a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd b/reports/writeups/calibration_experiments/_e16_sqrt_constant_lr.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd rename to reports/writeups/calibration_experiments/_e16_sqrt_constant_lr.Rmd diff --git a/reports/writeups/calibration_experiments/e17_late_decay.Rmd b/reports/writeups/calibration_experiments/_e17_late_decay.Rmd similarity index 99% rename from reports/writeups/calibration_experiments/e17_late_decay.Rmd rename to reports/writeups/calibration_experiments/_e17_late_decay.Rmd index f1bd0908..d3f31c64 100644 --- a/reports/writeups/calibration_experiments/e17_late_decay.Rmd +++ b/reports/writeups/calibration_experiments/_e17_late_decay.Rmd @@ -132,7 +132,7 @@ month_m <- ws_month_scores(fc %>% mutate(config = interaction(rate, decay, sep = tidyr::separate(config, c("rate", "decay"), sep = "[|]") %>% mutate(decay = factor(decay, levels = names(DECAYS)), h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) for (r in names(RATES)) { - cat("\n\n## ", r, "\n\n", sep = "") + cat("\n\n", cal_h(2), " ", r, "\n\n", sep = "") d <- month_m %>% filter(rate == r) share <- d %>% filter(decay == "no decay") print( diff --git a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd b/reports/writeups/calibration_experiments/_e18_ref_op_bridge.Rmd similarity index 82% rename from reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd rename to reports/writeups/calibration_experiments/_e18_ref_op_bridge.Rmd index d477a305..f24f5518 100644 --- a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd +++ b/reports/writeups/calibration_experiments/_e18_ref_op_bridge.Rmd @@ -52,8 +52,10 @@ learning rate's window, so eta is sized from the first live round (step 4 removes this). Burn-in outcomes never step the offsets. Step 5 changes only units, so that step 6 changes only the learning rate. -Two rate-scale constant rates are shown as references outside the chain: -rate 0.032 (E14) and rate 0.056 (E09's knee). +Outside the chain: two rate-scale constant rates as references, rate 0.032 +(E14) and rate 0.056 (E09's knee), and a branch, sqrt 0.018 warm, which is +step 6 with the 2023-24 season present and `slow_init = +"burn_in_quantile"` (the warm start of step 0, here on the single term). Steps 0, 3, 4 and 6 are the cached runs of E13 (REF-op warm), E02 (`reference + burn-in`), E01/E02 (`reference`) and E16 (sqrt 0.018), so they @@ -87,7 +89,8 @@ STEPS <- list( e14_const <- ws_configs(wi$rate_scales, burn_in = TRUE)[["E11 constant 0.1"]] REFS <- list( `ref: rate 0.032 (E14)` = list(cfg = utils::modifyList(e14_const, list(lr = 0.032)), burn_in = FALSE), - `ref: rate 0.056 (E09 knee)` = list(cfg = utils::modifyList(e14_const, list(lr = 0.056)), burn_in = FALSE) + `ref: rate 0.056 (E09 knee)` = list(cfg = utils::modifyList(e14_const, list(lr = 0.056)), burn_in = FALSE), + `branch: sqrt 0.018 warm` = list(cfg = utils::modifyList(STEPS[[7]]$cfg, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")), burn_in = TRUE) ) run <- function(spec) { fc <- if (spec$burn_in) fc_burn_in else wi$fc @@ -98,7 +101,7 @@ cals <- purrr::map(c(STEPS, REFS), run) ```{r metrics} SEASONS <- c(LIVE, "both") -STEP_COLS <- setNames(c("#000000", "#E69F00", "#56B4E9", "#009E73", "#CC79A7", "#0072B2", "#D55E00", "#999999", "#882255"), names(c(STEPS, REFS))) +STEP_COLS <- setNames(c("#000000", "#E69F00", "#56B4E9", "#009E73", "#CC79A7", "#0072B2", "#D55E00", "#999999", "#882255", "#AA4499"), names(c(STEPS, REFS))) step_m <- purrr::imap(cals, function(cal, nm) { fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) %>% filter(location != "US") ws_scores(fc) %>% @@ -199,7 +202,44 @@ ggplot(delta_long, aes(value, step, fill = step)) + labs(x = "change from the previous step (both seasons, states only)", y = NULL) ``` +# 3. State by state: REF-op against sqrt 0.018 + +How many of the 52 states (50 states, DC and PR) each config of a pair has +the lower WIS in, per season and horizon. States share the national wave, +so the counts are not independent tests; they show whether a pooled gap +comes from most states or from a few large ones. + +```{r state-wins} +PAIRS <- list( + c("0 REF-op", "6 sqrt constant 0.018 (E16)"), + c("0 REF-op", "branch: sqrt 0.018 warm"), + c("branch: sqrt 0.018 warm", "6 sqrt constant 0.018 (E16)") +) +state_wis <- cal_state_wis(cals[unique(unlist(PAIRS))]) +knitr::kable(cal_pair_wins(state_wis, PAIRS), caption = CAL_PAIR_WINS_CAPTION) +``` + +Per state, pooled over h0–h3: WIS reduction % of REF-op against sqrt 0.018 +(above the line, REF-op is better). + +```{r state-scatter, fig.height=5} +state_pct <- state_wis %>% + filter(config %in% PAIRS[[1]]) %>% + group_by(config, season, location) %>% + summarize(pct = 100 * (sum(wis_base) - sum(wis_cal)) / sum(wis_base), .groups = "drop") %>% + tidyr::pivot_wider(names_from = config, values_from = pct) +ggplot(state_pct, aes(.data[[PAIRS[[1]][2]]], .data[[PAIRS[[1]][1]]])) + + geom_abline(slope = 1, intercept = 0, colour = "grey50") + + geom_hline(yintercept = 0, colour = "grey85") + + geom_vline(xintercept = 0, colour = "grey85") + + geom_point(alpha = 0.8) + + facet_wrap(~season) + + labs(x = "sqrt 0.018: WIS reduction %, h0–h3", y = "REF-op: WIS reduction %, h0–h3") +``` + ```{r month-view-cfgs} +# Five configs, so the month plots stay readable; every config is in the tables above. +mv_plot_cfgs <- c("0 REF-op", "1 no warm start", "4 no burn-in season", "6 sqrt constant 0.018 (E16)", "branch: sqrt 0.018 warm") mv_cols <- STEP_COLS ``` diff --git a/reports/writeups/calibration_experiments/_month_view.Rmd b/reports/writeups/calibration_experiments/_month_view.Rmd index 4fbe0a30..8cfdd211 100644 --- a/reports/writeups/calibration_experiments/_month_view.Rmd +++ b/reports/writeups/calibration_experiments/_month_view.Rmd @@ -33,14 +33,14 @@ if (!is.null(get0("mv_plot_cfgs"))) { mv_plot_month <- mv_plot_month %>% filter(config %in% mv_plot_cfgs) %>% mutate(config = droplevels(config)) } for (s in sort(unique(mv_plot_month$season))) { - cat("\n\n## ", s, "\n\n", sep = "") + cat("\n\n", cal_h(2), " ", s, "\n\n", sep = "") print(cal_month_plot(mv_plot_month, s, colours = get0("mv_cols"), linetype = get0("mv_linetype"), colour_by = get0("mv_colour_by") %||% "config")) } ``` ```{r mview-table, results="asis"} if (!isFALSE(get0("mv_show_table"))) { - cat("\n\n## Coverage bias: season-pooled and month-average\n\n") + cat("\n\n", cal_h(2), " Coverage bias: season-pooled and month-average\n\n", sep = "") print(knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION)) } ``` @@ -58,7 +58,7 @@ if (mv_one_base) { mv_abs_base <- mv_abs %>% distinct(season, month, h = paste0("h", horizon), base = sprintf("%.3f", cov_base)) mv_abs_tbl <- mv_abs_tbl %>% left_join(mv_abs_base, by = c("season", "month", "h")) } -cat("\n\n## Monthly L1 coverage bias, absolute (h0 and h2)\n\n") +cat("\n\n", cal_h(2), " Monthly L1 coverage bias, absolute (h0 and h2)\n\n", sep = "") print(knitr::kable( mv_abs_tbl %>% arrange(h, season, month), caption = if (mv_one_base) { diff --git a/reports/writeups/calibration_experiments/_standard_views.Rmd b/reports/writeups/calibration_experiments/_standard_views.Rmd index 678bd4ec..293193c1 100644 --- a/reports/writeups/calibration_experiments/_standard_views.Rmd +++ b/reports/writeups/calibration_experiments/_standard_views.Rmd @@ -99,7 +99,7 @@ fan starts. ```{r v-state, results="asis", fig.height=4} for (nm in names(CAL_STATES)) { - cat("\n\n### ", nm, "\n\n", sep = "") + cat("\n\n", cal_h(3), " ", nm, "\n\n", sep = "") for (s in LIVE) print(cal_state_panel(cals[[focus]], CAL_STATES[[nm]], s, truth, vintages, title = paste(nm, s))) } ``` @@ -118,7 +118,7 @@ picks <- cal_gallery_pick(cals[[focus]], population) abbrev <- setNames(toupper(get_population_data()$state_id), get_population_data()$state_code) for (i in seq_len(nrow(picks))) { p <- picks[i, ] - if (i == 1L || p$phase != picks$phase[i - 1L]) cat("\n\n### ", as.character(p$phase), "\n\n", sep = "") + if (i == 1L || p$phase != picks$phase[i - 1L]) cat("\n\n", cal_h(3), " ", as.character(p$phase), "\n\n", sep = "") sub <- sprintf("%s (%s), %s, %s: WIS reduction %+.1f%% (%+.1f%% of its season's base WIS; positive = calibrated WIS lower)", abbrev[[p$location]], p$size, p$reference_date, p$kind, p$wis_pct, p$share) print(cal_gallery_panel(cals[[focus]], p$location, p$reference_date, truth, vintages, subtitle = sub)) } diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd deleted file mode 100644 index 46e86cc6..00000000 --- a/reports/writeups/calibration_experiments/e10_covid.Rmd +++ /dev/null @@ -1,93 +0,0 @@ ---- -title: "E10: covid at the reference configs" -date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" -output: - html_document: - code_folding: hide - toc: true - toc_float: true - toc_depth: 2 -params: - disease: "covid" - hub_dir: "../covid19-forecast-hub" - workers: 14 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} -``` - -Question: how do the two reference configs do on covid? - -Setup: - -- Disease: covid. Base forecaster: the submitted `CMU-TimeSeries` ensemble. -- Learning truth: NHSN covid vintages, re-run at every round as prod does - (E00). Scores against finalized covid NHSN. -- No burn-in season (covid submissions start 2024-11-23); both trackers start - from zero offsets. REF-op runs without its warm start, as in covid prod. -- Rounds 2024-11-30 to 2026-09-19. The 2024-11-23 round is excluded (made - from a bad NHSN release corrected the next day). -- Covid is submitted year-round; per-season views use August–July seasons. -- Horizons h−1 to h3. Headline tables: all hub locations; month view: states - only. - -| config | change from REF-paper | -|---|---| -| `REF-paper` | none (counts, adaptive learning rate) | -| `REF-op (no warm start)` | sqrt scale, learning-rate floor, slow term 0.003, leak 0.1; no warm start | - -WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated -WIS lower than base. L1 coverage bias: lower is better. - -```{r runs} -no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) -cals <- list( - `REF-paper` = do.call(run, c(list("paper"), no_burn_in)), - `REF-op (no warm start)` = do.call(run, c(list("op"), no_burn_in)) -) -season_of <- function(d) { - y <- as.integer(format(d, "%Y")) - (as.integer(format(d, "%m")) < 8) - paste0(y, "-", y + 1) -} -cals <- purrr::map(cals, function(cal) { - cal$forecasts <- cal$forecasts %>% mutate(season = season_of(reference_date)) - cal -}) -LIVE <- c("2024-2025", "2025-2026") -focus <- "REF-op (no warm start)" -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} -``` - -# Findings - -There are no uncertainty estimates; a WIS gap under 1 point or a month-avg -coverage gap under 0.01 is read as no clear difference. The pooled headline, -median-error and by-month tables run to 2026-09-19 to include the 2026 summer -wave; those Aug–Sep 2026 rounds are in no per-season table. - -- **h−1:** both configs improve WIS by about 10–11% in each season - (REF-paper 11.6/10.5, REF-op 8.9/10.8) and the absolute error of the median - by about 20%. Which config is ahead flips between seasons. -- **REF-paper at h1–h3:** −4.5/−7.3/−10.3 pooled, almost all from 2025-26 - (−10.2/−14.4/−19.3). In 2024-25 it is within 0.3 points of the base. -- **REF-op at h1–h3:** 0.0/−0.6/−1.3 pooled; +2.4/+2.8/+1.9 in 2024-25 and - −3.0/−4.0/−4.3 in 2025-26. No strong result: the sign depends on the - season. -- **Month-avg coverage (states, h−1…h3):** base 0.148/0.102/0.125/0.143/0.157; - REF-op 0.123/0.096/0.126/0.147/0.161 (lower at h−1 only); REF-paper - 0.111/0.094/0.126/0.155/0.171 (about 0.01 worse at h2/h3). Season-pooled, - the base looks calibrated at h1–h3, but pooling lets the two seasons' - errors cancel: within each season its bias is 0.072–0.078 (2024-25) and - 0.029–0.051 (2025-26). -- **By month at h3:** REF-op −16.5% in September and −12.0% in October - (together 20% of base WIS); REF-paper −42% to −49% in April–May (about 6%). diff --git a/reports/writeups/calibration_experiments/finalists.Rmd b/reports/writeups/calibration_experiments/finalists.Rmd new file mode 100644 index 00000000..dd9361f2 --- /dev/null +++ b/reports/writeups/calibration_experiments/finalists.Rmd @@ -0,0 +1,36 @@ +--- +title: "Finalists" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 3 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` + +The two operating-point candidates, REF-op (prod) and sqrt constant 0.018. E18 steps from one to the other one setting at a time on the clean flu replay, adds sqrt 0.018 with a warm start, and counts the states each one wins. E07b ranks the wider candidate set on the submitted ensemble and the replay. E15 shows where the WIS gains come from. Each section is a self-contained experiment with its own runs; +the ids match `notes/calibration-ledger.md`. + +```{r e18, results="asis"} +cal_section("_e18_ref_op_bridge.Rmd", list(workers = params$workers)) +``` + +```{r e07b, results="asis"} +cal_section("_e07b_candidates.Rmd", list(workers = params$workers)) +``` + +```{r e15, results="asis"} +cal_section("_e15_wis_sources.Rmd", list(workers = params$workers)) +``` diff --git a/reports/writeups/calibration_experiments/flu_covid.Rmd b/reports/writeups/calibration_experiments/flu_covid.Rmd new file mode 100644 index 00000000..b116b10f --- /dev/null +++ b/reports/writeups/calibration_experiments/flu_covid.Rmd @@ -0,0 +1,32 @@ +--- +title: "Flu and covid" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 3 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` + +The finalists on both diseases and both bases (the submitted ensemble and the clean `windowed_seasonal` replay). E10 runs them; E07 compares the two flu bases before and after calibration. Each section is a self-contained experiment with its own runs; +the ids match `notes/calibration-ledger.md`. + +```{r e10, results="asis"} +cal_section("_e10_flu_covid.Rmd", list(workers = params$workers)) +``` + +```{r e07, results="asis"} +cal_section("_e07_across_eras.Rmd", list(workers = params$workers)) +``` diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index fcf372d8..d41ada14 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -24,44 +24,72 @@ is read as no clear difference. ```{r rows, include=FALSE} out_dir <- here::here("reports/calibration_experiments") -link <- function(id, file) { +link <- function(id, file, anchor = "") { html <- paste0(file, ".html") if (!file.exists(file.path(out_dir, html))) return(paste(id, "(not rendered)")) + sprintf("[%s](%s%s)", id, html, if (nzchar(anchor)) paste0("#", anchor) else "") +} +nb_link <- function(title, file) { + html <- paste0(file, ".html") + if (!file.exists(file.path(out_dir, html))) return(paste(title, "(not rendered)")) stamp <- format(file.mtime(file.path(out_dir, html)), "%Y-%m-%d", tz = "America/Los_Angeles") - sprintf("[%s](%s) (%s)", id, html, stamp) + sprintf("[%s](%s) (rendered %s)", title, html, stamp) } ``` # Notebooks +Six notebooks, each with one section per experiment. The ids match +`notes/calibration-ledger.md`. + +## `r nb_link("Finalists", "finalists")` + | | base | question | what the numbers show | |---|---|---|---| -| `r link("E01", "e01_eta_settings")` | replay, cold | How do the adaptive learning-rate settings (multiplier, window, carry or reset) change WIS and coverage? | Multipliers 0.1 and 0.3 lose 7–77 points at h1–h3. 0.01 is 0.5–2.4 points above 0.03 at h1–h3; month-avg coverage shows no clear difference. Count scale is 2–5 points below sqrt. Window and carry vs reset: no strong result. | -| `r link("E03", "e03_scale")` | ensemble | Which scale should the offsets live on: counts, square root, or rate per 100k? | sqrt is 1.4–3.9 points above count pooled, mostly from 2025-26. Coverage: no strong result. Rate is identical to count in every cell. | -| `r link("E09", "e09_lr_delay")` | ensemble | With a constant learning rate on the rate scale, how do WIS and coverage change with the rate and with extra learning delay? | WIS peaks near 0.056–0.1 per 100k and falls fast past 0.3. Month-avg coverage is no lower than the base at h1–h3 for any constant rate. Each week of extra delay costs about 1 point at h−1. Stateful learning with no extra delay makes h−1 worse than the base. | -| `r link("E10", "e10_covid")` | covid ensemble | How do the two reference configs do on covid? | h−1 improves about 10–11% in both seasons. REF-paper loses 10–19% at h1–h3 in 2025-26. REF-op at h1–h3 is −4 to +3, sign depending on the season: no strong result. | -| `r link("E11", "e11_constant_lr")` | ensemble | Does a constant learning rate do better than the adaptive rate? | Constant 0.1 vs adaptive sqrt: the order flips by horizon (1–2 points either way). REF-op has the highest pooled WIS at every horizon, by up to 2 points. On month-avg coverage the constant rate is 0.003–0.022 worse. | -| `r link("E14", "e14_lr_by_season")` | replay, cold | How do WIS and coverage change with a constant learning rate, season by season? | Every rate from 0.0032 to 0.056 is at or above the base in both seasons, within 1.2 points of each other: no strong result for a best rate. From 0.1 WIS falls. Month-avg coverage for rates ≤ 0.032 stays within about 0.01 of the base. | -| `r link("E15", "e15_wis_sources")` | replay, cold | Where does the WIS change come from? | Offsets are 2–5% of the 90% interval width. REF-op's gain (+2.0 at h0, about 0 by h3) comes from raising the lower quantiles; a median shift alone matches it within 0.6 points. | -| `r link("E16", "e16_sqrt_constant_lr")` | replay, cold | Constant learning rate on the square-root scale vs the rate scale | sqrt 0.018 is about 1 point above rate 0.032 at h0 and within 0.2 at h1–h3 pooled. Against REF-op cold the order flips by season. Every tracker loses in March (up to −28% within the month at h2). | -| `r link("E17", "e17_late_decay")` | replay, cold | Does shrinking or switching off the offset from 1 March change WIS and coverage? | No strong result: pooled WIS gaps ≤ 0.7 points and month-avg coverage gaps ≤ 0.010. The switch-off's sqrt gain comes mostly from 2025-26; the date was chosen in-sample. | -| `r link("E18", "e18_ref_op_bridge")` | replay | Stepping from REF-op to sqrt constant 0.018 one setting at a time | REF-op vs sqrt 0.018: no strong result (0.3–0.6 points pooled, order flips by season). The warm start helps in 2024-25 only. Constant 0.018 is 1.7–2.3 points above cold adaptive at h2–h3. No step moves month-avg coverage by more than 0.008. | +| `r link("E18", "finalists", "e18")` | replay | Stepping from REF-op to sqrt constant 0.018 one setting at a time | REF-op vs sqrt 0.018: no strong result (0.3–0.6 points pooled, order flips by season). The warm start helps in 2024-25 only. Constant 0.018 is 1.7–2.3 points above cold adaptive at h2–h3. No step moves month-avg coverage by more than 0.008. | +| `r link("E07b", "finalists", "e07b")` | both | Do the operating-point candidates rank the same way on both bases? | The candidate order flips between seasons on both. | +| `r link("E15", "finalists", "e15")` | replay, cold | Where does the WIS change come from? | Offsets are 2–5% of the 90% interval width. REF-op's gain (+2.0 at h0, about 0 by h3) comes from raising the lower quantiles; a median shift alone matches it within 0.6 points. | -E12 and E13 (the clean replay without and with a 2023-24 burn-in) have no -notebook; they are `scripts/calibration/calibration_ws_replay.R`. +## `r nb_link("Flu and covid", "flu_covid")` -# Archived +| | base | question | what the numbers show | +|---|---|---|---| +| `r link("E10", "flu_covid", "e10")` | both diseases, both bases | How do the finalists do on flu and covid? (Part 2: covid ensemble at h−1–h3 with REF-paper.) | Part 2: h−1 improves about 10–11% in both seasons. REF-paper loses 10–19% at h1–h3 in 2025-26. REF-op at h1–h3 is −4 to +3, sign depending on the season. Part 1: not yet read. | +| `r link("E07", "flu_covid", "e07")` | both | Does calibration act the same on the ensemble and the clean replay? | Same direction on both; REF-op gains 0.7–1.1 points more on the ensemble. | + +## `r nb_link("Learning rate", "learning_rate")` + +| | base | question | what the numbers show | +|---|---|---|---| +| `r link("E01", "learning_rate", "e01")` | replay, cold | How do the adaptive learning-rate settings (multiplier, window, carry or reset) change WIS and coverage? | Multipliers 0.1 and 0.3 lose 7–77 points at h1–h3. 0.01 is 0.5–2.4 points above 0.03 at h1–h3; month-avg coverage shows no clear difference. Count scale is 2–5 points below sqrt. Window and carry vs reset: no strong result. | +| `r link("E09", "learning_rate", "e09")` | ensemble | With a constant learning rate on the rate scale, how do WIS and coverage change with the rate and with extra learning delay? | WIS peaks near 0.056–0.1 per 100k and falls fast past 0.3. Month-avg coverage is no lower than the base at h1–h3 for any constant rate. Each week of extra delay costs about 1 point at h−1. Stateful learning with no extra delay makes h−1 worse than the base. | +| `r link("E14", "learning_rate", "e14")` | replay, cold | How do WIS and coverage change with a constant learning rate, season by season? | Every rate from 0.0032 to 0.056 is at or above the base in both seasons, within 1.2 points of each other: no strong result for a best rate. From 0.1 WIS falls. Month-avg coverage for rates ≤ 0.032 stays within about 0.01 of the base. | +| `r link("E16", "learning_rate", "e16")` | replay, cold | Constant learning rate on the square-root scale vs the rate scale | sqrt 0.018 is about 1 point above rate 0.032 at h0 and within 0.2 at h1–h3 pooled. Against REF-op cold the order flips by season. Every tracker loses in March (up to −28% within the month at h2). | -Earlier experiments, kept for reference. Their questions are settled or -covered by the notebooks above. +## `r nb_link("Tracker structure", "tracker_structure")` | | base | question | what the numbers show | |---|---|---|---| -| `r link("E00", "e00_vintage_backtest")` | ensemble | Should the tracker learn from finalized data or from data as published? | Exact and final differ by 0.6 points or less, except REF-op h−1 (2.2, 2024-25). Learning once from the first report loses most of the h−1 gain. All other notebooks use the exact (prod) design. | -| `r link("E02", "e02_eta_variants")` | replay, cold | Spring switch-off, per-level learning rate, seasonal window | Switching off from 1 March or 15 February is 2–4 points above the reference at h1–h3 (dates in-sample; in 2025-26 about half the gain is December carry-over). Per-level about +1. Month-avg coverage: no clear difference. | -| `r link("E04", "e04_offset_structure")` | ensemble | Leak and slow term | The leak is +3.3 to +5.6 at h1–h3 in 2025-26 and −0.4 to −1.3 in 2024-25. The slow term without a warm start: no strong result. | -| `r link("E05", "e05_warm_start")` | ensemble | Warm start from the burn-in season | +1.7 to +3.2 at h2/h3 in 2024-25 for every tracker, within ±0.4 in 2025-26. | -| `r link("E07", "e07_across_eras")` | both | Does calibration act the same on the ensemble and the clean replay? | Same direction on both; REF-op gains 0.7–1.1 points more on the ensemble. The candidate order flips between seasons on both. | +| `r link("E03", "tracker_structure", "e03")` | ensemble | Which scale should the offsets live on: counts, square root, or rate per 100k? | sqrt is 1.4–3.9 points above count pooled, mostly from 2025-26. Coverage: no strong result. Rate is identical to count in every cell. | +| `r link("E04", "tracker_structure", "e04")` | ensemble | Leak and slow term | The leak is +3.3 to +5.6 at h1–h3 in 2025-26 and −0.4 to −1.3 in 2024-25. The slow term without a warm start: no strong result. | +| `r link("E05", "tracker_structure", "e05")` | ensemble | Warm start from the burn-in season | +1.7 to +3.2 at h2/h3 in 2024-25 for every tracker, within ±0.4 in 2025-26. | +| `r link("E11", "tracker_structure", "e11")` | ensemble | Does a constant learning rate do better than the adaptive rate? | Constant 0.1 vs adaptive sqrt: the order flips by horizon (1–2 points either way). REF-op has the highest pooled WIS at every horizon, by up to 2 points. On month-avg coverage the constant rate is 0.003–0.022 worse. | + +## `r nb_link("Late season", "late_season")` + +| | base | question | what the numbers show | +|---|---|---|---| +| `r link("E02", "late_season", "e02")` | replay, cold | Spring switch-off, per-level learning rate, seasonal window | Switching off from 1 March or 15 February is 2–4 points above the reference at h1–h3 (dates in-sample; in 2025-26 about half the gain is December carry-over). Per-level about +1. Month-avg coverage: no clear difference. | +| `r link("E17", "late_season", "e17")` | replay, cold | Does shrinking or switching off the offset from 1 March change WIS and coverage? | No strong result: pooled WIS gaps ≤ 0.7 points and month-avg coverage gaps ≤ 0.010. The switch-off's sqrt gain comes mostly from 2025-26; the date was chosen in-sample. | + +## `r nb_link("Learning truth", "learning_truth")` + +| | base | question | what the numbers show | +|---|---|---|---| +| `r link("E00", "learning_truth", "")` | ensemble | Should the tracker learn from finalized data or from data as published? | Exact and final differ by 0.6 points or less, except REF-op h−1 (2.2, 2024-25). Learning once from the first report loses most of the h−1 gain. All other notebooks use the exact (prod) design. | + +E12 and E13 (the clean replay without and with a 2023-24 burn-in) have no +notebook; they are `scripts/calibration/calibration_ws_replay.R`. # How to read the results diff --git a/reports/writeups/calibration_experiments/late_season.Rmd b/reports/writeups/calibration_experiments/late_season.Rmd new file mode 100644 index 00000000..20455970 --- /dev/null +++ b/reports/writeups/calibration_experiments/late_season.Rmd @@ -0,0 +1,32 @@ +--- +title: "Late season" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 3 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` + +Every tracker loses WIS after the peak. Both sections calibrate the clean replay, cold: E02 switches the offset off from a date (and tries a per-level and a seasonal learning rate); E17 shrinks the offset from 1 March instead. Each section is a self-contained experiment with its own runs; +the ids match `notes/calibration-ledger.md`. + +```{r e02, results="asis"} +cal_section("_e02_eta_variants.Rmd", list(workers = params$workers)) +``` + +```{r e17, results="asis"} +cal_section("_e17_late_decay.Rmd", list(workers = params$workers)) +``` diff --git a/reports/writeups/calibration_experiments/learning_rate.Rmd b/reports/writeups/calibration_experiments/learning_rate.Rmd new file mode 100644 index 00000000..c9a70f1a --- /dev/null +++ b/reports/writeups/calibration_experiments/learning_rate.Rmd @@ -0,0 +1,40 @@ +--- +title: "Learning rate" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 3 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` + +How large should the tracker's steps be? E01 varies the adaptive learning rate's settings on the clean replay; E09 sweeps a constant rate (and the extra revision delay) on the submitted ensemble; E14 and E16 sweep a constant rate on the replay, on the rate and square-root scales. Each section is a self-contained experiment with its own runs; +the ids match `notes/calibration-ledger.md`. + +```{r e01, results="asis"} +cal_section("_e01_eta_settings.Rmd", list(workers = params$workers)) +``` + +```{r e09, results="asis"} +cal_section("_e09_lr_delay.Rmd", list(workers = params$workers)) +``` + +```{r e14, results="asis"} +cal_section("_e14_lr_by_season.Rmd", list(workers = params$workers)) +``` + +```{r e16, results="asis"} +cal_section("_e16_sqrt_constant_lr.Rmd", list(workers = params$workers)) +``` diff --git a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd b/reports/writeups/calibration_experiments/learning_truth.Rmd similarity index 99% rename from reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd rename to reports/writeups/calibration_experiments/learning_truth.Rmd index 5ea3fe92..799a3bd6 100644 --- a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd +++ b/reports/writeups/calibration_experiments/learning_truth.Rmd @@ -1,5 +1,5 @@ --- -title: "E00: learning from vintages instead of finalized truth" +title: "Learning truth (E00): learning from vintages instead of finalized truth" date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" output: html_document: diff --git a/reports/writeups/calibration_experiments/tracker_structure.Rmd b/reports/writeups/calibration_experiments/tracker_structure.Rmd new file mode 100644 index 00000000..731ca65c --- /dev/null +++ b/reports/writeups/calibration_experiments/tracker_structure.Rmd @@ -0,0 +1,40 @@ +--- +title: "Tracker structure" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 3 +params: + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +``` + +What shape should the offset have? All four sections calibrate the submitted flu ensemble: the working scale (E03), the leak and the slow term (E04), the warm start from the burn-in season (E05), and a constant against an adaptive learning rate (E11). Each section is a self-contained experiment with its own runs; +the ids match `notes/calibration-ledger.md`. + +```{r e03, results="asis"} +cal_section("_e03_scale.Rmd", list(workers = params$workers)) +``` + +```{r e04, results="asis"} +cal_section("_e04_offset_structure.Rmd", list(workers = params$workers)) +``` + +```{r e05, results="asis"} +cal_section("_e05_warm_start.Rmd", list(workers = params$workers)) +``` + +```{r e11, results="asis"} +cal_section("_e11_constant_lr.Rmd", list(workers = params$workers)) +``` diff --git a/scripts/calibration/render_experiments.R b/scripts/calibration/render_experiments.R index bf07134e..7ccdc8a5 100644 --- a/scripts/calibration/render_experiments.R +++ b/scripts/calibration/render_experiments.R @@ -1,7 +1,7 @@ # Render calibration experiment notebooks into reports/calibration_experiments/ # and rebuild its index.html (an overview plus links to every rendered notebook). # -# Usage: Rscript scripts/calibration/render_experiments.R [e00_vintage_backtest ...] +# Usage: Rscript scripts/calibration/render_experiments.R [learning_rate ...] # With no arguments, renders every notebook; `--index` renders only the index. src_dir <- here::here("reports/writeups/calibration_experiments") @@ -14,7 +14,7 @@ notebooks <- if (identical(args, "--index")) { } else if (length(args) > 0) { file.path(src_dir, paste0(sub("[.]Rmd$", "", args), ".Rmd")) } else { - list.files(src_dir, pattern = "^e[0-9]+_.*[.]Rmd$", full.names = TRUE) + setdiff(list.files(src_dir, pattern = "^[^_].*[.]Rmd$", full.names = TRUE), file.path(src_dir, "index.Rmd")) } for (nb in notebooks) { rmarkdown::render(nb, output_dir = out_dir, envir = new.env(), quiet = TRUE) From dc7dbef8195e18e3d070c5ce27de7eafc9aa68ab Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:47:14 -0700 Subject: [PATCH 079/107] docs(roadmap): calibration finalists, burn-in no longer critical path, add evaluation-pipeline calibration item --- notes/ROADMAP.md | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index 526a4f7c..3ded1a2a 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -7,10 +7,11 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. ## Forecast evaluation -1. **Use the evaluation project as the calibration testbed** (parked - 2026-10-01 as validation for after the operating point is chosen; the - operating-point work runs on the hub harness, `notes/calibration-ledger.md`). - Done so far: a1–a4, b1–b4 (E12, E13). Calibrating needs full quantile forecasts over many rounds, +1. **Use the evaluation project as the calibration testbed.** The + operating-point work (hub harness, `notes/calibration-ledger.md`) is down + to two finalists, REF-op and sqrt constant 0.018, which no single-run + comparison separates (E18). Done so far: a1–a4, b1–b4 (E12, E13). Next: + c, then item 1f. Calibrating needs full quantile forecasts over many rounds, not just scores. The evaluation project (`flu_hosp_evaluation` / `covid_hosp_evaluation`) replays the current prod components and ensemble weekly since 2024-11-20, which is what calibration would see in @@ -61,9 +62,11 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. Decision (2026-10-01): calibrate h0 and up only. h−1 belongs to the revision-aware methods (item 3) and gets integrated with them later. - b. *2023-24 as a burn-in season.* The best configs so far (E05 - `warm + single`, REF-op) warm-start from a burn-in season, and the - replay has none. Delphi's `hhs` source + b. *2023-24 as a burn-in season.* REF-op warm-starts from a burn-in + season, and the replay has none. The warm start helps only in the + first live season (E05, E18), and sqrt 0.018 needs no burn-in, so if + it is chosen the open level and stitching questions below drop off the + critical path. Delphi's `hhs` source (`confirmed_admissions_{influenza,covid}_1d`) has real issue history for 2023-24 (CA 2023-12-01: issues 12-06, 12-08, 12-20, 12-22), until HHS reporting ended 2024-04-30. Steps, each checked before the next: @@ -114,9 +117,20 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. ensemble weights as well as exclusions, so "no hand edits" for `ensemble_mix` means one fixed weight block for every date. Decide which block (the default one at the top of the file, or the current - one), check the resolved weights on one date, then replay. Only - needed once a single component's calibration results look worth - extending to what we submit. + one), check the resolved weights on one date, then replay. Needed + now: the finalists are chosen on the single component, and what we + submit is the ensemble. + f. *Calibration in the evaluation pipeline.* The calibration targets in + `pipelines/{flu,covid}_hosp_prod.R` run in evaluation mode too, but + there they calibrate the submitted `CMU-TimeSeries` history from the + hub checkout plus only the current replayed round; learn from + `hhs_evaluation_data` (the latest version, so final truth, not exact); + hard-code REF-op; and are skipped when `EVALUATION_FORECASTERS` drops + an ensemble component. To evaluate calibration there: calibrate the + replayed forecasts over all dates, learn from per-round snapshots + (`calibrate_hub_forecasts_exact()`), take the config as a parameter so + both finalists run, and score states only. A burn-in for REF-op needs + the HHS stitching (b), which today exists only in the harness. **Later (not needed to evaluate calibration):** @@ -231,9 +245,9 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. no burn-in. - CMU-climate_baseline US forecast is too low (item 2b). - `us_method` spec column and removing the `usa` alias (item 2b). -- Calibration WIS summaries pool raw counts over all locations, so US is - about half of every "WIS change %" in calibration-ledger.md. b4 reports states - only as well; re-reading the older ledger waits. +- Ensemble-base calibration headline tables (E00, E03–E05, E09–E11) pool + raw counts over all locations, so US is about half of their "WIS change + %". The replay experiments report states only (except E12). ## Tech debt From fe0e6dc51b967fcf9fd590bf79ceaf5527abba08 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:47:55 -0700 Subject: [PATCH 080/107] docs(ilinet): compare v5 fluview_ilinet against epidatr fluview; v5 matches outside NY --- notes/ROADMAP.md | 5 + notes/ilinet-sources.md | 158 +++++++++ scripts/one_offs/compare_ilinet_sources.py | 356 +++++++++++++++++++++ 3 files changed, 519 insertions(+) create mode 100644 notes/ilinet-sources.md create mode 100644 scripts/one_offs/compare_ilinet_sources.py diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index 3ded1a2a..e6951bf6 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -149,6 +149,11 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. shares the scale question with NSSP-target scoring (item 2). Prototype on one forecaster and a few seasons; `calibration_ili_backfill.R` (ILI+, no vintages) is a reference, not the base to extend. + Source (checked 2026-10-02, `notes/ilinet-sources.md`): v5 + `fluview_ilinet` matches epidatr `fluview` at every issue outside NY + (99.99% of state rows), with version date = CDC release Friday. State + version history starts 2017w40 in both, so a vintage backtest has + 2017-18 on; use state `ili` (v5 has no state `wili`). 2. **NSSP-target backtesting is second-class and needs dedicated attention.** - Explore forecasts NHSN only: every family sets `outcome = "hhs"`. NSSP diff --git a/notes/ilinet-sources.md b/notes/ilinet-sources.md new file mode 100644 index 00000000..d7010e22 --- /dev/null +++ b/notes/ilinet-sources.md @@ -0,0 +1,158 @@ +# ILINet sources: epidata v4 `fluview` vs v5 `fluview_ilinet` + +Checks whether v5 can replace the v4 `fluview` endpoint as the issue-versioned ILINet archive for flu +backtests. Script: `scripts/one_offs/compare_ilinet_sources.py` (`fetch`, then `analyze`; the full table +dump is in `cache/ilinet/analysis_output.md`). Data pulled 2026-10-02. Latest issue in both: 202637 (report 2026-09-25). + +## The two sources + +**v4** `https://api.delphi.cmu.edu/epidata/fluview/` (`epidatr::pub_fluview`). It stores one row per +`(region, epiweek, issue)`, so every issue repeats the whole window of weeks that CDC re-published. Columns: +`wili, ili, num_ili, num_patients, num_providers, num_age_0..5, release_date, lag`. Regions: `nat`, `hhs1-10`, +`cen1-9`, states, `jfk` (NYC), `ny` (all of NY, which v4 builds itself). One call with `regions=ca`, +`epiweeks=199701-203053`, `issues=199701-203053` returns everything for a region (no row cap was hit at 93k rows). + +**v5** `https://delphi.cmu.edu/epidata/v5`, source `fluview_ilinet`. +- Signals: `ili, wili, num_ili, num_patients, num_providers`. Geo types: `nation` (`us`), `state`, `hhs`, `census_division`. +- Extra key `age_group` (`all`, plus `0-4, 5-24, 25-49, 25-64, 50-64, 65+` on `num_ili`). Filter on `age_group = all`. +- `fill_method` is `source`, except v5 `ny` (all of NY), which has fill_method `nyc_plus_ny_minus_nyc` because it is derived. +- `/archive/` stores changes only: you get a row when a `(signal, geo, reference_time, age_group)` value changes. + To rebuild an issue, carry the last change forward (as-of join). `/snapshot/?snapshot_date=D` does that on the server. +- `reference_time` is the Saturday that ends the MMWR week. `report_time` is a UTC timestamp, midnight except for the newest row. +- The deployed `/archive/` takes only `source, signal, geo_type, fill_method, extra_keys, limit, use_pagination, + report_time_query, columns, format, header`. The `geo_value` and `reference_times` filters in `../cast-api` are + not deployed yet, so pull a whole `(signal, geo_type)` at once. That's 10 requests and 36 MB of CSV for state plus nation. +- Auth: send `token: $DELPHI_EPIDATA_KEY` as a header (same key as v4). Anonymous archive calls are limited to 5/min and 60/hour. + +``` +GET /epidata/v5/archive/?source=fluview_ilinet&signal=ili&geo_type=state (header token: …) +GET /epidata/v5/metadata/?source=fluview_ilinet +``` + +### Geo mapping + +| v4 | v5 | note | +|---|---|---| +| `nat` | `us` (geo_type `nation`) | | +| `jfk` | `nyc` | NYC; v5 stops at week 2025-09-27 | +| `ny` | `ny` with fill_method `nyc_plus_ny_minus_nyc` | all of NY; v5 stops at week 2025-09-27 | +| none | `ny_minus_nyc` | upstate only; v5 continues past 2025-09 | +| other states, `dc pr vi` | same codes | | + +v4 `ny` and v5 `ny` are both sums of NYC and upstate, but they disagree slightly. Only 32% of `ili` and `num_patients` +values match exactly, with a median relative difference of 2e-5. The NYC component matches exactly, so v4's upstate +component (or the revision it summed) is a few patients off. Both sources stop publishing NYC and the NY total +after week 2025-39. Only v5 `ny_minus_nyc` continues, and I did not check whether it switches to the whole state. + +## Coverage + +| | v4 nat | v5 nat | v4 states | v5 states | +|---|---|---|---|---| +| geos | 1 | 1 | 54 | 54 + `ny_minus_nyc` | +| weeks | 1997-10-04 – 2026-09-19 | same | 2010-10-09 – 2026-09-19 | same | +| issues / reports | 200347 – 202637 | 2003-11-28 – 2026-09-25 | 201740 – 202637 | 2017-10-20 – 2026-09-25 | +| rows (ili) | 56,898 | 20,172 (changes) | 1,966,481 | 93,498 (changes) | +| state `wili` | n/a | n/a | yes, always equal to `ili` (100%) | absent | + +- Both sources have state issues only from 2017w40. Before that, state values exist only as the 2017w40 backfill. + "CA history back to 2010" therefore means weeks back to 2010, not issues. +- Both skip the same releases: 202539–202544 (October 2025, when CDC published nothing) and 202350 for every state except `pr`. + In 202350 (released 2023-12-22), both sources have only nat and `pr`. +- Value changes since 2019w40 are almost equal (state ili: 55,928 in v4 vs 56,915 in v5). v5 has slightly more + because of the extra `ny_minus_nyc` geo and a few off-cycle reports. +- In the latest version, these weeks exist in v4 only: + - `fl`: 574 weeks from 2010-10-09 to 2021-10-02 (v5 Florida starts at week 2021-10-09). + - `la`, `vi`, `pr`: 313, 261 and 156 old weeks (2010 to 2013–2016) that appear only in v4's 2017w40 backfill. + - `dc`: one week, 2022-02-26. +- `ny_minus_nyc` is in v5 only (833 weeks). + +## Version alignment + +**The v5 `report_time` date is the v4 `release_date`: the Friday CDC publishes FluView, normally the issue +week's Saturday plus 6 days.** 1,062 of 1,071 distinct v5 report dates equal a v4 national release date. In the +remaining 9, v4 stamped a release a day or more late for some geos (August 2025, 2017-10-24, 2014). Holiday-delayed +releases (9–10 days after week end, 34 issues) fall on the same date in both sources. + +Shift test (state + nat `ili`, issues ≥ 2019w40): agreement between v4 issue W and v5 read as of `release_date + k`: + +| k (days) | -7 | -1 | 0 | +1 | +6 | +7 | +|---|---|---|---|---|---|---| +| % within 5e-6 | 93.68 | 93.87 | **97.74** | 97.74 | 97.54 | 95.27 | + +(The ceiling of 97.7 comes from the NY sum difference. Without NY it is 99.99.) There is no off-by-one-week +offset. To reproduce v4 issue W from v5, take the last change with `report_time::date <= ew_end(W) + 6`, or +better, v4's own `release_date`, which covers delayed releases. In a backtest with a forecast date D, the v5 as-of-D +snapshot is the v4 data available on D, so no translation to issues is needed. + +## Value agreement + +Each v4 `(geo, week, issue)` row is compared with the v5 value as of that issue's release date. + +| geo | signal | n | % v5 missing | % exact | % within 5e-6 | max abs diff | +|---|---|---|---|---|---|---| +| nat | ili | 56,898 | 0 | 99.88 | 99.97 | 0.029 | +| nat | wili | 56,898 | 0 | 99.79 | 99.97 | 0.103 | +| nat | counts (each) | 56,898 | 0 | 99.97–99.98 | 99.97–99.98 | — | +| state | ili | 1,966,481 | 1.11 | 97.49 | 97.68 | 8.18 | +| state | num_ili | 1,966,481 | 1.11 | 98.86 | 98.86 | 1430 | +| state | num_patients | 1,966,481 | 1.11 | 97.67 | 97.67 | 104,269 | + +Excluding NY, the 2017w40–41 backfill issues and rows that v5 doesn't have: + +| geo | signal | n | % exact | max abs diff | +|---|---|---|---|---| +| nat | all 5 | 36,306 | 100 | 0 | +| state | ili | 1,888,666 | 99.99 | 8.18 (VT, below) | +| state | num_ili / num_patients / num_providers | 1,888,762 | 99.99 | 92 / 6886 / 14 | + +The two sources use the same units: ILI is a percent (e.g. 6.47), not a proportion. Neither rounds differently. +The national "exact vs within 5e-6" gap in 2009–2012 issues is float noise (≤1.3e-8). + +The residual mismatches, by cause: +1. **NY sum** (about 23.6k ili rows, median diff 7e-5): described in the geo mapping section. +2. **2017w40/41 backfill** (about 90 rows in 37 states and 14 nat weeks, weeks 2017-05 to 2017-10): v4's first state + issue (release_date 2017-10-24, filled in later) and nat 201741 differ by 0.002–1.2 ILI points from v5. +3. **VT, issues 202502–202513**: v4 has `num_patients = 10/11` for weeks 202440–202501, which pushes ili up to about + 8 points high. v5 never records those values and keeps 6375 until the zero in 202514, which both sources have. + v4 looks like it captured a corrupt CDC publication that v5 missed or rejected. +4. **DC week 2022-02-26**: v4 has `num_patients = 10` from 202208 to 202339, while v5 has 0 throughout. +5. **v5 missing** (1.1% of state rows): FL before 2021-10 (19.5k rows), and LA/PR/VI old weeks in v4's 201740 backfill. + A few hundred more rows come from 201740 issues released before v5's first report. + +## Revision spot checks + +The as-of trajectories of CA, TX and nat (wili) for weeks 201952, 202250 and 202350 match v4 exactly at every +issue: the same values, revised on the same Fridays. Examples: +- CA 201952: 3.803 → 4.924 → … → 4.699, with the last change in the 202026 release. +- CA 202350: first value in 202351 in both sources (202350 missing in both), last revision on 2024-10-04. +- nat wili 202350: 24 revisions through 202531, all identical. + +NY has the same revision timing with a constant offset of about 2e-4. The full trajectories are in `cache/ilinet/analysis_output.md`. + +## Finalized values (latest version, all weeks) + +| geo | signal | geo-weeks | v4 only | v5 only | % exact | max abs diff | +|---|---|---|---|---|---|---| +| nat | ili / wili / counts | 1417 | 0 | 0 | 99.7–100 (100 within 1e-8) | 1e-8 | +| state | ili | 45,356 | 1305 | 833 | 98.94 (99.3 within 5e-6) | 0.00096 (NY) | +| state | num_ili / num_patients / num_providers | 45,356 | 1304 | 833 | 99.99 / 99.3 / 100 | 2 / 56 / 0 | + +All finalized state differences are in NY, plus DC 2022-02-26. + +## Recommendation + +v5 can replace v4 for state and national ILINet backtests. Exact agreement is 99.99% outside NY, and version +dates and revision trajectories match one for one. Read v5 as of the forecast date, or as of the v4 `release_date` +to reproduce an issue. Caveats: +- **State wILI**: v5 has no state `wili`. In v4 it is a copy of `ili`, so use `ili` for states. +- **NY**: use v5 `ny` with fill_method `nyc_plus_ny_minus_nyc`. Small differences from v4 `ny` (≈2e-5 relative) + shift historical scores slightly. After 2025-09 there is no NYC or NY total in either source, so define what "NY" means from 2025-26 on. +- **Florida before 2021-10**: v4 has 574 weeks that v5 lacks. This only matters if a training window reaches before + 2021-22 for FL (it has no issue history before 2017w40 anyway). +- **State issue history starts 2017w40 in both sources.** For a state as-of before October 2017, neither source helps. +- **VT 2025 and DC 2022-02-26**: v4 contains bad values (num_patients = 10) that v5 doesn't have. v5 is the + cleaner archive here, but a backtest replayed on v5 won't reproduce whatever a 2025 forecast saw from v4 for VT + in January–March 2025. +- **API**: until `geo_value` and `reference_times` filters are deployed, pull whole `(signal, geo_type)` archives with + the token header (cheap: 36 MB) and do the as-of join locally (DuckDB `ASOF JOIN`). Don't use `/snapshot/` per + date (3/min anonymous). Not compared: HHS and census regions, and age-group counts (v5 has them only for national `num_ili`). diff --git a/scripts/one_offs/compare_ilinet_sources.py b/scripts/one_offs/compare_ilinet_sources.py new file mode 100644 index 00000000..9e957c28 --- /dev/null +++ b/scripts/one_offs/compare_ilinet_sources.py @@ -0,0 +1,356 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = ["duckdb", "typer"] +# /// +"""Compare versioned ILINet from epidata v4 `fluview` against v5 `fluview_ilinet`. + +Usage: + uv run scripts/one_offs/compare_ilinet_sources.py fetch # download into cache/ilinet/ + uv run scripts/one_offs/compare_ilinet_sources.py analyze # print markdown tables + +Needs DELPHI_EPIDATA_KEY in the environment (used for both APIs). Findings are in +notes/ilinet-sources.md. +""" + +import json +import os +import time +import urllib.parse +import urllib.request +from pathlib import Path + +import duckdb +import typer + +ROOT = Path(__file__).parents[2] +CACHE = ROOT / "cache" / "ilinet" +V4 = "https://api.delphi.cmu.edu/epidata/fluview/" +V5 = "https://delphi.cmu.edu/epidata/v5/archive/" +SIGNALS = ["wili", "ili", "num_ili", "num_patients", "num_providers"] +V5_GEO_TYPES = ["state", "nation"] +# v4 region codes for states + national. v4 `jfk` is NYC (v5 `nyc`) and v4 `ny` is all of NY +# (v5 `ny` with fill_method nyc_plus_ny_minus_nyc). v5 `ny_minus_nyc` has no v4 counterpart. +STATES = ( + "ak al ar az ca co ct dc de fl ga hi ia id il in ks ky la ma md me mi mn mo ms mt nc nd ne nh nj " + "nm nv ny nyc jfk oh ok or pa pr ri sc sd tn tx ut va vi vt wa wi wv wy" +).split() +V4_REGIONS = ["nat", *STATES] + +app = typer.Typer(add_completion=False) + + +def _get(url: str) -> bytes: + for attempt in range(4): + try: + with urllib.request.urlopen(url, timeout=600) as resp: + return resp.read() + except Exception as exc: # noqa: BLE001 + print(f" retry {attempt + 1} after {exc}") + time.sleep(10 * (attempt + 1)) + raise RuntimeError(f"failed: {url.split('?')[0]}") + + +@app.command() +def fetch() -> None: + """Download both sources into cache/ilinet/, skipping files already present.""" + key = os.environ["DELPHI_EPIDATA_KEY"] + CACHE.mkdir(parents=True, exist_ok=True) + for geo_type in V5_GEO_TYPES: + for signal in SIGNALS: + out = CACHE / f"v5_{geo_type}_{signal}.csv" + if out.exists(): + continue + qs = urllib.parse.urlencode( + {"source": "fluview_ilinet", "signal": signal, "geo_type": geo_type, "token": key} + ) + print(f"v5 {geo_type} {signal}") + out.write_bytes(_get(f"{V5}?{qs}")) + time.sleep(2) + for region in V4_REGIONS: + out = CACHE / f"v4_{region}.json" + if out.exists(): + continue + qs = urllib.parse.urlencode( + {"regions": region, "epiweeks": "199701-203053", "issues": "199701-203053", "api_key": key} + ) + print(f"v4 {region}") + body = json.loads(_get(f"{V4}?{qs}")) + if body["result"] != 1: + print(f" {region}: {body['message']}") + out.write_text(json.dumps(body.get("epidata", []))) + time.sleep(1) + + +def connect() -> duckdb.DuckDBPyConnection: + """Load both sources as long tables keyed by (geo, week_end, signal, version date).""" + con = duckdb.connect() + con.execute("SET TimeZone = 'UTC'") + con.execute(f""" + CREATE TABLE v5 AS + SELECT signal, + CAST(report_time AS TIMESTAMP) AS report_time, + CAST(report_time AS DATE) AS report_date, + CASE geo_value WHEN 'us' THEN 'nat' WHEN 'nyc' THEN 'jfk' ELSE geo_value END AS geo, + geo_type, + CAST(reference_time AS DATE) AS week_end, + value + FROM read_csv('{CACHE}/v5_*.csv', union_by_name = true) + WHERE age_group = 'all' + """) + unpivot = " UNION ALL ".join( + f"SELECT region AS geo, epiweek, issue, CAST(release_date AS DATE) AS raw_release_date, lag, " + f"'{s}' AS signal, CAST({s} AS DOUBLE) AS value FROM raw WHERE {s} IS NOT NULL" + for s in SIGNALS + ) + con.execute(f""" + CREATE TABLE raw AS SELECT * FROM read_json('{CACHE}/v4_*.json', format = 'array', + columns = {{region: 'VARCHAR', epiweek: 'INT', issue: 'INT', release_date: 'VARCHAR', lag: 'INT', + wili: 'DOUBLE', ili: 'DOUBLE', num_ili: 'DOUBLE', num_patients: 'DOUBLE', + num_providers: 'DOUBLE'}}); + -- Saturday ending MMWR week: week 1 starts on the Sunday on or before Jan 4. + CREATE MACRO ew_end(ew) AS CAST( + make_date(CAST(ew // 100 AS INT), 1, 4) + - to_days(CAST(dayofweek(make_date(CAST(ew // 100 AS INT), 1, 4)) AS INT)) + + to_days(CAST((ew % 100) * 7 - 1 AS INT)) AS DATE); + CREATE TABLE v4 AS + -- Some state rows lack release_date; those fall back to the Friday after the issue week. + SELECT u.*, ew_end(epiweek) AS week_end, ew_end(issue) AS issue_week_end, + coalesce(raw_release_date, ew_end(issue) + 6) AS release_date + FROM ({unpivot}) u; + """) + return con + + +def md(con: duckdb.DuckDBPyConnection, title: str, sql: str) -> None: + rel = con.sql(sql) + cols = rel.columns + rows = rel.fetchall() + print(f"\n### {title}\n") + print("| " + " | ".join(cols) + " |") + print("|" + "---|" * len(cols)) + for r in rows: + print("| " + " | ".join("" if v is None else (f"{v:.6g}" if isinstance(v, float) else str(v)) for v in r) + " |") + + +@app.command() +def analyze() -> None: + """Print the coverage, alignment, agreement, revision and finalized-value tables.""" + con = connect() + geo_class = "CASE WHEN geo = 'nat' THEN 'nat' ELSE 'state' END" + + # MMWR week sanity check: 202350 ends 2023-12-16. + assert str(con.sql("SELECT ew_end(202350)").fetchone()[0]) == "2023-12-16" + + md(con, "Coverage: v4 fluview (issue rows)", f""" + SELECT {geo_class} AS geo_class, signal, count(DISTINCT geo) AS n_geo, + min(week_end) AS first_week, max(week_end) AS last_week, + min(issue) AS first_issue, max(issue) AS last_issue, count(*) AS n_rows, + median(n) AS med_issues_per_geo_week + FROM (SELECT *, count(*) OVER (PARTITION BY geo, signal, epiweek) AS n FROM v4) + GROUP BY ALL ORDER BY ALL + """) + md(con, "Coverage: v5 fluview_ilinet (change rows)", f""" + SELECT {geo_class} AS geo_class, signal, count(DISTINCT geo) AS n_geo, + min(week_end) AS first_week, max(week_end) AS last_week, + min(report_date) AS first_report, max(report_date) AS last_report, count(*) AS n_rows, + median(n) AS med_versions_per_geo_week + FROM (SELECT *, count(*) OVER (PARTITION BY geo, signal, week_end) AS n FROM v5) + GROUP BY ALL ORDER BY ALL + """) + md(con, "Geos in one source only", """ + SELECT 'v4 only' AS side, string_agg(DISTINCT geo, ' ') AS geos FROM v4 WHERE geo NOT IN (SELECT geo FROM v5) + UNION ALL + SELECT 'v5 only', string_agg(DISTINCT geo, ' ') FROM v5 WHERE geo NOT IN (SELECT geo FROM v4) + """) + + # Version alignment: weekday of v5 report dates, and their relation to v4 release dates. + md(con, "v5 report_time weekday/hour distribution", """ + SELECT dayname(report_time) AS weekday, count(DISTINCT report_time) AS n_report_times, + min(report_date) AS first, max(report_date) AS last + FROM v5 GROUP BY ALL ORDER BY n_report_times DESC + """) + con.execute(""" + CREATE TABLE releases AS + SELECT issue, issue_week_end, min(release_date) AS release_date, max(release_date) AS max_release_date + FROM v4 WHERE geo = 'nat' GROUP BY ALL; + CREATE TABLE v5_reports AS + SELECT DISTINCT report_time, report_date FROM v5; + """) + md(con, "v4 release_date minus issue week end (days)", """ + SELECT release_date - issue_week_end AS days_after_week_end, count(*) AS n_issues, + min(issue) AS first_issue, max(issue) AS last_issue + FROM releases GROUP BY ALL ORDER BY n_issues DESC LIMIT 10 + """) + md(con, "v5 report dates matched to v4 release dates", """ + SELECT CASE WHEN r.release_date IS NOT NULL THEN 'equals a v4 release_date' + WHEN r2.release_date IS NOT NULL THEN 'within 6 days after a v4 release_date' + ELSE 'no nearby v4 release' END AS match, + count(*) AS n_report_times, min(v.report_date) AS first, max(v.report_date) AS last + FROM v5_reports v + LEFT JOIN releases r ON v.report_date = r.release_date + LEFT JOIN (SELECT DISTINCT release_date FROM releases) r2 + ON v.report_date > r2.release_date AND v.report_date <= r2.release_date + 6 AND r.release_date IS NULL + GROUP BY ALL ORDER BY ALL + """) + md(con, "v5 report times not on a v4 release date", """ + SELECT v.report_time, count(*) AS n_rows, min(week_end) AS min_week, max(week_end) AS max_week + FROM v5 v WHERE report_date NOT IN (SELECT release_date FROM releases) + GROUP BY ALL ORDER BY n_rows DESC LIMIT 15 + """) + + # v5 as of each v4 release date (v5 is change-only, so carry the last change forward). + con.execute(""" + CREATE TABLE paired AS + SELECT a.geo, a.signal, a.epiweek, a.week_end, a.issue, a.release_date, a.lag, + a.value AS v4_value, b.value AS v5_value, b.report_date AS v5_report_date + FROM v4 a ASOF LEFT JOIN v5 b + ON a.geo = b.geo AND a.signal = b.signal AND a.week_end = b.week_end + AND a.release_date >= b.report_date + """) + agree = f""" + SELECT {{grp}}, count(*) AS n, + round(100 * avg(CASE WHEN v5_value IS NULL THEN 1 ELSE 0 END), 2) AS pct_v5_missing, + round(100 * avg(CASE WHEN v4_value = v5_value THEN 1 ELSE 0 END), 2) AS pct_exact, + round(100 * avg(CASE WHEN abs(v4_value - v5_value) <= 5e-6 THEN 1 ELSE 0 END), 2) AS pct_within_5e6, + round(100 * avg(CASE WHEN abs(v4_value - v5_value) <= 1e-3 * greatest(abs(v4_value), 1) THEN 1 ELSE 0 END), 2) AS pct_within_0p1pct, + max(abs(v4_value - v5_value)) AS max_abs_diff + FROM paired {{where}} GROUP BY ALL ORDER BY ALL + """ + md(con, "Agreement at v4 (geo, week, issue), v5 as of the release date", agree.format( + grp=f"{geo_class} AS geo_class, signal", where="")) + md(con, "Agreement by issue season (ili, all geos)", agree.format( + grp="CASE WHEN issue % 100 >= 40 THEN issue // 100 ELSE issue // 100 - 1 END AS season_start", + where="WHERE signal = 'ili'")) + md(con, "Agreement on the lag-0 (first) issue", agree.format( + grp=f"{geo_class} AS geo_class, signal", where="WHERE lag = 0")) + md(con, "Agreement restricted to issues >= 2019w40 and v5 present", agree.format( + grp=f"{geo_class} AS geo_class, signal", where="WHERE issue >= 201940 AND v5_value IS NOT NULL")) + md(con, "Agreement excluding ny (v4 sums NY itself), the 2017w40-41 backfill issues, and rows v5 lacks", agree.format( + grp=f"{geo_class} AS geo_class, signal", where="WHERE geo != 'ny' AND issue > 201741 AND v5_value IS NOT NULL")) + md(con, "Largest mismatches (v5 present, diff > 5e-6)", """ + SELECT geo, signal, epiweek, issue, release_date, v5_report_date, v4_value, v5_value, + v4_value - v5_value AS diff + FROM paired WHERE v5_value IS NOT NULL AND abs(v4_value - v5_value) > 5e-6 + ORDER BY abs(v4_value - v5_value) / greatest(abs(v4_value), 1) DESC LIMIT 20 + """) + md(con, "Mismatch (diff > 5e-6, v5 present) counts by geo, top 15", """ + SELECT geo, count(*) AS n_mismatch, count(DISTINCT issue) AS n_issues, min(issue) AS first_issue, + max(issue) AS last_issue + FROM paired WHERE v5_value IS NOT NULL AND abs(v4_value - v5_value) > 5e-6 + GROUP BY ALL ORDER BY n_mismatch DESC LIMIT 15 + """) + # Shift test: does v5 as of release_date + k days agree better, i.e. is the version off by a week? + md(con, "Why v5 is missing at a v4 (geo, week, issue)", f""" + SELECT {geo_class} AS geo_class, + CASE WHEN p.release_date < f.first_report THEN 'v4 release before first v5 report of the week' + WHEN f.first_report IS NULL THEN 'week never in v5' + ELSE 'other' END AS reason, + count(*) AS n, min(p.issue) AS first_issue, max(p.issue) AS last_issue, + string_agg(DISTINCT p.geo, ' ') AS geos + FROM paired p LEFT JOIN (SELECT geo, signal, week_end, min(report_date) AS first_report FROM v5 GROUP BY ALL) f + USING (geo, signal, week_end) + WHERE p.v5_value IS NULL AND p.signal = 'ili' GROUP BY ALL ORDER BY ALL + """) + md(con, "Mismatches (ili, v5 present, diff > 5e-6) by geo", """ + SELECT geo, count(*) AS n_mismatch, count(DISTINCT issue) AS n_issues, min(issue) AS first_issue, + max(issue) AS last_issue, count(DISTINCT epiweek) AS n_weeks, min(epiweek) AS first_week, + max(epiweek) AS last_week, median(abs(v4_value - v5_value)) AS median_abs_diff + FROM paired WHERE signal = 'ili' AND v5_value IS NOT NULL AND abs(v4_value - v5_value) > 5e-6 + GROUP BY ALL ORDER BY n_mismatch DESC LIMIT 20 + """) + md(con, "Mismatches (ili, v5 present, diff > 5e-6, geo != ny) by issue, top 15", """ + SELECT issue, release_date, count(*) AS n_mismatch, count(DISTINCT geo) AS n_geo, + min(epiweek) AS first_week, max(epiweek) AS last_week, max(abs(v4_value - v5_value)) AS max_abs_diff + FROM paired WHERE signal = 'ili' AND geo != 'ny' AND v5_value IS NOT NULL AND abs(v4_value - v5_value) > 5e-6 + GROUP BY ALL ORDER BY n_mismatch DESC LIMIT 15 + """) + shift = " UNION ALL ".join( + f"""SELECT {k} AS k, abs(a.value - b.value) <= 5e-6 AS ok FROM + (SELECT *, release_date + {k} AS t FROM v4 WHERE signal = 'ili' AND issue >= 201940) a + ASOF LEFT JOIN v5 b ON a.geo = b.geo AND a.signal = b.signal AND a.week_end = b.week_end + AND a.t >= b.report_date""" + for k in [-7, -1, 0, 1, 6, 7] + ) + md(con, "Version shift test (ili, issues >= 2019w40): agreement when v5 is read k days after release", f""" + SELECT k, count(*) AS n, round(100 * avg(CASE WHEN ok THEN 1 ELSE 0 END), 2) AS pct_within_5e6 + FROM ({shift}) GROUP BY k ORDER BY k + """) + + # v5 change rows that v4 never shows as a value change on that release. + con.execute(""" + CREATE TABLE v4_changes AS + SELECT * FROM ( + SELECT *, lag(value) OVER (PARTITION BY geo, signal, epiweek ORDER BY issue) AS prev + FROM v4) WHERE prev IS NULL OR prev != value + """) + md(con, "Number of value changes per (geo, signal, week), issues >= 2019w40 vs v5 reports >= 2019-10-01", f""" + WITH a AS (SELECT {geo_class} AS g, signal, epiweek, count(*) AS n FROM v4_changes + WHERE issue >= 201940 GROUP BY ALL), + b AS (SELECT {geo_class} AS g, signal, week_end, count(*) AS n FROM v5 + WHERE report_date >= DATE '2019-10-01' GROUP BY ALL) + SELECT 'v4' AS src, g, signal, sum(n) AS n_changes, avg(n) AS mean_per_week FROM a GROUP BY ALL + UNION ALL SELECT 'v5', g, signal, sum(n), avg(n) FROM b GROUP BY ALL ORDER BY g, signal, src + """) + + # Finalized: latest version per (geo, signal, week). + con.execute(""" + CREATE TABLE fin AS + SELECT coalesce(a.geo, b.geo) AS geo, coalesce(a.signal, b.signal) AS signal, + coalesce(a.week_end, b.week_end) AS week_end, a.value AS v4_value, b.value AS v5_value + FROM (SELECT * FROM v4 QUALIFY row_number() OVER (PARTITION BY geo, signal, week_end ORDER BY issue DESC) = 1) a + FULL JOIN (SELECT * FROM v5 QUALIFY row_number() OVER (PARTITION BY geo, signal, week_end ORDER BY report_time DESC) = 1) b + USING (geo, signal, week_end) + """) + md(con, "Finalized (latest) values", f""" + SELECT {geo_class} AS geo_class, signal, count(*) AS n_geo_weeks, + sum(CASE WHEN v5_value IS NULL THEN 1 ELSE 0 END) AS v4_only, + sum(CASE WHEN v4_value IS NULL THEN 1 ELSE 0 END) AS v5_only, + round(100 * avg(CASE WHEN v4_value = v5_value THEN 1.0 WHEN v4_value IS NULL OR v5_value IS NULL THEN NULL ELSE 0 END), 2) AS pct_exact, + round(100 * avg(CASE WHEN abs(v4_value - v5_value) <= 5e-6 THEN 1.0 WHEN v4_value IS NULL OR v5_value IS NULL THEN NULL ELSE 0 END), 2) AS pct_within_5e6, + max(abs(v4_value - v5_value)) AS max_abs_diff + FROM fin GROUP BY ALL ORDER BY ALL + """) + md(con, "Finalized mismatches (diff > 5e-6), largest", """ + SELECT geo, signal, week_end, v4_value, v5_value, v4_value - v5_value AS diff FROM fin + WHERE abs(v4_value - v5_value) > 5e-6 + ORDER BY abs(v4_value - v5_value) / greatest(abs(v4_value), 1) DESC LIMIT 15 + """) + md(con, "Finalized coverage gaps by geo (weeks present in one source only)", f""" + SELECT geo, signal, sum(CASE WHEN v5_value IS NULL THEN 1 ELSE 0 END) AS v4_only, + min(CASE WHEN v5_value IS NULL THEN week_end END) AS v4_only_first, + max(CASE WHEN v5_value IS NULL THEN week_end END) AS v4_only_last, + sum(CASE WHEN v4_value IS NULL THEN 1 ELSE 0 END) AS v5_only, + min(CASE WHEN v4_value IS NULL THEN week_end END) AS v5_only_first, + max(CASE WHEN v4_value IS NULL THEN week_end END) AS v5_only_last + FROM fin WHERE signal = 'ili' GROUP BY ALL + HAVING v4_only + v5_only > 0 ORDER BY v4_only + v5_only DESC LIMIT 20 + """) + + md(con, "v4 state wili vs ili (v5 has no state wili)", """ + SELECT count(*) AS n, avg(CASE WHEN a.value = b.value THEN 1 ELSE 0 END) AS share_equal + FROM v4 a JOIN v4 b USING (geo, epiweek, issue) + WHERE a.signal = 'wili' AND b.signal = 'ili' AND a.geo != 'nat' + """) + md(con, "Issues (>= 2017w40) missing for some geos in v4", """ + WITH s AS (SELECT DISTINCT geo, issue FROM v4 WHERE signal = 'ili'), + i AS (SELECT DISTINCT issue FROM v4 WHERE issue >= 201740), + g AS (SELECT DISTINCT geo FROM s) + SELECT issue, count(*) AS n_geo_missing FROM g CROSS JOIN i ANTI JOIN s USING (geo, issue) + GROUP BY ALL ORDER BY issue + """) + + # Revision trajectories for spot checks: first 8 issues plus any issue where either source changes. + for geo, signal in [("ca", "ili"), ("ny", "ili"), ("tx", "ili"), ("nat", "wili")]: + for ew in [201952, 202250, 202350]: + md(con, f"Trajectory {geo} {signal} epiweek {ew}", f""" + SELECT issue, release_date, lag, v4_value, v5_value, v5_report_date, + round(v4_value - v5_value, 6) AS diff + FROM (SELECT *, lag(v4_value) OVER w AS p4, lag(v5_value) OVER w AS p5 FROM paired + WHERE geo = '{geo}' AND signal = '{signal}' AND epiweek = {ew} + WINDOW w AS (ORDER BY issue)) + WHERE lag <= 8 OR v4_value IS DISTINCT FROM p4 OR v5_value IS DISTINCT FROM p5 + ORDER BY issue + """) + +if __name__ == "__main__": + app() From 0fd7e40d8df82bb3ffa6e402408126e23b77ee76 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:11:07 -0700 Subject: [PATCH 081/107] docs(calibration): findings for E18 warm branch and E10 finalists; ledger records clean rerun and finalists summary --- notes/calibration-ledger.md | 46 +++++++++++++------ .../_e10_flu_covid.Rmd | 39 ++++++++++++++++ .../_e18_ref_op_bridge.Rmd | 16 +++++++ 3 files changed, 88 insertions(+), 13 deletions(-) diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index f6a6c06d..a765ba34 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -95,9 +95,27 @@ Coverage: - **Season-pooled coverage gains are cancellation.** For example, E11's "4–5× lower bias" is about 2× per season, and nothing month by month. -Operating point: nothing tested clearly beats the prod config (REF-op). -Choosing between the candidates above needs uncertainty estimates and more -seasons, not more single-run comparisons. +Finalists (E10 part 1, E18), states only, h1/h2/h3 pooled over both seasons: + +- **Flu: sqrt 0.018 with a warm start leads on both bases.** Ensemble + +2.5/+2.7/+4.3 vs REF-op warm +1.9/+1.5/+2.3; replay +1.6/+1.5/+2.7 vs + +1.1/+0.8/+1.2. On the ensemble all of its lead is 2024-25; in 2025-26 it + is −0.2/−0.9/−1.7, slightly below REF-op warm. Month-avg coverage: on the + ensemble REF-op warm is the only config below the base at every horizon; + sqrt warm is level with the base there and 0.009–0.017 above it at h2–h3 + on the replay. +- **Covid: sqrt 0.018 is worse than REF-op cold on the ensemble.** + −2.7/−3.7/−4.2 vs −0.1/−0.3/−0.8, from 2025-26 (−6.9 to −8.3 at h1–h3; + REF-op has the lower WIS in 44–45 of 52 states). At h−1 sqrt 0.018 gains + the most of any config, +14.9 vs REF-op +10.2 (E10 part 2). No config + helps covid at h1–h3 on either base. +- **The season flip is broad.** REF-op beats sqrt 0.018 cold in 41–45 of + 52 flu states in 2024-25 and 14–18 in 2025-26 (E18). + +Operating point: for covid nothing tested beats REF-op cold (prod) at +h0–h3. For flu, sqrt 0.018 warm is ahead of REF-op warm (prod) by about 1 +point at h1–h2 and 2 at h3 on the ensemble, all of it in 2024-25, with +month-avg coverage closer to the base than REF-op's. ## Reference configs @@ -146,7 +164,7 @@ gating), exact learning truth, spoiled submissions excluded. | E05 | `tracker_structure` | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | | E07 | `flu_covid` (E07), `finalists` (E07b, the candidates) | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | | | E09 | `learning_rate` | ensemble | constant rate × revision delay | knee 0.056: +3 to +5.5 over REF-paper at h1–h3; REF-op +1.2 to +2.4 over knee. Extra delay costs about 1 h−1 point per week. Above 0.1 loses steeply | month-avg: no constant rate beats base at h1–h3 | -| E10 | `flu_covid` | covid ensemble | REF-paper and REF-op on covid | h−1 +10 to +11. REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 at h1–h3 in 2025-26 | headline and month tables run to 2026-09-19, including the 2026 summer wave | +| E10 | `flu_covid` | flu + covid, ensemble + replay | the finalists on both diseases and bases (part 1); REF-paper, REF-op and sqrt 0.018 on the covid ensemble at h−1–h3 (part 2) | see "Finalists" above. Part 2: h−1 +10 to +15 (sqrt 0.018 largest). REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 and sqrt 0.018 −7.7 to −9.5 at h1–h3 in 2025-26 | part 1 is h0–h3 on rows both bases have; part 2 tables run to 2026-09-19, including the 2026 summer wave | | E11 | `tracker_structure` | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | | E12 | none (`calibration_ws_replay.R`) | replay, cold, flu + covid | first look at the clean replay | flu: REF-op cold leads constant 0.1 by 0–2.5. Covid: REF-op cold h0 +1.3 states, losses at h1–h3 in 2025-26 | **all locations** (US is 47% of WIS); covid h0 gains are mostly US | | E13 | none (`calibration_ws_replay.R burn_in`) | replay, flu + covid | 2023-24 HHS burn-in | flu warm +1 to +2.6 in 2024-25, ≤ 0 in 2025-26. Covid: warm worse at h0–h1, better at h2–h3 (+1.3, +4.4); constant 0.1 warm −8 to −27 | HHS used as-is (3–9% below NHSN); coverage is season-pooled only | @@ -154,7 +172,7 @@ gating), exact learning truth, spoiled submissions excluded. | E15 | `finalists` | replay, cold | where the WIS gain comes from | offsets a few % of interval width; REF-op gain from the lower half and the median shift | width-only variant's coverage not measured; split is not additive | | E16 | `learning_rate` | replay, cold | constant rate on sqrt scale | sqrt 0.018 about +1 over rate constants at h0; at h1–h3 within 0.2 pooled, up to +0.7 in 2025-26. vs REF-op cold: behind in 2024-25 h0–h1, ahead elsewhere; month-avg coverage tie | 0.018 picked in-sample | | E17 | `late_season` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | -| E18 | `finalists` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance) | +| E18 | `finalists` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008. Branch sqrt 0.018 warm: +3.4/+1.6/+1.6/+2.5 pooled, best of all, but month-avg coverage 0.011–0.018 worse than base at h2–h3 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance) | E06 (ILI+ burn-in) and E08 (old covid) were run on code or data later found broken and were not rebuilt. @@ -162,10 +180,14 @@ broken and were not rebuilt. ## Known issues - **Run cache ignores code.** `cal_run_cached()` keys on inputs and - arguments, not code. Some cached runs date from 2026-09-30 before the - exact-learning commit landed that evening. The E00 and E09 caches are - keyed by chunk code or string. Plan: clear the whole cache and re-run - everything exact from scratch. + arguments, not code; the E00 and E09 caches are keyed by chunk code or + name. Clear `cache/calibration/experiments/` after changing + `R/calibration/`. + +Checked on 2026-10-02: the whole cache was cleared and every experiment +re-run from scratch (E12, E13 and all six notebooks). Every table matched +the cached results cell for cell, so no result above came from a stale +run. The pre-rerun cache is in `cache/calibration/_stale_2026-10-02/`. Checked and resolved on 2026-10-02: @@ -186,7 +208,7 @@ Checked and resolved on 2026-10-02: |---|---| | E00, E01 | verified | | E02, E07, E09 | commented; changes made, not re-verified | -| E18 | changes made (reference rows), not reviewed | +| E10 part 1, E18 | changes made (finalists runs, state counts), not reviewed | | E03, E04, E05, E10, E11, E14, E15, E16, E17 | not reviewed | On 2026-10-02 every notebook's introduction and Findings were rewritten to @@ -280,11 +302,9 @@ Checked against the code on 2026-10-02. Repo-wide items are in | item | status | |---|---| | Uncertainty for WIS and coverage differences (bootstrap over rounds or locations) | open; needed before any ranking under ~2 points means anything | -| Clear the run cache and re-run everything exact | open | | Report ramp and post-peak months separately | open; only the month view exists | -| sqrt constant 0.018 with a warm start | open | | Shrink only the played offset after the peak, keep the stored state | open; `late_decay` shrinks the stored state | -| Covid with sqrt constant; covid at h−1/h0 only | open; "h0 only" can be read from existing columns | +| Disease-specific operating point (flu sqrt 0.018 warm vs REF-op; covid h−1-only sqrt 0.018) | open; E10 numbers in "Finalists" | | h−1 calibration | blocked on the revision-method backtests (ROADMAP 3) | | HHS burn-in level (3–9% below NHSN) | open; E13 used it as-is | | ILI+ burn-in on the fixed replay | open; replay rebuilt 2026-09-23, nothing re-run | diff --git a/reports/writeups/calibration_experiments/_e10_flu_covid.Rmd b/reports/writeups/calibration_experiments/_e10_flu_covid.Rmd index 54edf6d6..061e9537 100644 --- a/reports/writeups/calibration_experiments/_e10_flu_covid.Rmd +++ b/reports/writeups/calibration_experiments/_e10_flu_covid.Rmd @@ -159,6 +159,41 @@ for (d in names(FIN)) { } ``` +# Findings, part 1 + +States only, h0/h1/h2/h3, WIS reduction % (positive = calibrated WIS lower +than base). No uncertainty estimates; gaps under 1 point read as no clear +difference. + +- **Flu, ensemble:** sqrt 0.018 warm +4.4/+2.5/+2.7/+4.3 pooled, REF-op + warm +3.9/+1.9/+1.5/+2.3, both cold configs within ±0.7 at h1–h3. All of + the warm configs' lead is 2024-25 (sqrt warm +4.4/+4.0/+4.9/+7.9; REF-op + warm wins 49–51 of 52 states against sqrt cold). In 2025-26 every config + is within 1 point of the base at h1–h2, and both warm configs lose at h3 + (sqrt warm −1.7, REF-op warm −0.9). +- **Flu, replay:** as E18: sqrt 0.018 warm +3.4/+1.6/+1.5/+2.7, REF-op warm + +2.8/+1.1/+0.8/+1.2. Here sqrt warm stays positive in 2025-26 + (+3.3/+0.9/+0.8/+1.3) where on the ensemble it does not, so the replay + overstates its 2025-26 result for what we submit. +- **Flu coverage, month-avg:** on the ensemble, REF-op warm is the only + config below the base at every horizon (0.090/0.110/0.124/0.138 vs + 0.116/0.123/0.137/0.151); sqrt warm is close to the base at h1–h3 + (0.119/0.133/0.147). On the replay sqrt warm is 0.009–0.017 above the base + at h2–h3 (E18). +- **Covid, ensemble:** REF-op cold (covid prod) +0.8/−0.1/−0.3/−0.8 pooled; + sqrt 0.018 +0.1/−2.7/−3.7/−4.2. Both gain a little in 2024-25; in 2025-26 + sqrt 0.018 loses 6.9–8.3 at h1–h3 against REF-op's 2.0–2.9, and REF-op + has the lower WIS in 44–45 of 52 states at h1–h3. Month-avg coverage is + within 0.011 of the base for both, and 0.004–0.006 above it at h2–h3. +- **Covid, replay:** nothing is consistently positive beyond h0. Warm + starts (from the HHS backfill) lose at h0–h2 in 2024-25 (sqrt warm −1.3 + to −5.1) and gain at h3 in 2025-26 (+4.1, +5.0). Pooled, the best h3 is + REF-op warm +2.8, the best h1 sqrt 0.018 cold −1.8 and REF-op cold −1.1. +- **So:** on flu the warm sqrt 0.018 leads on both bases, but on the + ensemble its whole lead is the first live season. On covid sqrt 0.018 is + clearly worse than REF-op cold on what we submit, and no config tested + helps at h1–h3. + # Part 2: covid ensemble, h−1–h3 - Disease: covid. Base forecaster: the submitted `CMU-TimeSeries` ensemble. @@ -226,3 +261,7 @@ wave; those Aug–Sep 2026 rounds are in no per-season table. 0.029–0.051 (2025-26). - **By month at h3:** REF-op −16.5% in September and −12.0% in October (together 20% of base WIS); REF-paper −42% to −49% in April–May (about 6%). +- **sqrt 0.018 (added 2026-10-02):** the largest h−1 gain, +14.9 pooled + (+14.9/+14.1 by season) against REF-op's +10.2, and +24.4% median absolute + error. At h1–h3 it is −3.3/−4.2/−4.1 pooled and −9.5/−9.1/−7.7 in 2025-26, + worse than REF-op (−3.0/−4.0/−4.3) and better than REF-paper. diff --git a/reports/writeups/calibration_experiments/_e18_ref_op_bridge.Rmd b/reports/writeups/calibration_experiments/_e18_ref_op_bridge.Rmd index f24f5518..7354f634 100644 --- a/reports/writeups/calibration_experiments/_e18_ref_op_bridge.Rmd +++ b/reports/writeups/calibration_experiments/_e18_ref_op_bridge.Rmd @@ -277,3 +277,19 @@ h0/h1/h2/h3, both seasons pooled unless stated. - **References:** rate 0.032 +1.3/+0.7/+0.5/+0.4 and rate 0.056 +1.3/+0.5/+0.3/+0.0 pooled. Month-avg coverage: rate 0.032 equals the base; rate 0.056 is 0.008–0.014 above it. +- **Branch, sqrt 0.018 warm:** +3.4/+1.6/+1.6/+2.5 pooled, the highest of + any config here, 0.5–1.4 above REF-op and 0.8–1.9 above sqrt 0.018 cold. + In 2024-25 +3.4/+2.1/+2.2/+3.3 (above REF-op by 0.2–1.7); in 2025-26 + +3.4/+0.9/+0.8/+1.2 (above REF-op by 1.0–1.2, within 0.3 of sqrt 0.018 + cold at h1–h3). Only the h3 gap to REF-op exceeds 1 point pooled. +- **Branch coverage costs month-avg:** 0.098/0.126/0.164/0.195 pooled + against base 0.112/0.127/0.153/0.177 and REF-op 0.095/0.121/0.150/0.175, + so 0.011–0.018 worse than the base at h2–h3. In 2025-26 it is 0.026–0.029 + above the base at h2–h3. Season-pooled it looks best at h0–h2 + (0.040/0.034/0.032), which is the ramp/post-peak cancellation. +- **State by state:** in 2024-25 REF-op beats sqrt 0.018 cold in 41–45 of + 52 states at every horizon (median gap +1.5 to +1.7); in 2025-26 in only + 14–18 (median −0.4 to −2.0). So the season flip is broad, not a few large + states. The warm branch beats sqrt 0.018 cold in 37–50 states in 2024-25 + and 20–30 in 2025-26. Against REF-op it wins 19/21/26/41 of 52 states + in 2024-25 and 33/36/36/41 in 2025-26 (h0/h1/h2/h3). From 6a7195f58277c90df75e90d87529b9918dba1fd5 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:54:14 -0700 Subject: [PATCH 082/107] docs(calibration): roadmap and index state the finalists result --- notes/ROADMAP.md | 9 +++++---- reports/writeups/calibration_experiments/index.Rmd | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index e6951bf6..aedf4280 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -8,10 +8,11 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. ## Forecast evaluation 1. **Use the evaluation project as the calibration testbed.** The - operating-point work (hub harness, `notes/calibration-ledger.md`) is down - to two finalists, REF-op and sqrt constant 0.018, which no single-run - comparison separates (E18). Done so far: a1–a4, b1–b4 (E12, E13). Next: - c, then item 1f. Calibrating needs full quantile forecasts over many rounds, + operating-point work (hub harness, `notes/calibration-ledger.md`, + "Finalists") is down to a disease-specific choice: covid stays on REF-op + cold (sqrt 0.018 loses 3–4 points at h1–h3), flu is REF-op warm or sqrt + 0.018 warm (ahead by 1–2 points, all in 2024-25). Done so far: a1–a4, + b1–b4 (E12, E13). Next: c, then item 1f. Calibrating needs full quantile forecasts over many rounds, not just scores. The evaluation project (`flu_hosp_evaluation` / `covid_hosp_evaluation`) replays the current prod components and ensemble weekly since 2024-11-20, which is what calibration would see in diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index d41ada14..bfb4b033 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -46,7 +46,7 @@ Six notebooks, each with one section per experiment. The ids match | | base | question | what the numbers show | |---|---|---|---| -| `r link("E18", "finalists", "e18")` | replay | Stepping from REF-op to sqrt constant 0.018 one setting at a time | REF-op vs sqrt 0.018: no strong result (0.3–0.6 points pooled, order flips by season). The warm start helps in 2024-25 only. Constant 0.018 is 1.7–2.3 points above cold adaptive at h2–h3. No step moves month-avg coverage by more than 0.008. | +| `r link("E18", "finalists", "e18")` | replay | Stepping from REF-op to sqrt constant 0.018 one setting at a time | REF-op vs sqrt 0.018: no strong result (0.3–0.6 points pooled, order flips by season). The warm start helps in 2024-25 only. Constant 0.018 is 1.7–2.3 points above cold adaptive at h2–h3. No step moves month-avg coverage by more than 0.008. sqrt 0.018 warm (branch) is the best pooled (+3.4/+1.6/+1.6/+2.5) but 0.011–0.018 worse than the base on month-avg coverage at h2–h3. REF-op beats sqrt 0.018 cold in 41–45 of 52 states in 2024-25 and 14–18 in 2025-26. | | `r link("E07b", "finalists", "e07b")` | both | Do the operating-point candidates rank the same way on both bases? | The candidate order flips between seasons on both. | | `r link("E15", "finalists", "e15")` | replay, cold | Where does the WIS change come from? | Offsets are 2–5% of the 90% interval width. REF-op's gain (+2.0 at h0, about 0 by h3) comes from raising the lower quantiles; a median shift alone matches it within 0.6 points. | @@ -54,7 +54,7 @@ Six notebooks, each with one section per experiment. The ids match | | base | question | what the numbers show | |---|---|---|---| -| `r link("E10", "flu_covid", "e10")` | both diseases, both bases | How do the finalists do on flu and covid? (Part 2: covid ensemble at h−1–h3 with REF-paper.) | Part 2: h−1 improves about 10–11% in both seasons. REF-paper loses 10–19% at h1–h3 in 2025-26. REF-op at h1–h3 is −4 to +3, sign depending on the season. Part 1: not yet read. | +| `r link("E10", "flu_covid", "e10")` | both diseases, both bases | How do the finalists do on flu and covid? (Part 2: covid ensemble at h−1–h3 with REF-paper.) | Part 2: h−1 improves about 10–11% in both seasons. REF-paper loses 10–19% at h1–h3 in 2025-26. REF-op at h1–h3 is −4 to +3, sign depending on the season. Part 1: covid, nothing beats REF-op cold at h0–h3 and sqrt 0.018 loses 3–4 points at h1–h3; flu, sqrt 0.018 warm leads on both bases by 1–2 points at h1–h3, all in 2024-25 on the ensemble. sqrt 0.018 has the largest covid h−1 gain (+14.9). | | `r link("E07", "flu_covid", "e07")` | both | Does calibration act the same on the ensemble and the clean replay? | Same direction on both; REF-op gains 0.7–1.1 points more on the ensemble. | ## `r nb_link("Learning rate", "learning_rate")` From 1a11e92bb62b791ce603003af17f44b8c993c5a8 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:40:23 -0700 Subject: [PATCH 083/107] refactor(calibration): restore per-experiment notebooks with the index grouping them by topic; close stale open threads and record the provisional operating point --- Justfile | 2 +- R/calibration/views.R | 58 +--------------- notes/ROADMAP.md | 25 ++++--- notes/calibration-ledger.md | 69 +++++++++---------- ...ing_truth.Rmd => e00_vintage_backtest.Rmd} | 2 +- ..._eta_settings.Rmd => e01_eta_settings.Rmd} | 0 ..._eta_variants.Rmd => e02_eta_variants.Rmd} | 0 .../{_e03_scale.Rmd => e03_scale.Rmd} | 0 ...structure.Rmd => e04_offset_structure.Rmd} | 0 ..._e05_warm_start.Rmd => e05_warm_start.Rmd} | 0 ...07_across_eras.Rmd => e07_across_eras.Rmd} | 0 ...07b_candidates.Rmd => e07b_candidates.Rmd} | 0 .../{_e09_lr_delay.Rmd => e09_lr_delay.Rmd} | 0 .../{_e10_flu_covid.Rmd => e10_covid.Rmd} | 0 ...11_constant_lr.Rmd => e11_constant_lr.Rmd} | 0 ..._lr_by_season.Rmd => e14_lr_by_season.Rmd} | 0 ...15_wis_sources.Rmd => e15_wis_sources.Rmd} | 0 ...nstant_lr.Rmd => e16_sqrt_constant_lr.Rmd} | 0 ..._e17_late_decay.Rmd => e17_late_decay.Rmd} | 0 ...ef_op_bridge.Rmd => e18_ref_op_bridge.Rmd} | 0 .../calibration_experiments/finalists.Rmd | 36 ---------- .../calibration_experiments/flu_covid.Rmd | 32 --------- .../calibration_experiments/index.Rmd | 59 ++++++++-------- .../calibration_experiments/late_season.Rmd | 32 --------- .../calibration_experiments/learning_rate.Rmd | 40 ----------- .../tracker_structure.Rmd | 40 ----------- 26 files changed, 78 insertions(+), 317 deletions(-) rename reports/writeups/calibration_experiments/{learning_truth.Rmd => e00_vintage_backtest.Rmd} (99%) rename reports/writeups/calibration_experiments/{_e01_eta_settings.Rmd => e01_eta_settings.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e02_eta_variants.Rmd => e02_eta_variants.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e03_scale.Rmd => e03_scale.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e04_offset_structure.Rmd => e04_offset_structure.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e05_warm_start.Rmd => e05_warm_start.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e07_across_eras.Rmd => e07_across_eras.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e07b_candidates.Rmd => e07b_candidates.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e09_lr_delay.Rmd => e09_lr_delay.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e10_flu_covid.Rmd => e10_covid.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e11_constant_lr.Rmd => e11_constant_lr.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e14_lr_by_season.Rmd => e14_lr_by_season.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e15_wis_sources.Rmd => e15_wis_sources.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e16_sqrt_constant_lr.Rmd => e16_sqrt_constant_lr.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e17_late_decay.Rmd => e17_late_decay.Rmd} (100%) rename reports/writeups/calibration_experiments/{_e18_ref_op_bridge.Rmd => e18_ref_op_bridge.Rmd} (100%) delete mode 100644 reports/writeups/calibration_experiments/finalists.Rmd delete mode 100644 reports/writeups/calibration_experiments/flu_covid.Rmd delete mode 100644 reports/writeups/calibration_experiments/late_season.Rmd delete mode 100644 reports/writeups/calibration_experiments/learning_rate.Rmd delete mode 100644 reports/writeups/calibration_experiments/tracker_structure.Rmd diff --git a/Justfile b/Justfile index d3b91644..9d8e22b0 100644 --- a/Justfile +++ b/Justfile @@ -1,6 +1,6 @@ # R isn't installed on the host; run recipes inside the distrobox: `distrobox enter rocker -- just `. rscript := env_var_or_default("RSCRIPT", "Rscript") -# Render calibration experiment notebooks (all, or the named ones, e.g. finalists) and the index. +# Render calibration experiment notebooks (all, or the named ones, e.g. e18_ref_op_bridge) and the index. calibration-experiments *notebooks: {{rscript}} scripts/calibration/render_experiments.R {{notebooks}} diff --git a/R/calibration/views.R b/R/calibration/views.R index 3005cf52..256306c4 100644 --- a/R/calibration/views.R +++ b/R/calibration/views.R @@ -485,63 +485,11 @@ cal_month_plot <- function(month_tbl, season, colours = NULL, linetype = NULL, c CAL_NOTEBOOK_DIR <- "reports/writeups/calibration_experiments" -#' Markdown heading marks for a heading written by `cat()` inside a section: -#' level `n`, pushed down by the section's shift. +#' Markdown heading marks for a heading written by `cat()`: level `n`, pushed +#' down when knit inside `cal_month_section()`. #' @export cal_h <- function(n) strrep("#", n + getOption("cal.hshift", 0L)) -#' Knit a section notebook (`_eNN_*.Rmd`) into a combined notebook. -#' -#' The section's title becomes a heading with id `#eNN`, its own headings -#' (and those of the children it includes) move down `shift` levels, and it -#' runs in its own environment with `params` from its YAML, overridden by -#' `params`. Call from a chunk with `results = "asis"`. See -#' notes/calibration-ledger.md, "Notebooks". -#' @export -cal_section <- function(file, params = list(), shift = 1L, envir = parent.frame()) { - path <- here::here(CAL_NOTEBOOK_DIR, file) - yaml <- rmarkdown::yaml_front_matter(path) - id <- sub("^_", "", tools::file_path_sans_ext(file)) - lines <- cal_shift_headings(cal_expand_children(readLines(path)), shift) - env <- new.env(parent = envir) - env$params <- utils::modifyList(yaml$params %||% list(), params) - old <- options(cal.hshift = shift, knitr.duplicate.label = "allow") - old_chunk <- knitr::opts_chunk$get() - on.exit({ - options(old) - knitr::opts_chunk$restore(old_chunk) - }) - out <- knitr::knit_child( - text = lines, envir = env, quiet = TRUE, - options = list(fig.path = paste0(knitr::opts_chunk$get("fig.path"), id, "-")) - ) - cat("\n\n", strrep("#", shift), " ", yaml$title, " {#", sub("_.*", "", id), "}\n\n", out, "\n\n", sep = "") -} - -# Drop the YAML header and the root.dir setting (the combined notebook sets -# it), and inline `child=` chunks, recursively, so their headings can be -# shifted with the rest. -cal_expand_children <- function(lines) { - if (length(lines) > 0 && lines[[1]] == "---") { - end <- which(lines == "---")[2] - lines <- lines[-seq_len(end)] - } - lines <- lines[!grepl("^knitr::opts_knit\\$set\\(root.dir", lines)] - out <- character(0) - i <- 1L - while (i <= length(lines)) { - m <- regmatches(lines[[i]], regexec("^```\\{r child=(.*)\\}\\s*$", lines[[i]]))[[1]] - if (length(m) == 2) { - out <- c(out, cal_expand_children(readLines(eval(parse(text = m[[2]]))))) - i <- i + 2L - } else { - out <- c(out, lines[[i]]) - i <- i + 1L - } - } - out -} - # Add `shift` levels to every markdown heading outside code chunks. cal_shift_headings <- function(lines, shift) { in_code <- FALSE @@ -608,7 +556,7 @@ cal_season_of <- function(d) { } #' Knit the month view (`_month_view.Rmd`) in the calling environment under -#' its own heading, `level` levels below the section's top. For notebooks that +#' its own heading, `level` levels below the notebook's top. For notebooks that #' show more than one month view; uses `mv` and the optional `mv_*` settings #' from `envir`. Call from a chunk with `results = "asis"`. #' @export diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index aedf4280..dd6065b8 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -9,10 +9,11 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. 1. **Use the evaluation project as the calibration testbed.** The operating-point work (hub harness, `notes/calibration-ledger.md`, - "Finalists") is down to a disease-specific choice: covid stays on REF-op - cold (sqrt 0.018 loses 3–4 points at h1–h3), flu is REF-op warm or sqrt - 0.018 warm (ahead by 1–2 points, all in 2024-25). Done so far: a1–a4, - b1–b4 (E12, E13). Next: c, then item 1f. Calibrating needs full quantile forecasts over many rounds, + "Finalists") is down to a disease-specific choice. Provisional + (2026-10-02, to confirm after the notebook review, not implemented): + covid REF-op cold (sqrt 0.018 loses 3–4 points at h1–h3), flu REF-op warm + or sqrt 0.018 warm (ahead by 1–2 points, all in 2024-25). Done so far: + a1–a4, b1–b4 (E12, E13). Next: item 1f, in its own thread. Calibrating needs full quantile forecasts over many rounds, not just scores. The evaluation project (`flu_hosp_evaluation` / `covid_hosp_evaluation`) replays the current prod components and ensemble weekly since 2024-11-20, which is what calibration would see in @@ -114,13 +115,6 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. 2020-08 (issues before 2023-07 collapse to one version), stitched before NHSN's first version; a 2024-12-04 control matches the store exactly. Results: E13 in `notes/calibration-ledger.md`. - c. *The ensemble with fixed weights.* The geo-exclusions CSVs carry the - ensemble weights as well as exclusions, so "no hand edits" for - `ensemble_mix` means one fixed weight block for every date. Decide - which block (the default one at the top of the file, or the current - one), check the resolved weights on one date, then replay. Needed - now: the finalists are chosen on the single component, and what we - submit is the ensemble. f. *Calibration in the evaluation pipeline.* The calibration targets in `pipelines/{flu,covid}_hosp_prod.R` run in evaluation mode too, but there they calibrate the submitted `CMU-TimeSeries` history from the @@ -135,6 +129,15 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. **Later (not needed to evaluate calibration):** + c. *The ensemble with fixed weights.* The geo-exclusions CSVs carry the + ensemble weights as well as exclusions, so "no hand edits" for + `ensemble_mix` means one fixed weight block for every date. Decide + which block (the default one at the top of the file, or the current + one), check the resolved weights on one date, then replay. Not needed + to choose the operating point: E10 already scores the finalists on + the submitted ensemble, and E07 shows calibration acts the same way + on the ensemble and the single-component replay. Revisit if 1f needs + a clean ensemble history. d. *Hand edits on vs off as its own question.* A separate evaluation project (like the `_regr` ones) replaying with the edits, to measure whether the weekly substitutions and weight changes help. The only diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index a765ba34..f23d6e77 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -130,17 +130,11 @@ Both are `cal_ref_args()` in `R/calibration/views.R`; notebooks run them via `cal_run()`, which also picks the learning truth (`"exact"`: re-run from scratch each round on that round's data, as prod does). -**Notebooks.** Six notebooks in `reports/writeups/calibration_experiments/` -(`finalists`, `flu_covid`, `learning_rate`, `tracker_structure`, -`late_season`, `learning_truth`), rendered to `reports/calibration_experiments/` -with `just calibration-experiments [name]`. Each experiment is a section -file `_eNN_*.Rmd`; a notebook knits its sections with `cal_section()` -(`R/calibration/views.R`), which gives each its own environment, YAML -`params` and figure prefix, and shifts its headings down one level (headings -written with `cat()` use `cal_h()` so they shift too). The index links each -experiment as `.html#eNN`. `learning_truth` is E00 rendered on its -own, so that its knitr chunk cache (keyed on chunk options, including the -figure path) still hits. +**Notebooks.** One notebook per experiment, `eNN_*.Rmd` in +`reports/writeups/calibration_experiments/`, rendered to +`reports/calibration_experiments/` with `just calibration-experiments [name]`. +The index groups them by topic (finalists, flu and covid, learning rate, +tracker structure, late season, learning truth). Standard views (`_standard_views.Rmd`, `_month_view.Rmd`): V-head (WIS and coverage by horizon × season), V-month (by month of reference date, with @@ -156,23 +150,23 @@ gating), exact learning truth, spoiled submissions excluded. | id | notebook | base | question | what the numbers show | caveats | |---|---|---|---|---|---| -| E00 | `learning_truth` | ensemble | learn from final, vintage or exact data? | exact ≈ final within 0.6 points except REF-op h−1 (2.2, 2024-25 only). Vintage at settle 7 loses the h−1 gain. Settle 7 vs 14: REF-paper +1 to +3 at every horizon; REF-op tie (flips by season) | | -| E01 | `learning_rate` | replay, cold | adaptive eta settings | mult ≥ 0.1 costs 7–81 at h1–h3. 0.01 beats 0.03 by 0.5–2.4 at h1–h3 in both seasons; 0.03 better at h0 in 2024-25 (0.8). Window and carry vs reset: ≤ 2.4, mostly < 1. Count is 2–5 worse than sqrt | month-avg coverage of 0.01 vs 0.03 within noise | -| E02 | `late_season` | replay, cold | `off_after`, per-level eta, seasonal window | off 1 Mar / 15 Feb +2 to +4 at h1–h3 over the 0.03 reference, both seasons, no month-avg coverage cost. Per-level +1. Seasonal window, burn-in, off 1 Apr: < 1 | dates in-sample; half the 2025-26 gain is December carry-over; "seasonal window" also changes `lr_window` 20 → 10 | -| E03 | `tracker_structure` | ensemble | count vs sqrt vs rate | sqrt +1.4 to +3.9 over count pooled, mostly 2025-26. Coverage: neither better (sqrt worse per season pooled, better month-avg only in 2024-25) | rate identical to count in every cell (adaptive rate is scale invariant) | -| E04 | `tracker_structure` | ensemble | leak, slow term | leak +2.5 to +5.6 at h1–h3 in 2025-26, −0.4 to −1.3 in 2024-25; month-avg coverage flips the same way (±0.015). Slow term without warm start: ≤ 0.5 | | -| E05 | `tracker_structure` | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | -| E07 | `flu_covid` (E07), `finalists` (E07b, the candidates) | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | | -| E09 | `learning_rate` | ensemble | constant rate × revision delay | knee 0.056: +3 to +5.5 over REF-paper at h1–h3; REF-op +1.2 to +2.4 over knee. Extra delay costs about 1 h−1 point per week. Above 0.1 loses steeply | month-avg: no constant rate beats base at h1–h3 | -| E10 | `flu_covid` | flu + covid, ensemble + replay | the finalists on both diseases and bases (part 1); REF-paper, REF-op and sqrt 0.018 on the covid ensemble at h−1–h3 (part 2) | see "Finalists" above. Part 2: h−1 +10 to +15 (sqrt 0.018 largest). REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 and sqrt 0.018 −7.7 to −9.5 at h1–h3 in 2025-26 | part 1 is h0–h3 on rows both bases have; part 2 tables run to 2026-09-19, including the 2026 summer wave | -| E11 | `tracker_structure` | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | +| E00 | `e00_vintage_backtest` | ensemble | learn from final, vintage or exact data? | exact ≈ final within 0.6 points except REF-op h−1 (2.2, 2024-25 only). Vintage at settle 7 loses the h−1 gain. Settle 7 vs 14: REF-paper +1 to +3 at every horizon; REF-op tie (flips by season) | | +| E01 | `e01_eta_settings` | replay, cold | adaptive eta settings | mult ≥ 0.1 costs 7–81 at h1–h3. 0.01 beats 0.03 by 0.5–2.4 at h1–h3 in both seasons; 0.03 better at h0 in 2024-25 (0.8). Window and carry vs reset: ≤ 2.4, mostly < 1. Count is 2–5 worse than sqrt | month-avg coverage of 0.01 vs 0.03 within noise | +| E02 | `e02_eta_variants` | replay, cold | `off_after`, per-level eta, seasonal window | off 1 Mar / 15 Feb +2 to +4 at h1–h3 over the 0.03 reference, both seasons, no month-avg coverage cost. Per-level +1. Seasonal window, burn-in, off 1 Apr: < 1 | dates in-sample; half the 2025-26 gain is December carry-over; "seasonal window" also changes `lr_window` 20 → 10 | +| E03 | `e03_scale` | ensemble | count vs sqrt vs rate | sqrt +1.4 to +3.9 over count pooled, mostly 2025-26. Coverage: neither better (sqrt worse per season pooled, better month-avg only in 2024-25) | rate identical to count in every cell (adaptive rate is scale invariant) | +| E04 | `e04_offset_structure` | ensemble | leak, slow term | leak +2.5 to +5.6 at h1–h3 in 2025-26, −0.4 to −1.3 in 2024-25; month-avg coverage flips the same way (±0.015). Slow term without warm start: ≤ 0.5 | | +| E05 | `e05_warm_start` | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | +| E07 | `e07_across_eras`, `e07b_candidates` (the candidates) | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | | +| E09 | `e09_lr_delay` | ensemble | constant rate × revision delay | knee 0.056: +3 to +5.5 over REF-paper at h1–h3; REF-op +1.2 to +2.4 over knee. Extra delay costs about 1 h−1 point per week. Above 0.1 loses steeply | month-avg: no constant rate beats base at h1–h3 | +| E10 | `e10_covid` | flu + covid, ensemble + replay | the finalists on both diseases and bases (part 1); REF-paper, REF-op and sqrt 0.018 on the covid ensemble at h−1–h3 (part 2) | see "Finalists" above. Part 2: h−1 +10 to +15 (sqrt 0.018 largest). REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 and sqrt 0.018 −7.7 to −9.5 at h1–h3 in 2025-26 | part 1 is h0–h3 on rows both bases have; part 2 tables run to 2026-09-19, including the 2026 summer wave | +| E11 | `e11_constant_lr` | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | | E12 | none (`calibration_ws_replay.R`) | replay, cold, flu + covid | first look at the clean replay | flu: REF-op cold leads constant 0.1 by 0–2.5. Covid: REF-op cold h0 +1.3 states, losses at h1–h3 in 2025-26 | **all locations** (US is 47% of WIS); covid h0 gains are mostly US | | E13 | none (`calibration_ws_replay.R burn_in`) | replay, flu + covid | 2023-24 HHS burn-in | flu warm +1 to +2.6 in 2024-25, ≤ 0 in 2025-26. Covid: warm worse at h0–h1, better at h2–h3 (+1.3, +4.4); constant 0.1 warm −8 to −27 | HHS used as-is (3–9% below NHSN); coverage is season-pooled only | -| E14 | `learning_rate` | replay, cold | constant rate grid by season | every rate 0.0032–0.056 is ≥ 0 in both seasons and within 1.2 of each other; 0.018 vs 0.032 ≤ 0.3. Above 0.1 loses | month-avg: no rate ≤ 0.032 beats base beyond 0.01 | -| E15 | `finalists` | replay, cold | where the WIS gain comes from | offsets a few % of interval width; REF-op gain from the lower half and the median shift | width-only variant's coverage not measured; split is not additive | -| E16 | `learning_rate` | replay, cold | constant rate on sqrt scale | sqrt 0.018 about +1 over rate constants at h0; at h1–h3 within 0.2 pooled, up to +0.7 in 2025-26. vs REF-op cold: behind in 2024-25 h0–h1, ahead elsewhere; month-avg coverage tie | 0.018 picked in-sample | -| E17 | `late_season` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | -| E18 | `finalists` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008. Branch sqrt 0.018 warm: +3.4/+1.6/+1.6/+2.5 pooled, best of all, but month-avg coverage 0.011–0.018 worse than base at h2–h3 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance) | +| E14 | `e14_lr_by_season` | replay, cold | constant rate grid by season | every rate 0.0032–0.056 is ≥ 0 in both seasons and within 1.2 of each other; 0.018 vs 0.032 ≤ 0.3. Above 0.1 loses | month-avg: no rate ≤ 0.032 beats base beyond 0.01 | +| E15 | `e15_wis_sources` | replay, cold | where the WIS gain comes from | offsets a few % of interval width; REF-op gain from the lower half and the median shift | width-only variant's coverage not measured; split is not additive | +| E16 | `e16_sqrt_constant_lr` | replay, cold | constant rate on sqrt scale | sqrt 0.018 about +1 over rate constants at h0; at h1–h3 within 0.2 pooled, up to +0.7 in 2025-26. vs REF-op cold: behind in 2024-25 h0–h1, ahead elsewhere; month-avg coverage tie | 0.018 picked in-sample | +| E17 | `e17_late_decay` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | +| E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008. Branch sqrt 0.018 warm: +3.4/+1.6/+1.6/+2.5 pooled, best of all, but month-avg coverage 0.011–0.018 worse than base at h2–h3 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance) | E06 (ILI+ burn-in) and E08 (old covid) were run on code or data later found broken and were not rebuilt. @@ -185,9 +179,9 @@ broken and were not rebuilt. `R/calibration/`. Checked on 2026-10-02: the whole cache was cleared and every experiment -re-run from scratch (E12, E13 and all six notebooks). Every table matched +re-run from scratch (E12, E13 and every notebook). Every table matched the cached results cell for cell, so no result above came from a stale -run. The pre-rerun cache is in `cache/calibration/_stale_2026-10-02/`. +run. Checked and resolved on 2026-10-02: @@ -213,7 +207,7 @@ Checked and resolved on 2026-10-02: On 2026-10-02 every notebook's introduction and Findings were rewritten to state setup and measured sizes only; the rewritten text has not been -reviewed. The index lists E00, E02, E04, E05 and E07 as archived. +reviewed. ## Code, data and prod @@ -297,19 +291,20 @@ keyed on inputs and arguments but not code: clear the cache after changing ## Open threads Checked against the code on 2026-10-02. Repo-wide items are in -`notes/ROADMAP.md`. +`notes/ROADMAP.md`. Add a thread only if its answer could change the +operating point. | item | status | |---|---| | Uncertainty for WIS and coverage differences (bootstrap over rounds or locations) | open; needed before any ranking under ~2 points means anything | -| Report ramp and post-peak months separately | open; only the month view exists | | Shrink only the played offset after the peak, keep the stored state | open; `late_decay` shrinks the stored state | -| Disease-specific operating point (flu sqrt 0.018 warm vs REF-op; covid h−1-only sqrt 0.018) | open; E10 numbers in "Finalists" | -| h−1 calibration | blocked on the revision-method backtests (ROADMAP 3) | -| HHS burn-in level (3–9% below NHSN) | open; E13 used it as-is | -| ILI+ burn-in on the fixed replay | open; replay rebuilt 2026-09-23, nothing re-run | +| Disease-specific operating point | provisional (2026-10-02): covid REF-op cold; flu REF-op warm or sqrt 0.018 warm, to confirm after the notebook review. Not implemented | +| HHS burn-in level (3–9% below NHSN) | used as-is (E13, E18); settle it with item 1f if a warm flu config is chosen | | `windowed_seasonal_extra_sources` retrospective | parked (ROADMAP) | -| Gallery review (do losses sit at turning points?) | stratified picker exists; review not done | | Method ideas: cross-horizon gradients for staleness, data-driven phase gating, scale proxy | open; no code | -| Speed up exact runs (sort-based snapshot lookup, skip isoreg when ordered; checkpoint/resume in `qt_track()`) | open; no resumable state yet | | Collaborator email | unknown | + +Closed 2026-10-02 without further work: ramp vs post-peak split (the month +view shows it), h−1 calibration (left to the revision-aware methods, +ROADMAP 3), ILI+ burn-in (the HHS burn-in covers it), gallery review (review +uses the month view), speeding up exact runs. diff --git a/reports/writeups/calibration_experiments/learning_truth.Rmd b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd similarity index 99% rename from reports/writeups/calibration_experiments/learning_truth.Rmd rename to reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd index 799a3bd6..5ea3fe92 100644 --- a/reports/writeups/calibration_experiments/learning_truth.Rmd +++ b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd @@ -1,5 +1,5 @@ --- -title: "Learning truth (E00): learning from vintages instead of finalized truth" +title: "E00: learning from vintages instead of finalized truth" date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" output: html_document: diff --git a/reports/writeups/calibration_experiments/_e01_eta_settings.Rmd b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e01_eta_settings.Rmd rename to reports/writeups/calibration_experiments/e01_eta_settings.Rmd diff --git a/reports/writeups/calibration_experiments/_e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e02_eta_variants.Rmd rename to reports/writeups/calibration_experiments/e02_eta_variants.Rmd diff --git a/reports/writeups/calibration_experiments/_e03_scale.Rmd b/reports/writeups/calibration_experiments/e03_scale.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e03_scale.Rmd rename to reports/writeups/calibration_experiments/e03_scale.Rmd diff --git a/reports/writeups/calibration_experiments/_e04_offset_structure.Rmd b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e04_offset_structure.Rmd rename to reports/writeups/calibration_experiments/e04_offset_structure.Rmd diff --git a/reports/writeups/calibration_experiments/_e05_warm_start.Rmd b/reports/writeups/calibration_experiments/e05_warm_start.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e05_warm_start.Rmd rename to reports/writeups/calibration_experiments/e05_warm_start.Rmd diff --git a/reports/writeups/calibration_experiments/_e07_across_eras.Rmd b/reports/writeups/calibration_experiments/e07_across_eras.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e07_across_eras.Rmd rename to reports/writeups/calibration_experiments/e07_across_eras.Rmd diff --git a/reports/writeups/calibration_experiments/_e07b_candidates.Rmd b/reports/writeups/calibration_experiments/e07b_candidates.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e07b_candidates.Rmd rename to reports/writeups/calibration_experiments/e07b_candidates.Rmd diff --git a/reports/writeups/calibration_experiments/_e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e09_lr_delay.Rmd rename to reports/writeups/calibration_experiments/e09_lr_delay.Rmd diff --git a/reports/writeups/calibration_experiments/_e10_flu_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e10_flu_covid.Rmd rename to reports/writeups/calibration_experiments/e10_covid.Rmd diff --git a/reports/writeups/calibration_experiments/_e11_constant_lr.Rmd b/reports/writeups/calibration_experiments/e11_constant_lr.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e11_constant_lr.Rmd rename to reports/writeups/calibration_experiments/e11_constant_lr.Rmd diff --git a/reports/writeups/calibration_experiments/_e14_lr_by_season.Rmd b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e14_lr_by_season.Rmd rename to reports/writeups/calibration_experiments/e14_lr_by_season.Rmd diff --git a/reports/writeups/calibration_experiments/_e15_wis_sources.Rmd b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e15_wis_sources.Rmd rename to reports/writeups/calibration_experiments/e15_wis_sources.Rmd diff --git a/reports/writeups/calibration_experiments/_e16_sqrt_constant_lr.Rmd b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e16_sqrt_constant_lr.Rmd rename to reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd diff --git a/reports/writeups/calibration_experiments/_e17_late_decay.Rmd b/reports/writeups/calibration_experiments/e17_late_decay.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e17_late_decay.Rmd rename to reports/writeups/calibration_experiments/e17_late_decay.Rmd diff --git a/reports/writeups/calibration_experiments/_e18_ref_op_bridge.Rmd b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd similarity index 100% rename from reports/writeups/calibration_experiments/_e18_ref_op_bridge.Rmd rename to reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd diff --git a/reports/writeups/calibration_experiments/finalists.Rmd b/reports/writeups/calibration_experiments/finalists.Rmd deleted file mode 100644 index dd9361f2..00000000 --- a/reports/writeups/calibration_experiments/finalists.Rmd +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: "Finalists" -date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" -output: - html_document: - code_folding: hide - toc: true - toc_float: true - toc_depth: 3 -params: - workers: 14 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -``` - -The two operating-point candidates, REF-op (prod) and sqrt constant 0.018. E18 steps from one to the other one setting at a time on the clean flu replay, adds sqrt 0.018 with a warm start, and counts the states each one wins. E07b ranks the wider candidate set on the submitted ensemble and the replay. E15 shows where the WIS gains come from. Each section is a self-contained experiment with its own runs; -the ids match `notes/calibration-ledger.md`. - -```{r e18, results="asis"} -cal_section("_e18_ref_op_bridge.Rmd", list(workers = params$workers)) -``` - -```{r e07b, results="asis"} -cal_section("_e07b_candidates.Rmd", list(workers = params$workers)) -``` - -```{r e15, results="asis"} -cal_section("_e15_wis_sources.Rmd", list(workers = params$workers)) -``` diff --git a/reports/writeups/calibration_experiments/flu_covid.Rmd b/reports/writeups/calibration_experiments/flu_covid.Rmd deleted file mode 100644 index b116b10f..00000000 --- a/reports/writeups/calibration_experiments/flu_covid.Rmd +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: "Flu and covid" -date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" -output: - html_document: - code_folding: hide - toc: true - toc_float: true - toc_depth: 3 -params: - workers: 14 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -``` - -The finalists on both diseases and both bases (the submitted ensemble and the clean `windowed_seasonal` replay). E10 runs them; E07 compares the two flu bases before and after calibration. Each section is a self-contained experiment with its own runs; -the ids match `notes/calibration-ledger.md`. - -```{r e10, results="asis"} -cal_section("_e10_flu_covid.Rmd", list(workers = params$workers)) -``` - -```{r e07, results="asis"} -cal_section("_e07_across_eras.Rmd", list(workers = params$workers)) -``` diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index bfb4b033..00dd3ad6 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -24,69 +24,64 @@ is read as no clear difference. ```{r rows, include=FALSE} out_dir <- here::here("reports/calibration_experiments") -link <- function(id, file, anchor = "") { +link <- function(id, file) { html <- paste0(file, ".html") if (!file.exists(file.path(out_dir, html))) return(paste(id, "(not rendered)")) - sprintf("[%s](%s%s)", id, html, if (nzchar(anchor)) paste0("#", anchor) else "") -} -nb_link <- function(title, file) { - html <- paste0(file, ".html") - if (!file.exists(file.path(out_dir, html))) return(paste(title, "(not rendered)")) - stamp <- format(file.mtime(file.path(out_dir, html)), "%Y-%m-%d", tz = "America/Los_Angeles") - sprintf("[%s](%s) (rendered %s)", title, html, stamp) + stamp <- format(file.mtime(file.path(out_dir, html)), "%m-%d", tz = "America/Los_Angeles") + sprintf("[%s](%s) (%s)", id, html, stamp) } ``` # Notebooks -Six notebooks, each with one section per experiment. The ids match -`notes/calibration-ledger.md`. +One notebook per experiment, grouped by topic; the date after each link is +when it was last rendered. The ids match `notes/calibration-ledger.md`. -## `r nb_link("Finalists", "finalists")` +## Finalists | | base | question | what the numbers show | |---|---|---|---| -| `r link("E18", "finalists", "e18")` | replay | Stepping from REF-op to sqrt constant 0.018 one setting at a time | REF-op vs sqrt 0.018: no strong result (0.3–0.6 points pooled, order flips by season). The warm start helps in 2024-25 only. Constant 0.018 is 1.7–2.3 points above cold adaptive at h2–h3. No step moves month-avg coverage by more than 0.008. sqrt 0.018 warm (branch) is the best pooled (+3.4/+1.6/+1.6/+2.5) but 0.011–0.018 worse than the base on month-avg coverage at h2–h3. REF-op beats sqrt 0.018 cold in 41–45 of 52 states in 2024-25 and 14–18 in 2025-26. | -| `r link("E07b", "finalists", "e07b")` | both | Do the operating-point candidates rank the same way on both bases? | The candidate order flips between seasons on both. | -| `r link("E15", "finalists", "e15")` | replay, cold | Where does the WIS change come from? | Offsets are 2–5% of the 90% interval width. REF-op's gain (+2.0 at h0, about 0 by h3) comes from raising the lower quantiles; a median shift alone matches it within 0.6 points. | +| `r link("E18", "e18_ref_op_bridge")` | replay | Stepping from REF-op to sqrt constant 0.018 one setting at a time | REF-op vs sqrt 0.018: no strong result (0.3–0.6 points pooled, order flips by season). The warm start helps in 2024-25 only. Constant 0.018 is 1.7–2.3 points above cold adaptive at h2–h3. No step moves month-avg coverage by more than 0.008. sqrt 0.018 warm (branch) is the best pooled (+3.4/+1.6/+1.6/+2.5) but 0.011–0.018 worse than the base on month-avg coverage at h2–h3. REF-op beats sqrt 0.018 cold in 41–45 of 52 states in 2024-25 and 14–18 in 2025-26. | +| `r link("E07b", "e07b_candidates")` | both | Do the operating-point candidates rank the same way on both bases? | The candidate order flips between seasons on both. | +| `r link("E15", "e15_wis_sources")` | replay, cold | Where does the WIS change come from? | Offsets are 2–5% of the 90% interval width. REF-op's gain (+2.0 at h0, about 0 by h3) comes from raising the lower quantiles; a median shift alone matches it within 0.6 points. | -## `r nb_link("Flu and covid", "flu_covid")` +## Flu and covid | | base | question | what the numbers show | |---|---|---|---| -| `r link("E10", "flu_covid", "e10")` | both diseases, both bases | How do the finalists do on flu and covid? (Part 2: covid ensemble at h−1–h3 with REF-paper.) | Part 2: h−1 improves about 10–11% in both seasons. REF-paper loses 10–19% at h1–h3 in 2025-26. REF-op at h1–h3 is −4 to +3, sign depending on the season. Part 1: covid, nothing beats REF-op cold at h0–h3 and sqrt 0.018 loses 3–4 points at h1–h3; flu, sqrt 0.018 warm leads on both bases by 1–2 points at h1–h3, all in 2024-25 on the ensemble. sqrt 0.018 has the largest covid h−1 gain (+14.9). | -| `r link("E07", "flu_covid", "e07")` | both | Does calibration act the same on the ensemble and the clean replay? | Same direction on both; REF-op gains 0.7–1.1 points more on the ensemble. | +| `r link("E10", "e10_covid")` | both diseases, both bases | How do the finalists do on flu and covid? (Part 2: covid ensemble at h−1–h3 with REF-paper.) | Part 2: h−1 improves about 10–11% in both seasons. REF-paper loses 10–19% at h1–h3 in 2025-26. REF-op at h1–h3 is −4 to +3, sign depending on the season. Part 1: covid, nothing beats REF-op cold at h0–h3 and sqrt 0.018 loses 3–4 points at h1–h3; flu, sqrt 0.018 warm leads on both bases by 1–2 points at h1–h3, all in 2024-25 on the ensemble. sqrt 0.018 has the largest covid h−1 gain (+14.9). | +| `r link("E07", "e07_across_eras")` | both | Does calibration act the same on the ensemble and the clean replay? | Same direction on both; REF-op gains 0.7–1.1 points more on the ensemble. | -## `r nb_link("Learning rate", "learning_rate")` +## Learning rate | | base | question | what the numbers show | |---|---|---|---| -| `r link("E01", "learning_rate", "e01")` | replay, cold | How do the adaptive learning-rate settings (multiplier, window, carry or reset) change WIS and coverage? | Multipliers 0.1 and 0.3 lose 7–77 points at h1–h3. 0.01 is 0.5–2.4 points above 0.03 at h1–h3; month-avg coverage shows no clear difference. Count scale is 2–5 points below sqrt. Window and carry vs reset: no strong result. | -| `r link("E09", "learning_rate", "e09")` | ensemble | With a constant learning rate on the rate scale, how do WIS and coverage change with the rate and with extra learning delay? | WIS peaks near 0.056–0.1 per 100k and falls fast past 0.3. Month-avg coverage is no lower than the base at h1–h3 for any constant rate. Each week of extra delay costs about 1 point at h−1. Stateful learning with no extra delay makes h−1 worse than the base. | -| `r link("E14", "learning_rate", "e14")` | replay, cold | How do WIS and coverage change with a constant learning rate, season by season? | Every rate from 0.0032 to 0.056 is at or above the base in both seasons, within 1.2 points of each other: no strong result for a best rate. From 0.1 WIS falls. Month-avg coverage for rates ≤ 0.032 stays within about 0.01 of the base. | -| `r link("E16", "learning_rate", "e16")` | replay, cold | Constant learning rate on the square-root scale vs the rate scale | sqrt 0.018 is about 1 point above rate 0.032 at h0 and within 0.2 at h1–h3 pooled. Against REF-op cold the order flips by season. Every tracker loses in March (up to −28% within the month at h2). | +| `r link("E01", "e01_eta_settings")` | replay, cold | How do the adaptive learning-rate settings (multiplier, window, carry or reset) change WIS and coverage? | Multipliers 0.1 and 0.3 lose 7–77 points at h1–h3. 0.01 is 0.5–2.4 points above 0.03 at h1–h3; month-avg coverage shows no clear difference. Count scale is 2–5 points below sqrt. Window and carry vs reset: no strong result. | +| `r link("E09", "e09_lr_delay")` | ensemble | With a constant learning rate on the rate scale, how do WIS and coverage change with the rate and with extra learning delay? | WIS peaks near 0.056–0.1 per 100k and falls fast past 0.3. Month-avg coverage is no lower than the base at h1–h3 for any constant rate. Each week of extra delay costs about 1 point at h−1. Stateful learning with no extra delay makes h−1 worse than the base. | +| `r link("E14", "e14_lr_by_season")` | replay, cold | How do WIS and coverage change with a constant learning rate, season by season? | Every rate from 0.0032 to 0.056 is at or above the base in both seasons, within 1.2 points of each other: no strong result for a best rate. From 0.1 WIS falls. Month-avg coverage for rates ≤ 0.032 stays within about 0.01 of the base. | +| `r link("E16", "e16_sqrt_constant_lr")` | replay, cold | Constant learning rate on the square-root scale vs the rate scale | sqrt 0.018 is about 1 point above rate 0.032 at h0 and within 0.2 at h1–h3 pooled. Against REF-op cold the order flips by season. Every tracker loses in March (up to −28% within the month at h2). | -## `r nb_link("Tracker structure", "tracker_structure")` +## Tracker structure | | base | question | what the numbers show | |---|---|---|---| -| `r link("E03", "tracker_structure", "e03")` | ensemble | Which scale should the offsets live on: counts, square root, or rate per 100k? | sqrt is 1.4–3.9 points above count pooled, mostly from 2025-26. Coverage: no strong result. Rate is identical to count in every cell. | -| `r link("E04", "tracker_structure", "e04")` | ensemble | Leak and slow term | The leak is +3.3 to +5.6 at h1–h3 in 2025-26 and −0.4 to −1.3 in 2024-25. The slow term without a warm start: no strong result. | -| `r link("E05", "tracker_structure", "e05")` | ensemble | Warm start from the burn-in season | +1.7 to +3.2 at h2/h3 in 2024-25 for every tracker, within ±0.4 in 2025-26. | -| `r link("E11", "tracker_structure", "e11")` | ensemble | Does a constant learning rate do better than the adaptive rate? | Constant 0.1 vs adaptive sqrt: the order flips by horizon (1–2 points either way). REF-op has the highest pooled WIS at every horizon, by up to 2 points. On month-avg coverage the constant rate is 0.003–0.022 worse. | +| `r link("E03", "e03_scale")` | ensemble | Which scale should the offsets live on: counts, square root, or rate per 100k? | sqrt is 1.4–3.9 points above count pooled, mostly from 2025-26. Coverage: no strong result. Rate is identical to count in every cell. | +| `r link("E04", "e04_offset_structure")` | ensemble | Leak and slow term | The leak is +3.3 to +5.6 at h1–h3 in 2025-26 and −0.4 to −1.3 in 2024-25. The slow term without a warm start: no strong result. | +| `r link("E05", "e05_warm_start")` | ensemble | Warm start from the burn-in season | +1.7 to +3.2 at h2/h3 in 2024-25 for every tracker, within ±0.4 in 2025-26. | +| `r link("E11", "e11_constant_lr")` | ensemble | Does a constant learning rate do better than the adaptive rate? | Constant 0.1 vs adaptive sqrt: the order flips by horizon (1–2 points either way). REF-op has the highest pooled WIS at every horizon, by up to 2 points. On month-avg coverage the constant rate is 0.003–0.022 worse. | -## `r nb_link("Late season", "late_season")` +## Late season | | base | question | what the numbers show | |---|---|---|---| -| `r link("E02", "late_season", "e02")` | replay, cold | Spring switch-off, per-level learning rate, seasonal window | Switching off from 1 March or 15 February is 2–4 points above the reference at h1–h3 (dates in-sample; in 2025-26 about half the gain is December carry-over). Per-level about +1. Month-avg coverage: no clear difference. | -| `r link("E17", "late_season", "e17")` | replay, cold | Does shrinking or switching off the offset from 1 March change WIS and coverage? | No strong result: pooled WIS gaps ≤ 0.7 points and month-avg coverage gaps ≤ 0.010. The switch-off's sqrt gain comes mostly from 2025-26; the date was chosen in-sample. | +| `r link("E02", "e02_eta_variants")` | replay, cold | Spring switch-off, per-level learning rate, seasonal window | Switching off from 1 March or 15 February is 2–4 points above the reference at h1–h3 (dates in-sample; in 2025-26 about half the gain is December carry-over). Per-level about +1. Month-avg coverage: no clear difference. | +| `r link("E17", "e17_late_decay")` | replay, cold | Does shrinking or switching off the offset from 1 March change WIS and coverage? | No strong result: pooled WIS gaps ≤ 0.7 points and month-avg coverage gaps ≤ 0.010. The switch-off's sqrt gain comes mostly from 2025-26; the date was chosen in-sample. | -## `r nb_link("Learning truth", "learning_truth")` +## Learning truth | | base | question | what the numbers show | |---|---|---|---| -| `r link("E00", "learning_truth", "")` | ensemble | Should the tracker learn from finalized data or from data as published? | Exact and final differ by 0.6 points or less, except REF-op h−1 (2.2, 2024-25). Learning once from the first report loses most of the h−1 gain. All other notebooks use the exact (prod) design. | +| `r link("E00", "e00_vintage_backtest")` | ensemble | Should the tracker learn from finalized data or from data as published? | Exact and final differ by 0.6 points or less, except REF-op h−1 (2.2, 2024-25). Learning once from the first report loses most of the h−1 gain. All other notebooks use the exact (prod) design. | E12 and E13 (the clean replay without and with a 2023-24 burn-in) have no notebook; they are `scripts/calibration/calibration_ws_replay.R`. diff --git a/reports/writeups/calibration_experiments/late_season.Rmd b/reports/writeups/calibration_experiments/late_season.Rmd deleted file mode 100644 index 20455970..00000000 --- a/reports/writeups/calibration_experiments/late_season.Rmd +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: "Late season" -date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" -output: - html_document: - code_folding: hide - toc: true - toc_float: true - toc_depth: 3 -params: - workers: 14 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -``` - -Every tracker loses WIS after the peak. Both sections calibrate the clean replay, cold: E02 switches the offset off from a date (and tries a per-level and a seasonal learning rate); E17 shrinks the offset from 1 March instead. Each section is a self-contained experiment with its own runs; -the ids match `notes/calibration-ledger.md`. - -```{r e02, results="asis"} -cal_section("_e02_eta_variants.Rmd", list(workers = params$workers)) -``` - -```{r e17, results="asis"} -cal_section("_e17_late_decay.Rmd", list(workers = params$workers)) -``` diff --git a/reports/writeups/calibration_experiments/learning_rate.Rmd b/reports/writeups/calibration_experiments/learning_rate.Rmd deleted file mode 100644 index c9a70f1a..00000000 --- a/reports/writeups/calibration_experiments/learning_rate.Rmd +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: "Learning rate" -date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" -output: - html_document: - code_folding: hide - toc: true - toc_float: true - toc_depth: 3 -params: - workers: 14 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -``` - -How large should the tracker's steps be? E01 varies the adaptive learning rate's settings on the clean replay; E09 sweeps a constant rate (and the extra revision delay) on the submitted ensemble; E14 and E16 sweep a constant rate on the replay, on the rate and square-root scales. Each section is a self-contained experiment with its own runs; -the ids match `notes/calibration-ledger.md`. - -```{r e01, results="asis"} -cal_section("_e01_eta_settings.Rmd", list(workers = params$workers)) -``` - -```{r e09, results="asis"} -cal_section("_e09_lr_delay.Rmd", list(workers = params$workers)) -``` - -```{r e14, results="asis"} -cal_section("_e14_lr_by_season.Rmd", list(workers = params$workers)) -``` - -```{r e16, results="asis"} -cal_section("_e16_sqrt_constant_lr.Rmd", list(workers = params$workers)) -``` diff --git a/reports/writeups/calibration_experiments/tracker_structure.Rmd b/reports/writeups/calibration_experiments/tracker_structure.Rmd deleted file mode 100644 index 731ca65c..00000000 --- a/reports/writeups/calibration_experiments/tracker_structure.Rmd +++ /dev/null @@ -1,40 +0,0 @@ ---- -title: "Tracker structure" -date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" -output: - html_document: - code_folding: hide - toc: true - toc_float: true - toc_depth: 3 -params: - workers: 14 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) -suppressPackageStartupMessages(source(here::here("R/load_all.R"))) -``` - -What shape should the offset have? All four sections calibrate the submitted flu ensemble: the working scale (E03), the leak and the slow term (E04), the warm start from the burn-in season (E05), and a constant against an adaptive learning rate (E11). Each section is a self-contained experiment with its own runs; -the ids match `notes/calibration-ledger.md`. - -```{r e03, results="asis"} -cal_section("_e03_scale.Rmd", list(workers = params$workers)) -``` - -```{r e04, results="asis"} -cal_section("_e04_offset_structure.Rmd", list(workers = params$workers)) -``` - -```{r e05, results="asis"} -cal_section("_e05_warm_start.Rmd", list(workers = params$workers)) -``` - -```{r e11, results="asis"} -cal_section("_e11_constant_lr.Rmd", list(workers = params$workers)) -``` From f74340c555e7f39ef4097144fba49f290e9ff1bf Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:01:43 -0700 Subject: [PATCH 084/107] fix(calibration): e10 allows repeated month-view chunk labels --- reports/writeups/calibration_experiments/e10_covid.Rmd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd index 061e9537..2437fb11 100644 --- a/reports/writeups/calibration_experiments/e10_covid.Rmd +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -19,6 +19,8 @@ editor_options: knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +# The month view is knit several times, so its chunk labels repeat. +options(knitr.duplicate.label = "allow") ``` ```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} From ea72227dbd7e9190a8e2586110eae0cb5f4d3fb9 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:54:34 -0700 Subject: [PATCH 085/107] docs(calibration): ledger marks the ILI+ backfill script unused --- notes/calibration-ledger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index f23d6e77..b6001798 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -226,7 +226,7 @@ reviewed. `cache/calibration/ws_replay_scores_*.csv`. - `scripts/calibration/calibration_ili_backfill.R`: replays `windowed_seasonal` over the ILI+ state history (2010–2024) into - hub-schema parquets in `cache/calibration/`, for the ILI+ burn-in thread. + hub-schema parquets in `cache/calibration/`, for an ILI+ burn-in (thread closed 2026-10-02, unused). - The pre-experiment notebooks (`reports/writeups/calibration/`) and the old E07 scripts were deleted on 2026-10-02; they are in VCS history. Their only analyses not in the current suite are the staleness lagged From 3dae4788e608d29a9172d3a5370a8b1e82306466 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:42:26 -0700 Subject: [PATCH 086/107] feat(calibration): E19 script comparing tracker eta and offsets; ledger records the leak vs constant-rate mechanism --- notes/calibration-ledger.md | 24 ++++- .../calibration_tracker_internals.R | 92 +++++++++++++++++++ 2 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 scripts/calibration/calibration_tracker_internals.R diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index b6001798..87d11608 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -8,9 +8,10 @@ and projected to be monotone. Dropped hub rounds are in "Running things". Every number below was re-read from the rendered notebook (or, for E12/E13, -the score CSV) on 2026-10-02. Older notes, pre-fix tables and the -summary-of-reviews layers were removed; they are in VCS history at change -`uxposooz` (`notes/CALIBRATION.md`, `notes/calibration-review.md`, +the score CSV) on 2026-10-02; E19's come from its CSVs on 2026-10-06. +Older notes, pre-fix tables and the summary-of-reviews layers were +removed; they are in VCS history at change `uxposooz` +(`notes/CALIBRATION.md`, `notes/calibration-review.md`, `notes/calibration-summary-2026-10-01.md`). ## How to read the numbers @@ -35,7 +36,7 @@ summary-of-reviews layers were removed; they are in VCS history at change `off_after` dates were picked on the same two seasons they are scored on. - **Two base forecasters.** E00, E03–E05 and E09–E11 calibrate the submitted ensemble (h−1…h3, all locations in headline tables). E01, E02, - E07 and E12–E18 calibrate the clean `windowed_seasonal` replay (h0–h3, + E07 and E12–E19 calibrate the clean `windowed_seasonal` replay (h0–h3, states only, except E12). ## What the numbers support @@ -72,6 +73,18 @@ Small, or dependent on season (0–3 points): a 3–5 point loss at h1–h3 in 2025-26 and costs 0.4–1.3 in 2024-25 (E04, E05). On the clean replay it adds 0.5–1.3 at h1–h3 in both seasons and costs 0.4 at h0 (E18). +- **Why the leak and a small constant rate land close (E19).** Both limit + the adaptive step's late-season overreaction, in different ways. The + adaptive eta comes from base residuals, so the leak does not change it; + relative to interval width it rises from about 0.015 in January to 0.10 + at h3 in May 2025, 3–4× the constant rate's. Without the leak the offset + keeps growing into March–April, when the base over-predicts (h3 median + offset 0.24 of width in April 2025 vs REF-op 0.11). REF-op tracks the + ramp as fast (0.045 in February) and the leak empties it by May. Constant + 0.018 barely corrects the ramp (0.013 in February) and has little to + overshoot with. In April–May 2026 the base under-predicted and the leak + dropped an offset that was still right (0.01 vs 0.14 at h3), which fits + sqrt 0.018 leading in 2025-26. The link to WIS is inferred, not scored. - **Switching off from 1 Mar or 15 Feb.** On the clean replay: +2 to +4 at h1–h3 over sqrt adaptive 0.03 (E02), +0.4 to +0.7 over sqrt constant 0.018 (E17). The date was chosen after seeing the March losses. About half of @@ -167,6 +180,7 @@ gating), exact learning truth, spoiled submissions excluded. | E16 | `e16_sqrt_constant_lr` | replay, cold | constant rate on sqrt scale | sqrt 0.018 about +1 over rate constants at h0; at h1–h3 within 0.2 pooled, up to +0.7 in 2025-26. vs REF-op cold: behind in 2024-25 h0–h1, ahead elsewhere; month-avg coverage tie | 0.018 picked in-sample | | E17 | `e17_late_decay` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | | E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008. Branch sqrt 0.018 warm: +3.4/+1.6/+1.6/+2.5 pooled, best of all, but month-avg coverage 0.011–0.018 worse than base at h2–h3 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance) | +| E19 | none (`calibration_tracker_internals.R`) | replay, cold, final truth | why do the leak and constant 0.018 score alike? | REF-op cold and no-leak eta identical; adaptive eta / width peaks Apr–May (h3 0.10) vs constant ≤ 0.03. h3 median offset / width, Feb/Mar/Apr/May 2025: REF-op 0.045/0.078/0.114/0.009, no leak 0.048/0.109/0.237/0.199, constant 0.013/0.026/0.057/0.074; Apr/May 2026: 0.093/0.013, 0.165/0.149, 0.101/0.143. Share of truths above the base median at h3: Mar 0.27–0.43, Apr–May 2026 0.62–0.71 | final truth, not exact; h1–h3; median-level offset only; no WIS by month in the script | E06 (ILI+ burn-in) and E08 (old covid) were run on code or data later found broken and were not rebuilt. @@ -224,6 +238,8 @@ reviewed. - `R/calibration/views.R`: reference configs, cached runs, standard views. - `scripts/calibration/calibration_ws_replay.R`: E12/E13. Scores in `cache/calibration/ws_replay_scores_*.csv`. +- `scripts/calibration/calibration_tracker_internals.R`: E19. Per-round + eta and offsets in `cache/calibration/tracker_internals/`. - `scripts/calibration/calibration_ili_backfill.R`: replays `windowed_seasonal` over the ILI+ state history (2010–2024) into hub-schema parquets in `cache/calibration/`, for an ILI+ burn-in (thread closed 2026-10-02, unused). diff --git a/scripts/calibration/calibration_tracker_internals.R b/scripts/calibration/calibration_tracker_internals.R new file mode 100644 index 00000000..267042db --- /dev/null +++ b/scripts/calibration/calibration_tracker_internals.R @@ -0,0 +1,92 @@ +# E19: per-round learning rate and offsets for REF-op cold, sqrt adaptive +# without the leak, and sqrt constant 0.018 on the clean flu windowed_seasonal +# replay, h1–h3, states only. Learns from final truth (one run per config), so +# each round's internals come from a single tracker. Eta and offsets are +# divided by the base 90% interval width, in each config's working units, so +# configs with and without `scales` compare. Writes a per-round CSV, a monthly +# CSV and a plot to cache/calibration/tracker_internals/. Findings: E19 in +# notes/calibration-ledger.md. +# +# Usage: distrobox enter rocker -- Rscript scripts/calibration/calibration_tracker_internals.R + +suppressPackageStartupMessages({ + source(here::here("scripts/calibration/calibration_ws_replay.R")) + library(ggplot2) +}) +out_dir <- file.path(CH_CACHE_DIR, "tracker_internals") +dir.create(out_dir, showWarnings = FALSE, recursive = TRUE) + +wi <- ws_inputs("flu") +no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) +CONFIGS <- list( + `REF-op cold` = c(list(ref = "op"), no_burn_in), + `sqrt adaptive, no leak` = c(list(ref = "paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)), no_burn_in), + `sqrt constant 0.018` = c(list(ref = "paper", transform = "sqrt", scales = wi$rate_scales, lr = 0.018), no_burn_in) +) +scale_of <- wi$rate_scales %>% select(location, scale) + +rows <- purrr::imap(CONFIGS, function(cfg, nm) { + cli::cli_alert_info("running {nm}") + cal <- do.call(cal_run, c(list(wi$fc, wi$truth), cfg, list(learn = "final"))) + fc <- cal$forecasts %>% + filter(location != "US", !is_burn_in, horizon >= 1, !is.na(value_base)) + # Working-unit 90% width: sqrt of (count / scale), scale 1 unless the config sets `scales`. + s <- if (is.null(cfg$scales)) fc %>% distinct(location) %>% mutate(scale = 1) else scale_of + fc %>% + left_join(s, by = "location") %>% + group_by(location, horizon, reference_date) %>% + summarize( + width_count = value_base[level == 0.95] - value_base[level == 0.05], + width_work = sqrt(value_base[level == 0.95] / scale[1]) - sqrt(value_base[level == 0.05] / scale[1]), + eta = mean(lr_level), + fast_mid = fast[level == 0.5], + slow_mid = slow[level == 0.5], + offset_mid = offset[level == 0.5], + offset_abs = mean(abs(offset)), + truth = truth[1], + base_mid = value_base[level == 0.5], + .groups = "drop" + ) %>% + mutate( + config = nm, + eta = ifelse(eta == 0, NA_real_, eta), + # Unit-free: step size and offsets relative to the base 90% width. + eta_rel = eta / width_work, + fast_rel = fast_mid / width_work, + slow_rel = slow_mid / width_work, + offset_rel = offset_mid / width_count, + offset_abs_rel = offset_abs / width_count + ) +}) %>% bind_rows() + +per_round <- rows %>% + group_by(config, horizon, reference_date) %>% + summarize( + eta_rel = median(eta_rel, na.rm = TRUE), + fast_rel = median(fast_rel, na.rm = TRUE), + offset_rel = median(offset_rel, na.rm = TRUE), + offset_abs_rel = median(offset_abs_rel, na.rm = TRUE), + above_base_median = mean(truth > base_mid, na.rm = TRUE), + .groups = "drop" + ) +readr::write_csv(per_round, file.path(out_dir, "tracker_internals_per_round.csv")) + +month_tbl <- per_round %>% + mutate(season = season_of(reference_date), month = format(reference_date, "%b")) %>% + group_by(config, horizon, season, month) %>% + summarize(across(c(eta_rel, offset_rel, offset_abs_rel, above_base_median), \(x) mean(x, na.rm = TRUE)), first = min(reference_date), .groups = "drop") %>% + arrange(config, horizon, first) +readr::write_csv(month_tbl, file.path(out_dir, "tracker_internals_by_month.csv")) + +long <- per_round %>% + select(config, horizon, reference_date, `eta / width` = eta_rel, `median-level offset / width` = offset_rel, `mean |offset| / width` = offset_abs_rel) %>% + tidyr::pivot_longer(-c(config, horizon, reference_date)) +p <- ggplot(long, aes(reference_date, value, colour = config)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_line() + + facet_grid(name ~ paste0("h", horizon), scales = "free_y") + + labs(x = NULL, y = "median over states", colour = NULL) + + theme_bw() + + theme(legend.position = "bottom") +ggsave(file.path(out_dir, "tracker_internals.png"), p, width = 13, height = 8) +cli::cli_alert_success("wrote {out_dir}") From 041ae5004158f5d589d49afc13f814b831513e1f Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:11:05 -0700 Subject: [PATCH 087/107] refactor(calibration): notebooks lead with the month view and its by-horizon tables; drop findings and the standard views --- R/calibration/views.R | 17 +++ .../calibration_experiments/_month_view.Rmd | 48 ++---- .../_standard_views.Rmd | 128 ---------------- .../e00_vintage_backtest.Rmd | 32 ---- .../e01_eta_settings.Rmd | 26 ---- .../e02_eta_variants.Rmd | 31 ---- .../calibration_experiments/e03_scale.Rmd | 38 +---- .../e04_offset_structure.Rmd | 36 +---- .../e05_warm_start.Rmd | 40 +---- .../e07_across_eras.Rmd | 19 --- .../e07b_candidates.Rmd | 24 --- .../calibration_experiments/e09_lr_delay.Rmd | 38 ----- .../calibration_experiments/e10_covid.Rmd | 139 ++---------------- .../e11_constant_lr.Rmd | 34 +---- .../e14_lr_by_season.Rmd | 26 ---- .../e15_wis_sources.Rmd | 30 ---- .../e16_sqrt_constant_lr.Rmd | 35 ----- .../e17_late_decay.Rmd | 33 ----- .../e18_ref_op_bridge.Rmd | 48 ------ 19 files changed, 63 insertions(+), 759 deletions(-) delete mode 100644 reports/writeups/calibration_experiments/_standard_views.Rmd diff --git a/R/calibration/views.R b/R/calibration/views.R index 256306c4..1dc18949 100644 --- a/R/calibration/views.R +++ b/R/calibration/views.R @@ -413,6 +413,23 @@ cal_month_cov_table <- function(mv, seasons = NULL) { } +#' WIS reduction % per season and horizon as a wide table: configs as rows, +#' `season hX` as columns, `both` pooling the seasons. Same rows as the month +#' view (states only, burn-in dropped). +#' @export +cal_month_wis_table <- function(mv, seasons = NULL) { + m <- mv$month + if (!is.null(seasons)) m <- m %>% filter(.data$season %in% seasons) + bind_rows(m, m %>% mutate(season = "both")) %>% + mutate(season = factor(.data$season, levels = c(sort(unique(m$season)), "both"))) %>% + group_by(.data$config, .data$season, .data$horizon) %>% + summarize(pct = 100 * (sum(.data$wis_base) - sum(.data$wis_cal)) / sum(.data$wis_base), .groups = "drop") %>% + arrange(.data$season, .data$horizon) %>% + transmute(.data$config, col = paste0(.data$season, " h", .data$horizon), cell = sprintf("%+.1f", .data$pct)) %>% + tidyr::pivot_wider(names_from = "col", values_from = "cell") +} + + #' The month view plot for one season: rows are WIS reduction %, L1 coverage #' bias and median absolute-error reduction %; columns are horizons; one line #' per config. The base's coverage bias is grey (dashed unless `linetype` is diff --git a/reports/writeups/calibration_experiments/_month_view.Rmd b/reports/writeups/calibration_experiments/_month_view.Rmd index 8cfdd211..e3a72b28 100644 --- a/reports/writeups/calibration_experiments/_month_view.Rmd +++ b/reports/writeups/calibration_experiments/_month_view.Rmd @@ -1,10 +1,10 @@ # Month view: WIS, coverage, median error - + -The same scores split by calendar month of the reference date, one figure -per season, **states only (US excluded)**. Rows are the three metrics, -columns the horizons: +Scores by calendar month of the reference date, one figure per season, +**states only (US excluded)**. Rows are the three metrics, columns the +horizons: - **WIS reduction %**: `100 × (WIS_base − WIS_cal) / WIS_base` within the month, so positive = calibrated WIS lower than base. Grey bars are the @@ -15,9 +15,9 @@ columns the horizons: bias is computed within the month: the mean over the 23 levels of |share of outcomes at or below the quantile − level|. A season-pooled bias lets under-prediction on the ramp cancel over-prediction after the peak; the - monthly one does not. The absolute monthly values are in the table below - the plots: when the base's bias is already small, a small absolute change - shows as a large percentage. + monthly one does not. When the base's bias is already small, a small + absolute change shows as a large percentage; the month-average column of + the coverage table below gives the absolute values. - **Median AE reduction %**: the same as WIS reduction % for the absolute error of the 0.5 quantile, so positive = calibrated median closer to the outcome. @@ -38,33 +38,9 @@ for (s in sort(unique(mv_plot_month$season))) { } ``` -```{r mview-table, results="asis"} -if (!isFALSE(get0("mv_show_table"))) { - cat("\n\n", cal_h(2), " Coverage bias: season-pooled and month-average\n\n", sep = "") - print(knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION)) -} -``` - -```{r mview-abs, results="asis"} -# Absolute within-month coverage bias behind the plot's reduction %, at h0 and h2. -mv_abs <- mv_plot_month %>% filter(horizon %in% c(0, 2)) -# Configs on different base forecasters (E07) have different bases, so each cell then carries its own base. -mv_one_base <- mv_abs %>% distinct(season, month, horizon, cov_base) %>% count(season, month, horizon) %>% pull(n) %>% max() == 1L -mv_abs_tbl <- mv_abs %>% - transmute(season, month, h = paste0("h", horizon), config, - v = if (mv_one_base) sprintf("%.3f", cov_cal) else sprintf("%.3f (base %.3f)", cov_cal, cov_base)) %>% - tidyr::pivot_wider(names_from = config, values_from = v) -if (mv_one_base) { - mv_abs_base <- mv_abs %>% distinct(season, month, h = paste0("h", horizon), base = sprintf("%.3f", cov_base)) - mv_abs_tbl <- mv_abs_tbl %>% left_join(mv_abs_base, by = c("season", "month", "h")) -} -cat("\n\n", cal_h(2), " Monthly L1 coverage bias, absolute (h0 and h2)\n\n", sep = "") -print(knitr::kable( - mv_abs_tbl %>% arrange(h, season, month), - caption = if (mv_one_base) { - "Within-month L1 coverage bias (lower is better), calibrated configs and the uncalibrated base." - } else { - "Within-month L1 coverage bias (lower is better) of each config, with its own uncalibrated base in parentheses." - } -)) +```{r mview-tables, results="asis"} +cat("\n\n", cal_h(2), " By horizon\n\n", sep = "") +cat("WIS reduction % (positive = calibrated WIS lower than base), states only; `both` pools the seasons.\n\n") +print(knitr::kable(cal_month_wis_table(mv))) +print(knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION)) ``` diff --git a/reports/writeups/calibration_experiments/_standard_views.Rmd b/reports/writeups/calibration_experiments/_standard_views.Rmd deleted file mode 100644 index 293193c1..00000000 --- a/reports/writeups/calibration_experiments/_standard_views.Rmd +++ /dev/null @@ -1,128 +0,0 @@ -# Standard views - - - -Every score is over both live seasons, all -hub locations, against finalized truth; burn-in rounds are excluded. - -## WIS and coverage - -WIS reduction % (positive = calibrated WIS lower than base): - -```{r v-head} -head_all <- cal_headline(cals) -knitr::kable(cal_wide(head_all, "pct_improvement")) -``` - -L1 coverage bias (lower is better; mean over levels of |coverage − level|), with the base first: - -```{r v-head-cov} -knitr::kable(bind_rows( - cal_wide(head_all %>% filter(variant == names(cals)[1]) %>% mutate(variant = "base"), "cal_error_base", 3), - cal_wide(head_all %>% mutate(variant = as.character(variant)), "cal_error_cal", 3) -)) -``` - -The same bias for states only, season-pooled against month-average (see -"Month view" below), per season and for both seasons: - -```{r v-head-cov-month} -mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) -mv_show_table <- FALSE -knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) -``` - -WIS reduction % by season (positive = calibrated WIS lower than base): - -```{r v-head-season} -knitr::kable( - cal_wide(cal_headline(cals, by = "season") %>% filter(season %in% LIVE), "pct_improvement", id = c("variant", "season")) %>% - arrange(season, variant) -) -``` - -## Absolute error of the median - -Summed |truth − median|, reduction % vs base (positive = calibrated absolute error lower than base): - -```{r v-ae} -knitr::kable(cal_wide(cal_ae(cals), "ae_pct")) -``` - -## Where in the season - -WIS reduction % (positive = calibrated WIS lower than base) by month of the reference date (clipped at −60), h0 and h3, -and each month's share of the base WIS at h3. Months run July to June; only -months with forecasts are shown. - -```{r v-month, fig.height=4.5} -month_tbl <- purrr::imap(cals, function(cal, nm) cal_by_month(cal) %>% mutate(variant = nm)) %>% - bind_rows() %>% - mutate(variant = factor(variant, levels = names(cals)), month = droplevels(month)) -ggplot(month_tbl %>% filter(horizon %in% c(0, 3)), aes(month, pct, colour = variant, group = variant)) + - geom_hline(yintercept = 0, colour = "grey50") + - geom_line() + - geom_point() + - facet_wrap(~ paste0("h", horizon)) + - scale_colour_brewer(palette = "Dark2", name = NULL) + - coord_cartesian(ylim = c(-60, 30)) + - labs(x = "month of reference date", y = "WIS reduction % (positive = calibrated\nWIS lower than base)") + - theme(legend.position = "bottom", legend.direction = "vertical") -knitr::kable( - month_tbl %>% - filter(horizon == 3) %>% - transmute(variant, month, pct = round(pct, 1)) %>% - tidyr::pivot_wider(names_from = month, values_from = pct) %>% - bind_rows(month_tbl %>% filter(horizon == 3, variant == names(cals)[1]) %>% - transmute(variant = "share of base WIS %", month, pct = round(share, 1)) %>% - tidyr::pivot_wider(names_from = month, values_from = pct)) -) -``` - -## Fixed states, `r focus` {.tabset} - -The same states (US, CA, GA, MN, VT) in every experiment notebook. Every other -round's fan at h−1–h3: blue ribbon and line are the base 80% band and median, -orange lines the calibrated 10/50/90% quantiles, grey the finalized truth. - -The black line is the last two reported weeks of NHSN data as published when -that round was forecast. Its dot is normally the round's h−1 week, where its -fan starts. -Plotted rounds are two weeks apart, so each black line also runs across the -first weeks of the previous round's fan. Black sits below grey where first -reports were later revised up. A black line with no fan in front of it, or a -gap between fans, is a round with no submission (for flu, 2025-10-25 to -2025-11-15 and 2026-04-18). Around Christmas and New Year 2024 the Wednesday -releases slipped to Thursday and Friday, so the 2024-12-28 and 2025-01-04 -rounds saw one week less data than usual; their dot sits a week before the -fan starts. - -```{r v-state, results="asis", fig.height=4} -for (nm in names(CAL_STATES)) { - cat("\n\n", cal_h(3), " ", nm, "\n\n", sep = "") - for (s in LIVE) print(cal_state_panel(cals[[focus]], CAL_STATES[[nm]], s, truth, vintages, title = paste(nm, s))) -} -``` - -## Gallery: `r focus` {.tabset} - -For each season phase and location size (population above or below the -median), the forecast with the largest WIS gain and the one with the largest -loss, each from a different location. Forecasts are ranked by their -WIS reduction as a share of their own location-season's base WIS, so small states late in the season, where the base is -tiny, do not crowd the gallery. Ribbons are the 50% -and 90% bands, the dashed line is the forecast date. - -```{r v-gallery, results="asis", fig.height=3.5} -picks <- cal_gallery_pick(cals[[focus]], population) -abbrev <- setNames(toupper(get_population_data()$state_id), get_population_data()$state_code) -for (i in seq_len(nrow(picks))) { - p <- picks[i, ] - if (i == 1L || p$phase != picks$phase[i - 1L]) cat("\n\n", cal_h(3), " ", as.character(p$phase), "\n\n", sep = "") - sub <- sprintf("%s (%s), %s, %s: WIS reduction %+.1f%% (%+.1f%% of its season's base WIS; positive = calibrated WIS lower)", abbrev[[p$location]], p$size, p$reference_date, p$kind, p$wis_pct, p$share) - print(cal_gallery_panel(cals[[focus]], p$location, p$reference_date, truth, vintages, subtitle = sub)) -} -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` diff --git a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd index 5ea3fe92..1d3ae784 100644 --- a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd +++ b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd @@ -161,35 +161,3 @@ mv_plot_cfgs <- c("op/14/final", "op/14/exact", "op/7/vintage", "op/7/exact", "p ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` - -# Findings - -WIS reduction % is positive = calibrated WIS lower than base; coverage bias is -lower-is-better. A WIS gap under 1 point or a month-avg coverage gap under -0.01 is read as no clear difference. - -- **final vs exact.** REF-op at settle 14: h−1 +7.6 (final) vs +5.4 (exact), - a 2.2-point gap that comes from 2024-25 (11.2 vs 7.7; 2025-26 2.4 vs 2.1). - At h0–h3 the gap is 0.5 points or less. REF-paper: under 1 point at every - horizon (0.3–0.6). Month-avg coverage bias: no clear difference (REF-op - 0.106 vs 0.107 at h−1; REF-paper 0.122 vs 0.115). Season-pooled bias is - higher under exact (REF-paper h−1 0.044 vs 0.030), but pooling lets ramp - and post-peak errors cancel. -- **vintage vs exact.** At settle 7 the vintage mode loses most of the h−1 - gain: REF-op 0.0 vs +6.1, REF-paper −1.1 vs +4.5. Its month-avg coverage - bias at h−1 is worse than the base (REF-op 0.136, REF-paper 0.149, base - 0.134; exact 0.107 and 0.110). At settle 14 the gaps are smaller (REF-op h−1 - +4.9 vs +5.4). At REF-paper h1–h3 the vintage mode has higher WIS reduction - than exact (settle 14: −0.9/−1.6/−4.2 vs −2.0/−2.7/−5.3). -- **settle 7 vs 14 under exact.** REF-paper: settle 7 is 1.3–2.8 points - higher at every horizon, in both seasons. REF-op: no clear difference - pooled (0.0–0.7); in 2025-26 settle 14 is 0.5–0.6 points higher at h1–h3. - Month-avg coverage: no clear difference at either reference. -- **settle 7 vs 14 under vintage.** settle 14 is higher at h−1/h0 (REF-op - +4.9/+4.5 vs 0.0/+3.0). At REF-paper h1–h3 the order is the other way - (−0.1/−0.4/−2.3 at settle 7 vs −0.9/−1.6/−4.2 at settle 14). -- **Month-avg coverage at h2–h3.** Every REF-paper config is at or above the - base (h3 0.160–0.170 vs 0.151); every REF-op config is below it (h3 - 0.135–0.137). -- Offsets start from zero in November 2024 (the burn-in only warms the - learning rate), so early 2024-25 rounds of the REF-paper runs equal the base. diff --git a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd index 140a5cbf..d1d2f14d 100644 --- a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd +++ b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd @@ -261,29 +261,3 @@ mv_plot_cfgs <- c( ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` - -# Findings - -There are no uncertainty estimates; a WIS gap under 1 point or a month-avg -coverage gap under 0.01 is read as no clear difference. The cold start leaves -Nov–Dec 2024 (about 38% of 2024-25's base WIS at h2) equal to the base for -every config. States only, window 20, carry unless stated. - -- **Large multipliers:** 0.1 is −1.3/−6.8/−11.3/−16.7 pooled at h0–h3 and - 0.3 loses 14–77. Across multipliers WIS moves 16–81 points; across windows - 0.8–2.4; across season policies 0.3–1.7. -- **0.01 vs 0.03:** 0.01 is ahead at h1–h3 in both seasons (by 0.5–2.4); - 0.03 is ahead at h0 in 2024-25 (by 0.8). Pooled: 0.01 - +1.7/+0.7/+0.5/+0.5, 0.03 +1.9/−0.2/−1.1/−1.7. -- **Month-avg coverage (both seasons, h0…h3):** base 0.112/0.127/0.153/0.177; - 0.01 0.093/0.114/0.142/0.168; 0.03 0.095/0.120/0.150/0.176; 0.1 - 0.097/0.130/0.161/0.190. 0.01 vs 0.03: no clear difference. 0.1 is worse - than the base at h1–h3. Season-pooled, 0.03 and 0.1 look much better than - the base (0.017–0.045), but pooling lets ramp and post-peak errors cancel. -- **Window:** no strong result (under 1 point in most cells; up to 2.4 at h3, - 2025-26, driven by mult 0.3). -- **Carry vs reset (2025-26 only):** at mult ≤ 0.03 carry is 0.3–1.9 points - ahead on WIS; at 0.1 and 0.3 the order is mixed. Month-avg coverage: no - clear difference (reset lower by 0.002–0.008). -- **Count scale** (mult 0.03): −0.4/−2.5/−4.9/−7.0 pooled, 2–5 points below - sqrt at h1–h3 in both seasons. diff --git a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd index a3891186..bf4d0fb5 100644 --- a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd +++ b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd @@ -236,34 +236,3 @@ mv_cols <- CFG_COLS ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` - -# Findings - -There are no uncertainty estimates; a WIS gap under 1 point or a month-avg -coverage gap under 0.01 is read as no clear difference. The cold start leaves -Nov–Dec 2024 equal to the base for the cold configs. States only, pooled over -both seasons unless stated. - -- **Switching off from 1 March or 15 February:** +2.8/+1.7/+2.0/+2.5 and - +2.7/+2.1/+2.5/+3.2 at h0–h3, against the reference's - +1.9/−0.2/−1.1/−1.7, ahead at every horizon in both seasons. The two dates - differ by 0.1–0.7 (no clear difference). Both dates were chosen with these - seasons in view. -- **Where that gain sits:** in 2024-25 it is the March–April months, where - the reference loses (h2: −31% in March, −41% in April). In 2025-26 about - half of the h2 gain is December, where offsets frozen in February 2025 are - still in play (+13.6% vs +6.3%), partly offset by January and October. -- **Switching off from 1 April:** +1.6/−0.2/−0.7/−0.8, within 1 point of the - reference at h0–h2. -- **Per-level eta:** +2.1/+0.5/−0.0/−0.3, about 1 point above the reference - at h1–h3 in both seasons. -- **Seasonal window and burn-in:** against `reference + burn-in` - (+2.0/−0.0/−0.6/−1.0), the seasonal window is +2.0/−0.1/−0.8/−1.3 with carry - and +1.4/−0.2/−0.9/−1.6 with reset. The burn-in alone is +0.5 to +0.7 at - h2–h3 over the cold reference, in 2024-25 only. No strong result. -- **Month-avg coverage (h0…h3):** base 0.112/0.127/0.153/0.177; reference - 0.095/0.120/0.150/0.176; off 1 March 0.104/0.124/0.153/0.170; off 15 - February 0.106/0.126/0.153/0.169; per-level 0.092/0.117/0.147/0.173. No - clear difference between any variant and the reference. Season-pooled the - switch-offs look much worse than the reference (0.080–0.100 vs - 0.032–0.045), but pooling lets ramp and post-peak errors cancel. diff --git a/reports/writeups/calibration_experiments/e03_scale.Rmd b/reports/writeups/calibration_experiments/e03_scale.Rmd index 4a3ed2d4..81dab88a 100644 --- a/reports/writeups/calibration_experiments/e03_scale.Rmd +++ b/reports/writeups/calibration_experiments/e03_scale.Rmd @@ -37,8 +37,8 @@ Setup: zero at the first 2024-25 round (cold start). - Learning truth: exact (re-run at every round on the data published by then, as prod does; E00). Scores against finalized truth. -- Seasons scored: 2024-25 and 2025-26. Horizons h−1 to h3. Headline tables: - all hub locations including US; month view: states only. +- Seasons scored: 2024-25 and 2025-26. Horizons h−1 to h3. States only + (US excluded). | config | change from REF-paper | |---|---| @@ -55,37 +55,11 @@ cals <- list( sqrt = run("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)), rate = run("paper", scales = rate_scales, lr_args = list(mult = 0.03, floor = 1e-3)) ) -focus <- "sqrt" ``` -```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} +```{r month-view-data} +mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) ``` -# Findings - -WIS reduction % is positive = calibrated WIS lower than base; coverage bias is -lower-is-better. A WIS gap under 1 point or a month-avg coverage gap under -0.01 is read as no clear difference. - -- **sqrt vs count, WIS.** sqrt is 1.4 to 3.9 points higher pooled at h−1…h3 - (+5.1/+4.0/+0.7/−1.3/−1.4 vs +3.2/+1.9/−2.0/−2.7/−5.3). In 2025-26 the gap - is 1.5–6.8 points at every horizon. In 2024-25 it is 2.2 at h−1 and 3.5 at - h3, and under 1 point at h0–h2. -- **sqrt vs count, h3 by month.** The largest gaps are in months with little - base WIS: April −20.0 vs −122.2 and May +17.7 vs −63.4 (1.3% and 1.2% of - base WIS); sqrt is lower in October (−35.1 vs −26.9) and March (−42.1 vs - −26.0). Weighted by base WIS share, April–May and November–January each - account for about half of the 3.9-point h3 gap; March takes back about 0.9. -- **sqrt vs count, coverage.** Month-avg bias (states): sqrt is 0.013 and - 0.018 lower at h2 and h3, from 2024-25 only (h3 0.181 vs 0.223); in 2025-26 - there is no clear difference (h3 0.122 vs 0.120). At h3 neither is below the - base month-avg (sqrt 0.150, count 0.168, base 0.151). Season-pooled per - season, sqrt is higher than count (2024-25 by 0.02–0.05); pooled over both - seasons the two look alike, because pooling lets errors cancel across - seasons as well as within one. -- **rate vs count.** Identical in every table. With the adaptive learning rate, - dividing a location by its population rescales steps and offsets together. - A direct check (2026-10-02) confirmed `scales` is applied: with a constant - learning rate the two differ by up to 300 admissions. -- Offsets start from zero in November 2024, so 2024-25 includes several - rounds where every config equals the base. +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd index 38ef2362..3bb031b1 100644 --- a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd +++ b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd @@ -39,8 +39,8 @@ Setup: term). - Learning truth: exact (re-run at every round on the data published by then, as prod does; E00). Scores against finalized truth. -- Seasons scored: 2024-25 and 2025-26. Horizons h−1 to h3. Headline tables: - all hub locations including US; month view: states only. +- Seasons scored: 2024-25 and 2025-26. Horizons h−1 to h3. States only + (US excluded). | config | change from the sqrt reference | |---|---| @@ -57,35 +57,11 @@ cals <- list( `slow 0.003` = do.call(run, c(sqrt_args, list(lr_slow = list(mult = 0.003)))), `slow 0.003 + leak 0.1` = do.call(run, c(sqrt_args, list(lr_slow = list(mult = 0.003)), fast_decay = 0.1)) ) -focus <- "leak 0.1" ``` -```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} +```{r month-view-data} +mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) ``` -# Findings - -WIS reduction % is positive = calibrated WIS lower than base; coverage bias is -lower-is-better. A WIS gap under 1 point or a month-avg coverage gap under -0.01 is read as no clear difference. - -- **leak 0.1 vs single, WIS: the order flips by season.** Pooled, the leak is - 0.9–2.1 points higher at h1–h3 (+1.6/+0.8/+0.6 vs +0.7/−1.3/−1.4) and 0.3 - lower at h−1/h0. In 2025-26 it is 3.3–5.6 points higher at h1–h3. In - 2024-25 it is 0.4–1.3 points lower at h−1…h2 and 0.4 higher at h3. -- **leak 0.1 vs single, h3 by month.** Most of the pooled 2.0-point h3 gap - comes from March (−14.2 vs −42.1, 5.4% of base WIS) and February (+0.3 vs - −3.8). The leak is lower in December (+0.8 vs +1.7) and May (+5.7 vs +17.7). -- **leak 0.1 vs single, coverage: the order flips by season.** Month-avg bias - (states) over both seasons: no clear difference (0.115/0.102/0.115/0.132/0.143 - vs 0.108/0.102/0.117/0.130/0.150; base 0.134/0.116/0.123/0.137/0.151). Per - season, the leak is 0.014–0.017 higher in 2024-25 at h−1…h2 and 0.010–0.018 - lower in 2025-26 at h0…h3. Season-pooled bias roughly triples with the leak - (0.026–0.028 → 0.075–0.081 at h1–h3), but pooling lets errors cancel, and - `single`'s two-season value at h1 (0.026) is below either season's (0.089, - 0.038). -- **No strong result for the slow term.** `slow 0.003` vs `single`: at most - 0.2 points pooled, at most 0.5 per season. `slow 0.003 + leak 0.1` vs - `leak 0.1`: 0.1–0.5 points; month-avg coverage within 0.004. -- Offsets start from zero in November 2024, so early 2024-25 rounds of every - config equal the base. +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e05_warm_start.Rmd b/reports/writeups/calibration_experiments/e05_warm_start.Rmd index 13e45841..c7a9ad6a 100644 --- a/reports/writeups/calibration_experiments/e05_warm_start.Rmd +++ b/reports/writeups/calibration_experiments/e05_warm_start.Rmd @@ -36,8 +36,7 @@ errors, rather than from zero, change WIS and coverage for each tracker? 0.03, floor 0.001, window 20, offsets carried across seasons). - Learning truth: NHSN data as published each week, re-run from scratch each week (prod's design, E00). Scoring: finalized truth. -- Headline tables cover all hub locations; the month view and its tables - cover states only. +- Scores cover states only (US excluded). - The warm start sets the slow offset, per horizon and level, to the level-quantile of the burn-in season's errors pooled over locations, on the sqrt scale. @@ -67,42 +66,13 @@ cals <- list( `slow + leak` = do.call(run, c(sqrt_args, list(lr_slow = list(mult = 0.003)), fast_decay = 0.1)), `warm + slow + leak (REF-op)` = run("op") ) -focus <- "warm + slow + leak (REF-op)" # The month view drops the two leak-only runs to keep it readable. mv_plot_cfgs <- c("warm only", "single", "warm + single", "slow + leak", "warm + slow + leak (REF-op)") ``` -```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} +```{r month-view-data} +mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) ``` -# Findings - -WIS reduction % (positive = calibrated WIS lower than base) differences are -in points. L1 coverage bias (lower is better) leads with the month-avg, -states only. - -- **Warm start, WIS: depends on the season.** In 2024-25 it adds 1.7–3.2 - points at h2/h3 and 0.1–1.2 at h−1…h1 to every tracker. In 2025-26 warm - and cold differ by −0.4 to +0.4 at every horizon: no clear difference. - Part of the 2024-25 gain is against cold runs that play the base - forecast for the first live rounds. -- **Warm start, coverage.** In 2024-25 month-avg bias is 0.007–0.012 lower - for `warm + single` than `single`, and 0.011–0.017 lower for REF-op than - `slow + leak`, at h0–h3. In 2025-26 the pairs are within 0.008 in either - direction (`warm + single` is 0.002–0.004 higher than `single`): no clear - difference. Season-pooled bias makes `warm + single` look lowest - (0.013–0.020 at h0–h3, all locations), but pooling lets ramp - under-prediction cancel post-peak over-prediction. -- **Leak vs plain tracker.** In 2025-26 at h1–h3, `single` is - −1.8/−4.6/−3.5 and `leak 0.1` +1.5/+1.0/+0.9, 3.3–5.6 points apart. - In 2024-25 `single` is ahead by 1.3 points at h−1 and by under 1 point - elsewhere. Month-avg bias, both seasons: within 0.007 at every horizon. -- **Slow term (REF-op vs `warm + leak 0.1`).** No strong result: WIS within - 0.4 points at every horizon (`warm + leak 0.1` ahead at h2/h3), - month-avg bias within 0.003. -- **Warm start alone.** Pooled WIS +0.5 to +2.1 (2024-25 +0.7 to +3.5, - 2025-26 +0.1 to +1.3). Month-avg bias 0.007–0.017 below the base at - h−1…h3. -- **By month, h3, both seasons.** Every config loses in October (−12 to - −49%) and March (−11 to −51%), months holding 0.3% and 5.4% of the base - WIS; December–January (71% of it) gain 1–6%. +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e07_across_eras.Rmd b/reports/writeups/calibration_experiments/e07_across_eras.Rmd index a7747685..2f6b3e6a 100644 --- a/reports/writeups/calibration_experiments/e07_across_eras.Rmd +++ b/reports/writeups/calibration_experiments/e07_across_eras.Rmd @@ -278,22 +278,3 @@ knitr::kable(cal_month_cov_table(mv_trk), caption = CAL_MONTH_COV_CAPTION) ```{r table-all} knitr::kable(season_table(GEOS[["all"]])) ``` - -# Findings - -States only. WIS reduction % (positive = calibrated WIS lower than base) -differences are in points; L1 coverage bias is lower-is-better and leads -with the month-avg. - -- **Bases.** B/A base WIS is 0.93–1.01 in 2024-25 and 1.05–1.33 in 2025-26 - (largest at h0); per state B is lower in 29 of 52 states, then 7 of 52. - Month-avg base bias over h0–h3 is 0.116–0.151 for A and 0.112–0.177 for - B (both seasons), 0.089–0.117 vs 0.097–0.146 in 2025-26. Both bases have - over 86% of outcomes above the median in December and under 40% in March - 2025-26. They differ in November 2024-25 (51% vs 73% above), March–April - 2024-25 (59/75% vs 45/56%) and February 2025-26 (60% vs 71%). -- **Trackers.** On both, pooled WIS at h1–h3 orders operating > leaky > - single. Operating gains 0.7–1.1 points more on A than on B (A - +3.9/+1.9/+1.5/+2.3, B +2.8/+1.1/+0.8/+1.2), mostly 2024-25 h2/h3 - (+2.8/+4.3 vs +1.4/+1.7) and 2025-26 h0 (+4.3 vs +2.2). On B, leaky and - single are under 1 point apart except at h3 (1.2). diff --git a/reports/writeups/calibration_experiments/e07b_candidates.Rmd b/reports/writeups/calibration_experiments/e07b_candidates.Rmd index 34919c43..dd48fc45 100644 --- a/reports/writeups/calibration_experiments/e07b_candidates.Rmd +++ b/reports/writeups/calibration_experiments/e07b_candidates.Rmd @@ -211,27 +211,3 @@ mv_linetype <- "forecaster" ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` - -# Findings - -States only. WIS reduction % (positive = calibrated WIS lower than base) -differences are in points; L1 coverage bias is lower-is-better and leads -with the month-avg. - -- **Candidates, WIS: the order flips between seasons.** Mean over h0–h3, - REF-op warm leads in 2024-25 on both (A +3.44, B +2.07; next +0.79 and - +0.90). In 2025-26 the 1 March switch-off leads on both (A +1.96, B - +2.37) and REF-op warm is 4th on A, 5th on B (+0.69, +0.60). Pooled over - both seasons the other four candidates are within 0.2 points of each - other on A (+0.64 to +0.83) and within 0.9 on B (+0.63 to +1.48). -- **1 March switch-off in 2024-25.** No strong result: 0.47 points below - sqrt 0.018 on A and 0.24 above it on B. The rates (0.018, 0.032) and the - 1 March date were picked in E14/E16/E17 on these same seasons, on B. -- **Coverage, month-avg.** No strong result among candidates: over h0–h3, - A ranges 0.115–0.126 for four candidates (base 0.132) and B 0.130–0.136 - (base 0.142). Rate 0.032 is no better than the base on either (A 0.133, - B 0.142), and in 2025-26 it is above the base at every horizon (A by - 0.016–0.024, B by 0.007–0.017). Season-pooled, rate 0.032 has the lowest - 2025-26 bias on B (0.010–0.014), because pooling lets ramp and post-peak - errors cancel. Season-pooled coverage ranks agree between A and B over - both seasons (Spearman 0.97) but not in 2025-26 (0.31). diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd index 179664ff..84c008ee 100644 --- a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -352,41 +352,3 @@ mv_plot_cfgs <- c( ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` - -# Findings - -WIS reduction % (positive = calibrated WIS lower than base) differences -are in points. L1 coverage bias is lower-is-better; month-avg is states -only, season-pooled is all locations. Unless stated, +1 week, re-run. - -- **Constant rate.** WIS rises with the rate up to about 0.056–0.1 (knee: - +3.2/+2.5/+1.1/+0.5/+0.2 at h−1…h3) and falls fast past 0.3 (at 1.0: - −12 to −23 at h0–h3). The rate with the highest WIS per horizon is - 0.1/0.056/0.056/0.018/0.018, picked on the scored seasons; at h2, - 0.018–0.056 are within 0.1 point. -- **Knee vs REF-paper.** WIS: knee ahead by 3.1–5.5 points at h1–h3, 0.6 at - h0, tied at h−1. Coverage: no clear difference season-pooled (within - 0.007); month-avg REF-paper is lower at h−1…h1 by 0.007/0.015/0.009 and - the knee at h2/h3 by 0.001/0.004. -- **Month-avg coverage vs base.** Against the base - (0.134/0.116/0.123/0.137/0.151 at h−1…h3), the knee differs by - −0.012/+0.001/+0.006/+0.005/+0.013 and 0.1 by - −0.016/−0.002/+0.004/+0.010/+0.021: no lower than the base at h1–h3. REF-op is 0.012–0.027 below the base at every horizon. REF-op - differs from the sweep in scale, leak, slow offset and warm start, so - the comparison does not isolate any one of them. Season-pooled, the - knee's bias (0.026–0.040) is about half REF-op's (0.041–0.074), but - pooling lets ramp and post-peak errors cancel. -- **Extra delay, re-run.** At the knee, h−1 WIS is +4.1/+3.2/+2.1/+1.0 at - +0/+1/+2/+3 weeks and h0 +2.9/+2.5/+2.0/+1.5; at h1–h3 the delays are - within 1 point. Month-avg bias at the knee moves by under 0.01 between - +0 and +2 weeks. -- **Extra delay, stateful.** At +0 weeks the knee makes h−1 worse than the - base: WIS −0.9, season-pooled bias 0.142 vs 0.111, month-avg 0.151 vs - 0.134. At +1 week h−1 is +3.7, 0.066 pooled, 0.118 month-avg. At h0–h3, - +0 weeks has WIS within 0.6 points of +1 week. -- **Known issues.** `rate, adaptive` matches REF-paper to every printed - digit because the adaptive rate is scale invariant (`scales` is applied; - checked 2026-10-02). Runs are cached - by name, not by inputs or code (clear `cache/calibration/experiments/e09/` - after changes). In October 2025 (one round) the knee and 0.1 carry spring - offsets forward: h0 month bias 0.232/0.218 vs the base's 0.053. diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd index 2437fb11..301417e8 100644 --- a/reports/writeups/calibration_experiments/e10_covid.Rmd +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -30,8 +30,7 @@ Question: how do the two finalists, REF-op and sqrt constant 0.018, do on flu and covid, on the submitted ensemble and on the clean replay? Part 1 runs them on both diseases and both bases, h0–h3, on identical rows. -Part 2 is the covid ensemble at h−1–h3 with REF-paper for reference, and the -standard views. +Part 2 is the covid ensemble at h−1–h3 with REF-paper for reference. # Part 1: both diseases, both bases @@ -91,111 +90,22 @@ FIN <- purrr::map(c(flu = "flu", covid = "covid"), function(d) { }) list(cals = setNames(cals, paste(grid$base, grid$cand, sep = ", ")), grid = grid, cands = names(cands)) }) -fin_score <- function(cal) { - ws_scores(cal$forecasts %>% mutate(season = season_of(reference_date)) %>% filter(location != "US")) %>% - select(season, horizon, which, wis, cal_err) %>% - tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% - mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) -} -FIN_M <- purrr::map(FIN, function(f) { - purrr::map2(f$cals, seq_len(nrow(f$grid)), \(cal, i) fin_score(cal) %>% mutate(f$grid[i, ])) %>% - bind_rows() %>% - mutate( - base = factor(base, levels = c("ensemble", "replay")), cand = factor(cand, levels = f$cands), - season = factor(season, levels = c(LIVE, "both"), labels = c(LIVE, "both seasons")), - h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) - ) -}) -BASE_COLS <- c(ensemble = "#0072B2", replay = "#D55E00") -fin_table <- function(m) { - bind_rows( - m %>% distinct(base, season, h, .keep_all = TRUE) %>% mutate(cand = "base", cell = sprintf("WIS %.1f", wis_base)), - m %>% mutate(cell = sprintf("%+.1f", wis_pct)) - ) %>% - mutate(config = paste(base, cand, sep = ", ")) %>% - arrange(base, cand != "base", cand) %>% - transmute(config = factor(config, levels = unique(config)), col = paste(season, h), cell) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) -} -fin_plot <- function(m) { - ggplot(m, aes(cand, wis_pct, fill = base)) + - geom_hline(yintercept = 0, colour = "grey50") + - geom_col(position = position_dodge(width = 0.85, preserve = "single"), width = 0.8) + - facet_grid(season ~ h) + - scale_fill_manual(values = BASE_COLS, name = NULL) + - labs(x = NULL, y = "WIS reduction % (positive = calibrated WIS lower than base)") + - theme(legend.position = "bottom", axis.text.x = element_text(angle = 30, hjust = 1)) -} ``` -```{r fin-sections, results="asis", fig.height=6.5} -for (d in names(FIN)) { - f <- FIN[[d]] - m <- FIN_M[[d]] - cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") - cat("\n\n", cal_h(3), " WIS\n\n", sep = "") - cat("WIS reduction % (positive = calibrated WIS lower than base), states only; the base rows give the base WIS.\n\n") - print(knitr::kable(fin_table(m))) - print(fin_plot(m)) - cat("\n\n", cal_h(3), " Coverage\n\n", sep = "") - print(knitr::kable(cal_month_cov_table(cal_month_view(f$cals, seasons = LIVE)), caption = CAL_MONTH_COV_CAPTION)) - cat("\n\n", cal_h(3), " State by state\n\n", sep = "") - cat("States share the national wave, so the counts are not independent tests.\n\n") - op <- if (d == "flu") "REF-op warm" else "REF-op cold" - pairs <- unlist(purrr::map(c("ensemble", "replay"), function(b) { - nm <- function(x) paste(b, x, sep = ", ") - ps <- list(c(nm(op), nm("sqrt 0.018")), c(nm("sqrt 0.018 warm"), nm("sqrt 0.018"))) - purrr::keep(ps, \(p) all(p %in% names(f$cals))) - }), recursive = FALSE) - print(knitr::kable(cal_pair_wins(cal_state_wis(f$cals), pairs), caption = CAL_PAIR_WINS_CAPTION)) -} -``` +Per disease, a month view for each base. The ensemble and replay rows are +the same (round, location, horizon, level) cells, so the two bases' tables +compare like for like. ```{r fin-month, results="asis"} for (d in names(FIN)) { + cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") for (b in c("ensemble", "replay")) { mv <- cal_month_view(FIN[[d]]$cals[startsWith(names(FIN[[d]]$cals), paste0(b, ", "))], seasons = LIVE) - mv_show_table <- FALSE - cal_month_section(paste("Month view:", d, b), level = 2L) + cal_month_section(paste(d, b), level = 3L) } } ``` -# Findings, part 1 - -States only, h0/h1/h2/h3, WIS reduction % (positive = calibrated WIS lower -than base). No uncertainty estimates; gaps under 1 point read as no clear -difference. - -- **Flu, ensemble:** sqrt 0.018 warm +4.4/+2.5/+2.7/+4.3 pooled, REF-op - warm +3.9/+1.9/+1.5/+2.3, both cold configs within ±0.7 at h1–h3. All of - the warm configs' lead is 2024-25 (sqrt warm +4.4/+4.0/+4.9/+7.9; REF-op - warm wins 49–51 of 52 states against sqrt cold). In 2025-26 every config - is within 1 point of the base at h1–h2, and both warm configs lose at h3 - (sqrt warm −1.7, REF-op warm −0.9). -- **Flu, replay:** as E18: sqrt 0.018 warm +3.4/+1.6/+1.5/+2.7, REF-op warm - +2.8/+1.1/+0.8/+1.2. Here sqrt warm stays positive in 2025-26 - (+3.3/+0.9/+0.8/+1.3) where on the ensemble it does not, so the replay - overstates its 2025-26 result for what we submit. -- **Flu coverage, month-avg:** on the ensemble, REF-op warm is the only - config below the base at every horizon (0.090/0.110/0.124/0.138 vs - 0.116/0.123/0.137/0.151); sqrt warm is close to the base at h1–h3 - (0.119/0.133/0.147). On the replay sqrt warm is 0.009–0.017 above the base - at h2–h3 (E18). -- **Covid, ensemble:** REF-op cold (covid prod) +0.8/−0.1/−0.3/−0.8 pooled; - sqrt 0.018 +0.1/−2.7/−3.7/−4.2. Both gain a little in 2024-25; in 2025-26 - sqrt 0.018 loses 6.9–8.3 at h1–h3 against REF-op's 2.0–2.9, and REF-op - has the lower WIS in 44–45 of 52 states at h1–h3. Month-avg coverage is - within 0.011 of the base for both, and 0.004–0.006 above it at h2–h3. -- **Covid, replay:** nothing is consistently positive beyond h0. Warm - starts (from the HHS backfill) lose at h0–h2 in 2024-25 (sqrt warm −1.3 - to −5.1) and gain at h3 in 2025-26 (+4.1, +5.0). Pooled, the best h3 is - REF-op warm +2.8, the best h1 sqrt 0.018 cold −1.8 and REF-op cold −1.1. -- **So:** on flu the warm sqrt 0.018 leads on both bases, but on the - ensemble its whole lead is the first live season. On covid sqrt 0.018 is - clearly worse than REF-op cold on what we submit, and no config tested - helps at h1–h3. - # Part 2: covid ensemble, h−1–h3 - Disease: covid. Base forecaster: the submitted `CMU-TimeSeries` ensemble. @@ -206,8 +116,8 @@ difference. - Rounds 2024-11-30 to 2026-09-19. The 2024-11-23 round is excluded (made from a bad NHSN release corrected the next day). - Covid is submitted year-round; per-season views use August–July seasons. -- Horizons h−1 to h3. Headline tables: all hub locations; month view: states - only. +- Horizons h−1 to h3. Scores: states only (US excluded), the two live + seasons. | config | change from REF-paper | |---|---| @@ -234,36 +144,9 @@ cals <- purrr::map(cals, function(cal) { cal }) LIVE <- c("2024-2025", "2025-2026") -focus <- "REF-op (no warm start)" ``` -```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} +```{r p2-month-view, results="asis"} +mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) +cal_month_section("Month view", level = 2L) ``` - -# Findings, part 2 - -There are no uncertainty estimates; a WIS gap under 1 point or a month-avg -coverage gap under 0.01 is read as no clear difference. The pooled headline, -median-error and by-month tables run to 2026-09-19 to include the 2026 summer -wave; those Aug–Sep 2026 rounds are in no per-season table. - -- **h−1:** both configs improve WIS by about 10–11% in each season - (REF-paper 11.6/10.5, REF-op 8.9/10.8) and the absolute error of the median - by about 20%. Which config is ahead flips between seasons. -- **REF-paper at h1–h3:** −4.5/−7.3/−10.3 pooled, almost all from 2025-26 - (−10.2/−14.4/−19.3). In 2024-25 it is within 0.3 points of the base. -- **REF-op at h1–h3:** 0.0/−0.6/−1.3 pooled; +2.4/+2.8/+1.9 in 2024-25 and - −3.0/−4.0/−4.3 in 2025-26. No strong result: the sign depends on the - season. -- **Month-avg coverage (states, h−1…h3):** base 0.148/0.102/0.125/0.143/0.157; - REF-op 0.123/0.096/0.126/0.147/0.161 (lower at h−1 only); REF-paper - 0.111/0.094/0.126/0.155/0.171 (about 0.01 worse at h2/h3). Season-pooled, - the base looks calibrated at h1–h3, but pooling lets the two seasons' - errors cancel: within each season its bias is 0.072–0.078 (2024-25) and - 0.029–0.051 (2025-26). -- **By month at h3:** REF-op −16.5% in September and −12.0% in October - (together 20% of base WIS); REF-paper −42% to −49% in April–May (about 6%). -- **sqrt 0.018 (added 2026-10-02):** the largest h−1 gain, +14.9 pooled - (+14.9/+14.1 by season) against REF-op's +10.2, and +24.4% median absolute - error. At h1–h3 it is −3.3/−4.2/−4.1 pooled and −9.5/−9.1/−7.7 in 2025-26, - worse than REF-op (−3.0/−4.0/−4.3) and better than REF-paper. diff --git a/reports/writeups/calibration_experiments/e11_constant_lr.Rmd b/reports/writeups/calibration_experiments/e11_constant_lr.Rmd index 61151585..76196c56 100644 --- a/reports/writeups/calibration_experiments/e11_constant_lr.Rmd +++ b/reports/writeups/calibration_experiments/e11_constant_lr.Rmd @@ -36,8 +36,8 @@ Setup: effectively cold. - Learning truth: exact (re-run at every round on the data published by then; E00), one week of extra delay. Scores against finalized truth. -- Seasons scored: 2024-25 and 2025-26. Horizons h−1 to h3. Headline tables: - all hub locations; month view: states only. +- Seasons scored: 2024-25 and 2025-26. Horizons h−1 to h3. States only + (US excluded). | config | change from REF-paper (counts, adaptive 0.03 × p90 of the last 20 errors) | |---|---| @@ -59,33 +59,11 @@ cals <- list( `rate, constant 0.1` = run("paper", scales = rate_scales, lr = 0.1), `REF-op` = run("op") ) -focus <- "rate, constant 0.1" ``` -```{r child=here::here("reports/writeups/calibration_experiments/_standard_views.Rmd")} +```{r month-view-data} +mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) ``` -# Findings - -There are no uncertainty estimates; a WIS gap under 1 point or a month-avg -coverage gap under 0.01 is read as no clear difference. - -- **Constant 0.1 vs `sqrt, adaptive`:** WIS +3.6/+2.4/+0.7/+0.3/−0.5 vs - +5.1/+4.0/+0.7/−1.3/−1.4 at h−1…h3 pooled. The adaptive tracker is ahead - at h−1/h0 (1.5/1.6 points), the constant at h2/h3 (1.6/0.9). In 2025-26 the - gaps are larger in both directions (adaptive +4.6 at h0, constant +2.8 at - h2). The order depends on horizon. -- **Constant 0.1 vs `warm + sqrt, adaptive`:** the warm tracker is ahead in - every 2024-25 cell (by 0.8–2.7). In 2025-26 the constant is ahead at h2/h3 - (by 2.5/1.9). This compares a cold config with a warm one. -- **REF-op:** pooled WIS +5.4/+4.9/+2.4/+1.7/+2.2, the highest at every - horizon, by up to about 2 points over `warm + sqrt, adaptive`. Its 2025-26 - gains at h1–h3 are under 1.5 points. -- **Month-avg coverage (states, h−1…h3):** base 0.134/0.116/0.123/0.137/0.151; - constant 0.118/0.114/0.127/0.147/0.172; `sqrt, adaptive` - 0.108/0.102/0.117/0.130/0.150; `warm + sqrt, adaptive` - 0.107/0.100/0.116/0.128/0.146; REF-op 0.107/0.091/0.111/0.125/0.136. The - adaptive configs are within 0.01 of each other at most horizons. About half - of the constant's h0 gap comes from October 2025, a single sparse month - (0.3% of base WIS) with spring offsets carried over. Season-pooled the - constant has the lowest bias (0.017–0.032), but pooling lets errors cancel. +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd index 12a2fd64..06ae991a 100644 --- a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd +++ b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd @@ -224,29 +224,3 @@ mv_plot_cfgs <- c("rate 0.018", "rate 0.032", "rate 0.056", "rate 0.1", "REF-op ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` - -# Findings - -There are no uncertainty estimates; a WIS gap under 1 point or a month-avg -coverage gap under 0.01 is read as no clear difference. The cold start leaves -Nov–Dec 2024 equal to the base for every config, so 2024-25 gains are -diluted. The rate range was looked at on these same seasons. - -- **WIS up to 0.056:** every rate from 0.0032 to 0.056 is at or above the - base in both seasons (at most +1.2 in 2024-25, +1.9 in 2025-26), and the - rates in this range are within 1.2 points of each other (0.018 vs 0.032: - within 0.3). No strong result for a best rate. -- **WIS above 0.056:** from 0.1 WIS falls (2024-25 h2/h3 −0.5/−0.9; 2025-26 - h1–h3 −0.8 to −1.2); by 0.18 it reaches −3 to −4 at h3. -- **Month-avg coverage (states, both seasons, h0…h3):** base - 0.112/0.127/0.153/0.177; 0.01 0.099/0.115/0.144/0.170; 0.032 about equal to - the base; 0.056 and above 0.008–0.026 worse than the base at h1–h3. In - 2025-26 at h0, 0.032 is above the base only because of October 2025. - Season-pooled bias falls steadily with the rate (2025-26 reaches about 0.01 - by 0.032), but pooling lets ramp and post-peak errors cancel. -- **References:** at h0, `REF-op cold` and `sqrt, adaptive` gain +1.8 to - +2.2 in each season. At h2/h3 in 2024-25 they lose (REF-op −0.0/−0.8, - sqrt −1.4/−2.6). Against the constant rates 0.018–0.032 the gaps are under - 1 point in most other cells. -- **50% interval:** at 0.018–0.032 it covers 0.47–0.51 at h0/h2; from 0.1 up - it under-covers in 2024-25 (0.39–0.45 at h2). diff --git a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd index 3255914d..ec52bff5 100644 --- a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd +++ b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd @@ -294,33 +294,3 @@ mv <- cal_month_view(cals, seasons = LIVE) ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` - -# Findings - -There are no uncertainty estimates; a WIS gap under 1 point or a month-avg -coverage gap under 0.01 is read as no clear difference. Both configs are cold -and equal the base through December 2024 (about a third of 2024-25's base -WIS), so the pooled gains are diluted. There is no per-season split here. -States only, both seasons pooled, h0/h1/h2/h3. - -- **WIS:** REF-op +2.03/+0.59/+0.09/−0.17, E11 +0.73/−0.21/−0.70/−1.03. - REF-op is 1.3 points ahead at h0; at h1–h3 the gap is 0.8–0.9 (no clear - difference). -- **By level:** the six tail levels (0.01–0.05, 0.95–0.99) add at most 0.1 - point. REF-op's levels below the median give +1.36/+0.72/+0.59/+0.58 and - those above +0.54/−0.14/−0.48/−0.76. E11: below +1.00/+0.39/+0.12/−0.02, - above −0.31/−0.55/−0.73/−0.91. -- **Offset size:** median |offset| is 2–5% of the base 90% width at levels - 0.25–0.95, and 0.4–2.1% at the 0.05 level. The median calibrated 90% width - is 1.00–1.04 times the base. The 90th percentile of |offset| ÷ 90% width - (levels 0.25–0.95) is 0.07–0.20 for REF-op and 0.24–0.55 for E11. -- **Median shift vs width change:** for REF-op, median shift only gives - +2.29/+0.80/+0.47/+0.39 (0.2–0.6 above the full calibration) and width - change only −0.59 to −1.14. The interaction is +0.33 to +0.57 (E11: +0.43 - to +0.86), so the split is not additive. The coverage of the width-only - variant is not measured. -- **Month-avg coverage:** base 0.112/0.127/0.153/0.177; REF-op - 0.098/0.119/0.146/0.167 (0.007–0.014 lower); E11 0.116/0.143/0.174/0.203 - (0.004–0.026 higher than the base). Season-pooled, E11 looks lowest - (0.020–0.025, REF-op 0.077–0.085, base 0.105–0.127), but pooling lets ramp - and post-peak errors cancel. diff --git a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd index 45def696..7952939a 100644 --- a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd +++ b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd @@ -390,38 +390,3 @@ mv_cols <- MONTH_COLS ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` - -# Findings - -There are no uncertainty estimates; a WIS gap under 1 point or a month-avg -coverage gap under 0.01 is read as no clear difference. The cold start leaves -Nov–Dec 2024 equal to the base for every config. States only, h0/h1/h2/h3. - -- **Sweep:** sqrt 0.001–0.018 are at or above the base in every season × - horizon, often by +0.0 to +0.3. From 0.032 some cells turn negative - (2024-25 h3 −0.3); from 0.056 the losses grow. The rule selects 0.018 - (mean +1.15, worst +0.06); 0.01 is close (mean +1.02, worst +0.10). -- **sqrt 0.018 vs rate 0.032:** pooled +2.2/+0.8/+0.6/+0.6 vs - +1.3/+0.7/+0.5/+0.4. About 1 point apart at h0 (2024-25 +0.7, 2025-26 - +1.1); within 0.2 at h1–h3 pooled (no clear difference). At h3 sqrt is - ahead in 2025-26 (+0.7) and rate in 2024-25 (+0.1). -- **sqrt 0.018 vs REF-op cold:** the order flips by season. In 2024-25 - REF-op is ahead at h0–h1 (+2.2/+0.9 vs +1.7/+0.6) and behind at h2–h3 - (−0.0/−0.8 vs +0.3/+0.1); in 2025-26 sqrt is ahead at every horizon. - Pooled gaps are 0.2–0.8. -- **Month-avg coverage (both seasons):** base 0.112/0.127/0.153/0.177; sqrt - 0.018 0.094/0.116/0.143/0.169; REF-op 0.098/0.119/0.146/0.167; rate 0.018 - 0.102/0.120/0.147/0.172; rate 0.032 0.108/0.127/0.155/0.177. sqrt vs REF-op - and vs rate 0.018: no clear difference. sqrt vs rate 0.032: 0.008–0.014 - lower, all from 2025-26 (0.023–0.029 there; in 2024-25 rate 0.032 is - 0.003–0.008 lower). Rate 0.032's low season-pooled 2025-26 bias - (0.010–0.014) is cancellation; month-avg it is above the base. -- **Offset tail:** 90th percentile of |offset| ÷ base 90% width, all levels: - sqrt 0.018 0.092–0.111, rate 0.018 0.078–0.096, rate 0.032 0.124–0.153, - REF-op 0.074–0.114. sqrt is smaller than rate 0.032 at the 0.05 level and - larger at 0.95. -- **March at h2** (within the month, 2024-25 / 2025-26): rate 0.018 - −1.2/−10.6, rate 0.032 −2.2/−18.7, sqrt 0.018 −7.2/−27.9, REF-op - −19.8/−24.3. March is 7.5% and 4.2% of the base WIS, so these cost 0.1–1.5 - season points. In March the base already over-predicts (58% and 67% of - outcomes below its median at h2). diff --git a/reports/writeups/calibration_experiments/e17_late_decay.Rmd b/reports/writeups/calibration_experiments/e17_late_decay.Rmd index d3f31c64..3b828f5d 100644 --- a/reports/writeups/calibration_experiments/e17_late_decay.Rmd +++ b/reports/writeups/calibration_experiments/e17_late_decay.Rmd @@ -198,36 +198,3 @@ mv_cols <- setNames(DECAY_COLS, mv_plot_cfgs) ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` - -# Findings - -There are no uncertainty estimates; a WIS gap under 1 point or a month-avg -coverage gap under 0.01 is read as no clear difference. The cold start leaves -Nov–Dec 2024 equal to the base for every config. States only, h0/h1/h2/h3. - -- **Decays vs no decay, pooled WIS:** sqrt 0.018 loses 0.4–0.5 at h0 and is - within 0.2 at h1–h3; rate 0.018 loses 0.5–0.7 at h0 and 0.2–0.3 at h1–h3; - rate 0.032 loses 0.2–0.5 at h0 and at most 0.2 at h1–h3. No strong result. - For sqrt the order flips by season: in 2024-25 the decays are 0.1–0.3 - ahead at h1–h3; in 2025-26 they are 0.9–1.3 behind at h0. -- **`off_after` vs no decay, pooled WIS:** sqrt 0.018 +2.3/+1.2/+1.2/+1.3 vs - +2.2/+0.8/+0.6/+0.6 (0.1–0.7 apart); rate 0.032 −0.1 to +0.5; rate 0.018 - −0.5 to 0. No pooled gap reaches 1 point. The sqrt gain is mostly 2025-26 - (+3.4/+1.6/+1.9/+2.6 vs +3.0/+1.0/+1.1/+1.5). In 2024-25 the rate-scale - trackers are lower at h0 with the switch-off. The date was tuned - in-sample. -- **March and spring at h2:** the decays shrink the March loss (sqrt - 2025-26: −27.9 to −10.7/−6.3/−2.9; `off_after` −0.0), and also remove the - April–May gains (sqrt May 2025-26: +17.3 to +3.3/−0.0/−0.1; `off_after` - +0.0). -- **Month-avg coverage (both seasons):** sqrt 0.018 no decay - 0.094/0.116/0.143/0.169, decay 0.85 0.103/0.123/0.149/0.173, `off_after` - 0.104/0.121/0.150/0.170, base 0.112/0.127/0.153/0.177: gaps ≤ 0.010, no - clear difference. For rate 0.032, decay 0.85 is 0.005–0.007 lower than no - decay overall and 0.014–0.024 lower in 2025-26. Season-pooled, no decay is - 0.032–0.043 lower for sqrt, but pooling lets ramp and post-peak errors - cancel. -- **Carry-over:** `off_after` carries the pre-March offset into the next - ramp. In 2025-26 that ramp under-predicted too (Nov 2025 h0 month bias - 0.065 for `off_after` vs 0.096–0.103 for the decays), which a season that - ramps differently would not repeat. diff --git a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd index 7354f634..211e6223 100644 --- a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd +++ b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd @@ -245,51 +245,3 @@ mv_cols <- STEP_COLS ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` - -# Findings - -There are no uncertainty estimates; a WIS gap under 1 point or a month-avg -coverage gap under 0.01 is read as no clear difference. The cold steps equal -the base through Nov–Dec 2024; only the warm steps act there. States only, -h0/h1/h2/h3, both seasons pooled unless stated. - -- **Endpoints:** REF-op +2.8/+1.1/+0.9/+1.1, sqrt constant 0.018 - +2.2/+0.8/+0.6/+0.6 (0.3–0.6 apart, no clear difference). The order flips - by season: REF-op is ahead in 2024-25 (+3.2/+1.9/+1.6/+1.6 vs - +1.7/+0.6/+0.3/+0.1), sqrt 0.018 in 2025-26 (+3.0/+1.0/+1.1/+1.5 vs - +2.2/−0.1/−0.0/+0.2). -- **Warm start (step 1):** −0.9/−0.5/−0.6/−1.0 pooled when removed. In - 2024-25 it is worth 0.9–2.0 points; in 2025-26 removing it gains 0.2–0.7 - at h1–h3. -- **Slow term without warm start (step 2):** within ±0.2. No strong result. -- **Leak (step 3):** removing it costs 0.6–1.2 at h1–h3 and gains 0.4 at h0. -- **Burn-in season's eta warm-up (step 4):** −0.1 to −0.7, in 2024-25 only. - No strong result. -- **Per-100k units (step 5):** identical to step 4 in every reported digit. - The adaptive rate is scale invariant when the 1e-3 floor doesn't bind; a - direct check (2026-10-02) confirmed `scales` is applied. -- **Constant 0.018 vs cold adaptive (step 6):** +0.3/+1.0/+1.7/+2.3, ahead at - h2–h3 in both seasons. -- **Month-avg coverage:** no step changes it by more than 0.008. REF-op - 0.095/0.121/0.150/0.175, sqrt 0.018 0.094/0.116/0.143/0.169, base - 0.112/0.127/0.153/0.177. Season-pooled bias moves by up to 0.06 per step, - but pooling lets ramp and post-peak errors cancel. -- **References:** rate 0.032 +1.3/+0.7/+0.5/+0.4 and rate 0.056 - +1.3/+0.5/+0.3/+0.0 pooled. Month-avg coverage: rate 0.032 equals the base; - rate 0.056 is 0.008–0.014 above it. -- **Branch, sqrt 0.018 warm:** +3.4/+1.6/+1.6/+2.5 pooled, the highest of - any config here, 0.5–1.4 above REF-op and 0.8–1.9 above sqrt 0.018 cold. - In 2024-25 +3.4/+2.1/+2.2/+3.3 (above REF-op by 0.2–1.7); in 2025-26 - +3.4/+0.9/+0.8/+1.2 (above REF-op by 1.0–1.2, within 0.3 of sqrt 0.018 - cold at h1–h3). Only the h3 gap to REF-op exceeds 1 point pooled. -- **Branch coverage costs month-avg:** 0.098/0.126/0.164/0.195 pooled - against base 0.112/0.127/0.153/0.177 and REF-op 0.095/0.121/0.150/0.175, - so 0.011–0.018 worse than the base at h2–h3. In 2025-26 it is 0.026–0.029 - above the base at h2–h3. Season-pooled it looks best at h0–h2 - (0.040/0.034/0.032), which is the ramp/post-peak cancellation. -- **State by state:** in 2024-25 REF-op beats sqrt 0.018 cold in 41–45 of - 52 states at every horizon (median gap +1.5 to +1.7); in 2025-26 in only - 14–18 (median −0.4 to −2.0). So the season flip is broad, not a few large - states. The warm branch beats sqrt 0.018 cold in 37–50 states in 2024-25 - and 20–30 in 2025-26. Against REF-op it wins 19/21/26/41 of 52 states - in 2024-25 and 33/36/36/41 in 2025-26 (h0/h1/h2/h3). From 9a1528591f0c23d1d2c8827a1518f44a31206be3 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:15:15 -0700 Subject: [PATCH 088/107] refactor(calibration): trim each notebook to the month view, elbow plots and question-specific views; drop unused view helpers --- R/calibration/views.R | 100 -------- notes/calibration-ledger.md | 14 +- .../e00_vintage_backtest.Rmd | 60 +---- .../e01_eta_settings.Rmd | 76 +----- .../e02_eta_variants.Rmd | 153 +----------- .../e07_across_eras.Rmd | 139 ++--------- .../e07b_candidates.Rmd | 95 ++------ .../calibration_experiments/e09_lr_delay.Rmd | 136 ++++------- .../e14_lr_by_season.Rmd | 85 +------ .../e15_wis_sources.Rmd | 84 +------ .../e16_sqrt_constant_lr.Rmd | 224 +++--------------- .../e17_late_decay.Rmd | 99 +------- .../e18_ref_op_bridge.Rmd | 89 +------ 13 files changed, 169 insertions(+), 1185 deletions(-) diff --git a/R/calibration/views.R b/R/calibration/views.R index 1dc18949..2ac91ecd 100644 --- a/R/calibration/views.R +++ b/R/calibration/views.R @@ -50,58 +50,6 @@ cal_run <- function(forecasts, truth, ref = "paper", ..., learn = c("final", "vi } -#' V-head: WIS reduction % and L1 coverage bias by horizon, one row per -#' (variant, `by`, horizon). -#' -#' @param cals named list of [calibrate_hub_forecasts()] outputs. -#' @param from only score rounds with reference date on or after this date. -#' @export -cal_headline <- function(cals, by = character(0), from = NULL) { - purrr::imap(cals, function(cal, nm) { - fc <- cal$forecasts - if (!is.null(from)) fc <- fc %>% filter(.data$reference_date >= from) - left_join( - hub_quantile_loss(fc, by = by), - hub_coverage_summary(fc, by = by) %>% - select(all_of(c(by, "horizon")), "cal_error_base", "cal_error_cal"), - by = c(by, "horizon") - ) %>% mutate(variant = nm, .before = 1) - }) %>% - bind_rows() %>% - mutate(variant = factor(.data$variant, levels = names(cals))) -} - - -#' One column of a [cal_headline()] table, horizons across. -#' @export -cal_wide <- function(tbl, col, digits = 1, id = "variant") { - tbl %>% - select(all_of(id), "horizon", value = all_of(col)) %>% - mutate(value = round(.data$value, digits)) %>% - tidyr::pivot_wider(names_from = "horizon", values_from = "value", names_prefix = "h") -} - - -#' V-month: WIS reduction % by month of reference date, with each month's share -#' of the base WIS, per horizon. -#' @export -cal_by_month <- function(cal) { - pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) - cal$forecasts %>% - filter(!.data$is_burn_in, !is.na(.data$truth), !is.na(.data$value_base)) %>% - mutate(month = factor(format(.data$reference_date, "%b"), levels = month.abb[c(7:12, 1:6)])) %>% - group_by(.data$horizon, .data$month) %>% - summarize( - base = sum(pinball(.data$truth, .data$value_base, .data$level)), - cal = sum(pinball(.data$truth, .data$value_cal, .data$level)), - .groups = "drop" - ) %>% - group_by(.data$horizon) %>% - mutate(pct = 100 * (.data$base - .data$cal) / .data$base, share = 100 * .data$base / sum(.data$base)) %>% - ungroup() -} - - #' [cal_run()], cached on disk under a hash of its inputs. #' #' Keeps only the forecast columns the views use. Clear `cache_dir` after @@ -126,26 +74,6 @@ cal_run_cached <- function(forecasts, truth, ref = "paper", ..., learn = "exact" } -#' V-ae: summed absolute error of the median, change % vs base (positive is -#' better), per (variant, `by`, horizon). -#' @export -cal_ae <- function(cals, by = character(0)) { - purrr::imap(cals, function(cal, nm) { - cal$forecasts %>% - filter(!.data$is_burn_in, .data$level == 0.5, !is.na(.data$truth), !is.na(.data$value_base)) %>% - group_by(across(all_of(c(by, "horizon")))) %>% - summarize( - ae_base = sum(abs(.data$truth - .data$value_base)), - ae_cal = sum(abs(.data$truth - .data$value_cal)), - .groups = "drop" - ) %>% - mutate(variant = nm, .before = 1, ae_pct = 100 * (.data$ae_base - .data$ae_cal) / .data$ae_base) - }) %>% - bind_rows() %>% - mutate(variant = factor(.data$variant, levels = names(cals))) -} - - #' The fixed V-state locations, so state panels are comparable across #' experiments: national, then states from large to very small. #' @export @@ -538,34 +466,6 @@ cal_state_wis <- function(cals, seasons = c("2024-2025", "2025-2026")) { }) %>% bind_rows() } -#' For each pair `c(a, b)` of runs in `state_wis`: per (season, horizon), the -#' number of states where `a` has the lower WIS, and the median per-state gap -#' in WIS reduction points (positive = `a` better). A table for `knitr::kable()`. -#' @export -cal_pair_wins <- function(state_wis, pairs) { - labels <- purrr::map_chr(pairs, \(pr) paste(pr[1], "vs", pr[2])) - purrr::map2(pairs, labels, function(pr, lab) { - state_wis %>% - filter(.data$config %in% pr) %>% - select("season", "location", "horizon", "config", "wis_cal", "wis_base") %>% - tidyr::pivot_wider(names_from = "config", values_from = "wis_cal") %>% - transmute( - season = .data$season, horizon = .data$horizon, pair = lab, - a_wins = .data[[pr[1]]] < .data[[pr[2]]], - gap = 100 * (.data[[pr[2]]] - .data[[pr[1]]]) / .data$wis_base - ) - }) %>% - bind_rows() %>% - mutate(pair = factor(.data$pair, levels = labels)) %>% - mutate(h = paste0("h", .data$horizon)) %>% - group_by(across(all_of(c("pair", "season", "h")))) %>% - summarize(cell = sprintf("%d of %d (median %+.1f)", sum(.data$a_wins), n(), stats::median(.data$gap)), .groups = "drop") %>% - tidyr::pivot_wider(names_from = "h", values_from = "cell") %>% - arrange(.data$pair, .data$season) -} - -CAL_PAIR_WINS_CAPTION <- "States where the first config of the pair has the lower WIS, of the states scored, and the median per-state gap in WIS reduction points (positive = first config better)" - # August-July season label, as `season_of()` in the replay scripts. cal_season_of <- function(d) { y <- as.integer(format(d, "%Y")) - (as.integer(format(d, "%m")) < 8) diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 87d11608..93077e5f 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -149,12 +149,14 @@ scratch each round on that round's data, as prod does). The index groups them by topic (finalists, flu and covid, learning rate, tracker structure, late season, learning truth). -Standard views (`_standard_views.Rmd`, `_month_view.Rmd`): V-head (WIS and -coverage by horizon × season), V-month (by month of reference date, with -month-avg coverage), V-state (per-state season panels), V-gallery (a few -forecasts picked by `cal_gallery_pick()`, stratified by season phase × -location size), V-ae (median absolute error), V-curve (coverage vs WIS over a -swept parameter). +Each comparison notebook leads with the month view (`_month_view.Rmd`): +WIS, coverage bias and median error by month of reference date, then +by-horizon tables of WIS reduction % and season-pooled / month-avg coverage +bias. All scores are states only. Sweep notebooks (E01, E09, E14, E16) lead +with their coverage-vs-WIS curves and follow with a month view of a few rates +around the elbow. Per-state fan panels and the gallery of best and worst +forecasts are only in `prod_views.Rmd`, for the prod configs. Conclusions +live in this ledger, not in the notebooks. ## Experiments diff --git a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd index 1d3ae784..1d6bd160 100644 --- a/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd +++ b/reports/writeups/calibration_experiments/e00_vintage_backtest.Rmd @@ -38,8 +38,7 @@ Setup: zero at the first 2024-25 round (REF-op's slow term starts from its warm start). - Seasons scored: 2024-25 and 2025-26, burn-in rounds excluded. Horizons - h−1 to h3. Headline tables: all hub locations including US; month view: - states only. + h−1 to h3. States only (US excluded). - Scores are always against finalized truth. Configs are `ref/settle_days/learn`: @@ -76,56 +75,19 @@ names(cals) <- with(grid, paste(ref, settle_days, learn, sep = "/")) ```{r headline} LIVE <- c("2024-2025", "2025-2026") -head_tbl <- cal_headline(cals) %>% - tidyr::separate(variant, c("ref", "settle_days", "learn"), sep = "/", remove = FALSE) %>% - mutate(learn = factor(learn, levels = c("final", "vintage", "exact"))) first_version <- min(vintages$version) n_clamped <- sum(unique(hub_fc$reference_date) - 3L < first_version & hub_label_seasons(unique(hub_fc$reference_date))$season %in% LIVE) -``` - -`r n_clamped` live rounds are issued before the first vintage (2024-11-19); only -burn-in rounds use that vintage in place of their own. - -# Headline - -WIS reduction % by horizon (positive = calibrated WIS lower than base): - -```{r wis-table} -knitr::kable(cal_wide(head_tbl, "pct_improvement", id = c("ref", "settle_days", "learn")) %>% arrange(ref, settle_days, learn)) -``` - -L1 coverage bias (lower is better; mean over levels of |coverage − level|); the `base` row is -the uncalibrated forecast: - -```{r cal-table} -knitr::kable( - bind_rows( - cal_wide(head_tbl, "cal_error_cal", 3, id = c("ref", "settle_days", "learn")), - cal_wide(head_tbl %>% filter(learn == "final", ref == "paper", settle_days == "14") %>% mutate(ref = "base", settle_days = "", learn = ""), "cal_error_base", 3, id = c("ref", "settle_days", "learn")) - ) %>% arrange(ref, settle_days, learn) -) -``` - -The same bias for states only, both seasons, season-pooled against -month-average (see "Month view" below). Configs are `ref/settle_days/learn`: - -```{r cal-table-month} mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) -mv_show_table <- FALSE -knitr::kable(cal_month_cov_table(mv, seasons = "both"), caption = CAL_MONTH_COV_CAPTION) +# Runs drawn in the month view; the tables keep all twelve. +mv_plot_cfgs <- c("op/14/final", "op/14/exact", "op/7/vintage", "op/7/exact", "paper/14/final", "paper/14/exact") ``` -WIS reduction % by season (positive = calibrated WIS lower than base): +`r n_clamped` live rounds are issued before the first vintage (2024-11-19); only +burn-in rounds use that vintage in place of their own. Configs are +`ref/settle_days/learn`. -```{r season-table} -knitr::kable( - cal_wide( - cal_headline(cals, by = "season") %>% filter(season %in% LIVE) %>% - tidyr::separate(variant, c("ref", "settle_days", "learn"), sep = "/"), - "pct_improvement", id = c("ref", "settle_days", "learn", "season") - ) %>% arrange(ref, settle_days, season, learn) -) +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` # Vintage vs exact: difference in the calibrated median @@ -153,11 +115,3 @@ ggplot(drift, aes(month, 100 * rel, colour = factor(horizon), group = horizon)) facet_wrap(~config) + labs(x = NULL, y = "median |vintage − exact| / base median (%)", colour = "horizon") ``` - -```{r month-view-cfgs} -# Runs drawn in the month view. -mv_plot_cfgs <- c("op/14/final", "op/14/exact", "op/7/vintage", "op/7/exact", "paper/14/final", "paper/14/exact") -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` diff --git a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd index d1d2f14d..5288a51b 100644 --- a/reports/writeups/calibration_experiments/e01_eta_settings.Rmd +++ b/reports/writeups/calibration_experiments/e01_eta_settings.Rmd @@ -43,7 +43,7 @@ Setup: - Grid: `lr_mult` {0.3, 0.1, 0.03, 0.01} × `lr_window` {8, 20, Inf} × `season_policy` {carry, reset}, 24 runs, plus one count-scale run at mult 0.03, window 20, carry. -- Tables lead with states only (US excluded). Each season is scored on its +- Scores: states only (US excluded). Each season is scored on its own rounds. WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated @@ -71,14 +71,14 @@ count_ref <- run(list(ref = "paper", burn_in_seasons = character(0), slow_init = ``` ```{r metrics} -GEOS <- c(states = "states only (US excluded)", all = "all locations") +GEOS <- c(states = "states only (US excluded)") SEASONS <- c(LIVE, "both") score <- function(cal) { fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% bind_rows() %>% - select(geos, season, horizon, which, wis, cal_err, cov50, cov90) %>% - tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err, cov50, cov90)) %>% + select(geos, season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) } label <- function(df) { @@ -125,18 +125,10 @@ curve_plot <- function(g) { } ``` -## States only - ```{r curve-states, fig.height=8} curve_plot(GEOS[["states"]]) ``` -## All locations - -```{r curve-all, fig.height=8} -curve_plot(GEOS[["all"]]) -``` - # 2. Which setting matters? For each season and horizon, how far WIS reduction % moves when one setting @@ -190,69 +182,13 @@ ggplot(long, aes(mult, value, colour = window_f, linetype = season)) + theme(legend.position = "bottom") ``` -# 4. Tables - -WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage -bias (lower is better) per config (rows) and season × horizon (columns). The base row shows the base's coverage bias; the count-scale -run is last. - -```{r tables} -season_table <- function(g) { - rows <- bind_rows( - sweep_m %>% filter(mult == MULTS[1], window == 20, policy == "carry") %>% mutate(config = "base", wis_pct = 0, cal_err_cal = cal_err_base), - sweep_m %>% arrange(desc(mult), window, policy) %>% mutate(config = sprintf("sqrt mult %s, window %s, %s", mult, as.character(window), policy)), - count_m %>% mutate(config = "count mult 0.03, window 20, carry") - ) %>% - filter(geos == g) %>% - mutate(config = factor(config, levels = unique(config))) - rows %>% - transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(config) -} -``` - -## States only - -```{r table-states} -knitr::kable(season_table(GEOS[["states"]])) -``` - -L1 coverage bias, season-pooled against month-average (see "Month view" below): - -```{r table-states-month} +```{r month-view-cfgs} +# The plots show the multipliers around the elbow and the season policy at window 20, plus the count scale; the tables cover every config. mv_cals <- c( setNames(sweep, with(grid, sprintf("sqrt mult %s, window %s, %s", mult, as.character(window), policy))), list(`count mult 0.03, window 20, carry` = count_ref) ) mv <- cal_month_view(mv_cals, seasons = LIVE) -mv_show_table <- FALSE -knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) -``` - -## All locations - -```{r table-all} -knitr::kable(season_table(GEOS[["all"]])) -``` - -## 50% and 90% interval coverage, states only, carry - -```{r table-cov} -bind_rows( - st %>% filter(mult == MULTS[1], window == 20, policy == "carry") %>% mutate(config = "base", cov50_cal = cov50_base, cov90_cal = cov90_base), - st %>% filter(policy == "carry") %>% arrange(desc(mult), window) %>% mutate(config = sprintf("mult %s, window %s", mult, as.character(window))) -) %>% - filter(h %in% c("h0", "h2"), season != "both seasons") %>% - mutate(config = factor(config, levels = unique(config))) %>% - transmute(config, col = paste(season, h), cell = sprintf("%.2f / %.2f", cov50_cal, cov90_cal)) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(config) %>% - knitr::kable() -``` - -```{r month-view-cfgs} -# The multipliers and the season policy the Findings compare, at window 20, plus the count-scale run. mv_plot_cfgs <- c( "sqrt mult 0.1, window 20, carry", "sqrt mult 0.03, window 20, carry", "sqrt mult 0.01, window 20, carry", "sqrt mult 0.03, window 20, reset", "count mult 0.03, window 20, carry" diff --git a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd index bf4d0fb5..6777f433 100644 --- a/reports/writeups/calibration_experiments/e02_eta_variants.Rmd +++ b/reports/writeups/calibration_experiments/e02_eta_variants.Rmd @@ -48,7 +48,7 @@ Setup: with the 2023-24 burn-in (the HHS-stitched backfill from E13), which only warms eta. The reference is re-run with the same burn-in (`reference + burn-in`) for comparison. -- Tables lead with states only (US excluded). +- Scores: states only (US excluded). WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated WIS lower than base. L1 coverage bias: lower is better. @@ -84,155 +84,16 @@ run <- function(spec) { cals <- purrr::map(CFGS, run) ``` -```{r metrics} -GEOS <- c(states = "states only (US excluded)", all = "all locations") -SEASONS <- c(LIVE, "both") -score <- function(cal) { - fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) - purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% - bind_rows() %>% - select(geos, season, horizon, which, wis, cal_err, cov50, cov90) %>% - tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err, cov50, cov90)) %>% - mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) -} -m <- purrr::imap(cals, \(cal, nm) score(cal) %>% mutate(config = nm)) %>% - bind_rows() %>% - mutate( - config = factor(config, levels = names(CFGS)), - geos = factor(geos, levels = names(GEOS), labels = GEOS), - season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), - h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) - ) +```{r month-view-cfgs} CFG_COLS <- setNames(c("black", "#56B4E9", "#0072B2", "#003F6B", "#CC79A7", "grey55", "#E69F00", "#D55E00"), names(CFGS)) -``` - -# 1. Overall, by season - -WIS reduction % (positive = calibrated WIS lower than base) per config, one panel per season and horizon, states only. The -seasonal-window bars compare with "reference + burn-in", the others with -"reference". - -```{r head-plot, fig.height=6} -ggplot(m %>% filter(geos == GEOS[["states"]]), aes(config, wis_pct, fill = config)) + - geom_hline(yintercept = 0, colour = "grey50") + - geom_col(width = 0.8) + - facet_grid(season ~ h) + - scale_fill_manual(values = CFG_COLS, name = NULL) + - labs(x = NULL, y = "WIS reduction % (positive = calibrated WIS lower than base)") + - theme(legend.position = "bottom", axis.text.x = element_blank(), axis.ticks.x = element_blank()) -``` - -WIS reduction % against L1 coverage bias, one point per config, states only. -The open circle is the base (at its coverage bias and zero WIS reduction); up -and to the left is better. The seasonal-window configs compare with -"reference + burn-in", the others with "reference". - -```{r scatter, fig.height=7} -d <- m %>% filter(geos == GEOS[["states"]]) -base <- d %>% filter(config == "reference") %>% distinct(season, h, cal_err_base) -ggplot(d, aes(cal_err_cal, wis_pct, colour = config)) + - geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + - geom_point(data = base, aes(cal_err_base, 0, shape = "base (uncalibrated)"), colour = "black", fill = "white", size = 2.2, inherit.aes = FALSE) + - geom_point(size = 2) + - facet_grid(season ~ h) + - scale_colour_manual(values = CFG_COLS, name = NULL) + - scale_shape_manual(values = c(`base (uncalibrated)` = 21), name = NULL) + - expand_limits(x = 0) + - labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)") + - theme(legend.position = "bottom") -``` - -Per config (rows) and season × horizon (columns): first WIS reduction % -(positive = calibrated WIS lower than base), then L1 coverage bias (lower is -better), with the base's coverage bias in the base row. - -```{r tables} -season_table <- function(g, value, fmt) { - bind_rows( - m %>% filter(config == "reference") %>% mutate(config = "base", wis_pct = 0, cal_err_cal = cal_err_base), - m - ) %>% - filter(geos == g) %>% - mutate(config = factor(config, levels = c("base", names(CFGS)))) %>% - transmute(config, col = paste(season, h), cell = sprintf(fmt, .data[[value]])) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(config) -} -wis_table <- function(g) { - knitr::kable(season_table(g, "wis_pct", "%+.1f"), caption = paste0("WIS reduction %, ", g, "; positive = calibrated WIS lower than base")) -} -cov_table <- function(g) { - knitr::kable(season_table(g, "cal_err_cal", "%.3f"), caption = paste0("L1 coverage bias (lower is better), ", g)) -} -``` - -## States only - -```{r table-states} -wis_table(GEOS[["states"]]) -cov_table(GEOS[["states"]]) -``` - -L1 coverage bias, season-pooled against month-average (see "Month view" below): - -```{r table-states-month} mv <- cal_month_view(cals, seasons = LIVE) -mv_show_table <- FALSE -knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) -``` - -## All locations - -```{r table-all} -wis_table(GEOS[["all"]]) -cov_table(GEOS[["all"]]) -``` - -## 50% and 90% interval coverage, states only - -```{r table-cov} -bind_rows( - m %>% filter(config == "reference") %>% mutate(config = "base", cov50_cal = cov50_base, cov90_cal = cov90_base), - m -) %>% - filter(geos == GEOS[["states"]], h %in% c("h0", "h2"), season != "both seasons") %>% - mutate(config = factor(config, levels = c("base", names(CFGS)))) %>% - transmute(config, col = paste(season, h), cell = sprintf("%.2f / %.2f", cov50_cal, cov90_cal)) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(config) %>% - knitr::kable() -``` - -# 2. Month by month - -The month plots are in "Month view" below. h2 by month, states only: WIS -reduction % (positive = calibrated WIS lower than base) (share of outcomes -below the calibrated median). The burn-in configs are left out except the -seasonal window with reset. - -```{r month-table} -MONTH_CFGS <- c("reference", "off after 1 Mar", "off after 15 Feb", "per-level eta", "seasonal window, reset") -fc_m <- purrr::imap(cals[MONTH_CFGS], \(cal, nm) cal$forecasts %>% mutate(config = nm)) %>% - bind_rows() %>% - mutate(season = season_of(reference_date), config = factor(config, levels = MONTH_CFGS)) %>% - filter(season %in% LIVE, location != "US") -month_m <- ws_month_scores(fc_m) %>% mutate(h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) -share <- month_m %>% filter(config == "reference") -month_m %>% - filter(h == "h2") %>% - transmute(season, month, config, v = sprintf("%+.1f (%.2f)", pct, below_med_cal)) %>% - tidyr::pivot_wider(names_from = config, values_from = v) %>% - left_join(share %>% filter(h == "h2") %>% transmute(season, month, `base share %` = round(share, 1), `base below` = round(below_med_base, 2)), - by = c("season", "month")) %>% - arrange(season, month) %>% - knitr::kable() -``` - -```{r month-view-cfgs} -# Same configs as the h2 table. -mv_plot_cfgs <- MONTH_CFGS +# The plots leave out the burn-in configs except the seasonal window with reset; the tables keep every config. +mv_plot_cfgs <- c("reference", "off after 1 Mar", "off after 15 Feb", "per-level eta", "seasonal window, reset") mv_cols <- CFG_COLS ``` +The seasonal-window configs compare with "reference + burn-in", the others +with "reference". + ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` diff --git a/reports/writeups/calibration_experiments/e07_across_eras.Rmd b/reports/writeups/calibration_experiments/e07_across_eras.Rmd index 2f6b3e6a..6cec0e72 100644 --- a/reports/writeups/calibration_experiments/e07_across_eras.Rmd +++ b/reports/writeups/calibration_experiments/e07_across_eras.Rmd @@ -34,7 +34,7 @@ submitted ensemble as for one fixed forecaster? |---|---| | rows | identical (round, location, horizon, level) rows for A and B: hub submission rounds, submissions excluded from the hub reads dropped from both | | seasons | 2023-24 burn-in (27 rounds), 2024-25 (27) and 2025-26 (28) scored | -| locations | all 53; results lead with states only | +| locations | all 53 learned; scores are states only | | horizons | h0–h3 (the ensemble's h−1 is a different model) | | learning truth | NHSN as published each week, re-run from scratch each week, one week of extra revision delay | | scoring truth | the store's NHSN truth | @@ -88,14 +88,14 @@ purrr::imap(FCS, \(fc, nm) fc %>% mutate(season = season_of(reference_date)) %>% ``` ```{r metrics} -GEOS <- c(states = "states only (US excluded)", all = "all locations") +GEOS <- c(states = "states only (US excluded)") SEASONS <- c(LIVE, "both") score <- function(cal) { fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% bind_rows() %>% - select(geos, season, horizon, which, wis, cal_err, cov50, cov90, above_median) %>% - tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err, cov50, cov90)) %>% + select(geos, season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) } m <- purrr::map2(cals, seq_len(nrow(grid)), \(cal, i) score(cal) %>% mutate(grid[i, ])) %>% @@ -119,60 +119,11 @@ the ratio B/A (below 1: `windowed_seasonal` is better). ```{r base-wis} m %>% filter(tracker == "single") %>% - select(geos, season, h, forecaster, wis_base) %>% + select(season, h, forecaster, wis_base) %>% tidyr::pivot_wider(names_from = forecaster, values_from = wis_base) %>% - transmute(geos, season, h, A = .data[[names(FCS)[1]]], B = .data[[names(FCS)[2]]], `B/A` = B / A) %>% - arrange(geos, season, h) %>% - knitr::kable(digits = c(0, 0, 0, 1, 1, 3), caption = "Base WIS, A ensemble vs B windowed_seasonal") -``` - -Per state: base WIS pooled over h0–h3, A against B (log scales; below the -line, B is better). - -```{r base-wis-state, fig.height=5} -base_fc <- purrr::map(which(grid$tracker == "single"), \(i) cals[[i]]$forecasts %>% mutate(forecaster = grid$forecaster[i])) %>% - bind_rows() %>% - mutate(season = season_of(reference_date)) %>% - filter(season %in% LIVE, !is.na(truth), !is.na(value_base)) -pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) -loc_wis <- base_fc %>% - filter(location != "US") %>% - group_by(forecaster, season, location) %>% - summarize(wis = 2 * mean(pinball(truth, value_base, level)), .groups = "drop") %>% - tidyr::pivot_wider(names_from = forecaster, values_from = wis) %>% - transmute(season, location, A = .data[[names(FCS)[1]]], B = .data[[names(FCS)[2]]], ratio = B / A) -ggplot(loc_wis, aes(A, B)) + - geom_abline(slope = 1, intercept = 0, colour = "grey50") + - geom_point(alpha = 0.8) + - facet_wrap(~season) + - scale_x_log10() + - scale_y_log10() + - labs(x = "A ensemble base WIS", y = "B windowed_seasonal base WIS") -``` - -```{r base-wis-ratio} -loc_wis %>% - group_by(season) %>% - summarize( - states = n(), `B better` = sum(ratio < 1), q10 = quantile(ratio, 0.1), q25 = quantile(ratio, 0.25), - median = median(ratio), q75 = quantile(ratio, 0.75), q90 = quantile(ratio, 0.9) - ) %>% - knitr::kable(digits = 2, caption = "Per-state B/A base WIS ratio, h0–h3 pooled") -``` - -## Base miscalibration - -How miscalibrated each forecaster is before calibration, states only: WIS, -L1 coverage bias, and the share of outcomes above the median (0.5 is -calibrated; above 0.5 means the forecaster under-predicts). - -```{r bases} -m %>% - filter(geos == GEOS[["states"]], tracker == "single") %>% - transmute(forecaster, season, h, cell = sprintf("%.1f / %.3f / %.2f", wis_base, cal_err_base, above_median)) %>% - tidyr::pivot_wider(names_from = h, values_from = cell) %>% - arrange(season, forecaster) %>% - knitr::kable(caption = "Base WIS / L1 coverage bias (lower is better) / share above the median, states only") + transmute(season, h, A = .data[[names(FCS)[1]]], B = .data[[names(FCS)[2]]], `B/A` = B / A) %>% + arrange(season, h) %>% + knitr::kable(digits = c(0, 0, 1, 1, 3), caption = "Base WIS, A ensemble vs B windowed_seasonal, states only") ``` ## Base bias by month @@ -181,6 +132,10 @@ Share of outcomes above the base median and L1 coverage bias, by calendar month of the reference date, states only. ```{r base-month, fig.height=7} +base_fc <- purrr::map(which(grid$tracker == "single"), \(i) cals[[i]]$forecasts %>% mutate(forecaster = grid$forecaster[i])) %>% + bind_rows() %>% + mutate(season = season_of(reference_date)) %>% + filter(season %in% LIVE, !is.na(truth), !is.na(value_base)) month_base <- function(d) { d %>% group_by(across(c(any_of(c("forecaster", "season", "horizon", "month")), level))) %>% @@ -205,76 +160,14 @@ mb %>% theme(legend.position = "bottom") ``` -Pooled over h0–h3: share above the base median / L1 coverage bias (lower is better), A vs B. - -```{r base-month-table} -month_base(base_fc_states %>% select(-horizon)) %>% - transmute(season, month, forecaster, cell = sprintf("%.2f / %.3f", above, cal_err)) %>% - tidyr::pivot_wider(names_from = forecaster, values_from = cell) %>% - left_join(base_fc_states %>% filter(level == 0.5) %>% count(season, month, forecaster) %>% filter(forecaster == names(FCS)[1]) %>% select(season, month, forecasts = n), by = c("season", "month")) %>% - arrange(season, month) %>% - knitr::kable(caption = "Share above base median / L1 coverage bias (lower is better), states only, h0–h3 pooled") -``` - # 2. What calibration does to each -WIS reduction % (positive = calibrated WIS lower than base) per tracker, one panel per season and horizon, states only. - -```{r head-plot, fig.height=6} -ggplot(m %>% filter(geos == GEOS[["states"]]), aes(tracker, wis_pct, fill = forecaster)) + - geom_hline(yintercept = 0, colour = "grey50") + - geom_col(position = position_dodge(width = 0.85), width = 0.8) + - facet_grid(season ~ h) + - scale_fill_manual(values = FC_COLS, name = NULL) + - labs(x = NULL, y = "WIS reduction % (positive = calibrated WIS lower than base)") + - theme(legend.position = "bottom") -``` - -L1 coverage bias (lower is better), calibrated (filled) against the base (open), states only. - -```{r cov-plot, fig.height=6} -d <- m %>% filter(geos == GEOS[["states"]]) -ggplot(d, aes(tracker, cal_err_cal, colour = forecaster)) + - geom_point(aes(y = cal_err_base), shape = 1, size = 2.4, position = position_dodge(width = 0.6)) + - geom_point(size = 2.4, position = position_dodge(width = 0.6)) + - facet_grid(season ~ h) + - scale_colour_manual(values = FC_COLS, name = NULL) + - expand_limits(y = 0) + - labs(x = NULL, y = "L1 coverage bias (lower is better; open: base)") + - theme(legend.position = "bottom") -``` - -WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better): - -```{r tables} -season_table <- function(g) { - bind_rows( - m %>% filter(tracker == "single") %>% mutate(tracker = "base", wis_pct = 0, cal_err_cal = cal_err_base), - m - ) %>% - filter(geos == g) %>% - mutate(config = factor(paste(forecaster, tracker, sep = ", "), levels = paste(rep(names(FCS), each = 4), c("base", names(TRACKERS)), sep = ", "))) %>% - transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(config) -} -``` - -## States only +WIS reduction % (positive = calibrated WIS lower than base) and L1 coverage +bias (lower is better), states only. The month view of these runs is in +E07b. ```{r table-states} -knitr::kable(season_table(GEOS[["states"]])) -``` - -L1 coverage bias, season-pooled against month-average (the month view is in E07b): - -```{r table-states-month} mv_trk <- cal_month_view(setNames(cals, paste(grid$forecaster, grid$tracker, sep = ", ")), seasons = LIVE) +knitr::kable(cal_month_wis_table(mv_trk)) knitr::kable(cal_month_cov_table(mv_trk), caption = CAL_MONTH_COV_CAPTION) ``` - -## All locations - -```{r table-all} -knitr::kable(season_table(GEOS[["all"]])) -``` diff --git a/reports/writeups/calibration_experiments/e07b_candidates.Rmd b/reports/writeups/calibration_experiments/e07b_candidates.Rmd index dd48fc45..204873a6 100644 --- a/reports/writeups/calibration_experiments/e07b_candidates.Rmd +++ b/reports/writeups/calibration_experiments/e07b_candidates.Rmd @@ -23,7 +23,7 @@ ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplo Question: do the operating-point candidates rank the same way on the submitted ensemble as on one fixed forecaster? -**Setup.** As in E07 (the sections above): A is the submitted flu ensemble, +**Setup.** As in E07: A is the submitted flu ensemble, B the clean `windowed_seasonal` replay with the 2023-24 HHS backfill as its burn-in season, on identical (round, location, horizon, level) rows, h0–h3, exact learning truth with one week of extra delay. @@ -69,23 +69,22 @@ ccals <- purrr::pmap(cgrid, function(forecaster, cand) { ``` ```{r metrics} -GEOS <- c(states = "states only (US excluded)", all = "all locations") +GEOS <- c(states = "states only (US excluded)") SEASONS <- c(LIVE, "both") score <- function(cal) { fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% bind_rows() %>% - select(geos, season, horizon, which, wis, cal_err, cov50, cov90, above_median) %>% - tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err, cov50, cov90)) %>% + select(geos, season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) } -FC_COLS <- c(`A ensemble` = "#0072B2", `B windowed_seasonal` = "#D55E00") ``` # Candidates on both The candidates from E14, E16 and E17, each run on both forecasters on the -same rows: REF-op warm (as in section 2) and cold, sqrt-scale constant +same rows: REF-op warm (as in E07) and cold, sqrt-scale constant 0.018 (E16), rate-scale constant 0.032 per 100k (E14), and sqrt 0.018 switched off from 1 March (`off_after = "03-01"`, E17). The cold ones learn from the live seasons only. @@ -101,32 +100,21 @@ cm <- purrr::map2(ccals, seq_len(nrow(cgrid)), \(cal, i) score(cal) %>% mutate(c ) ``` -WIS reduction % (positive = calibrated WIS lower than base) per candidate, states only. - -```{r cand-plot, fig.height=6.5} -ggplot(cm %>% filter(geos == GEOS[["states"]]), aes(cand, wis_pct, fill = forecaster)) + - geom_hline(yintercept = 0, colour = "grey50") + - geom_col(position = position_dodge(width = 0.85), width = 0.8) + - facet_grid(season ~ h) + - scale_fill_manual(values = FC_COLS, name = NULL) + - labs(x = NULL, y = "WIS reduction % (positive = calibrated WIS lower than base)") + - theme(legend.position = "bottom", axis.text.x = element_text(angle = 30, hjust = 1)) +```{r month-view-cfgs} +mv <- cal_month_view(setNames(ccals, paste(cgrid$forecaster, cgrid$cand, sep = ", ")), seasons = LIVE) +# Three candidates on both forecasters: colour is the candidate, linetype the forecaster (also for the grey base lines). +MV_CANDS <- c(`REF-op warm` = "black", `sqrt 0.018` = "#0072B2", `sqrt 0.018, off 1 Mar` = "#D55E00") +mv_plot_cfgs <- paste(rep(names(FCS), each = length(MV_CANDS)), names(MV_CANDS), sep = ", ") +mv_cols <- MV_CANDS +mv$month <- mv$month %>% mutate(forecaster = sub(",.*", "", config), cand = factor(sub("^[^,]*, ", "", config), levels = names(CANDS))) +mv_colour_by <- "cand" +mv_linetype <- "forecaster" ``` -L1 coverage bias (lower is better), calibrated (filled) against the base (open), states only. - -```{r cand-cov, fig.height=6.5} -ggplot(cm %>% filter(geos == GEOS[["states"]]), aes(cand, cal_err_cal, colour = forecaster)) + - geom_point(aes(y = cal_err_base), shape = 1, size = 2.4, position = position_dodge(width = 0.6)) + - geom_point(size = 2.4, position = position_dodge(width = 0.6)) + - facet_grid(season ~ h) + - scale_colour_manual(values = FC_COLS, name = NULL) + - expand_limits(y = 0) + - labs(x = NULL, y = "L1 coverage bias (lower is better; open: base)") + - theme(legend.position = "bottom", axis.text.x = element_text(angle = 30, hjust = 1)) +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` -## Rankings on A and B +# Rankings on A and B Candidates ordered by mean WIS reduction % (positive = calibrated WIS lower than base) over h0–h3, separately on each forecaster, states only. `rank A` / `rank B` are the positions (1 is the @@ -160,54 +148,3 @@ cm %>% ) %>% knitr::kable(digits = 2, caption = "Spearman correlation of A vs B over (candidate, horizon) cells") ``` - -## Tables - -WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better). - -```{r cand-tables} -cand_table <- function(g) { - bind_rows( - cm %>% filter(cand == names(CANDS)[1]) %>% mutate(cand = "base", wis_pct = 0, cal_err_cal = cal_err_base), - cm - ) %>% - filter(geos == g) %>% - mutate(config = factor(paste(forecaster, cand, sep = ", "), levels = paste(rep(names(FCS), each = length(CANDS) + 1), c("base", names(CANDS)), sep = ", "))) %>% - transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(config) -} -``` - -### States only - -```{r cand-table-states} -knitr::kable(cand_table(GEOS[["states"]])) -``` - -L1 coverage bias, season-pooled against month-average (see "Month view" below): - -```{r cand-table-states-month} -mv <- cal_month_view(setNames(ccals, paste(cgrid$forecaster, cgrid$cand, sep = ", ")), seasons = LIVE) -mv_show_table <- FALSE -knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) -``` - -### All locations - -```{r cand-table-all} -knitr::kable(cand_table(GEOS[["all"]])) -``` - -```{r month-view-cfgs} -# Three candidates on both forecasters: colour is the candidate, linetype the forecaster (also for the grey base lines). -MV_CANDS <- c(`REF-op warm` = "black", `sqrt 0.018` = "#0072B2", `sqrt 0.018, off 1 Mar` = "#D55E00") -mv_plot_cfgs <- paste(rep(names(FCS), each = length(MV_CANDS)), names(MV_CANDS), sep = ", ") -mv_cols <- MV_CANDS -mv$month <- mv$month %>% mutate(forecaster = sub(",.*", "", config), cand = factor(sub("^[^,]*, ", "", config), levels = names(CANDS))) -mv_colour_by <- "cand" -mv_linetype <- "forecaster" -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd index 84c008ee..6af3899b 100644 --- a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -35,7 +35,7 @@ the learning rate and with the extra delay before an outcome is learned? |---|---| | disease, base | flu, the submitted hub ensemble (`CMU-TimeSeries`) | | seasons | 2023-24 burn-in (its outcomes take no steps; offsets start at zero); 2024-25 and 2025-26 scored | -| locations, horizons | all 53 hub locations, h−1 to h3; the month view is states only | +| locations, horizons | learns on all 53 hub locations; scores are states only (US excluded), h−1 to h3 | | scoring truth | finalized NHSN | | tracker | REF-paper (one offset per level, carried across seasons) on the rate scale | | learning rate | constant, 0.001 to about 3.2 per 100k per step (15 values, a quarter-decade apart) | @@ -138,10 +138,10 @@ refs <- purrr::pmap(ref_grid, function(learn, settle_days) { ```{r metrics, cache=FALSE} pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) -# Per (location, horizon) and pooled over locations: WIS reduction % and L1 +# Per (location, horizon) and pooled over states: WIS reduction % and L1 # coverage bias, for the calibrated forecast and the base. metrics <- function(cal) { - fc <- cal$forecasts %>% filter(!is_burn_in, !is.na(truth), !is.na(value_base)) + fc <- cal$forecasts %>% filter(!is_burn_in, !is.na(truth), !is.na(value_base), location != "US") one <- function(df, by) { df %>% group_by(across(all_of(c(by, "horizon", "level")))) %>% @@ -176,61 +176,6 @@ sweep_m <- label_panels(sweep_m) ref_m <- label_panels(ref_m) ``` -# Bridge from REF-paper - -One change at a time at one week of extra delay, re-run design: first the -rate scale with the adaptive learning rate (its minimum step drops from 0.1 -admissions to 0.001 per 100k, since that minimum is in the working units), -then the constant learning rate with the lowest pooled WIS at each horizon. -REF-op is for orientation only; it differs in several settings. - -```{r bridge} -best_const <- sweep_m %>% - filter(location == "pooled", learn == "exact", settle_days == 14L) %>% - group_by(horizon) %>% - slice_max(wis_pct, n = 1, with_ties = FALSE) %>% - ungroup() %>% - mutate(ref = "rate, constant lr (highest WIS per horizon)") -bridge <- bind_rows( - ref_m %>% filter(learn == "exact", settle_days == 14L), - best_const -) %>% - mutate(ref = factor(ref, levels = c("REF-paper (count, adaptive)", "rate, adaptive", "rate, constant lr (highest WIS per horizon)", "REF-op"))) -base_row <- bridge %>% filter(ref == "REF-paper (count, adaptive)") %>% mutate(ref = "base", wis_pct = 0, bias_cal = bias_base) -tbl <- bind_rows(base_row, bridge) %>% - transmute(config = ref, horizon, cell = sprintf("%+.1f%% / %.3f", wis_pct, bias_cal)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = cell) -knitr::kable(tbl, caption = "WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), pooled over locations") -knitr::kable( - best_const %>% select(horizon, lr) %>% mutate(lr = signif(lr, 2)) %>% tidyr::pivot_wider(names_from = horizon, values_from = lr), - caption = "Constant learning rate with the lowest pooled WIS (rate units per step)" -) -``` - -L1 coverage bias for states only, both seasons, season-pooled against -month-average (see "Month view" below), for the reference configs, the knee -(0.056) and 0.1 at several delays and both designs: - -```{r bridge-month} -sweep_run <- function(learn, settle_days, lr) { - sweep[[which(sweep_grid$learn == learn & sweep_grid$settle_days == settle_days & abs(log10(sweep_grid$lr / lr)) < 1e-6)]] -} -ref_run <- function(learn, settle_days, nm) refs[[which(ref_grid$learn == learn & ref_grid$settle_days == settle_days)]][[nm]] -mv_cals <- list( - `REF-paper, re-run +1 wk` = ref_run("exact", 14L, "REF-paper (count, adaptive)"), - `REF-op, re-run +1 wk` = ref_run("exact", 14L, "REF-op"), - `knee 0.056, re-run +0 wk` = sweep_run("exact", 7L, 10^-1.25), - `knee 0.056, re-run +1 wk` = sweep_run("exact", 14L, 10^-1.25), - `knee 0.056, re-run +2 wk` = sweep_run("exact", 21L, 10^-1.25), - `0.1, re-run +1 wk` = sweep_run("exact", 14L, 10^-1), - `knee 0.056, stateful +0 wk` = sweep_run("vintage", 7L, 10^-1.25), - `knee 0.056, stateful +1 wk` = sweep_run("vintage", 14L, 10^-1.25) -) -mv <- cal_month_view(mv_cals, seasons = c("2024-2025", "2025-2026"), horizons = NULL) -mv_show_table <- FALSE -knitr::kable(cal_month_cov_table(mv, seasons = "both"), caption = CAL_MONTH_COV_CAPTION) -``` - # Coverage bias vs WIS, per location Each thin grey line is one location traced across learning rates; the bold @@ -314,41 +259,58 @@ ggplot(pooled_all, aes(bias_cal, wis_pct, colour = extra, linetype = design, gro theme(legend.position = "bottom", legend.box = "vertical") ``` -# Tables - -Pooled WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better) for every learning rate, by horizon -(columns) and extra delay (row blocks). - -```{r table-fn} -sweep_table <- function(which_learn) { - sweep_m %>% - filter(location == "pooled", learn == which_learn) %>% - transmute(extra, lr = signif(lr, 2), horizon, cell = sprintf("%+.1f / %.3f", wis_pct, bias_cal)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = cell) %>% - arrange(extra, lr) %>% - knitr::kable() +```{r month-view-cfgs} +sweep_run <- function(learn, settle_days, lr) { + sweep[[which(sweep_grid$learn == learn & sweep_grid$settle_days == settle_days & abs(log10(sweep_grid$lr / lr)) < 1e-6)]] } +ref_run <- function(learn, settle_days, nm) refs[[which(ref_grid$learn == learn & ref_grid$settle_days == settle_days)]][[nm]] +mv_cals <- list( + `REF-paper, re-run +1 wk` = ref_run("exact", 14L, "REF-paper (count, adaptive)"), + `REF-op, re-run +1 wk` = ref_run("exact", 14L, "REF-op"), + `0.032, re-run +1 wk` = sweep_run("exact", 14L, 10^-1.5), + `knee 0.056, re-run +0 wk` = sweep_run("exact", 7L, 10^-1.25), + `knee 0.056, re-run +1 wk` = sweep_run("exact", 14L, 10^-1.25), + `knee 0.056, re-run +2 wk` = sweep_run("exact", 21L, 10^-1.25), + `0.1, re-run +1 wk` = sweep_run("exact", 14L, 10^-1), + `0.18, re-run +1 wk` = sweep_run("exact", 14L, 10^-0.75), + `knee 0.056, stateful +0 wk` = sweep_run("vintage", 7L, 10^-1.25), + `knee 0.056, stateful +1 wk` = sweep_run("vintage", 14L, 10^-1.25) +) +mv <- cal_month_view(mv_cals, seasons = c("2024-2025", "2025-2026"), horizons = NULL) +# The plots show REF-op and the constant rates around the knee at one week of delay; the tables also cover the other delays and designs. +mv_plot_cfgs <- c("REF-op, re-run +1 wk", "0.032, re-run +1 wk", "knee 0.056, re-run +1 wk", "0.1, re-run +1 wk", "0.18, re-run +1 wk") ``` -## Re-run design (prod) - -```{r table-exact} -sweep_table("exact") +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` -## Stateful design +# Bridge from REF-paper -```{r table-vintage} -sweep_table("vintage") -``` +One change at a time at one week of extra delay, re-run design: first the +rate scale with the adaptive learning rate (its minimum step drops from 0.1 +admissions to 0.001 per 100k, since that minimum is in the working units), +then the constant learning rate with the lowest pooled WIS at each horizon. +REF-op is for orientation only; it differs in several settings. -```{r month-view-cfgs} -# The configs the Findings compare: the references and the knee at one week of delay, and the knee in the stateful design. -mv_plot_cfgs <- c( - "REF-paper, re-run +1 wk", "REF-op, re-run +1 wk", "knee 0.056, re-run +1 wk", "0.1, re-run +1 wk", - "knee 0.056, stateful +0 wk", "knee 0.056, stateful +1 wk" +```{r bridge} +best_const <- sweep_m %>% + filter(location == "pooled", learn == "exact", settle_days == 14L) %>% + group_by(horizon) %>% + slice_max(wis_pct, n = 1, with_ties = FALSE) %>% + ungroup() %>% + mutate(ref = "rate, constant lr (highest WIS per horizon)") +bridge <- bind_rows( + ref_m %>% filter(learn == "exact", settle_days == 14L), + best_const +) %>% + mutate(ref = factor(ref, levels = c("REF-paper (count, adaptive)", "rate, adaptive", "rate, constant lr (highest WIS per horizon)", "REF-op"))) +base_row <- bridge %>% filter(ref == "REF-paper (count, adaptive)") %>% mutate(ref = "base", wis_pct = 0, bias_cal = bias_base) +tbl <- bind_rows(base_row, bridge) %>% + transmute(config = ref, horizon, cell = sprintf("%+.1f%% / %.3f", wis_pct, bias_cal)) %>% + tidyr::pivot_wider(names_from = horizon, values_from = cell) +knitr::kable(tbl, caption = "WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), pooled over states") +knitr::kable( + best_const %>% select(horizon, lr) %>% mutate(lr = signif(lr, 2)) %>% tidyr::pivot_wider(names_from = horizon, values_from = lr), + caption = "Constant learning rate with the lowest pooled WIS (rate units per step)" ) ``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` diff --git a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd index 06ae991a..ac2432b2 100644 --- a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd +++ b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd @@ -37,8 +37,8 @@ Setup: decade apart. 0.1 is E11's config, though E11 used the submitted ensemble. - References on the same data: `REF-op cold` (prod's tracker without its warm start) and `sqrt, adaptive` (E13's "E05 single cold"). -- Tables lead with states only (US excluded; it is about 45% of the - all-locations WIS). +- Scores: states only (US excluded; it is about 45% of the all-locations + WIS). WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated WIS lower than base. L1 coverage bias: lower is better. @@ -59,15 +59,15 @@ refs <- purrr::map(ref_cfgs, run) ``` ```{r metrics} -GEOS <- c(states = "states only (US excluded)", all = "all locations") +GEOS <- c(states = "states only (US excluded)") SEASONS <- c(LIVE, "both") # ws_scores() per location subset; wis_pct is the WIS reduction % vs base (positive = calibrated WIS lower). score <- function(cal) { fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% bind_rows() %>% - select(geos, season, horizon, which, wis, cal_err, cov50, cov90) %>% - tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err, cov50, cov90)) %>% + select(geos, season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) } label <- function(df) { @@ -114,18 +114,10 @@ curve_plot <- function(g) { } ``` -## States only - ```{r curve-states} curve_plot(GEOS[["states"]]) ``` -## All locations - -```{r curve-all} -curve_plot(GEOS[["all"]]) -``` - # Each metric against the learning rate The same runs with the learning rate on the x axis, so the seasons can be @@ -155,71 +147,10 @@ ggplot(long, aes(lr, value, colour = season)) + theme(legend.position = "bottom") ``` -# Tables - -WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage -bias (lower is better), per learning rate (rows) and season × horizon -(columns). The base row shows the base's coverage bias. The -references follow the sweep. - -```{r tables} -season_table <- function(g) { - rows <- bind_rows( - sweep_m %>% filter(lr == LRS[1]) %>% mutate(config = "base", wis_pct = 0, cal_err_cal = cal_err_base), - sweep_m %>% mutate(config = format(lr, drop0trailing = TRUE)), - ref_m %>% mutate(config = ref) - ) %>% - filter(geos == g) %>% - mutate(config = factor(config, levels = unique(config))) - rows %>% - transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(config) -} -``` - -## States only - -```{r table-states} -knitr::kable(season_table(GEOS[["states"]])) -``` - -L1 coverage bias, season-pooled against month-average (see "Month view" below): - -```{r table-states-month} -mv <- cal_month_view(c(setNames(sweep, paste("rate", format(LRS, drop0trailing = TRUE))), refs), seasons = LIVE) -mv_show_table <- FALSE -knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) -``` - -## All locations - -```{r table-all} -knitr::kable(season_table(GEOS[["all"]])) -``` - -## 50% and 90% interval coverage, states only - -E12 found the constant rate at 0.1 under-covers the 50% interval. Coverage -of the 50% / 90% intervals, per season, at h0 and h2. - -```{r table-cov} -bind_rows( - sweep_m %>% filter(lr == LRS[1]) %>% mutate(config = "base", cov50_cal = cov50_base, cov90_cal = cov90_base), - sweep_m %>% mutate(config = format(lr, drop0trailing = TRUE)), - ref_m %>% mutate(config = ref) -) %>% - filter(geos == GEOS[["states"]], h %in% c("h0", "h2"), season != "both seasons") %>% - mutate(config = factor(config, levels = unique(config))) %>% - transmute(config, col = paste(season, h), cell = sprintf("%.2f / %.2f", cov50_cal, cov90_cal)) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(config) %>% - knitr::kable() -``` - ```{r month-view-cfgs} -# The good range (0.018–0.032), the knee and E11's 0.1, and the two references. -mv_plot_cfgs <- c("rate 0.018", "rate 0.032", "rate 0.056", "rate 0.1", "REF-op cold", "sqrt, adaptive") +mv <- cal_month_view(c(setNames(sweep, paste("rate", format(LRS, drop0trailing = TRUE))), refs), seasons = LIVE) +# The plots show the rates around the elbow and REF-op cold; the tables cover every rate. +mv_plot_cfgs <- c("rate 0.018", "rate 0.032", "rate 0.056", "rate 0.1", "REF-op cold") ``` ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} diff --git a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd index ec52bff5..182487ad 100644 --- a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd +++ b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd @@ -30,7 +30,7 @@ Setup: - Learning truth: exact (re-run at every round on the data published by then; E00), one week of extra delay. Cold start, no burn-in season. - Scores against finalized truth over 2024-25 and 2025-26, pooled; states - only first (US excluded), then all locations. + only (US excluded). - Configs, both from E12: - **REF-op cold**: prod's tracker (sqrt, adaptive rate, slow term, leak) without its warm start. @@ -72,13 +72,14 @@ cals <- purrr::map(cfgs, function(cfg) { do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) }) pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) -GEOS <- c(states = "states only (US excluded)", all = "all locations") +GEOS <- c(states = "states only (US excluded)") fc <- purrr::imap(cals, function(cal, nm) cal$forecasts %>% mutate(config = nm)) %>% bind_rows() %>% mutate(season = season_of(reference_date), config = factor(config, levels = names(cals))) %>% filter(season %in% LIVE, !is.na(truth), !is.na(value_base)) -# One copy of each row per location subset. -fc <- bind_rows(fc %>% filter(location != "US") %>% mutate(geos = "states"), fc %>% mutate(geos = "all")) %>% +fc <- fc %>% + filter(location != "US") %>% + mutate(geos = "states") %>% mutate(geos = factor(geos, levels = names(GEOS), labels = GEOS)) ``` @@ -110,18 +111,10 @@ level_plot <- function(g) { } ``` -## States only - ```{r by-level-states, fig.height=6} level_plot(GEOS[["states"]]) ``` -## All locations - -```{r by-level-all, fig.height=6} -level_plot(GEOS[["all"]]) -``` - ## Grouped: tails, shoulders, centre The same pieces summed over groups of levels: **tails** are the six levels @@ -149,21 +142,6 @@ group_tbl <- function(g) { arrange(config, part) } knitr::kable(group_tbl(GEOS[["states"]]), caption = "States only: WIS reduction % by group of levels (positive = calibrated WIS lower than base)") -knitr::kable(group_tbl(GEOS[["all"]]), caption = "All locations: WIS reduction % by group of levels (positive = calibrated WIS lower than base)") -``` - -Change in pinball loss at each level relative to that level's own base -loss (not a share of WIS), states only. - -```{r level-pct} -knitr::kable( - by_level %>% - filter(geos == GEOS[["states"]], level %in% c(0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99)) %>% - transmute(config, level, horizon = paste0("h", horizon), v = round(level_pct, 1)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = v) %>% - arrange(config, level), - caption = "States only: pinball loss reduction % at selected levels, vs that level's base loss (positive = calibrated loss lower than base)" -) ``` # 2. Offset size against the forecast's spread @@ -203,27 +181,6 @@ ggplot(off_q, aes(level, med, colour = config, fill = config)) + theme(legend.position = "bottom") ``` -Typical size, ignoring direction: median of |offset| ÷ the base 90% and -50% interval widths at selected levels, with the share of forecasts whose -offset exceeds a tenth of the 90% width. - -```{r offset-table} -off_tbl <- function(g) { - off %>% - filter(geos == g, level %in% c(0.05, 0.25, 0.5, 0.75, 0.95)) %>% - group_by(config, horizon, level) %>% - summarize( - `|off|/w90 median` = median(abs(r90)), `|off|/w90 p90` = quantile(abs(r90), 0.9), - `|off|/w50 median` = median(abs(r50)), `share |off| > 0.1 w90` = mean(abs(r90) > 0.1), - .groups = "drop" - ) %>% - mutate(horizon = paste0("h", horizon), across(where(is.double) & !level, \(x) round(x, 3))) %>% - arrange(config, horizon, level) -} -knitr::kable(off_tbl(GEOS[["states"]]), caption = "States only") -knitr::kable(off_tbl(GEOS[["all"]]) %>% filter(level %in% c(0.05, 0.5, 0.95)), caption = "All locations, three levels") -``` - # 3. Median shift vs width change ```{r split} @@ -262,35 +219,4 @@ split_tbl <- function(g) { arrange(config, variant) } knitr::kable(split_tbl(GEOS[["states"]]), caption = "States only: WIS reduction % (positive = calibrated WIS lower than base)") -knitr::kable(split_tbl(GEOS[["all"]]), caption = "All locations: WIS reduction % (positive = calibrated WIS lower than base)") -``` - -How the calibrated widths compare with the base: the median over forecasts -of calibrated ÷ base interval width, states only. 1 means unchanged. - -```{r width-ratio} -wr <- split %>% - filter(geos == GEOS[["states"]], level %in% c(0.05, 0.25, 0.75, 0.95)) %>% - select(config, location, horizon, reference_date, level, value_base, value_cal) %>% - tidyr::pivot_wider(names_from = level, values_from = c(value_base, value_cal)) %>% - mutate( - r90 = (value_cal_0.95 - value_cal_0.05) / (value_base_0.95 - value_base_0.05), - r50 = (value_cal_0.75 - value_cal_0.25) / (value_base_0.75 - value_base_0.25) - ) %>% - filter(is.finite(r90), is.finite(r50)) %>% - group_by(config, horizon) %>% - summarize( - `90% width ratio, median` = median(r90), `90% width ratio, middle 80%` = sprintf("%.2f–%.2f", quantile(r90, 0.1), quantile(r90, 0.9)), - `50% width ratio, median` = median(r50), `50% width ratio, middle 80%` = sprintf("%.2f–%.2f", quantile(r50, 0.1), quantile(r50, 0.9)), - .groups = "drop" - ) %>% - mutate(horizon = paste0("h", horizon), across(where(is.double), \(x) round(x, 2))) -knitr::kable(wr) -``` - -```{r month-view-cfgs} -mv <- cal_month_view(cals, seasons = LIVE) -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` diff --git a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd index 7952939a..3d3d1852 100644 --- a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd +++ b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd @@ -30,8 +30,8 @@ Setup: - Learning truth: exact (re-run at every round on the data published by then; E00), one week of extra delay. Cold start, no burn-in season. - The tracker runs over both seasons, so 2025-26 starts from the offsets - 2024-25 left behind; each season is scored on its own rounds. Results lead - with states only (US excluded). + 2024-25 left behind; each season is scored on its own rounds. Scores: + states only (US excluded). - Configs: - `sqrt `: the plain tracker with a constant learning rate on the square root of admissions per 100k; 11 rates from 0.001 to 0.32, a @@ -65,15 +65,15 @@ refop <- run(all_cfgs[["REF-op cold"]]) ``` ```{r metrics} -GEOS <- c(states = "states only (US excluded)", all = "all locations") +GEOS <- c(states = "states only (US excluded)") SEASONS <- c(LIVE, "both") SEASON_COLS <- c(`2024-2025` = "#0072B2", `2025-2026` = "#D55E00", `both seasons` = "grey45") score <- function(cal) { fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% bind_rows() %>% - select(geos, season, horizon, which, wis, cal_err, cov50, cov90) %>% - tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err, cov50, cov90)) %>% + select(geos, season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) } label <- function(df) { @@ -130,18 +130,10 @@ curve_plot <- function(g) { } ``` -## States only - ```{r curve-states} curve_plot(GEOS[["states"]]) ``` -## All locations - -```{r curve-all} -curve_plot(GEOS[["all"]]) -``` - ## Each metric against the learning rate, sqrt scale Top: WIS reduction % (positive = calibrated WIS lower than base). Bottom: L1 coverage bias (lower is better), each season's base dotted. @@ -170,38 +162,17 @@ ggplot(long, aes(lr, value, colour = season)) + theme(legend.position = "bottom") ``` -## Tables - -WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better) per config (rows) and season × horizon -(columns). The sqrt sweep, then the rate-scale rates 0.018 and 0.032 (E14) -and REF-op cold. +## Selecting a sqrt rate -```{r tables} -season_table <- function(g, rate_keep = c(0.018, 0.032, 0.056, 0.1)) { - rows <- bind_rows( - sweep_m %>% filter(scale == "sqrt", lr == SQRT_LRS[1]) %>% mutate(config = "base", wis_pct = 0, cal_err_cal = cal_err_base), - sweep_m %>% filter(scale == "sqrt") %>% mutate(config = paste("sqrt", format(lr, drop0trailing = TRUE))), - sweep_m %>% filter(scale == "rate", lr %in% rate_keep) %>% mutate(config = paste("rate", format(lr, drop0trailing = TRUE))), - ref_m %>% mutate(config = ref) - ) %>% - filter(geos == g) %>% - mutate(config = factor(config, levels = unique(config))) - rows %>% - transmute(config, col = paste(season, h), cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal)) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(config) -} -``` - -### States only +For each sqrt rate, the worst and the mean WIS reduction % (positive = calibrated WIS lower than base) over the two seasons +and h0–h3 (states only). The selected rate has the highest mean among those +whose worst case is still a gain (E14's rule). -```{r table-states} -knitr::kable(season_table(GEOS[["states"]])) +```{r best} +knitr::kable(best_tbl %>% mutate(across(c(worst, mean), \(x) round(x, 2)))) ``` -L1 coverage bias, season-pooled against month-average (see "Month view" below): - -```{r table-states-month} +```{r month-view-cfgs} lr_lab <- function(lr) vapply(lr, \(x) format(x, drop0trailing = TRUE), character(1)) rate_keep <- RATE_LRS %in% c(0.018, 0.032, 0.056, 0.1) mv <- cal_month_view(c( @@ -209,184 +180,49 @@ mv <- cal_month_view(c( setNames(rate_runs[rate_keep], paste("rate", lr_lab(RATE_LRS[rate_keep]))), list(`REF-op cold` = refop) ), seasons = LIVE) -mv_show_table <- FALSE -knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) -``` - -### All locations - -```{r table-all} -knitr::kable(season_table(GEOS[["all"]])) -``` - -### Selecting a sqrt rate - -For each sqrt rate, the worst and the mean WIS reduction % (positive = calibrated WIS lower than base) over the two seasons -and h0–h3 (states only). The selected rate has the highest mean among those -whose worst case is still a gain (E14's rule). - -```{r best} -knitr::kable(best_tbl %>% mutate(across(c(worst, mean), \(x) round(x, 2)))) +# The plots show the sqrt rates around the elbow, rate 0.032 (E14) and REF-op cold; the tables cover every sqrt rate. +MONTH_COLS <- c(`sqrt 0.01` = "#56B4E9", `sqrt 0.018` = "#CC79A7", `sqrt 0.032` = "#0072B2", `rate 0.032` = "#009E73", `REF-op cold` = "#D55E00") +mv_plot_cfgs <- names(MONTH_COLS) +mv_cols <- MONTH_COLS ``` -### 50% and 90% interval coverage, states only - -```{r table-cov} -bind_rows( - sweep_m %>% filter(scale == "sqrt", lr == SQRT_LRS[1]) %>% mutate(config = "base", cov50_cal = cov50_base, cov90_cal = cov90_base), - sweep_m %>% filter(scale == "sqrt", lr >= 0.0032, lr <= 0.056) %>% mutate(config = paste("sqrt", format(lr, drop0trailing = TRUE))), - sweep_m %>% filter(scale == "rate", lr %in% c(0.018, 0.032)) %>% mutate(config = paste("rate", format(lr, drop0trailing = TRUE))), - ref_m %>% mutate(config = ref) -) %>% - filter(geos == GEOS[["states"]], h %in% c("h0", "h2"), season != "both seasons") %>% - mutate(config = factor(config, levels = unique(config))) %>% - transmute(config, col = paste(season, h), cell = sprintf("%.2f / %.2f", cov50_cal, cov90_cal)) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(config) %>% - knitr::kable() +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` -# 2. Offset tail +# Mean offset at the median over time -E15's statistic: for each forecast and level, |offset| ÷ the base 90% -interval width (q95 − q05), and its 90th percentile over forecasts (location -× round, both seasons, states only). "all levels" pools the 23 levels. A -smaller number means fewer offsets that are large next to the interval. +The calibrated minus base median, in admissions per 100k, averaged over +states at each round, h0 and h2. The bottom row is the mean base median per +100k at h0, to place the season peak. -```{r tail} -pick <- list( - `sqrt best` = sqrt_runs[[which(SQRT_LRS == BEST_SQRT)]], - `rate 0.018` = rate_runs[[which(RATE_LRS == 0.018)]], - `rate 0.032` = rate_runs[[which(RATE_LRS == 0.032)]], - `rate 0.1` = rate_runs[[which(RATE_LRS == 0.1)]], - `REF-op cold` = refop -) -names(pick)[1] <- paste("sqrt", BEST_SQRT) -fc_pick <- purrr::imap(pick, \(cal, nm) cal$forecasts %>% mutate(config = nm)) %>% +```{r offset-data} +OFFSET_CFGS <- list(`sqrt 0.018` = sqrt_runs[[which(SQRT_LRS == 0.018)]], `rate 0.032` = rate_runs[[which(RATE_LRS == 0.032)]], `REF-op cold` = refop) +fc_pick <- purrr::imap(OFFSET_CFGS, \(cal, nm) cal$forecasts %>% mutate(config = nm)) %>% bind_rows() %>% - mutate(season = season_of(reference_date), config = factor(config, levels = names(pick))) %>% + mutate(season = season_of(reference_date), config = factor(config, levels = names(OFFSET_CFGS))) %>% filter(season %in% LIVE, location != "US", !is.na(value_base)) -w90 <- fc_pick %>% - filter(config == names(pick)[1], level %in% c(0.05, 0.95)) %>% - select(location, horizon, reference_date, level, value_base) %>% - tidyr::pivot_wider(names_from = level, values_from = value_base, names_prefix = "q") %>% - transmute(location, horizon, reference_date, w90 = q0.95 - q0.05) -ratio <- fc_pick %>% - inner_join(w90, by = c("location", "horizon", "reference_date")) %>% - filter(w90 > 0) %>% - mutate(r = abs(value_cal - value_base) / w90) -tail_tbl <- bind_rows( - ratio %>% filter(level %in% c(0.05, 0.25, 0.5, 0.75, 0.95)) %>% mutate(lv = as.character(level)), - ratio %>% mutate(lv = "all levels") -) %>% - group_by(config, horizon, lv) %>% - summarize(p90 = quantile(r, 0.9), .groups = "drop") %>% - mutate(lv = factor(lv, levels = c("0.05", "0.25", "0.5", "0.75", "0.95", "all levels"))) -knitr::kable( - tail_tbl %>% - transmute(config, lv, h = paste0("h", horizon), p90 = round(p90, 3)) %>% - tidyr::pivot_wider(names_from = h, values_from = p90) %>% - arrange(lv, config), - caption = "States only: 90th percentile of |offset| / base 90% width" -) ``` -# 3. Month by month - -Five forecasts per state-round are compared: the base, the rate-scale rates -0.018 and 0.032, the selected sqrt rate and REF-op cold. Months are calendar -months of the reference date. States only, h0–h3. Each cell holds about -150–260 state forecasts, so single months are noisy. - -```{r month-data} -MONTH_COLS <- setNames(c("#009E73", "#0072B2", "#CC79A7", "#D55E00"), names(pick)[c(2, 3, 1, 5)]) -month_m <- ws_month_scores(fc_pick %>% filter(config %in% names(MONTH_COLS)) %>% mutate(config = factor(config, levels = names(MONTH_COLS)))) %>% - mutate(h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) -``` - -WIS, coverage bias and median error by month for these four trackers are in -"Month view" at the end. Below: the share of outcomes below the median, the -offset over time and an h2 table. - -## Coverage by month - -Top: L1 coverage bias (lower is better) per month, base in black. Bottom: share of outcomes -below the median; 0.5 is calibrated, above 0.5 means the median sits too -high (over-prediction). Base in black. - -```{r month-cov, fig.height=8} -cov_long <- bind_rows( - month_m %>% transmute(config = as.character(config), season, h, month, `L1 coverage bias (lower is better)` = cal_err_cal, `share below the median` = below_med_cal), - month_m %>% filter(config == names(MONTH_COLS)[1]) %>% - transmute(config = "base", season, h, month, `L1 coverage bias (lower is better)` = cal_err_base, `share below the median` = below_med_base) -) %>% - tidyr::pivot_longer(c(`L1 coverage bias (lower is better)`, `share below the median`), names_to = "metric") %>% - mutate(config = factor(config, levels = c("base", names(MONTH_COLS)))) -half <- tibble(metric = "share below the median", y = 0.5) -ggplot(cov_long, aes(month, value, colour = config, group = config)) + - geom_hline(data = half, aes(yintercept = y), colour = "grey50", linetype = "dashed") + - geom_line() + - geom_point(size = 1) + - facet_grid(metric + season ~ h, scales = "free", space = "free_x", labeller = labeller(metric = label_wrap_gen(20))) + - scale_colour_manual(values = c(base = "black", MONTH_COLS), name = NULL) + - labs(x = "month of reference date", y = NULL) + - theme(legend.position = "bottom") -``` - -## Mean offset at the median over time - -The calibrated minus base median, in admissions per 100k, averaged over -states at each round, h0 and h2. The bottom row is the mean base median per -100k at h0, to place the season peak. - ```{r offset-time, fig.height=7} pop <- wi$rate_scales %>% select(location, scale) med <- fc_pick %>% - filter(level == 0.5, horizon %in% c(0, 2), config %in% names(MONTH_COLS)) %>% + filter(level == 0.5, horizon %in% c(0, 2), config %in% names(OFFSET_CFGS)) %>% left_join(pop, by = "location") %>% group_by(config, season, horizon, reference_date) %>% summarize(off = mean((value_cal - value_base) / scale), base = mean(value_base / scale), truth = mean(truth / scale, na.rm = TRUE), .groups = "drop") off_long <- bind_rows( med %>% transmute(config, season, reference_date, row = paste0("offset, h", horizon), value = off), - med %>% filter(horizon == 0, config == names(MONTH_COLS)[1]) %>% transmute(config = "base median", season, reference_date, row = "base median h0", value = base), - med %>% filter(horizon == 0, config == names(MONTH_COLS)[1]) %>% transmute(config = "truth", season, reference_date, row = "base median h0", value = truth) + med %>% filter(horizon == 0, config == names(OFFSET_CFGS)[1]) %>% transmute(config = "base median", season, reference_date, row = "base median h0", value = base), + med %>% filter(horizon == 0, config == names(OFFSET_CFGS)[1]) %>% transmute(config = "truth", season, reference_date, row = "base median h0", value = truth) ) %>% - mutate(config = factor(config, levels = c(names(MONTH_COLS), "base median", "truth")), row = factor(row, levels = c("offset, h0", "offset, h2", "base median h0"))) + mutate(config = factor(config, levels = c(names(OFFSET_CFGS), "base median", "truth")), row = factor(row, levels = c("offset, h0", "offset, h2", "base median h0"))) ggplot(off_long, aes(reference_date, value, colour = config)) + geom_hline(yintercept = 0, colour = "grey60") + geom_line() + geom_point(size = 0.7) + facet_grid(row ~ season, scales = "free") + - scale_colour_manual(values = c(MONTH_COLS, `base median` = "grey30", truth = "grey65"), name = NULL) + + scale_colour_manual(values = c(MONTH_COLS[names(OFFSET_CFGS)], `base median` = "grey30", truth = "grey65"), name = NULL) + scale_x_date(date_breaks = "1 month", date_labels = "%b") + labs(x = "reference date", y = "admissions per 100k (mean over states)") + theme(legend.position = "bottom") ``` - -## Month table, h2 - -WIS reduction % (positive = calibrated WIS lower than base) per month (states only, h2), with the base's share of the -season's WIS and the share of outcomes below the base and calibrated medians. - -```{r month-table} -month_m %>% - filter(h == "h2") %>% - transmute(season, month, config, cell = sprintf("%+.1f (%.2f)", pct, below_med_cal)) %>% - tidyr::pivot_wider(names_from = config, values_from = cell) %>% - left_join( - month_m %>% filter(h == "h2", config == names(MONTH_COLS)[1]) %>% - transmute(season, month, `base share %` = round(share, 1), `base below median` = round(below_med_base, 2)), - by = c("season", "month") - ) %>% - arrange(season, month) %>% - knitr::kable(caption = "WIS reduction %, positive = calibrated WIS lower than base (share of outcomes below the calibrated median)") -``` - -```{r month-view-cfgs} -# The four trackers of section 3. -mv_plot_cfgs <- names(MONTH_COLS) -mv_cols <- MONTH_COLS -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` diff --git a/reports/writeups/calibration_experiments/e17_late_decay.Rmd b/reports/writeups/calibration_experiments/e17_late_decay.Rmd index 3b828f5d..6cb84209 100644 --- a/reports/writeups/calibration_experiments/e17_late_decay.Rmd +++ b/reports/writeups/calibration_experiments/e17_late_decay.Rmd @@ -79,95 +79,17 @@ fc <- purrr::map2(cals, seq_len(nrow(grid)), function(cal, i) { DECAY_COLS <- setNames(c("black", "#56B4E9", "#0072B2", "#D55E00", "#009E73"), names(DECAYS)) ``` -# 1. Overall, by season - -WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), states only. - -```{r head} -head_m <- fc %>% - group_by(rate, decay) %>% - group_modify(\(d, k) ws_scores(d)) %>% - ungroup() %>% - select(rate, decay, season, horizon, which, wis, cal_err) %>% - tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% - mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base, season = factor(season, levels = c(LIVE, "both"), labels = c(LIVE, "both seasons"))) -base_row <- head_m %>% - filter(rate == names(RATES)[1], decay == "no decay") %>% - transmute(config = "base", season, horizon, cell = sprintf("%+.1f / %.3f", 0, cal_err_base)) -bind_rows(base_row, head_m %>% transmute(config = paste(rate, decay, sep = ", "), season, horizon, cell = sprintf("%+.1f / %.3f", wis_pct, cal_err_cal))) %>% - mutate(config = factor(config, levels = unique(config))) %>% - transmute(config, col = paste0(season, " h", horizon), cell) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - knitr::kable() -``` - -L1 coverage bias, season-pooled against month-average (see "Month view" below): - -```{r head-month} +```{r month-view-cfgs} mv <- cal_month_view(setNames(cals, paste(grid$rate, grid$decay, sep = ", ")), seasons = LIVE) -mv_show_table <- FALSE -knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) -``` - -The same WIS reductions as a plot, one panel per season and horizon. - -```{r head-plot, fig.height=5.5} -ggplot(head_m, aes(rate, wis_pct, fill = decay)) + - geom_hline(yintercept = 0, colour = "grey50") + - geom_col(position = position_dodge(width = 0.85), width = 0.8) + - facet_grid(season ~ paste0("h", horizon)) + - scale_fill_manual(values = DECAY_COLS, name = NULL) + - labs(x = NULL, y = "WIS reduction % (positive = calibrated WIS lower than base)") + - theme(legend.position = "bottom", axis.text.x = element_text(angle = 20, hjust = 1)) -``` - -# 2. Month by month - -WIS reduction % (positive = calibrated WIS lower than base) by calendar month of the reference date, one figure per -constant rate; rows are seasons, columns horizons. Grey bars are each -month's share of the season's base WIS. Clipped at −40%. - -```{r month, results="asis", fig.height=5.5} -month_m <- ws_month_scores(fc %>% mutate(config = interaction(rate, decay, sep = "|"))) %>% - tidyr::separate(config, c("rate", "decay"), sep = "[|]") %>% - mutate(decay = factor(decay, levels = names(DECAYS)), h = factor(paste0("h", horizon), levels = paste0("h", 0:3))) -for (r in names(RATES)) { - cat("\n\n", cal_h(2), " ", r, "\n\n", sep = "") - d <- month_m %>% filter(rate == r) - share <- d %>% filter(decay == "no decay") - print( - ggplot(d, aes(month, pct, colour = decay, group = decay)) + - geom_col(data = share, aes(month, share), inherit.aes = FALSE, fill = "grey85", width = 0.7) + - geom_hline(yintercept = 0, colour = "grey40") + - geom_line() + - geom_point(size = 1.1) + - facet_grid(season ~ h, scales = "free_x", space = "free_x") + - scale_colour_manual(values = DECAY_COLS, name = NULL) + - coord_cartesian(ylim = c(-40, 30)) + - labs(x = "month of reference date", y = "WIS reduction % (positive = calibrated\nWIS lower than base); grey: % of season base WIS", title = r) + - theme(legend.position = "bottom") - ) -} +# The plots show sqrt 0.018 with each decay setting; the tables cover all three trackers. +mv_plot_cfgs <- paste(names(RATES)[3], names(DECAYS), sep = ", ") +mv_cols <- setNames(DECAY_COLS, mv_plot_cfgs) ``` -## Share of outcomes below the median, March–May - -0.5 is calibrated; above 0.5 the median is too high. h2, states only. - -```{r month-below} -month_m %>% - filter(h == "h2", month %in% c("Mar", "Apr", "May")) %>% - transmute(rate, season, month, decay, v = sprintf("%+.1f (%.2f)", pct, below_med_cal)) %>% - tidyr::pivot_wider(names_from = decay, values_from = v) %>% - left_join( - month_m %>% filter(h == "h2", decay == "no decay") %>% transmute(rate, season, month, `base below median` = round(below_med_base, 2)), - by = c("rate", "season", "month") - ) %>% - arrange(rate, season, month) %>% - knitr::kable(caption = "h2: WIS reduction %, positive = calibrated WIS lower than base (share of outcomes below the calibrated median)") +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` -# 3. Mean offset at the median over time +# Mean offset at the median over time Calibrated minus base median in admissions per 100k, averaged over states at each round, h2. @@ -189,12 +111,3 @@ fc %>% labs(x = "reference date", y = "mean offset at the median, per 100k") + theme(legend.position = "bottom") ``` - -```{r month-view-cfgs} -# sqrt 0.018 with each decay setting; section 2 has the WIS months for the rate-scale trackers. -mv_plot_cfgs <- paste(names(RATES)[3], names(DECAYS), sep = ", ") -mv_cols <- setNames(DECAY_COLS, mv_plot_cfgs) -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` diff --git a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd index 211e6223..232c3c7a 100644 --- a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd +++ b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd @@ -116,46 +116,18 @@ step_m <- purrr::imap(cals, function(cal, nm) { h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) ) mv <- cal_month_view(cals, seasons = LIVE) -step_table <- function(col, fmt, base_col = NULL) { - rows <- step_m %>% mutate(step = as.character(step), v = .data[[col]]) - if (!is.null(base_col)) { - rows <- bind_rows(step_m %>% filter(step == names(STEPS)[1]) %>% mutate(step = "base", v = .data[[base_col]]), rows) - } - rows %>% - mutate(step = factor(step, levels = unique(step))) %>% - arrange(season, h) %>% - transmute(step, col = paste(season, h), cell = sprintf(fmt, v)) %>% - tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(step) -} ``` -# 1. Season scores per step - -States only. Each season is scored on its own rounds; "both seasons" pools -them. The tracker runs over both seasons, so 2025-26 starts from the offsets -2024-25 left behind. - -## WIS reduction % (positive = calibrated WIS lower than base) - -```{r table-wis} -knitr::kable(step_table("wis_pct", "%+.1f"), caption = "WIS reduction % (positive = calibrated WIS lower than base), states only") -``` - -## L1 coverage bias (lower is better), season-pooled - -```{r table-cov} -knitr::kable(step_table("cal_err_cal", "%.3f", base_col = "cal_err_base"), caption = "L1 coverage bias (lower is better), season-pooled, states only") +```{r month-view-cfgs} +# Five configs, so the month plots stay readable; the tables below them cover every config. +mv_plot_cfgs <- c("0 REF-op", "1 no warm start", "4 no burn-in season", "6 sqrt constant 0.018 (E16)", "branch: sqrt 0.018 warm") +mv_cols <- STEP_COLS ``` -## L1 coverage bias (lower is better), season-pooled and month-average - -```{r table-month-cov} -mv_show_table <- FALSE -knitr::kable(cal_month_cov_table(mv), caption = CAL_MONTH_COV_CAPTION) +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` -# 2. What each step changes +# What each step changes The change each step makes to the step before it, both seasons pooled, states only. WIS: change in WIS reduction % (positive = the step lowers @@ -185,50 +157,20 @@ knitr::kable( ) ``` -```{r delta-plot, fig.height=6.5} -delta_long <- delta %>% - transmute(step, h, - `WIS reduction %, change\n(positive = step lowers WIS)` = d_wis, - `month-avg L1 coverage bias,\nchange (negative = better)` = d_mavg, - `season-pooled L1 coverage bias,\nchange (negative = better)` = d_pooled - ) %>% - tidyr::pivot_longer(-c(step, h), names_to = "metric") %>% - mutate(metric = factor(metric, levels = unique(metric)), step = factor(step, levels = rev(names(STEPS)[-1]))) -ggplot(delta_long, aes(value, step, fill = step)) + - geom_vline(xintercept = 0, colour = "grey50", linewidth = 0.3) + - geom_col(width = 0.7, show.legend = FALSE) + - scale_fill_manual(values = STEP_COLS) + - facet_grid(h ~ metric, scales = "free_x") + - labs(x = "change from the previous step (both seasons, states only)", y = NULL) -``` - -# 3. State by state: REF-op against sqrt 0.018 - -How many of the 52 states (50 states, DC and PR) each config of a pair has -the lower WIS in, per season and horizon. States share the national wave, -so the counts are not independent tests; they show whether a pooled gap -comes from most states or from a few large ones. - -```{r state-wins} -PAIRS <- list( - c("0 REF-op", "6 sqrt constant 0.018 (E16)"), - c("0 REF-op", "branch: sqrt 0.018 warm"), - c("branch: sqrt 0.018 warm", "6 sqrt constant 0.018 (E16)") -) -state_wis <- cal_state_wis(cals[unique(unlist(PAIRS))]) -knitr::kable(cal_pair_wins(state_wis, PAIRS), caption = CAL_PAIR_WINS_CAPTION) -``` +# State by state: REF-op against sqrt 0.018 Per state, pooled over h0–h3: WIS reduction % of REF-op against sqrt 0.018 (above the line, REF-op is better). ```{r state-scatter, fig.height=5} +PAIR <- c("0 REF-op", "6 sqrt constant 0.018 (E16)") +state_wis <- cal_state_wis(cals[PAIR]) state_pct <- state_wis %>% - filter(config %in% PAIRS[[1]]) %>% + filter(config %in% PAIR) %>% group_by(config, season, location) %>% summarize(pct = 100 * (sum(wis_base) - sum(wis_cal)) / sum(wis_base), .groups = "drop") %>% tidyr::pivot_wider(names_from = config, values_from = pct) -ggplot(state_pct, aes(.data[[PAIRS[[1]][2]]], .data[[PAIRS[[1]][1]]])) + +ggplot(state_pct, aes(.data[[PAIR[2]]], .data[[PAIR[1]]])) + geom_abline(slope = 1, intercept = 0, colour = "grey50") + geom_hline(yintercept = 0, colour = "grey85") + geom_vline(xintercept = 0, colour = "grey85") + @@ -236,12 +178,3 @@ ggplot(state_pct, aes(.data[[PAIRS[[1]][2]]], .data[[PAIRS[[1]][1]]])) + facet_wrap(~season) + labs(x = "sqrt 0.018: WIS reduction %, h0–h3", y = "REF-op: WIS reduction %, h0–h3") ``` - -```{r month-view-cfgs} -# Five configs, so the month plots stay readable; every config is in the tables above. -mv_plot_cfgs <- c("0 REF-op", "1 no warm start", "4 no burn-in season", "6 sqrt constant 0.018 (E16)", "branch: sqrt 0.018 warm") -mv_cols <- STEP_COLS -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` From eb86502f4bb2143119dd6c1a07f8bcd2cf906a2b Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:16:17 -0700 Subject: [PATCH 089/107] feat(calibration): prod_views notebook with state fans and gallery for the prod configs; index lists questions only --- .../calibration_experiments/index.Rmd | 77 +++++++------ .../calibration_experiments/prod_views.Rmd | 105 ++++++++++++++++++ 2 files changed, 146 insertions(+), 36 deletions(-) create mode 100644 reports/writeups/calibration_experiments/prod_views.Rmd diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index 00dd3ad6..8c2e7c61 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -18,9 +18,9 @@ Tibshirani, 2025) learns from past coverage misses. Each notebook changes one setting of that tracker and compares the result with the uncalibrated forecast. -The summaries below give sizes, not verdicts. There are no uncertainty -estimates, so a WIS gap under 1 point or a month-avg coverage gap under 0.01 -is read as no clear difference. +There are no uncertainty estimates, so a WIS gap under 1 point or a +month-avg coverage gap under 0.01 is read as no clear difference. +Conclusions are kept in `notes/calibration-ledger.md`. ```{r rows, include=FALSE} out_dir <- here::here("reports/calibration_experiments") @@ -39,49 +39,55 @@ when it was last rendered. The ids match `notes/calibration-ledger.md`. ## Finalists -| | base | question | what the numbers show | -|---|---|---|---| -| `r link("E18", "e18_ref_op_bridge")` | replay | Stepping from REF-op to sqrt constant 0.018 one setting at a time | REF-op vs sqrt 0.018: no strong result (0.3–0.6 points pooled, order flips by season). The warm start helps in 2024-25 only. Constant 0.018 is 1.7–2.3 points above cold adaptive at h2–h3. No step moves month-avg coverage by more than 0.008. sqrt 0.018 warm (branch) is the best pooled (+3.4/+1.6/+1.6/+2.5) but 0.011–0.018 worse than the base on month-avg coverage at h2–h3. REF-op beats sqrt 0.018 cold in 41–45 of 52 states in 2024-25 and 14–18 in 2025-26. | -| `r link("E07b", "e07b_candidates")` | both | Do the operating-point candidates rank the same way on both bases? | The candidate order flips between seasons on both. | -| `r link("E15", "e15_wis_sources")` | replay, cold | Where does the WIS change come from? | Offsets are 2–5% of the 90% interval width. REF-op's gain (+2.0 at h0, about 0 by h3) comes from raising the lower quantiles; a median shift alone matches it within 0.6 points. | +| | base | question | +|---|---|---| +| `r link("E18", "e18_ref_op_bridge")` | replay | Which setting accounts for the gap between REF-op and sqrt constant 0.018, stepping from one to the other a setting at a time? | +| `r link("E07b", "e07b_candidates")` | both | Do the operating-point candidates rank the same way on both bases? | +| `r link("E15", "e15_wis_sources")` | replay, cold | Where does the WIS change come from? | + +## Prod configs + +`r link("Prod views", "prod_views")`: per-state forecast fans and a gallery +of the largest gains and losses, for flu REF-op warm and covid REF-op cold +on the submitted ensemble. ## Flu and covid -| | base | question | what the numbers show | -|---|---|---|---| -| `r link("E10", "e10_covid")` | both diseases, both bases | How do the finalists do on flu and covid? (Part 2: covid ensemble at h−1–h3 with REF-paper.) | Part 2: h−1 improves about 10–11% in both seasons. REF-paper loses 10–19% at h1–h3 in 2025-26. REF-op at h1–h3 is −4 to +3, sign depending on the season. Part 1: covid, nothing beats REF-op cold at h0–h3 and sqrt 0.018 loses 3–4 points at h1–h3; flu, sqrt 0.018 warm leads on both bases by 1–2 points at h1–h3, all in 2024-25 on the ensemble. sqrt 0.018 has the largest covid h−1 gain (+14.9). | -| `r link("E07", "e07_across_eras")` | both | Does calibration act the same on the ensemble and the clean replay? | Same direction on both; REF-op gains 0.7–1.1 points more on the ensemble. | +| | base | question | +|---|---|---| +| `r link("E10", "e10_covid")` | both diseases, both bases | How do the finalists do on flu and covid? (Part 2: covid ensemble at h−1–h3 with REF-paper.) | +| `r link("E07", "e07_across_eras")` | both | Does calibration act the same on the ensemble and the clean replay? | ## Learning rate -| | base | question | what the numbers show | -|---|---|---|---| -| `r link("E01", "e01_eta_settings")` | replay, cold | How do the adaptive learning-rate settings (multiplier, window, carry or reset) change WIS and coverage? | Multipliers 0.1 and 0.3 lose 7–77 points at h1–h3. 0.01 is 0.5–2.4 points above 0.03 at h1–h3; month-avg coverage shows no clear difference. Count scale is 2–5 points below sqrt. Window and carry vs reset: no strong result. | -| `r link("E09", "e09_lr_delay")` | ensemble | With a constant learning rate on the rate scale, how do WIS and coverage change with the rate and with extra learning delay? | WIS peaks near 0.056–0.1 per 100k and falls fast past 0.3. Month-avg coverage is no lower than the base at h1–h3 for any constant rate. Each week of extra delay costs about 1 point at h−1. Stateful learning with no extra delay makes h−1 worse than the base. | -| `r link("E14", "e14_lr_by_season")` | replay, cold | How do WIS and coverage change with a constant learning rate, season by season? | Every rate from 0.0032 to 0.056 is at or above the base in both seasons, within 1.2 points of each other: no strong result for a best rate. From 0.1 WIS falls. Month-avg coverage for rates ≤ 0.032 stays within about 0.01 of the base. | -| `r link("E16", "e16_sqrt_constant_lr")` | replay, cold | Constant learning rate on the square-root scale vs the rate scale | sqrt 0.018 is about 1 point above rate 0.032 at h0 and within 0.2 at h1–h3 pooled. Against REF-op cold the order flips by season. Every tracker loses in March (up to −28% within the month at h2). | +| | base | question | +|---|---|---| +| `r link("E01", "e01_eta_settings")` | replay, cold | How do the adaptive learning-rate settings (multiplier, window, carry or reset) change WIS and coverage? | +| `r link("E09", "e09_lr_delay")` | ensemble | With a constant learning rate on the rate scale, how do WIS and coverage change with the rate and with extra learning delay? | +| `r link("E14", "e14_lr_by_season")` | replay, cold | How do WIS and coverage change with a constant learning rate, season by season? | +| `r link("E16", "e16_sqrt_constant_lr")` | replay, cold | How does a constant learning rate on the square-root scale compare with the rate scale? | ## Tracker structure -| | base | question | what the numbers show | -|---|---|---|---| -| `r link("E03", "e03_scale")` | ensemble | Which scale should the offsets live on: counts, square root, or rate per 100k? | sqrt is 1.4–3.9 points above count pooled, mostly from 2025-26. Coverage: no strong result. Rate is identical to count in every cell. | -| `r link("E04", "e04_offset_structure")` | ensemble | Leak and slow term | The leak is +3.3 to +5.6 at h1–h3 in 2025-26 and −0.4 to −1.3 in 2024-25. The slow term without a warm start: no strong result. | -| `r link("E05", "e05_warm_start")` | ensemble | Warm start from the burn-in season | +1.7 to +3.2 at h2/h3 in 2024-25 for every tracker, within ±0.4 in 2025-26. | -| `r link("E11", "e11_constant_lr")` | ensemble | Does a constant learning rate do better than the adaptive rate? | Constant 0.1 vs adaptive sqrt: the order flips by horizon (1–2 points either way). REF-op has the highest pooled WIS at every horizon, by up to 2 points. On month-avg coverage the constant rate is 0.003–0.022 worse. | +| | base | question | +|---|---|---| +| `r link("E03", "e03_scale")` | ensemble | Which scale should the offsets live on: counts, square root, or rate per 100k? | +| `r link("E04", "e04_offset_structure")` | ensemble | Does a leak or a second, slow offset term change WIS and coverage? | +| `r link("E05", "e05_warm_start")` | ensemble | Does a warm start from the burn-in season change WIS and coverage for each tracker? | +| `r link("E11", "e11_constant_lr")` | ensemble | Does a constant learning rate do better than the adaptive rate? | ## Late season -| | base | question | what the numbers show | -|---|---|---|---| -| `r link("E02", "e02_eta_variants")` | replay, cold | Spring switch-off, per-level learning rate, seasonal window | Switching off from 1 March or 15 February is 2–4 points above the reference at h1–h3 (dates in-sample; in 2025-26 about half the gain is December carry-over). Per-level about +1. Month-avg coverage: no clear difference. | -| `r link("E17", "e17_late_decay")` | replay, cold | Does shrinking or switching off the offset from 1 March change WIS and coverage? | No strong result: pooled WIS gaps ≤ 0.7 points and month-avg coverage gaps ≤ 0.010. The switch-off's sqrt gain comes mostly from 2025-26; the date was chosen in-sample. | +| | base | question | +|---|---|---| +| `r link("E02", "e02_eta_variants")` | replay, cold | Do a spring switch-off, a per-level learning rate or a seasonal window change WIS and coverage? | +| `r link("E17", "e17_late_decay")` | replay, cold | Does shrinking or switching off the offset from 1 March change WIS and coverage? | ## Learning truth -| | base | question | what the numbers show | -|---|---|---|---| -| `r link("E00", "e00_vintage_backtest")` | ensemble | Should the tracker learn from finalized data or from data as published? | Exact and final differ by 0.6 points or less, except REF-op h−1 (2.2, 2024-25). Learning once from the first report loses most of the h−1 gain. All other notebooks use the exact (prod) design. | +| | base | question | +|---|---|---| +| `r link("E00", "e00_vintage_backtest")` | ensemble | Should the tracker learn from finalized data or from data as published? | E12 and E13 (the clean replay without and with a 2023-24 burn-in) have no notebook; they are `scripts/calibration/calibration_ws_replay.R`. @@ -101,11 +107,10 @@ notebook; they are `scripts/calibration/calibration_ws_replay.R`. the ramp cancel over-prediction after the peak, and usually looks much better. Month-level noise is about 0.01. - **Two bases.** "Ensemble" notebooks calibrate our submitted forecasts - (h−1 to h3, headline tables over all hub locations). "Replay" notebooks - calibrate a clean weekly replay of `windowed_seasonal` with no hand edits - (h0–h3, states only). -- **Scoring.** Every score is against finalized NHSN data over the live - seasons (2024-25 and 2025-26). A burn-in season (2023-24 for flu) only warms + (h−1 to h3). "Replay" notebooks calibrate a clean weekly replay of + `windowed_seasonal` with no hand edits (h0–h3). +- **Scoring.** Every score is states only (US excluded), against finalized + NHSN data over the live seasons (2024-25 and 2025-26). A burn-in season (2023-24 for flu) only warms up the learning rate and is not scored. - **Cold start.** Without a warm start, offsets start at zero, and every config equals the base through Nov–Dec 2024, so 2024-25 gains are diluted. diff --git a/reports/writeups/calibration_experiments/prod_views.Rmd b/reports/writeups/calibration_experiments/prod_views.Rmd new file mode 100644 index 00000000..fbbcabde --- /dev/null +++ b/reports/writeups/calibration_experiments/prod_views.Rmd @@ -0,0 +1,105 @@ +--- +title: "Prod configs: forecasts by state" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + flu_hub_dir: "../FluSight-forecast-hub" + covid_hub_dir: "../covid19-forecast-hub" + workers: 14 +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +What the calibrated forecasts look like next to the submitted ones, for the +configs running in prod: + +- **Flu**: REF-op warm (warm start from the 2023-24 burn-in season). +- **Covid**: REF-op cold (no burn-in season; covid submissions start + 2024-11-23). + +Both calibrate the submitted `CMU-TimeSeries` ensemble, learn from NHSN data +as published (re-run every round, as prod does; E00), and are drawn against +finalized truth. Spoiled submissions are dropped. Seasons run August–July; +only the two live seasons (2024-25 and 2025-26) are shown. + +```{r load} +suppressPackageStartupMessages(source(here::here("R/load_all.R"))) +LIVE <- c("2024-2025", "2025-2026") +# Same inputs and arguments as the E05/E11 (flu) and E10 (covid) runs, so the cached runs are reused. +load_prod <- function(disease, hub_dir, run_args) { + archive <- get_nhsn_data_archive(disease) + truth <- nhsn_read_truth(disease, archive) + vintages <- nhsn_read_vintages(disease, archive) + hub_target <- c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET)[[disease]] + hub_fc <- suppressMessages(hub_read_forecasts(hub_dir = here::here(hub_dir), target = hub_target)) + population <- get_population_data() %>% + distinct(state_code, .keep_all = TRUE) %>% + filter(state_code %in% unique(hub_fc$location)) %>% + select(location = state_code, population) + cal <- do.call(cal_run_cached, c(list(hub_fc, truth, "op"), run_args, list(vintages = vintages, workers = params$workers))) + cal$forecasts <- cal$forecasts %>% mutate(season = cal_season_of(reference_date)) + list(cal = cal, truth = truth, vintages = vintages, population = population) +} +PROD <- list( + flu = load_prod("flu", params$flu_hub_dir, list()), + covid = load_prod("covid", params$covid_hub_dir, list(burn_in_seasons = character(0), slow_init = NULL)) +) +abbrev <- setNames(toupper(get_population_data()$state_id), get_population_data()$state_code) +``` + +```{r sections, results="asis", fig.height=4} +for (d in names(PROD)) { + p <- PROD[[d]] + cat("\n\n# ", d, "\n\n", sep = "") + + cat("\n\n## Fixed states {.tabset}\n\n") + cat( + "The same states (US, CA, GA, MN, VT) for both diseases. Every other", + "round's fan at h−1–h3: blue ribbon and line are the base 80% band and", + "median, orange lines the calibrated 10/50/90% quantiles, grey the", + "finalized truth. The black line is the last two reported weeks of NHSN", + "data as published when that round was forecast; its dot is normally the", + "round's h−1 week, where its fan starts. Black sits below grey where first", + "reports were later revised up. A black line with no fan in front of it is", + "a round with no submission.\n\n" + ) + for (nm in names(CAL_STATES)) { + cat("\n\n### ", nm, "\n\n", sep = "") + for (s in LIVE) print(cal_state_panel(p$cal, CAL_STATES[[nm]], s, p$truth, p$vintages, title = paste(d, nm, s))) + } + + cat("\n\n## Gallery {.tabset}\n\n") + cat( + "For each season phase and location size (population above or below the", + "median), the forecast with the largest WIS gain and the one with the", + "largest loss, each from a different location. Forecasts are ranked by", + "their WIS reduction as a share of their own location-season's base WIS,", + "so small states late in the season, where the base is tiny, do not crowd", + "the gallery. Ribbons are the 50% and 90% bands, the dashed line is the", + "forecast date.\n\n" + ) + cal_live <- p$cal + cal_live$forecasts <- cal_live$forecasts %>% filter(season %in% LIVE) + picks <- cal_gallery_pick(cal_live, p$population) + for (i in seq_len(nrow(picks))) { + pk <- picks[i, ] + if (i == 1L || pk$phase != picks$phase[i - 1L]) cat("\n\n### ", as.character(pk$phase), "\n\n", sep = "") + sub <- sprintf( + "%s (%s), %s, %s: WIS reduction %+.1f%% (%+.1f%% of its season's base WIS; positive = calibrated WIS lower)", + abbrev[[pk$location]], pk$size, pk$reference_date, pk$kind, pk$wis_pct, pk$share + ) + print(cal_gallery_panel(p$cal, pk$location, pk$reference_date, p$truth, p$vintages, subtitle = sub)) + } +} +``` From 10889b215a8b4bc7e1f1a4fc58ade7f8bf6fa6c7 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:44:30 -0700 Subject: [PATCH 090/107] refactor(calibration): drop E09's bridge from REF-paper --- .../calibration_experiments/e09_lr_delay.Rmd | 36 +------------------ 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd index 6af3899b..09f98977 100644 --- a/reports/writeups/calibration_experiments/e09_lr_delay.Rmd +++ b/reports/writeups/calibration_experiments/e09_lr_delay.Rmd @@ -65,7 +65,6 @@ week. **Reference configs** (marked on the plots): - **REF-paper**: the paper's tracker on counts, adaptive learning rate. -- **rate, adaptive**: REF-paper on the rate scale (floor 0.001 per 100k). - **REF-op**: prod's config: REF-paper on the sqrt scale, plus a 10% per-round leak, a slow offset and a warm start from the burn-in season. @@ -123,14 +122,12 @@ sweep <- purrr::pmap(sweep_grid, function(learn, settle_days, lr) { do.call(cached_run, c(list(key, "paper", settle_days = settle_days, lr = lr, learn = learn), rate)) }) -# Reference points and the bridge from REF-paper, per (learn, settle_days). +# Reference points per (learn, settle_days). ref_grid <- tidyr::expand_grid(learn = c("exact", "vintage"), settle_days = SETTLE) refs <- purrr::pmap(ref_grid, function(learn, settle_days) { s <- sprintf("%s_s%d", learn, settle_days) list( `REF-paper (count, adaptive)` = cached_run(paste0("paper_", s), "paper", settle_days = settle_days, learn = learn), - `rate, adaptive` = do.call(cached_run, c(list(paste0("rate_adaptive_", s), "paper", - settle_days = settle_days, learn = learn, lr_args = list(mult = 0.03, floor = 1e-3)), rate)), `REF-op` = cached_run(paste0("op_", s), "op", settle_days = settle_days, learn = learn) ) }) @@ -283,34 +280,3 @@ mv_plot_cfgs <- c("REF-op, re-run +1 wk", "0.032, re-run +1 wk", "knee 0.056, re ```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} ``` - -# Bridge from REF-paper - -One change at a time at one week of extra delay, re-run design: first the -rate scale with the adaptive learning rate (its minimum step drops from 0.1 -admissions to 0.001 per 100k, since that minimum is in the working units), -then the constant learning rate with the lowest pooled WIS at each horizon. -REF-op is for orientation only; it differs in several settings. - -```{r bridge} -best_const <- sweep_m %>% - filter(location == "pooled", learn == "exact", settle_days == 14L) %>% - group_by(horizon) %>% - slice_max(wis_pct, n = 1, with_ties = FALSE) %>% - ungroup() %>% - mutate(ref = "rate, constant lr (highest WIS per horizon)") -bridge <- bind_rows( - ref_m %>% filter(learn == "exact", settle_days == 14L), - best_const -) %>% - mutate(ref = factor(ref, levels = c("REF-paper (count, adaptive)", "rate, adaptive", "rate, constant lr (highest WIS per horizon)", "REF-op"))) -base_row <- bridge %>% filter(ref == "REF-paper (count, adaptive)") %>% mutate(ref = "base", wis_pct = 0, bias_cal = bias_base) -tbl <- bind_rows(base_row, bridge) %>% - transmute(config = ref, horizon, cell = sprintf("%+.1f%% / %.3f", wis_pct, bias_cal)) %>% - tidyr::pivot_wider(names_from = horizon, values_from = cell) -knitr::kable(tbl, caption = "WIS reduction % (positive = calibrated WIS lower than base) / L1 coverage bias (lower is better), pooled over states") -knitr::kable( - best_const %>% select(horizon, lr) %>% mutate(lr = signif(lr, 2)) %>% tidyr::pivot_wider(names_from = horizon, values_from = lr), - caption = "Constant learning rate with the lowest pooled WIS (rate units per step)" -) -``` From 7c02a3e06c9db18ac4f4a9321d8f2ca03b3789c8 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:45:06 -0700 Subject: [PATCH 091/107] refactor(calibration): fold E14 into E16, which now covers the rate scale season by season --- notes/calibration-ledger.md | 4 +- .../e14_lr_by_season.Rmd | 157 ------------------ .../e16_sqrt_constant_lr.Rmd | 112 +++++++------ .../e17_late_decay.Rmd | 4 +- 4 files changed, 69 insertions(+), 208 deletions(-) delete mode 100644 reports/writeups/calibration_experiments/e14_lr_by_season.Rmd diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 93077e5f..71a17382 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -152,7 +152,7 @@ tracker structure, late season, learning truth). Each comparison notebook leads with the month view (`_month_view.Rmd`): WIS, coverage bias and median error by month of reference date, then by-horizon tables of WIS reduction % and season-pooled / month-avg coverage -bias. All scores are states only. Sweep notebooks (E01, E09, E14, E16) lead +bias. All scores are states only. Sweep notebooks (E01, E09, E16) lead with their coverage-vs-WIS curves and follow with a month view of a few rates around the elbow. Per-state fan panels and the gallery of best and worst forecasts are only in `prod_views.Rmd`, for the prod configs. Conclusions @@ -177,7 +177,7 @@ gating), exact learning truth, spoiled submissions excluded. | E11 | `e11_constant_lr` | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | | E12 | none (`calibration_ws_replay.R`) | replay, cold, flu + covid | first look at the clean replay | flu: REF-op cold leads constant 0.1 by 0–2.5. Covid: REF-op cold h0 +1.3 states, losses at h1–h3 in 2025-26 | **all locations** (US is 47% of WIS); covid h0 gains are mostly US | | E13 | none (`calibration_ws_replay.R burn_in`) | replay, flu + covid | 2023-24 HHS burn-in | flu warm +1 to +2.6 in 2024-25, ≤ 0 in 2025-26. Covid: warm worse at h0–h1, better at h2–h3 (+1.3, +4.4); constant 0.1 warm −8 to −27 | HHS used as-is (3–9% below NHSN); coverage is season-pooled only | -| E14 | `e14_lr_by_season` | replay, cold | constant rate grid by season | every rate 0.0032–0.056 is ≥ 0 in both seasons and within 1.2 of each other; 0.018 vs 0.032 ≤ 0.3. Above 0.1 loses | month-avg: no rate ≤ 0.032 beats base beyond 0.01 | +| E14 | folded into `e16_sqrt_constant_lr` | replay, cold | constant rate grid by season | every rate 0.0032–0.056 is ≥ 0 in both seasons and within 1.2 of each other; 0.018 vs 0.032 ≤ 0.3. Above 0.1 loses | month-avg: no rate ≤ 0.032 beats base beyond 0.01 | | E15 | `e15_wis_sources` | replay, cold | where the WIS gain comes from | offsets a few % of interval width; REF-op gain from the lower half and the median shift | width-only variant's coverage not measured; split is not additive | | E16 | `e16_sqrt_constant_lr` | replay, cold | constant rate on sqrt scale | sqrt 0.018 about +1 over rate constants at h0; at h1–h3 within 0.2 pooled, up to +0.7 in 2025-26. vs REF-op cold: behind in 2024-25 h0–h1, ahead elsewhere; month-avg coverage tie | 0.018 picked in-sample | | E17 | `e17_late_decay` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | diff --git a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd b/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd deleted file mode 100644 index ac2432b2..00000000 --- a/reports/writeups/calibration_experiments/e14_lr_by_season.Rmd +++ /dev/null @@ -1,157 +0,0 @@ ---- -title: "E14: constant learning rate, season by season" -date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" -output: - html_document: - code_folding: hide - toc: true - toc_float: true - toc_depth: 2 -params: - workers: 14 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) -``` - -Question: how do WIS and coverage change with a constant learning rate, -season by season? - -Setup: - -- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (E12: - substitutions off, all 53 locations, the 55 hub rounds; 27 in 2024-25 and - 28 in 2025-26). Horizons h0–h3. -- Learning truth: exact (re-run at every round on the data published by then; - E00), one week of extra delay. -- Cold start: no burn-in season, offsets start at zero in November 2024. The - tracker runs over both seasons, so 2025-26 starts from the offsets 2024-25 - left behind; each season is scored on its own rounds. -- Swept: the constant learning rate of the plain tracker on the rate scale - (admissions per 100k per step), 12 values from 0.0032 to 1.8, a quarter - decade apart. 0.1 is E11's config, though E11 used the submitted ensemble. -- References on the same data: `REF-op cold` (prod's tracker without its warm - start) and `sqrt, adaptive` (E13's "E05 single cold"). -- Scores: states only (US excluded; it is about 45% of the all-locations - WIS). - -WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated -WIS lower than base. L1 coverage bias: lower is better. - -```{r load} -source(here::here("scripts/calibration/calibration_ws_replay.R")) -wi <- ws_inputs("flu") -LRS <- signif(10^seq(-2.5, 0.25, by = 0.25), 2) -all_cfgs <- ws_configs(wi$rate_scales, burn_in = TRUE) -sweep_cfgs <- purrr::map(LRS, function(lr) utils::modifyList(all_cfgs[["E11 constant 0.1"]], list(lr = lr))) -ref_cfgs <- list(`REF-op cold` = all_cfgs[["REF-op cold"]], `sqrt, adaptive` = all_cfgs[["E05 single cold"]]) -# Cached under cache/calibration/experiments/runs; lr 0.1 and the references are E12/E13's runs. -run <- function(cfg) { - do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) -} -sweep <- purrr::map(sweep_cfgs, run) -refs <- purrr::map(ref_cfgs, run) -``` - -```{r metrics} -GEOS <- c(states = "states only (US excluded)") -SEASONS <- c(LIVE, "both") -# ws_scores() per location subset; wis_pct is the WIS reduction % vs base (positive = calibrated WIS lower). -score <- function(cal) { - fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) - purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% - bind_rows() %>% - select(geos, season, horizon, which, wis, cal_err) %>% - tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% - mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) -} -label <- function(df) { - df %>% mutate( - geos = factor(geos, levels = names(GEOS), labels = GEOS), - season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), - h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) - ) -} -sweep_m <- purrr::map2(sweep, LRS, function(cal, lr) score(cal) %>% mutate(lr = lr)) %>% bind_rows() %>% label() -ref_m <- purrr::imap(refs, function(cal, nm) score(cal) %>% mutate(ref = nm)) %>% bind_rows() %>% label() -SEASON_COLS <- c(`2024-2025` = "#0072B2", `2025-2026` = "#D55E00", `both seasons` = "grey45") -``` - -# Coverage bias vs WIS, by season - -One line per season, tracing the learning rate from 0.0032 (near the -base, the open circle) to 1.8. Labels mark every half decade. The grey -dashed line pools both seasons, as E09 did. Triangles are REF-op cold, -squares the adaptive sqrt tracker, in the season's colour. The y axis is -clipped at −10%. - -```{r curve-fn} -LABELLED <- c(0.01, 0.032, 0.1, 0.32, 1) -curve_plot <- function(g) { - d <- sweep_m %>% filter(geos == g) %>% arrange(lr) - r <- ref_m %>% filter(geos == g) - base <- d %>% distinct(h, season, cal_err_base) - ggplot(d, aes(cal_err_cal, wis_pct, colour = season)) + - geom_hline(yintercept = 0, colour = "grey60", linewidth = 0.3) + - geom_path(aes(linetype = season == "both seasons"), linewidth = 0.8) + - geom_point(size = 1) + - geom_text(data = d %>% filter(lr %in% LABELLED, season != "both seasons"), - aes(label = format(lr, drop0trailing = TRUE)), size = 2.6, vjust = -0.8, show.legend = FALSE) + - geom_point(data = base, aes(cal_err_base, 0), shape = 21, fill = "white", size = 2.2) + - geom_point(data = r %>% filter(season != "both seasons"), aes(shape = ref), size = 2.4) + - scale_colour_manual(values = SEASON_COLS, name = NULL) + - scale_linetype_manual(values = c(`FALSE` = "solid", `TRUE` = "22"), guide = "none") + - scale_shape_manual(values = c(`REF-op cold` = 17, `sqrt, adaptive` = 15), name = NULL) + - facet_wrap(~h, nrow = 1) + - coord_cartesian(ylim = c(max(-10, min(d$wis_pct)), max(c(d$wis_pct, r$wis_pct)) + 0.5)) + - labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)", title = g) + - theme(legend.position = "bottom") -} -``` - -```{r curve-states} -curve_plot(GEOS[["states"]]) -``` - -# Each metric against the learning rate - -The same runs with the learning rate on the x axis, so the seasons can be -compared at the same rate. Top: WIS reduction % (positive = calibrated WIS lower than -base). Bottom: L1 coverage bias (lower is better), with each season's base as a dotted line. States only. - -```{r vs-lr, fig.height=6.5} -long <- sweep_m %>% - filter(geos == GEOS[["states"]], season != "both seasons") %>% - transmute(h, season, lr, `WIS reduction % (positive = calibrated WIS lower than base)` = wis_pct, `L1 coverage bias (lower is better)` = cal_err_cal) %>% - tidyr::pivot_longer(c(`WIS reduction % (positive = calibrated WIS lower than base)`, `L1 coverage bias (lower is better)`), names_to = "metric") %>% - mutate(metric = factor(metric, levels = c("WIS reduction % (positive = calibrated WIS lower than base)", "L1 coverage bias (lower is better)"))) -base_lines <- sweep_m %>% - filter(geos == GEOS[["states"]], season != "both seasons") %>% - distinct(h, season, value = cal_err_base) %>% - mutate(metric = factor("L1 coverage bias (lower is better)", levels = levels(long$metric))) -zero <- tibble(metric = factor("WIS reduction % (positive = calibrated WIS lower than base)", levels = levels(long$metric)), value = 0) -ggplot(long, aes(lr, value, colour = season)) + - geom_hline(data = zero, aes(yintercept = value), colour = "grey60", linewidth = 0.3) + - geom_hline(data = base_lines, aes(yintercept = value, colour = season), linetype = "dotted") + - geom_line() + - geom_point(size = 1) + - scale_x_log10(breaks = c(0.003, 0.01, 0.03, 0.1, 0.3, 1), labels = c("0.003", "0.01", "0.03", "0.1", "0.3", "1")) + - scale_colour_manual(values = SEASON_COLS, name = NULL) + - facet_grid(metric ~ h, scales = "free_y", labeller = labeller(metric = label_wrap_gen(28))) + - labs(x = "constant learning rate (per 100k per step)", y = NULL) + - theme(legend.position = "bottom") -``` - -```{r month-view-cfgs} -mv <- cal_month_view(c(setNames(sweep, paste("rate", format(LRS, drop0trailing = TRUE))), refs), seasons = LIVE) -# The plots show the rates around the elbow and REF-op cold; the tables cover every rate. -mv_plot_cfgs <- c("rate 0.018", "rate 0.032", "rate 0.056", "rate 0.1", "REF-op cold") -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` diff --git a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd index 3d3d1852..04a84bc7 100644 --- a/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd +++ b/reports/writeups/calibration_experiments/e16_sqrt_constant_lr.Rmd @@ -1,5 +1,5 @@ --- -title: "E16: constant learning rate on the square-root scale, and month by month" +title: "E16: constant learning rate by season, sqrt and rate scales" date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" output: html_document: @@ -19,9 +19,9 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` -Question: how does a constant-rate tracker on the square-root scale compare -with the same tracker on the rate scale and with REF-op cold, across learning -rates and by month? +Question: how do WIS and coverage change with a constant learning rate, +season by season, on the square-root scale and on the rate scale, and how do +they compare with REF-op cold? Setup: @@ -38,11 +38,14 @@ Setup: quarter decade apart, in sqrt(per-100k) units (not comparable number for number with the rate scale). The grid was set after a first look at results. - - `rate `: E14's rate-scale runs; 0.018 and 0.032 are shown as - references. + - `rate `: the same tracker on admissions per 100k; 12 rates from + 0.0032 to 1.8, a quarter decade apart. 0.1 is E11's config, though E11 + used the submitted ensemble. - `REF-op cold`: prod's tracker without its warm start. -- A sqrt rate is selected below by E14's rule (highest mean gain with no loss - in any season × horizon). The selection is in-sample. + - `sqrt, adaptive`: REF-paper on the sqrt scale with its adaptive learning + rate (E05's `single`, cold). +- A sqrt rate is selected below as the highest mean gain with no loss in any + season × horizon. The selection is in-sample. WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated WIS lower than base. L1 coverage bias: lower is better. @@ -55,13 +58,14 @@ SQRT_LRS <- signif(10^seq(-3, -0.5, by = 0.25), 2) all_cfgs <- ws_configs(wi$rate_scales, burn_in = FALSE) rate_cfg <- function(lr) utils::modifyList(all_cfgs[["E11 constant 0.1"]], list(lr = lr)) sqrt_cfg <- function(lr) utils::modifyList(all_cfgs[["E11 constant 0.1"]], list(lr = lr, transform = "sqrt")) -# Cached under cache/calibration/experiments/runs; the rate sweep and REF-op cold are E14's runs. +# Cached under cache/calibration/experiments/runs; E17, E18 and E07 reuse some of these runs. run <- function(cfg) { do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) } rate_runs <- purrr::map(RATE_LRS, \(lr) run(rate_cfg(lr))) sqrt_runs <- purrr::map(SQRT_LRS, \(lr) run(sqrt_cfg(lr))) refop <- run(all_cfgs[["REF-op cold"]]) +sqrt_adaptive <- run(ws_configs(wi$rate_scales, burn_in = TRUE)[["E05 single cold"]]) ``` ```{r metrics} @@ -87,7 +91,10 @@ sweep_m <- bind_rows( purrr::map2(rate_runs, RATE_LRS, \(cal, lr) score(cal) %>% mutate(scale = "rate", lr = lr)), purrr::map2(sqrt_runs, SQRT_LRS, \(cal, lr) score(cal) %>% mutate(scale = "sqrt", lr = lr)) ) %>% label() -ref_m <- score(refop) %>% mutate(ref = "REF-op cold") %>% label() +ref_m <- bind_rows( + score(refop) %>% mutate(ref = "REF-op cold"), + score(sqrt_adaptive) %>% mutate(ref = "sqrt, adaptive") +) %>% label() # Best sqrt rate: the highest mean WIS reduction among rates that improve WIS in both seasons at h0–h3 (states only). best_tbl <- sweep_m %>% filter(scale == "sqrt", geos == GEOS[["states"]], season %in% LIVE) %>% @@ -99,9 +106,10 @@ BEST_SQRT <- best_tbl %>% filter(worst > 0) %>% slice_max(mean, n = 1) %>% pull( # 1. Coverage bias vs WIS, by season and scale One curve per season and scale, tracing the learning rate from smallest -(near the base, the open circle) to largest. Solid lines are the rate scale -(E14), dashed lines the sqrt scale. Labels mark a few rates on each curve. -Triangles are REF-op cold. The y axis is clipped at −10%. The sqrt rate +(near the base, the open circle) to largest. Solid lines are the rate scale, +dashed lines the sqrt scale. Labels mark a few rates on each curve. +Triangles are REF-op cold, squares the adaptive sqrt tracker. The y axis is +clipped at −10%. The sqrt rate selected by the rule below is **`r BEST_SQRT`**. ```{r curve-fn} @@ -118,14 +126,14 @@ curve_plot <- function(g) { geom_text(data = lab, aes(label = paste0(ifelse(scale == "sqrt", "s", "r"), format(lr, drop0trailing = TRUE))), size = 2.3, vjust = -0.8, show.legend = FALSE) + geom_point(data = base, aes(cal_err_base, 0), shape = 21, fill = "white", size = 2.2) + - geom_point(data = r, shape = 17, size = 2.6) + + geom_point(data = r, aes(shape = ref), size = 2.6) + scale_colour_manual(values = SEASON_COLS, name = NULL) + scale_linetype_manual(values = c(rate = "solid", sqrt = "22"), name = "scale") + - scale_shape_manual(values = c(rate = 16, sqrt = 1), name = "scale") + + scale_shape_manual(values = c(rate = 16, sqrt = 1, `REF-op cold` = 17, `sqrt, adaptive` = 15), name = NULL) + facet_wrap(~h, nrow = 1) + coord_cartesian(ylim = c(max(-10, min(d$wis_pct)), max(c(d$wis_pct, r$wis_pct)) + 0.5)) + labs(x = "L1 coverage bias (lower is better)", y = "WIS reduction % (positive = calibrated WIS lower than base)", title = g, - caption = "labels: r = rate-scale learning rate (per 100k), s = sqrt-scale learning rate; triangles: REF-op cold") + + caption = "labels: r = rate-scale learning rate (per 100k), s = sqrt-scale learning rate") + theme(legend.position = "bottom") } ``` @@ -134,39 +142,50 @@ curve_plot <- function(g) { curve_plot(GEOS[["states"]]) ``` -## Each metric against the learning rate, sqrt scale - -Top: WIS reduction % (positive = calibrated WIS lower than base). Bottom: L1 coverage bias (lower is better), each season's base dotted. -States only. - -```{r vs-lr, fig.height=6.5} -long <- sweep_m %>% - filter(scale == "sqrt", geos == GEOS[["states"]], season != "both seasons") %>% - transmute(h, season, lr, `WIS reduction % (positive = calibrated WIS lower than base)` = wis_pct, `L1 coverage bias (lower is better)` = cal_err_cal) %>% - tidyr::pivot_longer(c(`WIS reduction % (positive = calibrated WIS lower than base)`, `L1 coverage bias (lower is better)`), names_to = "metric") %>% - mutate(metric = factor(metric, levels = c("WIS reduction % (positive = calibrated WIS lower than base)", "L1 coverage bias (lower is better)"))) -base_lines <- sweep_m %>% - filter(scale == "sqrt", geos == GEOS[["states"]], season != "both seasons") %>% - distinct(h, season, value = cal_err_base) %>% - mutate(metric = factor("L1 coverage bias (lower is better)", levels = levels(long$metric))) -zero <- tibble(metric = factor("WIS reduction % (positive = calibrated WIS lower than base)", levels = levels(long$metric)), value = 0) -ggplot(long, aes(lr, value, colour = season)) + - geom_hline(data = zero, aes(yintercept = value), colour = "grey60", linewidth = 0.3) + - geom_hline(data = base_lines, aes(yintercept = value, colour = season), linetype = "dotted") + - geom_line() + - geom_point(size = 1) + - scale_x_log10(breaks = c(0.001, 0.003, 0.01, 0.03, 0.1, 0.3), labels = c("0.001", "0.003", "0.01", "0.03", "0.1", "0.3")) + - scale_colour_manual(values = SEASON_COLS, name = NULL) + - facet_grid(metric ~ h, scales = "free_y", labeller = labeller(metric = label_wrap_gen(28))) + - labs(x = "constant learning rate (sqrt of per-100k units per step)", y = NULL) + - theme(legend.position = "bottom") +## Each metric against the learning rate + +One panel set per scale; the two scales' rates are in different units. +Top: WIS reduction % (positive = calibrated WIS lower than base). Bottom: L1 +coverage bias (lower is better), each season's base dotted. States only. + +```{r vs-lr-fn} +vs_lr_plot <- function(sc, breaks, xlab) { + long <- sweep_m %>% + filter(scale == sc, geos == GEOS[["states"]], season != "both seasons") %>% + transmute(h, season, lr, `WIS reduction % (positive = calibrated WIS lower than base)` = wis_pct, `L1 coverage bias (lower is better)` = cal_err_cal) %>% + tidyr::pivot_longer(c(`WIS reduction % (positive = calibrated WIS lower than base)`, `L1 coverage bias (lower is better)`), names_to = "metric") %>% + mutate(metric = factor(metric, levels = c("WIS reduction % (positive = calibrated WIS lower than base)", "L1 coverage bias (lower is better)"))) + base_lines <- sweep_m %>% + filter(scale == sc, geos == GEOS[["states"]], season != "both seasons") %>% + distinct(h, season, value = cal_err_base) %>% + mutate(metric = factor("L1 coverage bias (lower is better)", levels = levels(long$metric))) + zero <- tibble(metric = factor("WIS reduction % (positive = calibrated WIS lower than base)", levels = levels(long$metric)), value = 0) + ggplot(long, aes(lr, value, colour = season)) + + geom_hline(data = zero, aes(yintercept = value), colour = "grey60", linewidth = 0.3) + + geom_hline(data = base_lines, aes(yintercept = value, colour = season), linetype = "dotted") + + geom_line() + + geom_point(size = 1) + + scale_x_log10(breaks = breaks, labels = format(breaks, drop0trailing = TRUE)) + + scale_colour_manual(values = SEASON_COLS, name = NULL) + + facet_grid(metric ~ h, scales = "free_y", labeller = labeller(metric = label_wrap_gen(28))) + + labs(x = xlab, y = NULL, title = paste(sc, "scale")) + + theme(legend.position = "bottom") +} +``` + +```{r vs-lr-sqrt, fig.height=6.5} +vs_lr_plot("sqrt", c(0.001, 0.003, 0.01, 0.03, 0.1, 0.3), "constant learning rate (sqrt of per-100k units per step)") +``` + +```{r vs-lr-rate, fig.height=6.5} +vs_lr_plot("rate", c(0.003, 0.01, 0.03, 0.1, 0.3, 1), "constant learning rate (per 100k per step)") ``` ## Selecting a sqrt rate For each sqrt rate, the worst and the mean WIS reduction % (positive = calibrated WIS lower than base) over the two seasons and h0–h3 (states only). The selected rate has the highest mean among those -whose worst case is still a gain (E14's rule). +whose worst case is still a gain. ```{r best} knitr::kable(best_tbl %>% mutate(across(c(worst, mean), \(x) round(x, 2)))) @@ -174,13 +193,12 @@ knitr::kable(best_tbl %>% mutate(across(c(worst, mean), \(x) round(x, 2)))) ```{r month-view-cfgs} lr_lab <- function(lr) vapply(lr, \(x) format(x, drop0trailing = TRUE), character(1)) -rate_keep <- RATE_LRS %in% c(0.018, 0.032, 0.056, 0.1) mv <- cal_month_view(c( setNames(sqrt_runs, paste("sqrt", lr_lab(SQRT_LRS))), - setNames(rate_runs[rate_keep], paste("rate", lr_lab(RATE_LRS[rate_keep]))), - list(`REF-op cold` = refop) + setNames(rate_runs, paste("rate", lr_lab(RATE_LRS))), + list(`REF-op cold` = refop, `sqrt, adaptive` = sqrt_adaptive) ), seasons = LIVE) -# The plots show the sqrt rates around the elbow, rate 0.032 (E14) and REF-op cold; the tables cover every sqrt rate. +# The plots show the sqrt rates around the elbow, rate 0.032 and REF-op cold; the tables cover every rate on both scales. MONTH_COLS <- c(`sqrt 0.01` = "#56B4E9", `sqrt 0.018` = "#CC79A7", `sqrt 0.032` = "#0072B2", `rate 0.032` = "#009E73", `REF-op cold` = "#D55E00") mv_plot_cfgs <- names(MONTH_COLS) mv_cols <- MONTH_COLS diff --git a/reports/writeups/calibration_experiments/e17_late_decay.Rmd b/reports/writeups/calibration_experiments/e17_late_decay.Rmd index 6cb84209..f3c506ce 100644 --- a/reports/writeups/calibration_experiments/e17_late_decay.Rmd +++ b/reports/writeups/calibration_experiments/e17_late_decay.Rmd @@ -29,7 +29,7 @@ Setup: locations, the 55 hub rounds). Horizons h0–h3. - Learning truth: exact (re-run at every round on the data published by then; E00), one week of extra delay. Cold start, no burn-in season. States only. -- Three constant-rate trackers: rate 0.018 and rate 0.032 per 100k (E14), +- Three constant-rate trackers: rate 0.018 and rate 0.032 per 100k (E16), and sqrt 0.018 (E16). Each crossed with five settings: - **no decay**. - **`late_decay`** with factor 0.85, 0.7 or 0.5: from 1 March to 30 June @@ -65,7 +65,7 @@ DECAYS <- list( `off after 1 Mar` = list(off_after = "03-01") ) grid <- tidyr::expand_grid(rate = names(RATES), decay = names(DECAYS)) -# Cached under cache/calibration/experiments/runs; the no-decay runs are E14/E16's. +# Cached under cache/calibration/experiments/runs; the no-decay runs are E16's. cals <- purrr::pmap(grid, function(rate, decay) { cfg <- c(utils::modifyList(base_cfg, RATES[[rate]]), DECAYS[[decay]]) do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) From 0fb15edff9702ac4126edf70df4b12e3529fbfd1 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:46:15 -0700 Subject: [PATCH 092/107] refactor(calibration): move the E07b ranking into E10 for both diseases and drop REF-paper from E10 part 2; finalist runs are a shared helper --- notes/calibration-ledger.md | 2 +- .../e07_across_eras.Rmd | 3 +- .../e07b_candidates.Rmd | 150 ------------------ .../calibration_experiments/e10_covid.Rmd | 102 ++++++------ scripts/calibration/calibration_ws_replay.R | 49 ++++++ 5 files changed, 108 insertions(+), 198 deletions(-) delete mode 100644 reports/writeups/calibration_experiments/e07b_candidates.Rmd diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 71a17382..6bd4ee41 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -171,7 +171,7 @@ gating), exact learning truth, spoiled submissions excluded. | E03 | `e03_scale` | ensemble | count vs sqrt vs rate | sqrt +1.4 to +3.9 over count pooled, mostly 2025-26. Coverage: neither better (sqrt worse per season pooled, better month-avg only in 2024-25) | rate identical to count in every cell (adaptive rate is scale invariant) | | E04 | `e04_offset_structure` | ensemble | leak, slow term | leak +2.5 to +5.6 at h1–h3 in 2025-26, −0.4 to −1.3 in 2024-25; month-avg coverage flips the same way (±0.015). Slow term without warm start: ≤ 0.5 | | | E05 | `e05_warm_start` | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | -| E07 | `e07_across_eras`, `e07b_candidates` (the candidates) | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | | +| E07 | `e07_across_eras`; the candidate ranking (formerly E07b) is in `e10_covid` | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | | | E09 | `e09_lr_delay` | ensemble | constant rate × revision delay | knee 0.056: +3 to +5.5 over REF-paper at h1–h3; REF-op +1.2 to +2.4 over knee. Extra delay costs about 1 h−1 point per week. Above 0.1 loses steeply | month-avg: no constant rate beats base at h1–h3 | | E10 | `e10_covid` | flu + covid, ensemble + replay | the finalists on both diseases and bases (part 1); REF-paper, REF-op and sqrt 0.018 on the covid ensemble at h−1–h3 (part 2) | see "Finalists" above. Part 2: h−1 +10 to +15 (sqrt 0.018 largest). REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 and sqrt 0.018 −7.7 to −9.5 at h1–h3 in 2025-26 | part 1 is h0–h3 on rows both bases have; part 2 tables run to 2026-09-19, including the 2026 summer wave | | E11 | `e11_constant_lr` | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | diff --git a/reports/writeups/calibration_experiments/e07_across_eras.Rmd b/reports/writeups/calibration_experiments/e07_across_eras.Rmd index 6cec0e72..7ea83037 100644 --- a/reports/writeups/calibration_experiments/e07_across_eras.Rmd +++ b/reports/writeups/calibration_experiments/e07_across_eras.Rmd @@ -163,8 +163,7 @@ mb %>% # 2. What calibration does to each WIS reduction % (positive = calibrated WIS lower than base) and L1 coverage -bias (lower is better), states only. The month view of these runs is in -E07b. +bias (lower is better), states only. ```{r table-states} mv_trk <- cal_month_view(setNames(cals, paste(grid$forecaster, grid$tracker, sep = ", ")), seasons = LIVE) diff --git a/reports/writeups/calibration_experiments/e07b_candidates.Rmd b/reports/writeups/calibration_experiments/e07b_candidates.Rmd deleted file mode 100644 index 204873a6..00000000 --- a/reports/writeups/calibration_experiments/e07b_candidates.Rmd +++ /dev/null @@ -1,150 +0,0 @@ ---- -title: "E07b: operating-point candidates on the ensemble and the replay" -date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" -output: - html_document: - code_folding: hide - toc: true - toc_float: true - toc_depth: 2 -params: - hub_dir: "../FluSight-forecast-hub" - workers: 14 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) -``` - -Question: do the operating-point candidates rank the same way on the -submitted ensemble as on one fixed forecaster? - -**Setup.** As in E07: A is the submitted flu ensemble, -B the clean `windowed_seasonal` replay with the 2023-24 HHS backfill as its -burn-in season, on identical (round, location, horizon, level) rows, h0–h3, -exact learning truth with one week of extra delay. - -Candidates: REF-op warm and cold; constant rate 0.018 on the sqrt of the -per-100k rate (E16); constant 0.032 per 100k on the rate scale (E14); sqrt -0.018 switched off from 1 March (`off_after = "03-01"`, E17). The cold -candidates see the live seasons only and start from zero offsets. - -"WIS reduction %" is `100 × (WIS_base − WIS_cal) / WIS_base`, so -**positive = calibrated WIS lower than base**. "L1 coverage bias" (lower is -better) is the mean over the 23 levels of |empirical coverage − level|. - -```{r load} -source(here::here("scripts/calibration/calibration_ws_replay.R")) -wi <- ws_inputs("flu") -hub_fc <- suppressMessages(hub_read_forecasts(hub_dir = here::here(params$hub_dir), target = HUB_FLU_TARGET)) %>% - filter(horizon %in% 0:3) -ws_fc <- bind_rows(ws_burn_in_forecasts("flu"), wi$fc) -# Keep only (round, location, horizon, level) rows both forecasters have. -key <- c("reference_date", "location", "horizon", "level") -common <- inner_join(distinct(hub_fc, across(all_of(key))), distinct(ws_fc, across(all_of(key))), by = key) -FCS <- list( - `A ensemble` = semi_join(hub_fc, common, by = key), - `B windowed_seasonal` = semi_join(ws_fc, common, by = key) -) -# Cold candidates see only the live seasons, so they start from zero offsets. -const_cfg <- ws_configs(wi$rate_scales, burn_in = FALSE)[["E11 constant 0.1"]] -sqrt_018 <- utils::modifyList(const_cfg, list(lr = 0.018, transform = "sqrt")) -CANDS <- list( - `REF-op warm` = list(cfg = list(ref = "op"), warm = TRUE), - `REF-op cold` = list(cfg = list(ref = "op", burn_in_seasons = character(0), slow_init = NULL), warm = FALSE), - `sqrt 0.018` = list(cfg = sqrt_018, warm = FALSE), - `rate 0.032` = list(cfg = utils::modifyList(const_cfg, list(lr = 0.032)), warm = FALSE), - `sqrt 0.018, off 1 Mar` = list(cfg = c(sqrt_018, list(off_after = "03-01")), warm = FALSE) -) -FCS_LIVE <- purrr::map(FCS, \(fc) filter(fc, season_of(reference_date) %in% LIVE)) -cgrid <- tidyr::expand_grid(forecaster = names(FCS), cand = names(CANDS)) -ccals <- purrr::pmap(cgrid, function(forecaster, cand) { - fc <- if (CANDS[[cand]]$warm) FCS[[forecaster]] else FCS_LIVE[[forecaster]] - do.call(cal_run_cached, c(list(fc, wi$truth), CANDS[[cand]]$cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) -}) -``` - -```{r metrics} -GEOS <- c(states = "states only (US excluded)") -SEASONS <- c(LIVE, "both") -score <- function(cal) { - fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) - purrr::imap(WS_GEOS[names(GEOS)], function(keep, g) ws_scores(fc %>% filter(keep(location))) %>% mutate(geos = g)) %>% - bind_rows() %>% - select(geos, season, horizon, which, wis, cal_err) %>% - tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% - mutate(wis_pct = 100 * (wis_base - wis_cal) / wis_base) -} -``` - -# Candidates on both - -The candidates from E14, E16 and E17, each run on both forecasters on the -same rows: REF-op warm (as in E07) and cold, sqrt-scale constant -0.018 (E16), rate-scale constant 0.032 per 100k (E14), and sqrt 0.018 -switched off from 1 March (`off_after = "03-01"`, E17). The cold ones learn -from the live seasons only. - -```{r cand-metrics} -cm <- purrr::map2(ccals, seq_len(nrow(cgrid)), \(cal, i) score(cal) %>% mutate(cgrid[i, ])) %>% - bind_rows() %>% - mutate( - forecaster = factor(forecaster, levels = names(FCS)), cand = factor(cand, levels = names(CANDS)), - geos = factor(geos, levels = names(GEOS), labels = GEOS), - season = factor(season, levels = SEASONS, labels = c(LIVE, "both seasons")), - h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) - ) -``` - -```{r month-view-cfgs} -mv <- cal_month_view(setNames(ccals, paste(cgrid$forecaster, cgrid$cand, sep = ", ")), seasons = LIVE) -# Three candidates on both forecasters: colour is the candidate, linetype the forecaster (also for the grey base lines). -MV_CANDS <- c(`REF-op warm` = "black", `sqrt 0.018` = "#0072B2", `sqrt 0.018, off 1 Mar` = "#D55E00") -mv_plot_cfgs <- paste(rep(names(FCS), each = length(MV_CANDS)), names(MV_CANDS), sep = ", ") -mv_cols <- MV_CANDS -mv$month <- mv$month %>% mutate(forecaster = sub(",.*", "", config), cand = factor(sub("^[^,]*, ", "", config), levels = names(CANDS))) -mv_colour_by <- "cand" -mv_linetype <- "forecaster" -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` - -# Rankings on A and B - -Candidates ordered by mean WIS reduction % (positive = calibrated WIS lower than base) over h0–h3, separately on each -forecaster, states only. `rank A` / `rank B` are the positions (1 is the -largest WIS reduction; `cov rank` 1 is the lowest L1 coverage bias); Spearman ρ compares the two orders per season, over all -(candidate, horizon) cells. - -```{r cand-rank} -rk <- cm %>% - filter(geos == GEOS[["states"]]) %>% - group_by(forecaster, season, cand) %>% - summarize(wis = mean(wis_pct), cov = mean(cal_err_cal), .groups = "drop") %>% - group_by(forecaster, season) %>% - mutate(rank = rank(-wis), cov_rank = rank(cov)) %>% - ungroup() %>% - mutate(fc = ifelse(forecaster == names(FCS)[1], "A", "B")) %>% - select(season, cand, fc, wis, rank, cov, cov_rank) %>% - tidyr::pivot_wider(names_from = fc, values_from = c(wis, rank, cov, cov_rank)) %>% - arrange(season, rank_A) -rk %>% - transmute(season, cand, `WIS reduction % A` = wis_A, `rank A` = rank_A, `WIS reduction % B` = wis_B, `rank B` = rank_B, - `cov bias A` = cov_A, `cov rank A` = cov_rank_A, `cov bias B` = cov_B, `cov rank B` = cov_rank_B) %>% - knitr::kable(digits = c(0, 0, 2, 0, 2, 0, 3, 0, 3, 0), caption = "Mean over h0–h3, states only. WIS reduction %: positive = calibrated WIS lower than base. Coverage bias is L1 coverage bias (lower is better)") -cm %>% - filter(geos == GEOS[["states"]]) %>% - select(season, cand, h, forecaster, wis_pct, cal_err_cal) %>% - tidyr::pivot_wider(names_from = forecaster, values_from = c(wis_pct, cal_err_cal)) %>% - group_by(season) %>% - summarize( - `ρ WIS cells` = cor(.data[[paste0("wis_pct_", names(FCS)[1])]], .data[[paste0("wis_pct_", names(FCS)[2])]], method = "spearman"), - `ρ coverage cells` = cor(.data[[paste0("cal_err_cal_", names(FCS)[1])]], .data[[paste0("cal_err_cal_", names(FCS)[2])]], method = "spearman") - ) %>% - knitr::kable(digits = 2, caption = "Spearman correlation of A vs B over (candidate, horizon) cells") -``` diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd index 301417e8..8498f6b1 100644 --- a/reports/writeups/calibration_experiments/e10_covid.Rmd +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -30,7 +30,8 @@ Question: how do the two finalists, REF-op and sqrt constant 0.018, do on flu and covid, on the submitted ensemble and on the clean replay? Part 1 runs them on both diseases and both bases, h0–h3, on identical rows. -Part 2 is the covid ensemble at h−1–h3 with REF-paper for reference. +Part 1 also checks whether the finalists rank the same way on both bases. +Part 2 is the covid ensemble at h−1–h3. # Part 1: both diseases, both bases @@ -51,45 +52,7 @@ Setup: ```{r fin-runs} source(here::here("scripts/calibration/calibration_ws_replay.R")) -key <- c("reference_date", "location", "horizon", "level") -fin_inputs <- function(d) { - wi <- ws_inputs(d) - # Right after ws_inputs(): both read the harness globals that ch_use() sets. - burn <- ws_burn_in_forecasts(d) - hub <- suppressMessages(hub_read_forecasts( - hub_dir = here::here(HUB_DIRS[[d]]), target = c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET)[[d]] - )) %>% filter(horizon %in% 0:3) - ws <- bind_rows(burn, wi$fc) - common <- inner_join(distinct(hub, across(all_of(key))), distinct(ws, across(all_of(key))), by = key) - fcs <- list(ensemble = semi_join(hub, common, by = key), replay = semi_join(ws, common, by = key)) - warm <- fcs - if (!any(season_of(fcs$ensemble$reference_date) == "2023-2024")) { - warm <- list(replay = bind_rows(burn, fcs$replay)) - } - list(wi = wi, cold = purrr::map(fcs, \(fc) filter(fc, season_of(reference_date) %in% LIVE)), warm = warm) -} -fin_cands <- function(rate_scales) { - const <- ws_configs(rate_scales, burn_in = FALSE)[["E11 constant 0.1"]] - sqrt_018 <- utils::modifyList(const, list(lr = 0.018, transform = "sqrt")) - list( - `REF-op warm` = list(cfg = list(ref = "op"), warm = TRUE), - `REF-op cold` = list(cfg = list(ref = "op", burn_in_seasons = character(0), slow_init = NULL), warm = FALSE), - `sqrt 0.018` = list(cfg = sqrt_018, warm = FALSE), - `sqrt 0.018 warm` = list(cfg = utils::modifyList(sqrt_018, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")), warm = TRUE) - ) -} -FIN <- purrr::map(c(flu = "flu", covid = "covid"), function(d) { - inp <- fin_inputs(d) - cands <- fin_cands(inp$wi$rate_scales) - grid <- tidyr::expand_grid(base = c("ensemble", "replay"), cand = names(cands)) %>% - filter(!cands[cand] %>% purrr::map_lgl("warm") | base %in% names(inp$warm)) - cals <- purrr::pmap(grid, function(base, cand) { - cc <- cands[[cand]] - fc <- if (cc$warm) inp$warm[[base]] else inp$cold[[base]] - do.call(cal_run_cached, c(list(fc, inp$wi$truth), cc$cfg, list(learn = "exact", vintages = inp$wi$vintages, workers = params$workers))) - }) - list(cals = setNames(cals, paste(grid$base, grid$cand, sep = ", ")), grid = grid, cands = names(cands)) -}) +FIN <- purrr::map(c(flu = "flu", covid = "covid"), \(d) fin_runs(d, workers = params$workers)) ``` Per disease, a month view for each base. The ensemble and replay rows are @@ -106,6 +69,57 @@ for (d in names(FIN)) { } ``` +## Do the finalists rank the same way on both bases? + +Per disease, the candidates run on both bases (covid has no warm runs on the +ensemble), ordered by mean WIS reduction % (positive = calibrated WIS lower +than base) over h0–h3, separately on each base, states only. `rank` 1 is the +largest WIS reduction; `cov rank` 1 is the lowest month-avg L1 coverage +bias. Spearman ρ compares the two bases over all (candidate, horizon) cells +of a season. + +```{r fin-rank, results="asis"} +for (d in names(FIN)) { + mv_all <- cal_month_view(FIN[[d]]$cals, seasons = LIVE) + wis <- bind_rows(mv_all$month, mv_all$month %>% mutate(season = "both")) %>% + group_by(config, season, horizon) %>% + summarize(wis = 100 * (sum(wis_base) - sum(wis_cal)) / sum(wis_base), .groups = "drop") + cells <- mv_all$cov %>% + transmute(config, season = as.character(season), horizon, cov = mavg_cal) %>% + inner_join(wis, by = c("config", "season", "horizon")) %>% + tidyr::separate(config, c("base", "cand"), sep = ", ", extra = "merge") + shared <- cells %>% distinct(base, cand) %>% count(cand) %>% filter(n == 2) %>% pull(cand) + cells <- cells %>% filter(cand %in% shared) + cat("\n\n", cal_h(3), " ", d, "\n\n", sep = "") + cells %>% + group_by(base, season, cand) %>% + summarize(wis = mean(wis), cov = mean(cov), .groups = "drop") %>% + group_by(base, season) %>% + mutate(rank = rank(-wis), cov_rank = rank(cov)) %>% + ungroup() %>% + tidyr::pivot_wider(names_from = base, values_from = c(wis, rank, cov, cov_rank)) %>% + arrange(season, rank_ensemble) %>% + transmute(season, cand, + `WIS % ensemble` = round(wis_ensemble, 2), `rank ensemble` = rank_ensemble, + `WIS % replay` = round(wis_replay, 2), `rank replay` = rank_replay, + `cov ensemble` = round(cov_ensemble, 3), `cov rank ensemble` = cov_rank_ensemble, + `cov replay` = round(cov_replay, 3), `cov rank replay` = cov_rank_replay + ) %>% + knitr::kable(caption = "Mean over h0–h3, states only. Coverage is month-avg L1 coverage bias (lower is better)") %>% + print() + cells %>% + select(season, cand, horizon, base, wis, cov) %>% + tidyr::pivot_wider(names_from = base, values_from = c(wis, cov)) %>% + group_by(season) %>% + summarize( + `ρ WIS cells` = cor(wis_ensemble, wis_replay, method = "spearman"), + `ρ coverage cells` = cor(cov_ensemble, cov_replay, method = "spearman") + ) %>% + knitr::kable(digits = 2, caption = "Spearman correlation of ensemble vs replay over (candidate, horizon) cells") %>% + print() +} +``` + # Part 2: covid ensemble, h−1–h3 - Disease: covid. Base forecaster: the submitted `CMU-TimeSeries` ensemble. @@ -119,10 +133,9 @@ for (d in names(FIN)) { - Horizons h−1 to h3. Scores: states only (US excluded), the two live seasons. -| config | change from REF-paper | +| config | tracker | |---|---| -| `REF-paper` | none (counts, adaptive learning rate) | -| `REF-op (no warm start)` | sqrt scale, learning-rate floor, slow term 0.003, leak 0.1; no warm start | +| `REF-op cold` | covid prod: sqrt scale, adaptive learning rate with floor, slow term 0.003, leak 0.1; no warm start | | `sqrt 0.018` | sqrt of the per-100k rate, constant learning rate 0.018 (E16), single term | WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated @@ -131,8 +144,7 @@ WIS lower than base. L1 coverage bias: lower is better. ```{r runs} no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) cals <- list( - `REF-paper` = do.call(run, c(list("paper"), no_burn_in)), - `REF-op (no warm start)` = do.call(run, c(list("op"), no_burn_in)), + `REF-op cold` = do.call(run, c(list("op"), no_burn_in)), `sqrt 0.018` = do.call(run, c(list("paper", transform = "sqrt", lr = 0.018, scales = rate_scales), no_burn_in)) ) season_of <- function(d) { diff --git a/scripts/calibration/calibration_ws_replay.R b/scripts/calibration/calibration_ws_replay.R index a719c450..3965d836 100644 --- a/scripts/calibration/calibration_ws_replay.R +++ b/scripts/calibration/calibration_ws_replay.R @@ -163,6 +163,55 @@ ws_month_scores <- function(fc) { ungroup() } +# The finalists' inputs for one disease: the submitted ensemble and the replay +# on the (round, location, horizon, level) rows both have, h0–h3. Cold runs see +# the live seasons only; warm runs also see the 2023-24 burn-in season (covid +# submissions start 2024-11-23, so covid has warm runs on the replay only). +fin_inputs <- function(disease) { + key <- c("reference_date", "location", "horizon", "level") + wi <- ws_inputs(disease) + # Right after ws_inputs(): both read the harness globals that ch_use() sets. + burn <- ws_burn_in_forecasts(disease) + hub <- suppressMessages(hub_read_forecasts( + hub_dir = here::here(HUB_DIRS[[disease]]), target = c(flu = HUB_FLU_TARGET, covid = HUB_COVID_TARGET)[[disease]] + )) %>% filter(horizon %in% 0:3) + ws <- bind_rows(burn, wi$fc) + common <- inner_join(distinct(hub, across(all_of(key))), distinct(ws, across(all_of(key))), by = key) + fcs <- list(ensemble = semi_join(hub, common, by = key), replay = semi_join(ws, common, by = key)) + warm <- fcs + if (!any(season_of(fcs$ensemble$reference_date) == "2023-2024")) { + warm <- list(replay = bind_rows(burn, fcs$replay)) + } + list(wi = wi, cold = purrr::map(fcs, \(fc) filter(fc, season_of(reference_date) %in% LIVE)), warm = warm) +} + +# The finalist configs: REF-op and sqrt constant 0.018 per 100k (E16), each cold and warm. +fin_candidates <- function(rate_scales) { + const <- ws_configs(rate_scales, burn_in = FALSE)[["E11 constant 0.1"]] + sqrt_018 <- utils::modifyList(const, list(lr = 0.018, transform = "sqrt")) + list( + `REF-op warm` = list(cfg = list(ref = "op"), warm = TRUE), + `REF-op cold` = list(cfg = list(ref = "op", burn_in_seasons = character(0), slow_init = NULL), warm = FALSE), + `sqrt 0.018` = list(cfg = sqrt_018, warm = FALSE), + `sqrt 0.018 warm` = list(cfg = utils::modifyList(sqrt_018, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")), warm = TRUE) + ) +} + +# Every finalist on each of `bases` that has its inputs, cached. Returns the +# inputs, the runs named ", ", their grid and the candidate names. +fin_runs <- function(disease, bases = c("ensemble", "replay"), workers = WORKERS) { + inp <- fin_inputs(disease) + cands <- fin_candidates(inp$wi$rate_scales) + grid <- tidyr::expand_grid(base = bases, cand = names(cands)) %>% + filter(!purrr::map_lgl(cands[cand], "warm") | base %in% names(inp$warm)) + cals <- purrr::pmap(grid, function(base, cand) { + cc <- cands[[cand]] + fc <- if (cc$warm) inp$warm[[base]] else inp$cold[[base]] + do.call(cal_run_cached, c(list(fc, inp$wi$truth), cc$cfg, list(learn = "exact", vintages = inp$wi$vintages, workers = workers))) + }) + list(inputs = inp, cals = setNames(cals, paste(grid$base, grid$cand, sep = ", ")), grid = grid, cands = names(cands)) +} + # Location subsets scored separately: raw-count pooling makes US about half of # the all-locations WIS. WS_GEOS <- list(all = \(loc) rep(TRUE, length(loc)), states = \(loc) loc != "US", us = \(loc) loc == "US") From 1ff8af47d03673b0d3c66d805b97e48be66fd68a Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:46:25 -0700 Subject: [PATCH 093/107] refactor(calibration): drop E18's rate-scale references outside the chain --- .../calibration_experiments/e18_ref_op_bridge.Rmd | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd index 232c3c7a..9eff5bf9 100644 --- a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd +++ b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd @@ -52,9 +52,7 @@ learning rate's window, so eta is sized from the first live round (step 4 removes this). Burn-in outcomes never step the offsets. Step 5 changes only units, so that step 6 changes only the learning rate. -Outside the chain: two rate-scale constant rates as references, rate 0.032 -(E14) and rate 0.056 (E09's knee), and a branch, sqrt 0.018 warm, which is -step 6 with the 2023-24 season present and `slow_init = +Outside the chain: a branch, sqrt 0.018 warm, which is step 6 with the 2023-24 season present and `slow_init = "burn_in_quantile"` (the warm start of step 0, here on the single term). Steps 0, 3, 4 and 6 are the cached runs of E13 (REF-op warm), E02 @@ -85,11 +83,8 @@ STEPS <- list( `5 per-100k units` = list(cfg = c(sqrt_adaptive, list(scales = wi$rate_scales)), burn_in = FALSE), `6 sqrt constant 0.018 (E16)` = list(cfg = utils::modifyList(const_cfg, list(lr = 0.018, transform = "sqrt")), burn_in = FALSE) ) -# Rate-scale constant rates for comparison, outside the chain: E14's best and E09's knee, as E14 runs them. -e14_const <- ws_configs(wi$rate_scales, burn_in = TRUE)[["E11 constant 0.1"]] +# Outside the chain: step 6 with a warm start. REFS <- list( - `ref: rate 0.032 (E14)` = list(cfg = utils::modifyList(e14_const, list(lr = 0.032)), burn_in = FALSE), - `ref: rate 0.056 (E09 knee)` = list(cfg = utils::modifyList(e14_const, list(lr = 0.056)), burn_in = FALSE), `branch: sqrt 0.018 warm` = list(cfg = utils::modifyList(STEPS[[7]]$cfg, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")), burn_in = TRUE) ) run <- function(spec) { @@ -101,7 +96,7 @@ cals <- purrr::map(c(STEPS, REFS), run) ```{r metrics} SEASONS <- c(LIVE, "both") -STEP_COLS <- setNames(c("#000000", "#E69F00", "#56B4E9", "#009E73", "#CC79A7", "#0072B2", "#D55E00", "#999999", "#882255", "#AA4499"), names(c(STEPS, REFS))) +STEP_COLS <- setNames(c("#000000", "#E69F00", "#56B4E9", "#009E73", "#CC79A7", "#0072B2", "#D55E00", "#AA4499"), names(c(STEPS, REFS))) step_m <- purrr::imap(cals, function(cal, nm) { fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) %>% filter(location != "US") ws_scores(fc) %>% From cdeb63ba92ac7e3b3397dd5f2a78fadd029c7a5f Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:47:11 -0700 Subject: [PATCH 094/107] refactor(calibration): E15 breaks down the finalists' WIS changes on the replay, for flu and covid --- .../e15_wis_sources.Rmd | 248 ++++++++---------- 1 file changed, 110 insertions(+), 138 deletions(-) diff --git a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd index 182487ad..d2a74b6e 100644 --- a/reports/writeups/calibration_experiments/e15_wis_sources.Rmd +++ b/reports/writeups/calibration_experiments/e15_wis_sources.Rmd @@ -19,26 +19,29 @@ knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` -Question: how does the WIS change split across quantile levels, how large -are the offsets next to the base forecast's spread, and how much of the -change comes from moving the median versus changing widths? +Question: for each finalist, how does the WIS change split across quantile +levels, how large are the offsets next to the base forecast's spread, and +how much of the change comes from moving the median versus changing widths? Setup: -- Disease: flu. Base forecaster: the clean `windowed_seasonal` replay (E12: - substitutions off, all 53 locations, the 55 hub rounds). Horizons h0–h3. +- Diseases: flu and covid, one section each. +- Base forecaster: the clean `windowed_seasonal` replay, on the (round, + location, horizon, level) rows the submitted ensemble also has, h0–h3 + (E10's replay runs). - Learning truth: exact (re-run at every round on the data published by then; - E00), one week of extra delay. Cold start, no burn-in season. + E00), one week of extra delay. - Scores against finalized truth over 2024-25 and 2025-26, pooled; states only (US excluded). -- Configs, both from E12: - - **REF-op cold**: prod's tracker (sqrt, adaptive rate, slow term, leak) - without its warm start. - - **E11 constant 0.1**: the plain tracker on the rate scale (per 100k), - constant learning rate 0.1. +- Configs, E10's finalists: + - **REF-op warm**: flu prod; warm start from the 2023-24 burn-in season. + - **REF-op cold**: covid prod; no burn-in season. + - **sqrt 0.018**: constant learning rate 0.018 on the square root of + admissions per 100k (E16), cold. + - **sqrt 0.018 warm**: the same with the burn-in season and a warm start. WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated -WIS lower than base. L1 coverage bias: lower is better. +WIS lower than base. **Three ways to cut the WIS reduction.** @@ -66,74 +69,49 @@ WIS lower than base. L1 coverage bias: lower is better. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) -wi <- ws_inputs("flu") -cfgs <- ws_configs(wi$rate_scales, burn_in = FALSE)[c("REF-op cold", "E11 constant 0.1")] -cals <- purrr::map(cfgs, function(cfg) { - do.call(cal_run_cached, c(list(wi$fc, wi$truth), cfg, list(learn = "exact", vintages = wi$vintages, workers = params$workers))) -}) pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) -GEOS <- c(states = "states only (US excluded)") -fc <- purrr::imap(cals, function(cal, nm) cal$forecasts %>% mutate(config = nm)) %>% - bind_rows() %>% - mutate(season = season_of(reference_date), config = factor(config, levels = names(cals))) %>% - filter(season %in% LIVE, !is.na(truth), !is.na(value_base)) -fc <- fc %>% - filter(location != "US") %>% - mutate(geos = "states") %>% - mutate(geos = factor(geos, levels = names(GEOS), labels = GEOS)) +FIN_COLS <- c(`REF-op warm` = "#000000", `REF-op cold` = "#D55E00", `sqrt 0.018` = "#0072B2", `sqrt 0.018 warm` = "#56B4E9") +fin_fc <- function(d) { + f <- fin_runs(d, bases = "replay", workers = params$workers) + purrr::imap(f$cals, \(cal, nm) cal$forecasts %>% mutate(config = sub("^replay, ", "", nm))) %>% + bind_rows() %>% + mutate(season = season_of(reference_date), config = factor(config, levels = names(FIN_COLS))) %>% + filter(season %in% LIVE, !is_burn_in, !is.na(truth), !is.na(value_base), location != "US") +} +FC <- purrr::map(c(flu = "flu", covid = "covid"), fin_fc) ``` -# 1. WIS reduction by quantile level - -Each bar is one level's share of the total WIS reduction % (positive = -calibrated WIS lower than base); the bars in a panel add up to the panel's total, printed in the -strip. - -```{r by-level} -by_level <- fc %>% - group_by(geos, config, horizon, level) %>% - summarize(base = sum(pinball(truth, value_base, level)), cal = sum(pinball(truth, value_cal, level)), .groups = "drop") %>% - group_by(geos, config, horizon) %>% - mutate(contrib = 100 * (base - cal) / sum(base), level_pct = 100 * (base - cal) / base, total = sum(contrib)) %>% - ungroup() -level_plot <- function(g) { - d <- by_level %>% - filter(geos == g) %>% - mutate(panel = sprintf("h%d, total %+.1f%%", horizon, total)) +```{r fns} +# 1. Each level's change in pinball loss as a share of the base WIS. +by_level <- function(fc) { + fc %>% + group_by(config, horizon, level) %>% + summarize(base = sum(pinball(truth, value_base, level)), cal = sum(pinball(truth, value_cal, level)), .groups = "drop") %>% + group_by(config, horizon) %>% + mutate(contrib = 100 * (base - cal) / sum(base), total = sum(contrib)) %>% + ungroup() +} +level_plot <- function(bl) { + d <- bl %>% mutate(panel = sprintf("h%d, total %+.1f%%", horizon, total)) ggplot(d, aes(factor(level), contrib, fill = config)) + geom_hline(yintercept = 0, colour = "grey50") + - geom_col(position = position_dodge(width = 0.8), width = 0.75) + + geom_col(width = 0.75) + facet_wrap(~ paste(config, panel, sep = ": "), ncol = 4) + - scale_fill_manual(values = c(`REF-op cold` = "#D55E00", `E11 constant 0.1` = "#0072B2"), guide = "none") + + scale_fill_manual(values = FIN_COLS, guide = "none") + scale_x_discrete(labels = function(x) ifelse(x %in% c("0.01", "0.05", "0.25", "0.5", "0.75", "0.95", "0.99"), x, "")) + - labs(x = "quantile level", y = "share of WIS reduction, % of base WIS\n(positive = calibrated WIS lower than base)", title = g) + + labs(x = "quantile level", y = "share of WIS reduction, % of base WIS\n(positive = calibrated WIS lower than base)") + theme(axis.text.x = element_text(angle = 90, vjust = 0.5, size = 7)) } -``` - -```{r by-level-states, fig.height=6} -level_plot(GEOS[["states"]]) -``` - -## Grouped: tails, shoulders, centre - -The same pieces summed over groups of levels: **tails** are the six levels -at or beyond 0.05/0.95, **shoulders** the ten levels 0.1–0.3 and 0.7–0.9, -**centre** the seven levels 0.35–0.65. Below and above the median are -split, to show whether the gain comes from moving the lower or the upper -side. - -```{r by-group} -grouped <- by_level %>% - mutate( - band = case_when(level <= 0.05 | level >= 0.95 ~ "tails", level <= 0.3 | level >= 0.7 ~ "shoulders", TRUE ~ "centre"), - side = case_when(level < 0.5 ~ "lower", level > 0.5 ~ "upper", TRUE ~ "median") - ) -group_tbl <- function(g) { +group_tbl <- function(bl) { + grouped <- bl %>% + mutate( + band = case_when(level <= 0.05 | level >= 0.95 ~ "tails", level <= 0.3 | level >= 0.7 ~ "shoulders", TRUE ~ "centre"), + side = case_when(level < 0.5 ~ "lower", level > 0.5 ~ "upper", TRUE ~ "median") + ) bind_rows( - grouped %>% group_by(config, horizon, part = band) %>% filter(geos == g) %>% summarize(v = sum(contrib), .groups = "drop"), - grouped %>% group_by(config, horizon, part = paste(side, "side")) %>% filter(geos == g) %>% summarize(v = sum(contrib), .groups = "drop"), - grouped %>% group_by(config, horizon) %>% filter(geos == g) %>% summarize(v = sum(contrib), .groups = "drop") %>% mutate(part = "total") + grouped %>% group_by(config, horizon, part = band) %>% summarize(v = sum(contrib), .groups = "drop"), + grouped %>% group_by(config, horizon, part = paste(side, "side")) %>% summarize(v = sum(contrib), .groups = "drop"), + grouped %>% group_by(config, horizon) %>% summarize(v = sum(contrib), .groups = "drop") %>% mutate(part = "total") ) %>% mutate(part = factor(part, levels = c("tails", "shoulders", "centre", "lower side", "median side", "upper side", "total"), labels = c("tails", "shoulders", "centre", "below the median", "the median level", "above the median", "total"))) %>% @@ -141,74 +119,51 @@ group_tbl <- function(g) { tidyr::pivot_wider(names_from = horizon, values_from = v) %>% arrange(config, part) } -knitr::kable(group_tbl(GEOS[["states"]]), caption = "States only: WIS reduction % by group of levels (positive = calibrated WIS lower than base)") -``` - -# 2. Offset size against the forecast's spread - -```{r offsets} -widths <- fc %>% - filter(config == names(cals)[1], level %in% c(0.05, 0.25, 0.75, 0.95)) %>% - select(geos, location, horizon, reference_date, level, value_base) %>% - tidyr::pivot_wider(names_from = level, values_from = value_base, names_prefix = "q") %>% - transmute(geos, location, horizon, reference_date, w90 = q0.95 - q0.05, w50 = q0.75 - q0.25) -off <- fc %>% - inner_join(widths, by = c("geos", "location", "horizon", "reference_date")) %>% - filter(w90 > 0, w50 > 0) %>% - mutate(offset = value_cal - value_base, r90 = offset / w90, r50 = offset / w50) -n_dropped <- fc %>% distinct(geos, location, horizon, reference_date) %>% nrow() - off %>% distinct(geos, location, horizon, reference_date) %>% nrow() -``` - -Offsets are signed: positive means the calibrated quantile is above the -base one. The line is the median over forecasts (location × round) and the -band the middle 80%, offset ÷ base 90% interval width, states only. -`r n_dropped` forecast rows with a zero-width base interval are left out. - -```{r offset-plot, fig.height=6} -off_q <- off %>% - filter(geos == GEOS[["states"]]) %>% - group_by(config, horizon, level) %>% - summarize(med = median(r90), lo = quantile(r90, 0.1), hi = quantile(r90, 0.9), .groups = "drop") -ggplot(off_q, aes(level, med, colour = config, fill = config)) + - geom_hline(yintercept = 0, colour = "grey50") + - geom_ribbon(aes(ymin = lo, ymax = hi), alpha = 0.15, colour = NA) + - geom_line() + - geom_point(size = 0.8) + - facet_wrap(~ paste0("h", horizon), nrow = 1) + - scale_colour_manual(values = c(`REF-op cold` = "#D55E00", `E11 constant 0.1` = "#0072B2"), name = NULL) + - scale_fill_manual(values = c(`REF-op cold` = "#D55E00", `E11 constant 0.1` = "#0072B2"), name = NULL) + - labs(x = "quantile level", y = "offset / base (q95 − q05)") + - theme(legend.position = "bottom") -``` - -# 3. Median shift vs width change - -```{r split} -med <- fc %>% - filter(level == 0.5) %>% - select(geos, config, location, horizon, reference_date, med_base = value_base, med_cal = value_cal) -split <- fc %>% - left_join(med, by = c("geos", "config", "location", "horizon", "reference_date")) %>% - mutate( - median_only = pmax(value_base + (med_cal - med_base), 0), - width_only = pmax(med_base + (value_cal - med_cal), 0) - ) -split_scores <- split %>% - group_by(geos, config, horizon) %>% - summarize( - base = sum(pinball(truth, value_base, level)), - full = sum(pinball(truth, value_cal, level)), - `median shift only` = sum(pinball(truth, median_only, level)), - `width change only` = sum(pinball(truth, width_only, level)), - .groups = "drop" - ) %>% - tidyr::pivot_longer(c(full, `median shift only`, `width change only`), names_to = "variant", values_to = "wis") %>% - mutate(pct = 100 * (base - wis) / base) -split_tbl <- function(g) { - d <- split_scores %>% filter(geos == g) +# 2. Signed offset ÷ the base 90% interval width, per forecast and level. +offset_plot <- function(fc) { + ref <- levels(droplevels(fc$config))[1] + widths <- fc %>% + filter(config == ref, level %in% c(0.05, 0.95)) %>% + select(location, horizon, reference_date, level, value_base) %>% + tidyr::pivot_wider(names_from = level, values_from = value_base, names_prefix = "q") %>% + transmute(location, horizon, reference_date, w90 = q0.95 - q0.05) + off_q <- fc %>% + inner_join(widths, by = c("location", "horizon", "reference_date")) %>% + filter(w90 > 0) %>% + mutate(r90 = (value_cal - value_base) / w90) %>% + group_by(config, horizon, level) %>% + summarize(med = median(r90), lo = quantile(r90, 0.1), hi = quantile(r90, 0.9), .groups = "drop") + ggplot(off_q, aes(level, med, colour = config, fill = config)) + + geom_hline(yintercept = 0, colour = "grey50") + + geom_ribbon(aes(ymin = lo, ymax = hi), alpha = 0.12, colour = NA) + + geom_line() + + geom_point(size = 0.8) + + facet_wrap(~ paste0("h", horizon), nrow = 2) + + scale_colour_manual(values = FIN_COLS, name = NULL, aesthetics = c("colour", "fill")) + + labs(x = "quantile level", y = "offset / base (q95 − q05)") + + theme(legend.position = "bottom") +} +# 3. Score the median shift and the width change on their own. +split_tbl <- function(fc) { + med <- fc %>% + filter(level == 0.5) %>% + select(config, location, horizon, reference_date, med_base = value_base, med_cal = value_cal) + scores <- fc %>% + left_join(med, by = c("config", "location", "horizon", "reference_date")) %>% + mutate(median_only = pmax(value_base + (med_cal - med_base), 0), width_only = pmax(med_base + (value_cal - med_cal), 0)) %>% + group_by(config, horizon) %>% + summarize( + base = sum(pinball(truth, value_base, level)), + full = sum(pinball(truth, value_cal, level)), + `median shift only` = sum(pinball(truth, median_only, level)), + `width change only` = sum(pinball(truth, width_only, level)), + .groups = "drop" + ) %>% + tidyr::pivot_longer(c(full, `median shift only`, `width change only`), names_to = "variant", values_to = "wis") %>% + mutate(pct = 100 * (base - wis) / base) bind_rows( - d %>% select(config, variant, horizon, pct), - d %>% + scores %>% select(config, variant, horizon, pct), + scores %>% group_by(config, horizon) %>% summarize(pct = pct[variant == "full"] - pct[variant == "median shift only"] - pct[variant == "width change only"], .groups = "drop") %>% mutate(variant = "interaction (full − the two parts)") @@ -218,5 +173,22 @@ split_tbl <- function(g) { tidyr::pivot_wider(names_from = horizon, values_from = pct) %>% arrange(config, variant) } -knitr::kable(split_tbl(GEOS[["states"]]), caption = "States only: WIS reduction % (positive = calibrated WIS lower than base)") +``` + +```{r sections, results="asis", fig.height=9} +for (d in names(FC)) { + fc <- FC[[d]] + bl <- by_level(fc) + cat("\n\n# ", d, "\n\n", sep = "") + cat("\n\n## WIS reduction by quantile level\n\n") + cat("Each bar is one level's share of the total WIS reduction %; the bars in a panel add up to the panel's total, printed in the strip.\n\n") + print(level_plot(bl)) + cat("\n\nThe same pieces summed over groups of levels: **tails** are the six levels at or beyond 0.05/0.95, **shoulders** the ten levels 0.1–0.3 and 0.7–0.9, **centre** the seven levels 0.35–0.65. Below and above the median are split, to show whether the gain comes from moving the lower or the upper side.\n\n") + print(knitr::kable(group_tbl(bl), caption = paste0(d, ": WIS reduction % by group of levels (positive = calibrated WIS lower than base)"))) + cat("\n\n## Offset size against the forecast's spread\n\n") + cat("Offsets are signed: positive means the calibrated quantile is above the base one. The line is the median over forecasts (location × round) and the band the middle 80%, offset ÷ base 90% interval width. Forecasts with a zero-width base interval are left out.\n\n") + print(offset_plot(fc)) + cat("\n\n## Median shift vs width change\n\n") + print(knitr::kable(split_tbl(fc), caption = paste0(d, ": WIS reduction % (positive = calibrated WIS lower than base)"))) +} ``` From bac60ccef180d5dd63b4a3cb5fd67479630c6d67 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:47:18 -0700 Subject: [PATCH 095/107] refactor(calibration): prod_views shows states only --- .../writeups/calibration_experiments/prod_views.Rmd | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/reports/writeups/calibration_experiments/prod_views.Rmd b/reports/writeups/calibration_experiments/prod_views.Rmd index fbbcabde..71db3d65 100644 --- a/reports/writeups/calibration_experiments/prod_views.Rmd +++ b/reports/writeups/calibration_experiments/prod_views.Rmd @@ -30,7 +30,7 @@ configs running in prod: Both calibrate the submitted `CMU-TimeSeries` ensemble, learn from NHSN data as published (re-run every round, as prod does; E00), and are drawn against -finalized truth. Spoiled submissions are dropped. Seasons run August–July; +finalized truth. Spoiled submissions are dropped. States only (US excluded). Seasons run August–July; only the two live seasons (2024-25 and 2025-26) are shown. ```{r load} @@ -56,6 +56,7 @@ PROD <- list( covid = load_prod("covid", params$covid_hub_dir, list(burn_in_seasons = character(0), slow_init = NULL)) ) abbrev <- setNames(toupper(get_population_data()$state_id), get_population_data()$state_code) +STATES <- CAL_STATES[names(CAL_STATES) != "US"] ``` ```{r sections, results="asis", fig.height=4} @@ -65,7 +66,7 @@ for (d in names(PROD)) { cat("\n\n## Fixed states {.tabset}\n\n") cat( - "The same states (US, CA, GA, MN, VT) for both diseases. Every other", + "The same states (CA, GA, MN, VT) for both diseases. Every other", "round's fan at h−1–h3: blue ribbon and line are the base 80% band and", "median, orange lines the calibrated 10/50/90% quantiles, grey the", "finalized truth. The black line is the last two reported weeks of NHSN", @@ -74,9 +75,9 @@ for (d in names(PROD)) { "reports were later revised up. A black line with no fan in front of it is", "a round with no submission.\n\n" ) - for (nm in names(CAL_STATES)) { + for (nm in names(STATES)) { cat("\n\n### ", nm, "\n\n", sep = "") - for (s in LIVE) print(cal_state_panel(p$cal, CAL_STATES[[nm]], s, p$truth, p$vintages, title = paste(d, nm, s))) + for (s in LIVE) print(cal_state_panel(p$cal, STATES[[nm]], s, p$truth, p$vintages, title = paste(d, nm, s))) } cat("\n\n## Gallery {.tabset}\n\n") @@ -90,8 +91,8 @@ for (d in names(PROD)) { "forecast date.\n\n" ) cal_live <- p$cal - cal_live$forecasts <- cal_live$forecasts %>% filter(season %in% LIVE) - picks <- cal_gallery_pick(cal_live, p$population) + cal_live$forecasts <- cal_live$forecasts %>% filter(season %in% LIVE, location != "US") + picks <- cal_gallery_pick(cal_live, p$population %>% filter(location != "US")) for (i in seq_len(nrow(picks))) { pk <- picks[i, ] if (i == 1L || pk$phase != picks$phase[i - 1L]) cat("\n\n### ", as.character(pk$phase), "\n\n", sep = "") From 77fcae5e385210989c040f735240fffb0cb249a0 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:47:28 -0700 Subject: [PATCH 096/107] docs(calibration): index regroups the finalists after folding E07b and E14 --- reports/writeups/calibration_experiments/index.Rmd | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index 8c2e7c61..f929633a 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -41,9 +41,9 @@ when it was last rendered. The ids match `notes/calibration-ledger.md`. | | base | question | |---|---|---| +| `r link("E10", "e10_covid")` | both diseases, both bases | How do the finalists do on flu and covid, and do they rank the same way on the ensemble and the replay? (Part 2: covid ensemble at h−1–h3.) | | `r link("E18", "e18_ref_op_bridge")` | replay | Which setting accounts for the gap between REF-op and sqrt constant 0.018, stepping from one to the other a setting at a time? | -| `r link("E07b", "e07b_candidates")` | both | Do the operating-point candidates rank the same way on both bases? | -| `r link("E15", "e15_wis_sources")` | replay, cold | Where does the WIS change come from? | +| `r link("E15", "e15_wis_sources")` | replay, both diseases | Where does each finalist's WIS change come from? | ## Prod configs @@ -51,11 +51,10 @@ when it was last rendered. The ids match `notes/calibration-ledger.md`. of the largest gains and losses, for flu REF-op warm and covid REF-op cold on the submitted ensemble. -## Flu and covid +## Bases | | base | question | |---|---|---| -| `r link("E10", "e10_covid")` | both diseases, both bases | How do the finalists do on flu and covid? (Part 2: covid ensemble at h−1–h3 with REF-paper.) | | `r link("E07", "e07_across_eras")` | both | Does calibration act the same on the ensemble and the clean replay? | ## Learning rate @@ -64,8 +63,7 @@ on the submitted ensemble. |---|---|---| | `r link("E01", "e01_eta_settings")` | replay, cold | How do the adaptive learning-rate settings (multiplier, window, carry or reset) change WIS and coverage? | | `r link("E09", "e09_lr_delay")` | ensemble | With a constant learning rate on the rate scale, how do WIS and coverage change with the rate and with extra learning delay? | -| `r link("E14", "e14_lr_by_season")` | replay, cold | How do WIS and coverage change with a constant learning rate, season by season? | -| `r link("E16", "e16_sqrt_constant_lr")` | replay, cold | How does a constant learning rate on the square-root scale compare with the rate scale? | +| `r link("E16", "e16_sqrt_constant_lr")` | replay, cold | How do WIS and coverage change with a constant learning rate, season by season, on the square-root and rate scales? | ## Tracker structure @@ -90,7 +88,8 @@ on the submitted ensemble. | `r link("E00", "e00_vintage_backtest")` | ensemble | Should the tracker learn from finalized data or from data as published? | E12 and E13 (the clean replay without and with a 2023-24 burn-in) have no -notebook; they are `scripts/calibration/calibration_ws_replay.R`. +notebook; they are `scripts/calibration/calibration_ws_replay.R`. E14 is +part of E16, and E07b's ranking is part of E10. # How to read the results From b0799e37bc4726b29ba234054dbc50a99e2c7d95 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:32:40 -0700 Subject: [PATCH 097/107] feat(calibration): E18 adds matched-pair state scatters (warm vs warm, cold vs cold) --- .../e18_ref_op_bridge.Rmd | 64 ++++++++++++++----- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd index 9eff5bf9..cff2c1af 100644 --- a/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd +++ b/reports/writeups/calibration_experiments/e18_ref_op_bridge.Rmd @@ -53,7 +53,10 @@ removes this). Burn-in outcomes never step the offsets. Step 5 changes only units, so that step 6 changes only the learning rate. Outside the chain: a branch, sqrt 0.018 warm, which is step 6 with the 2023-24 season present and `slow_init = -"burn_in_quantile"` (the warm start of step 0, here on the single term). +"burn_in_quantile"` (the warm start of step 0, here on the single term); and +REF-op cold (prod's covid config: step 0 without the warm start or the +2023-24 season), so the state scatters below can pair warm with warm and +cold with cold. Steps 0, 3, 4 and 6 are the cached runs of E13 (REF-op warm), E02 (`reference + burn-in`), E01/E02 (`reference`) and E16 (sqrt 0.018), so they @@ -83,9 +86,10 @@ STEPS <- list( `5 per-100k units` = list(cfg = c(sqrt_adaptive, list(scales = wi$rate_scales)), burn_in = FALSE), `6 sqrt constant 0.018 (E16)` = list(cfg = utils::modifyList(const_cfg, list(lr = 0.018, transform = "sqrt")), burn_in = FALSE) ) -# Outside the chain: step 6 with a warm start. +# Outside the chain: step 6 with a warm start, and REF-op without one (E16's cached run). REFS <- list( - `branch: sqrt 0.018 warm` = list(cfg = utils::modifyList(STEPS[[7]]$cfg, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")), burn_in = TRUE) + `branch: sqrt 0.018 warm` = list(cfg = utils::modifyList(STEPS[[7]]$cfg, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")), burn_in = TRUE), + `REF-op cold` = list(cfg = ws_configs(wi$rate_scales, burn_in = FALSE)[["REF-op cold"]], burn_in = FALSE) ) run <- function(spec) { fc <- if (spec$burn_in) fc_burn_in else wi$fc @@ -96,7 +100,7 @@ cals <- purrr::map(c(STEPS, REFS), run) ```{r metrics} SEASONS <- c(LIVE, "both") -STEP_COLS <- setNames(c("#000000", "#E69F00", "#56B4E9", "#009E73", "#CC79A7", "#0072B2", "#D55E00", "#AA4499"), names(c(STEPS, REFS))) +STEP_COLS <- setNames(c("#000000", "#E69F00", "#56B4E9", "#009E73", "#CC79A7", "#0072B2", "#D55E00", "#AA4499", "#999999"), names(c(STEPS, REFS))) step_m <- purrr::imap(cals, function(cal, nm) { fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) %>% filter(location != "US") ws_scores(fc) %>% @@ -152,24 +156,52 @@ knitr::kable( ) ``` -# State by state: REF-op against sqrt 0.018 +# State by state: REF-op against sqrt 0.018, matched on the warm start Per state, pooled over h0–h3: WIS reduction % of REF-op against sqrt 0.018 -(above the line, REF-op is better). - -```{r state-scatter, fig.height=5} -PAIR <- c("0 REF-op", "6 sqrt constant 0.018 (E16)") -state_wis <- cal_state_wis(cals[PAIR]) -state_pct <- state_wis %>% - filter(config %in% PAIR) %>% +(above the line, REF-op is better). Three pairs: the chain's ends (REF-op +warm vs sqrt 0.018 cold), then warm vs warm and cold vs cold. Cold configs +play the base through Nov–Dec 2024 (35–40% of that season's base WIS), so +only the matched pairs compare the trackers rather than the start. + +```{r state-scatter, fig.height=4.5} +PAIRS <- list( + `chain ends: REF-op warm vs sqrt 0.018 cold` = c("0 REF-op", "6 sqrt constant 0.018 (E16)"), + `warm vs warm` = c("0 REF-op", "branch: sqrt 0.018 warm"), + `cold vs cold` = c("REF-op cold", "6 sqrt constant 0.018 (E16)") +) +state_pct <- cal_state_wis(cals[unique(unlist(PAIRS))]) %>% group_by(config, season, location) %>% - summarize(pct = 100 * (sum(wis_base) - sum(wis_cal)) / sum(wis_base), .groups = "drop") %>% - tidyr::pivot_wider(names_from = config, values_from = pct) -ggplot(state_pct, aes(.data[[PAIR[2]]], .data[[PAIR[1]]])) + + summarize(pct = 100 * (sum(wis_base) - sum(wis_cal)) / sum(wis_base), .groups = "drop") +pair_pts <- purrr::imap(PAIRS, function(p, nm) { + state_pct %>% + filter(config %in% p) %>% + mutate(which = ifelse(config == p[1], "refop", "sqrt")) %>% + select(-config) %>% + tidyr::pivot_wider(names_from = which, values_from = pct) %>% + mutate(pair = nm) +}) %>% + bind_rows() %>% + mutate(pair = factor(pair, levels = names(PAIRS))) +ggplot(pair_pts, aes(sqrt, refop)) + geom_abline(slope = 1, intercept = 0, colour = "grey50") + geom_hline(yintercept = 0, colour = "grey85") + geom_vline(xintercept = 0, colour = "grey85") + geom_point(alpha = 0.8) + - facet_wrap(~season) + + facet_grid(season ~ pair) + labs(x = "sqrt 0.018: WIS reduction %, h0–h3", y = "REF-op: WIS reduction %, h0–h3") ``` + +States where REF-op has the lower WIS, out of the states scored, per pair +and season. + +```{r state-counts} +knitr::kable( + pair_pts %>% + group_by(pair, season) %>% + summarize(refop_better = sum(refop > sqrt), states = n(), .groups = "drop") %>% + transmute(pair, season, cell = sprintf("%d / %d", refop_better, states)) %>% + tidyr::pivot_wider(names_from = season, values_from = cell), + caption = "States (of those scored) where REF-op beats sqrt 0.018, h0–h3 pooled" +) +``` From 970c097b68fe62a28d699110feb36fda9c08826d Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:34:29 -0700 Subject: [PATCH 098/107] feat(calibration): shared grid configs and E20, the tracker structure grid on both learning-rate anchors --- .../e20_structure_grid.Rmd | 180 ++++++++++++++++++ scripts/calibration/calibration_ws_replay.R | 104 ++++++++++ 2 files changed, 284 insertions(+) create mode 100644 reports/writeups/calibration_experiments/e20_structure_grid.Rmd diff --git a/reports/writeups/calibration_experiments/e20_structure_grid.Rmd b/reports/writeups/calibration_experiments/e20_structure_grid.Rmd new file mode 100644 index 00000000..91b925a9 --- /dev/null +++ b/reports/writeups/calibration_experiments/e20_structure_grid.Rmd @@ -0,0 +1,180 @@ +--- +title: "E20: warm start and leak on both learning-rate anchors" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 + disease: flu +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: do the warm start and the leak change WIS and coverage in the same +way on the adaptive learning rate and on the constant one? E04 and E05 asked +this on the ensemble with the adaptive rate only; E18 found that the leak's +effect depends on how large the adaptive step gets late in the season, so the +two anchors may answer differently. + +Setup: + +- Disease: `r params$disease`. Base forecaster: the clean `windowed_seasonal` + replay (53 locations, the hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. +- Warm configs run with the 2023-24 season present (the HHS-stitched backfill + from E13) and start the offset from that season's error quantiles; cold + configs start at zero with no burn-in rounds, so they play the base through + Nov–Dec 2024. +- Scores over the two live seasons, each on its own rounds; states only (US + excluded). + +The grid is 2 × 2 × 2 (E18 step 2 shows the slow term does nothing, so it is +left out), plus prod's two configs: + +| factor | levels | +|---|---| +| anchor | `adaptive 0.03`: sqrt scale, eta = 0.03 × the 90th percentile of the last 20 rounds' absolute errors, floor 0.001 (E01's elbow). `constant 0.018`: sqrt of admissions per 100k, eta = 0.018 (E16's elbow) | +| start | `cold`: offsets start at zero. `warm`: `slow_init = "burn_in_quantile"` from the 2023-24 season | +| leak | `no leak`. `leak 0.1`: the offset shrinks by 10% each round | +| references | `REF-op warm` (flu prod: adaptive, warm, leak, plus a slow term); `REF-op cold` (covid prod) | + +WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated +WIS lower than base. L1 coverage bias: lower is better. "Month-avg" coverage +bias is computed within each calendar month and then averaged. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs(params$disease) +fc_burn_in <- bind_rows(ws_burn_in_forecasts(params$disease), wi$fc) +SPECS <- c(ws_grid_configs(wi$rate_scales), ws_prod_specs(wi$rate_scales)) +cals <- purrr::map(SPECS, ws_run_spec, wi = wi, fc_burn_in = fc_burn_in, workers = params$workers) +grid_names <- names(ws_grid_configs(wi$rate_scales)) +split_name <- function(d) { + d %>% + mutate(config = as.character(config)) %>% + tidyr::separate_wider_delim(config, " | ", names = c("anchor", "start", "leak"), cols_remove = FALSE, too_few = "align_start") +} +``` + +```{r metrics} +m <- ws_wis_by_season(cals) +mv <- cal_month_view(cals, seasons = LIVE) +cov <- mv$cov %>% + transmute(config, season, h = factor(paste0("h", horizon), levels = paste0("h", 0:3)), pooled = pooled_cal, mavg = mavg_cal) +``` + +# WIS by season and horizon + +```{r wis-table} +knitr::kable( + m %>% + transmute(config, col = paste(season, h), cell = sprintf("%+.1f", wis_pct)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell), + caption = "WIS reduction % (positive = calibrated WIS lower than base), states only; `both` pools the seasons" +) +``` + +# What each factor changes, per anchor + +The change from adding the warm start (warm − cold) at each leak level, and +from adding the leak (leak − no leak) at each start, per anchor. WIS: change +in WIS reduction % (positive = the factor lowers WIS). Coverage: change in +month-avg L1 coverage bias (negative = the factor improves coverage). + +```{r factor-deltas} +g <- m %>% + filter(config %in% grid_names) %>% + select(config, season, h, wis_pct) %>% + left_join(cov, by = c("config", "season", "h")) %>% + split_name() +contrast <- function(d, factor, hi, lo, held) { + d %>% + select(anchor, start, leak, season, h, wis_pct, mavg) %>% + tidyr::pivot_wider(names_from = all_of(factor), values_from = c(wis_pct, mavg)) %>% + transmute( + anchor, season, h, + factor = paste0("+ ", hi, " (", .data[[held]], ")"), + d_wis = .data[[paste0("wis_pct_", hi)]] - .data[[paste0("wis_pct_", lo)]], + d_mavg = .data[[paste0("mavg_", hi)]] - .data[[paste0("mavg_", lo)]] + ) +} +deltas <- bind_rows( + contrast(g, "start", "warm", "cold", "leak"), + contrast(g, "leak", "leak 0.1", "no leak", "start") +) +knitr::kable( + deltas %>% + transmute(anchor, factor, col = paste(season, h), cell = sprintf("%+.1f / %+.3f", d_wis, d_mavg)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + arrange(anchor, factor), + caption = "Change in WIS reduction % / month-avg L1 coverage bias from adding the factor, per anchor and season" +) +``` + +```{r factor-plot, fig.height=5} +ggplot(deltas %>% filter(season != "both"), aes(h, d_wis, colour = anchor, group = anchor)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_line() + + geom_point() + + facet_grid(season ~ factor) + + scale_colour_manual(values = c(`adaptive 0.03` = "#E69F00", `constant 0.018` = "#0072B2")) + + labs(x = NULL, y = "change in WIS reduction %", colour = NULL) + + theme(legend.position = "bottom") +``` + +# Anchor against anchor, state by state + +Per state, pooled over h0–h3: WIS reduction % of the adaptive anchor against +the constant one at the same start and leak (above the line, adaptive is +better). + +```{r state-scatter, fig.height=5.5} +state_pct <- cal_state_wis(cals[grid_names]) %>% + group_by(config, season, location) %>% + summarize(pct = 100 * (sum(wis_base) - sum(wis_cal)) / sum(wis_base), .groups = "drop") %>% + split_name() %>% + select(-config) %>% + tidyr::pivot_wider(names_from = anchor, values_from = pct) %>% + mutate(cell = paste(start, leak, sep = ", ")) +ggplot(state_pct, aes(`constant 0.018`, `adaptive 0.03`)) + + geom_abline(slope = 1, intercept = 0, colour = "grey50") + + geom_hline(yintercept = 0, colour = "grey85") + + geom_vline(xintercept = 0, colour = "grey85") + + geom_point(alpha = 0.8) + + facet_grid(season ~ cell) + + labs(x = "constant 0.018: WIS reduction %, h0–h3", y = "adaptive 0.03: WIS reduction %, h0–h3") +``` + +```{r state-counts} +knitr::kable( + state_pct %>% + group_by(cell, season) %>% + summarize(adaptive_better = sum(`adaptive 0.03` > `constant 0.018`), states = n(), .groups = "drop") %>% + transmute(cell, season, v = sprintf("%d / %d", adaptive_better, states)) %>% + tidyr::pivot_wider(names_from = season, values_from = v), + caption = "States (of those scored) where the adaptive anchor beats the constant one, h0–h3 pooled" +) +``` + +```{r month-view-cfgs} +# The month plots show the no-leak grid and flu prod; the tables keep every config. +mv_plot_cfgs <- c(grid_names[c(1, 3, 5, 7)], "REF-op warm") +mv_cols <- setNames( + c("#E69F00", "#D55E00", "#0072B2", "#56B4E9", "#000000"), + mv_plot_cfgs +) +``` + +```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +``` diff --git a/scripts/calibration/calibration_ws_replay.R b/scripts/calibration/calibration_ws_replay.R index 3965d836..419a2448 100644 --- a/scripts/calibration/calibration_ws_replay.R +++ b/scripts/calibration/calibration_ws_replay.R @@ -54,6 +54,110 @@ ws_configs <- function(rate_scales, burn_in = BURN_IN) { )) } +#' The structure grid (E19, E20): two learning-rate anchors, each cold or +#' warm-started, each with or without the leak. Anchors: `adaptive 0.03` is +#' E01's sqrt adaptive rate (E18 step 4); `constant 0.018` is E16's constant +#' rate on sqrt per 100k. Each spec is `list(cfg, burn_in)` as the E18 steps +#' are, named `" | | "`. The no-leak cold and warm specs +#' are written as E16 and E18 wrote them, so their cached runs are reused. +ws_grid_configs <- function(rate_scales) { + anchors <- list( + `adaptive 0.03` = list( + ref = "paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3), + burn_in_seasons = character(0), slow_init = NULL, lr_window = 20, season_policy = "carry" + ), + `constant 0.018` = utils::modifyList(ws_configs(rate_scales, burn_in = FALSE)[["E11 constant 0.1"]], list(lr = 0.018, transform = "sqrt")) + ) + starts <- list(cold = NULL, warm = list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")) + leaks <- list(`no leak` = NULL, `leak 0.1` = list(fast_decay = 0.1)) + specs <- list() + for (a in names(anchors)) { + for (s in names(starts)) { + for (l in names(leaks)) { + cfg <- anchors[[a]] + if (!is.null(starts[[s]])) cfg <- utils::modifyList(cfg, starts[[s]]) + if (!is.null(leaks[[l]])) cfg <- utils::modifyList(cfg, leaks[[l]]) + specs[[paste(a, s, l, sep = " | ")]] <- list(cfg = cfg, burn_in = s == "warm") + } + } + } + specs +} + +#' Prod's two configs as grid-style specs: REF-op warm (flu prod) and REF-op +#' cold (covid prod). +ws_prod_specs <- function(rate_scales) { + list( + `REF-op warm` = list(cfg = ws_configs(rate_scales, burn_in = TRUE)[["REF-op warm"]], burn_in = TRUE), + `REF-op cold` = list(cfg = ws_configs(rate_scales, burn_in = FALSE)[["REF-op cold"]], burn_in = FALSE) + ) +} + +#' Run one spec (`list(cfg, burn_in)`) through the cache, with the burn-in +#' rounds prepended when the spec asks for them. +ws_run_spec <- function(spec, wi, fc_burn_in, workers) { + fc <- if (spec$burn_in) fc_burn_in else wi$fc + do.call(cal_run_cached, c(list(fc, wi$truth), spec$cfg, list(learn = "exact", vintages = wi$vintages, workers = workers))) +} + +#' WIS reduction % and L1 coverage bias per config, season (live seasons and +#' "both") and horizon, states only, as a long tibble with `config`, `season`, +#' `h` ("h0".."h3"), `wis_pct`, `cal_err_base`, `cal_err_cal`. +ws_wis_by_season <- function(cals) { + purrr::imap(cals, function(cal, nm) { + fc <- cal$forecasts %>% mutate(season = season_of(reference_date)) %>% filter(location != "US") + ws_scores(fc) %>% + select(season, horizon, which, wis, cal_err) %>% + tidyr::pivot_wider(names_from = which, values_from = c(wis, cal_err)) %>% + mutate(config = nm, wis_pct = 100 * (wis_base - wis_cal) / wis_base) + }) %>% + bind_rows() %>% + mutate( + config = factor(config, levels = names(cals)), + season = factor(season, levels = c(LIVE, "both")), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) +} + +#' Per-round tracker internals from an exact run, states only, h0–h3: the +#' learning rate and the median-level offsets relative to the base 90% interval +#' width (in the config's working units, so configs with and without `scales` +#' compare), plus the share of truths above the base median. Under exact +#' learning each round's rows come from that round's own re-run, so the state +#' is the one that played. Medians over states per (horizon, reference_date). +ws_internals <- function(cal, cfg, rate_scales) { + fc <- cal$forecasts %>% + filter(location != "US", !is_burn_in, horizon >= 0, !is.na(value_base)) + s <- if (is.null(cfg$scales)) fc %>% distinct(location) %>% mutate(scale = 1) else rate_scales %>% select(location, scale) + tf <- if (identical(cfg$transform, "sqrt")) sqrt else identity + fc %>% + left_join(s, by = "location") %>% + group_by(location, horizon, reference_date) %>% + summarize( + width_count = value_base[level == 0.95] - value_base[level == 0.05], + width_work = tf(value_base[level == 0.95] / scale[1]) - tf(value_base[level == 0.05] / scale[1]), + eta = mean(lr_level), + offset_mid = offset[level == 0.5], + offset_abs = mean(abs(offset)), + truth = truth[1], + base_mid = value_base[level == 0.5], + .groups = "drop" + ) %>% + mutate( + eta_rel = ifelse(eta == 0, NA_real_, eta) / width_work, + offset_rel = offset_mid / width_count, + offset_abs_rel = offset_abs / width_count + ) %>% + group_by(horizon, reference_date) %>% + summarize( + eta_rel = median(eta_rel, na.rm = TRUE), + offset_rel = median(offset_rel, na.rm = TRUE), + offset_abs_rel = median(offset_abs_rel, na.rm = TRUE), + above_base_median = mean(truth > base_mid, na.rm = TRUE), + .groups = "drop" + ) +} + # 2023-24 burn-in rounds, backfilled on the HHS-stitched archive (cached). ws_burn_in_forecasts <- function(disease) { sched <- ch_schedule(through = BURN_IN_DATES[["through"]], from = as.Date(BURN_IN_DATES[["from"]])) From b8b4fb887faa0418b7746ea49a7c6cd42e4b4a85 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:36:13 -0700 Subject: [PATCH 099/107] refactor(calibration): E19 becomes a notebook: tracker internals on the structure grid, plus carry vs reset --- R/calibration/views.R | 14 +- .../e19_tracker_internals.Rmd | 242 ++++++++++++++++++ .../calibration_tracker_internals.R | 92 ------- scripts/calibration/calibration_ws_replay.R | 7 +- 4 files changed, 256 insertions(+), 99 deletions(-) create mode 100644 reports/writeups/calibration_experiments/e19_tracker_internals.Rmd delete mode 100644 scripts/calibration/calibration_tracker_internals.R diff --git a/R/calibration/views.R b/R/calibration/views.R index 2ac91ecd..2719bf8c 100644 --- a/R/calibration/views.R +++ b/R/calibration/views.R @@ -52,21 +52,27 @@ cal_run <- function(forecasts, truth, ref = "paper", ..., learn = c("final", "vi #' [cal_run()], cached on disk under a hash of its inputs. #' -#' Keeps only the forecast columns the views use. Clear `cache_dir` after +#' Keeps only the forecast columns the views use, plus the tracker's per-row +#' state (`offset`, `fast`, `slow`, `lr_level`) when `internals` is set; a +#' cached run without them is re-run and overwritten. Clear `cache_dir` after #' changing code in `R/calibration/`: the key covers the arguments and data, #' not the code. #' @export cal_run_cached <- function(forecasts, truth, ref = "paper", ..., learn = "exact", vintages = NULL, workers = 1L, - cache_dir = here::here("cache/calibration/experiments/runs")) { + internals = FALSE, cache_dir = here::here("cache/calibration/experiments/runs")) { key <- rlang::hash(list(forecasts, truth, ref, list(...), learn, if (learn != "final") vintages)) path <- file.path(cache_dir, paste0(key, ".rds")) + internal_cols <- c("offset", "fast", "slow", "lr_level") if (file.exists(path)) { - return(readRDS(path)) + cal <- readRDS(path) + if (!internals || all(internal_cols %in% names(cal$forecasts))) { + return(cal) + } } cal <- cal_run(forecasts, truth, ref, ..., learn = learn, vintages = vintages, workers = workers) cal <- list(forecasts = cal$forecasts %>% select( "location", "horizon", "reference_date", "target_end_date", "season", "level", - "value_base", "value_cal", "truth", "is_burn_in" + "value_base", "value_cal", "truth", "is_burn_in", any_of(if (internals) internal_cols else character(0)) )) dir.create(cache_dir, showWarnings = FALSE, recursive = TRUE) saveRDS(cal, path) diff --git a/reports/writeups/calibration_experiments/e19_tracker_internals.Rmd b/reports/writeups/calibration_experiments/e19_tracker_internals.Rmd new file mode 100644 index 00000000..eee19776 --- /dev/null +++ b/reports/writeups/calibration_experiments/e19_tracker_internals.Rmd @@ -0,0 +1,242 @@ +--- +title: "E19: tracker internals by month, and carry vs reset, on both anchors" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + workers: 14 + disease: flu +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: what does each tracker do month by month? How large is its step, +how large is the offset it plays, and does the WIS change in a month follow +from those together with the base's bias in that month? And how much of the +second season's result is the offset carried in from the first? + +Setup: + +- Disease: `r params$disease`. Base forecaster: the clean `windowed_seasonal` + replay (53 locations, the hub rounds). Horizons h0–h3. +- Learning truth: exact (re-run at every round on the data published by then; + E00), one week of extra delay. Each round's offsets come from that round's + own re-run, so they are the state that played. The step size is taken + from one pass on final truth per config, because in a round's own re-run + that round's outcomes have not yet stepped the tracker. +- Configs: the E20 grid ({adaptive 0.03, constant 0.018} × {cold, warm} × + {no leak, leak 0.1}), prod's REF-op warm and REF-op cold, and two + `season_policy = "reset"` runs (each anchor, cold, no leak), which zero the + offsets at the start of 2025-26. +- Scores over the two live seasons, each on its own rounds; states only (US + excluded). + +Internals are unit-free: the learning rate and the median-level offset are +divided by the base's 90% interval width, the rate in the config's working +units (sqrt of counts, or of counts per 100k) and the offset in counts. Each +value is the median over states for a (horizon, round), then averaged within +the calendar month. "Share above base median" is the fraction of state +outcomes above the base's median, the base's own bias that month. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs(params$disease) +fc_burn_in <- bind_rows(ws_burn_in_forecasts(params$disease), wi$fc) +grid <- ws_grid_configs(wi$rate_scales) +cold_no_leak <- grid[c("adaptive 0.03 | cold | no leak", "constant 0.018 | cold | no leak")] +RESETS <- purrr::map(cold_no_leak, function(s) list(cfg = utils::modifyList(s$cfg, list(season_policy = "reset")), burn_in = FALSE)) +names(RESETS) <- sub(" \\| no leak$", " | reset", names(RESETS)) +SPECS <- c(grid, ws_prod_specs(wi$rate_scales), RESETS) +# Internals are not in the standard cache, so cached runs without them are re-run here. +cals <- purrr::map(SPECS, ws_run_spec, wi = wi, fc_burn_in = fc_burn_in, workers = params$workers, internals = TRUE) +# An exact run's round-t rows come from the re-run at t, before round t's outcomes step the tracker, +# so their step size is zero. The step size comes from one pass on final truth per config instead. +cals_final <- purrr::map(SPECS, function(spec) { + fc <- if (spec$burn_in) fc_burn_in else wi$fc + do.call(cal_run, c(list(fc, wi$truth), spec$cfg, list(learn = "final"))) +}) +ANCHOR_COLS <- c(`adaptive 0.03` = "#E69F00", `constant 0.018` = "#0072B2", `REF-op` = "#000000") +anchor_of <- function(config) dplyr::case_when( + grepl("^adaptive", config) ~ "adaptive 0.03", + grepl("^constant", config) ~ "constant 0.018", + TRUE ~ "REF-op" +) +``` + +```{r internals} +internals <- purrr::imap(SPECS, function(spec, nm) { + eta <- ws_internals(cals_final[[nm]], spec$cfg, wi$rate_scales) %>% select(horizon, reference_date, eta_rel) + ws_internals(cals[[nm]], spec$cfg, wi$rate_scales) %>% + select(-eta_rel) %>% + left_join(eta, by = c("horizon", "reference_date")) %>% + mutate(config = nm) +}) %>% + bind_rows() %>% + mutate( + config = factor(config, levels = names(SPECS)), + season = season_of(reference_date), + month = factor(format(reference_date, "%b"), levels = month.abb[c(8:12, 1:7)]), + h = factor(paste0("h", horizon), levels = paste0("h", 0:3)) + ) %>% + filter(season %in% LIVE) +mv <- cal_month_view(cals, seasons = LIVE) +by_month <- internals %>% + group_by(config, season, month, h, horizon) %>% + summarize(across(c(eta_rel, offset_rel, offset_abs_rel, above_base_median), \(x) mean(x, na.rm = TRUE)), .groups = "drop") %>% + left_join(mv$month %>% select(config, season, month, horizon, wis_pct, share), by = c("config", "season", "month", "horizon")) %>% + mutate(anchor = anchor_of(as.character(config))) +# Internals are only shown for the no-leak grid, prod and the resets; the leak runs' offsets are in the tables. +PLOT_CFGS <- c(names(grid)[c(1, 3, 5, 7)], "REF-op warm", "REF-op cold") +``` + +# Per round + +Median over states of the step size and the played offset, both relative to +the base's 90% width, per round. Warm configs start with an offset; cold +ones at zero. + +```{r per-round, fig.height=8} +long <- internals %>% + filter(config %in% PLOT_CFGS) %>% + select(config, h, reference_date, `eta / width` = eta_rel, `median-level offset / width` = offset_rel, `mean |offset| / width` = offset_abs_rel) %>% + tidyr::pivot_longer(-c(config, h, reference_date)) +ggplot(long, aes(reference_date, value, colour = config)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_line() + + facet_grid(name ~ h, scales = "free_y") + + labs(x = NULL, y = "median over states", colour = NULL) + + theme(legend.position = "bottom") + + guides(colour = guide_legend(nrow = 2)) +``` + +# By month + +One row per config, horizon and season-month: step size, offset, the base's +bias that month, and the month's WIS change. The grey bar is the month's +share of the season's base WIS. + +```{r by-month-plot, fig.height=9} +pm <- by_month %>% filter(config %in% PLOT_CFGS) +for (s in LIVE) { + d <- pm %>% filter(season == s) + long <- d %>% + select(config, h, month, `eta / width` = eta_rel, `offset / width` = offset_rel, `share above base median` = above_base_median, `WIS reduction %` = wis_pct) %>% + tidyr::pivot_longer(-c(config, h, month)) %>% + mutate(name = factor(name, levels = c("eta / width", "offset / width", "share above base median", "WIS reduction %"))) + bars <- d %>% distinct(h, month, share) %>% mutate(name = factor("WIS reduction %", levels = levels(long$name))) + print( + ggplot(long, aes(month, value, colour = config, group = config)) + + geom_col(data = bars, aes(month, share), inherit.aes = FALSE, fill = "grey85", width = 0.7) + + geom_hline(data = tibble(name = factor("share above base median", levels = levels(long$name)), y = 0.5), aes(yintercept = y), colour = "grey60") + + geom_hline(data = tibble(name = factor(c("offset / width", "WIS reduction %"), levels = levels(long$name)), y = 0), aes(yintercept = y), colour = "grey60") + + geom_line() + + geom_point(size = 1) + + facet_grid(name ~ h, scales = "free_y") + + labs(title = s, x = NULL, y = NULL, colour = NULL) + + theme(legend.position = "bottom") + + guides(colour = guide_legend(nrow = 2)) + ) +} +``` + +## Tables + +```{r by-month-tables, results="asis"} +month_cell <- function(d, col, fmt) { + d %>% + transmute(config, h, col = paste(season, month), v = sprintf(fmt, .data[[col]])) %>% + tidyr::pivot_wider(names_from = col, values_from = v) %>% + arrange(h, config) +} +for (spec in list( + c("eta_rel", "%.3f", "Step size / base 90% width (mean over rounds of the state median)"), + c("offset_rel", "%+.3f", "Median-level offset / base 90% width"), + c("wis_pct", "%+.1f", "WIS reduction % within the month (positive = calibrated WIS lower than base)") +)) { + cat("\n\n", cal_h(3), " ", spec[3], "\n\n", sep = "") + print(knitr::kable(month_cell(by_month, spec[1], spec[2]))) +} +cat("\n\n", cal_h(3), " Share of state outcomes above the base median (the base's bias; same for every config)\n\n", sep = "") +print(knitr::kable( + by_month %>% + filter(config == names(SPECS)[1]) %>% + transmute(h, col = paste(season, month), v = sprintf("%.2f", above_base_median)) %>% + tidyr::pivot_wider(names_from = col, values_from = v) +)) +``` + +# Offset against WIS, month by month + +Each point is a config in one season-month at one horizon: the median-level +offset it played against the base's bias that month, coloured by the +month's WIS change. If a positive offset only pays when the base +under-predicts, the gains sit in the upper right and lower left. + +```{r offset-vs-bias, fig.height=4.5} +ggplot(by_month %>% filter(config %in% names(grid)), aes(above_base_median, offset_rel, colour = wis_pct)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_vline(xintercept = 0.5, colour = "grey60") + + geom_point(alpha = 0.8) + + facet_grid(season ~ h) + + scale_colour_gradient2(low = "#D55E00", mid = "grey80", high = "#009E73", midpoint = 0) + + labs(x = "share of outcomes above the base median", y = "median-level offset / width", colour = "WIS reduction %") +``` + +# Carry vs reset + +The offset each cold, no-leak tracker brings into 2025-26, and what resetting +it to zero at the season boundary does to the season's WIS. E01 did this for +the adaptive rate; the constant one is new. + +```{r carry-reset} +pairs <- tibble::tibble( + anchor = c("adaptive 0.03", "constant 0.018"), + carry = names(cold_no_leak), + reset = names(RESETS) +) +m <- ws_wis_by_season(cals[c(pairs$carry, pairs$reset)]) +cr <- pairs %>% + tidyr::pivot_longer(c(carry, reset), names_to = "policy", values_to = "config") %>% + left_join(m %>% mutate(config = as.character(config)), by = "config") +knitr::kable( + cr %>% + transmute(anchor, policy, col = paste(season, h), v = sprintf("%+.1f", wis_pct)) %>% + tidyr::pivot_wider(names_from = col, values_from = v), + caption = "WIS reduction %, states only, by season and horizon" +) +first_live <- internals %>% + filter(season == LIVE[2], config %in% pairs$carry) %>% + group_by(config, h) %>% + slice_min(reference_date, n = 1, with_ties = FALSE) %>% + ungroup() +knitr::kable( + first_live %>% + transmute(config, h, v = sprintf("%+.3f", offset_rel)) %>% + tidyr::pivot_wider(names_from = h, values_from = v), + caption = sprintf("Median-level offset / width at the first %s round, carried in from %s", LIVE[2], LIVE[1]) +) +``` + +```{r carry-reset-month, fig.height=4} +cr_month <- by_month %>% + filter(config %in% c(pairs$carry, pairs$reset), season == LIVE[2]) %>% + mutate(policy = ifelse(config %in% pairs$reset, "reset", "carry")) +ggplot(cr_month, aes(month, wis_pct, colour = anchor, linetype = policy, group = config)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_line() + + geom_point(size = 1) + + facet_wrap(~h, nrow = 1) + + scale_colour_manual(values = ANCHOR_COLS) + + labs(title = paste(LIVE[2], "by month"), x = NULL, y = "WIS reduction %", colour = NULL, linetype = NULL) + + theme(legend.position = "bottom") +``` diff --git a/scripts/calibration/calibration_tracker_internals.R b/scripts/calibration/calibration_tracker_internals.R deleted file mode 100644 index 267042db..00000000 --- a/scripts/calibration/calibration_tracker_internals.R +++ /dev/null @@ -1,92 +0,0 @@ -# E19: per-round learning rate and offsets for REF-op cold, sqrt adaptive -# without the leak, and sqrt constant 0.018 on the clean flu windowed_seasonal -# replay, h1–h3, states only. Learns from final truth (one run per config), so -# each round's internals come from a single tracker. Eta and offsets are -# divided by the base 90% interval width, in each config's working units, so -# configs with and without `scales` compare. Writes a per-round CSV, a monthly -# CSV and a plot to cache/calibration/tracker_internals/. Findings: E19 in -# notes/calibration-ledger.md. -# -# Usage: distrobox enter rocker -- Rscript scripts/calibration/calibration_tracker_internals.R - -suppressPackageStartupMessages({ - source(here::here("scripts/calibration/calibration_ws_replay.R")) - library(ggplot2) -}) -out_dir <- file.path(CH_CACHE_DIR, "tracker_internals") -dir.create(out_dir, showWarnings = FALSE, recursive = TRUE) - -wi <- ws_inputs("flu") -no_burn_in <- list(burn_in_seasons = character(0), slow_init = NULL) -CONFIGS <- list( - `REF-op cold` = c(list(ref = "op"), no_burn_in), - `sqrt adaptive, no leak` = c(list(ref = "paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)), no_burn_in), - `sqrt constant 0.018` = c(list(ref = "paper", transform = "sqrt", scales = wi$rate_scales, lr = 0.018), no_burn_in) -) -scale_of <- wi$rate_scales %>% select(location, scale) - -rows <- purrr::imap(CONFIGS, function(cfg, nm) { - cli::cli_alert_info("running {nm}") - cal <- do.call(cal_run, c(list(wi$fc, wi$truth), cfg, list(learn = "final"))) - fc <- cal$forecasts %>% - filter(location != "US", !is_burn_in, horizon >= 1, !is.na(value_base)) - # Working-unit 90% width: sqrt of (count / scale), scale 1 unless the config sets `scales`. - s <- if (is.null(cfg$scales)) fc %>% distinct(location) %>% mutate(scale = 1) else scale_of - fc %>% - left_join(s, by = "location") %>% - group_by(location, horizon, reference_date) %>% - summarize( - width_count = value_base[level == 0.95] - value_base[level == 0.05], - width_work = sqrt(value_base[level == 0.95] / scale[1]) - sqrt(value_base[level == 0.05] / scale[1]), - eta = mean(lr_level), - fast_mid = fast[level == 0.5], - slow_mid = slow[level == 0.5], - offset_mid = offset[level == 0.5], - offset_abs = mean(abs(offset)), - truth = truth[1], - base_mid = value_base[level == 0.5], - .groups = "drop" - ) %>% - mutate( - config = nm, - eta = ifelse(eta == 0, NA_real_, eta), - # Unit-free: step size and offsets relative to the base 90% width. - eta_rel = eta / width_work, - fast_rel = fast_mid / width_work, - slow_rel = slow_mid / width_work, - offset_rel = offset_mid / width_count, - offset_abs_rel = offset_abs / width_count - ) -}) %>% bind_rows() - -per_round <- rows %>% - group_by(config, horizon, reference_date) %>% - summarize( - eta_rel = median(eta_rel, na.rm = TRUE), - fast_rel = median(fast_rel, na.rm = TRUE), - offset_rel = median(offset_rel, na.rm = TRUE), - offset_abs_rel = median(offset_abs_rel, na.rm = TRUE), - above_base_median = mean(truth > base_mid, na.rm = TRUE), - .groups = "drop" - ) -readr::write_csv(per_round, file.path(out_dir, "tracker_internals_per_round.csv")) - -month_tbl <- per_round %>% - mutate(season = season_of(reference_date), month = format(reference_date, "%b")) %>% - group_by(config, horizon, season, month) %>% - summarize(across(c(eta_rel, offset_rel, offset_abs_rel, above_base_median), \(x) mean(x, na.rm = TRUE)), first = min(reference_date), .groups = "drop") %>% - arrange(config, horizon, first) -readr::write_csv(month_tbl, file.path(out_dir, "tracker_internals_by_month.csv")) - -long <- per_round %>% - select(config, horizon, reference_date, `eta / width` = eta_rel, `median-level offset / width` = offset_rel, `mean |offset| / width` = offset_abs_rel) %>% - tidyr::pivot_longer(-c(config, horizon, reference_date)) -p <- ggplot(long, aes(reference_date, value, colour = config)) + - geom_hline(yintercept = 0, colour = "grey60") + - geom_line() + - facet_grid(name ~ paste0("h", horizon), scales = "free_y") + - labs(x = NULL, y = "median over states", colour = NULL) + - theme_bw() + - theme(legend.position = "bottom") -ggsave(file.path(out_dir, "tracker_internals.png"), p, width = 13, height = 8) -cli::cli_alert_success("wrote {out_dir}") diff --git a/scripts/calibration/calibration_ws_replay.R b/scripts/calibration/calibration_ws_replay.R index 419a2448..dd9f8efa 100644 --- a/scripts/calibration/calibration_ws_replay.R +++ b/scripts/calibration/calibration_ws_replay.R @@ -94,10 +94,11 @@ ws_prod_specs <- function(rate_scales) { } #' Run one spec (`list(cfg, burn_in)`) through the cache, with the burn-in -#' rounds prepended when the spec asks for them. -ws_run_spec <- function(spec, wi, fc_burn_in, workers) { +#' rounds prepended when the spec asks for them. `internals` keeps the +#' tracker's per-row state (E19). +ws_run_spec <- function(spec, wi, fc_burn_in, workers, internals = FALSE) { fc <- if (spec$burn_in) fc_burn_in else wi$fc - do.call(cal_run_cached, c(list(fc, wi$truth), spec$cfg, list(learn = "exact", vintages = wi$vintages, workers = workers))) + do.call(cal_run_cached, c(list(fc, wi$truth), spec$cfg, list(learn = "exact", vintages = wi$vintages, workers = workers, internals = internals))) } #' WIS reduction % and L1 coverage bias per config, season (live seasons and From 79cd04486db9750c4ca0b6dbabf4fb138ee4ff3d Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:37:31 -0700 Subject: [PATCH 100/107] feat(calibration): E21, the two live seasons' data: curves, revisions, learning-truth gap and the HHS burn-in stitch --- .../e21_data_by_season.Rmd | 268 ++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 reports/writeups/calibration_experiments/e21_data_by_season.Rmd diff --git a/reports/writeups/calibration_experiments/e21_data_by_season.Rmd b/reports/writeups/calibration_experiments/e21_data_by_season.Rmd new file mode 100644 index 00000000..3afeea2d --- /dev/null +++ b/reports/writeups/calibration_experiments/e21_data_by_season.Rmd @@ -0,0 +1,268 @@ +--- +title: "E21: the two live seasons' data" +date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" +output: + html_document: + code_folding: hide + toc: true + toc_float: true + toc_depth: 2 +params: + disease: flu +editor_options: + chunk_output_type: console +--- + +```{r setup, echo=FALSE} +knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +knitr::opts_knit$set(root.dir = here::here()) +ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) +``` + +Question: how do 2024-25 and 2025-26 differ as data? Most tracker rankings +flip between the two seasons (ledger), so before reading a ranking as a +property of the tracker, this notebook describes what each season gave the +tracker to learn from: the shape of the season, how much the published +values were later revised, how far the value the tracker learns from sits +from the final one, and how the HHS-stitched 2023-24 burn-in compares with +NHSN. + +Setup: + +- Disease: `r params$disease`. Truth and vintages are the replay's + (`ws_inputs()`): the NHSN archive on hub coordinates, final values for + scoring, every vintage for learning. +- Seasons are August–July years. Hub rounds are the CMU-TimeSeries + submission rounds; states only (US excluded) unless stated. + +```{r load} +source(here::here("scripts/calibration/calibration_ws_replay.R")) +wi <- ws_inputs(params$disease) +states <- function(d) d %>% filter(location != "US") +truth <- states(wi$truth) %>% mutate(season = season_of(target_end_date)) +vint <- states(wi$vintages) %>% mutate(season = season_of(target_end_date)) +rounds <- wi$fc %>% distinct(reference_date) %>% mutate(season = season_of(reference_date)) %>% filter(season %in% LIVE) +pop <- wi$rate_scales %>% select(location, scale) +SEASON_COLS <- c(`2024-2025` = "#D55E00", `2025-2026` = "#0072B2") +``` + +# Season shape + +```{r shape-table} +natl <- truth %>% + filter(season %in% LIVE) %>% + group_by(season, target_end_date) %>% + summarize(v = sum(truth, na.rm = TRUE), .groups = "drop") +peaks <- natl %>% group_by(season) %>% slice_max(v, n = 1, with_ties = FALSE) %>% select(season, peak_week = target_end_date, peak = v) +knitr::kable( + rounds %>% + group_by(season) %>% + summarize(rounds = n(), first_round = min(reference_date), last_round = max(reference_date), .groups = "drop") %>% + left_join(peaks, by = "season") %>% + left_join(natl %>% group_by(season) %>% summarize(total = sum(v), .groups = "drop"), by = "season") %>% + mutate(across(c(peak, total), \(x) round(x))), + caption = "Hub rounds, peak week and size (states summed), and season total admissions" +) +``` + +```{r shape-plot, fig.height=4} +natl_wk <- natl %>% + group_by(season) %>% + mutate(week = as.integer(target_end_date - min(target_end_date)) %/% 7L + 1L) %>% + ungroup() +round_wk <- rounds %>% + left_join(natl %>% group_by(season) %>% summarize(start = min(target_end_date), .groups = "drop"), by = "season") %>% + mutate(week = as.integer(reference_date - start) %/% 7L + 1L) +ggplot(natl_wk, aes(week, v, colour = season)) + + geom_rug(data = round_wk, aes(week, colour = season), inherit.aes = FALSE, sides = "b", alpha = 0.6) + + geom_line() + + scale_colour_manual(values = SEASON_COLS) + + labs(x = "week of the season (from the first August week)", y = "admissions, states summed", colour = NULL, + subtitle = "Ticks along the bottom are hub rounds") + + theme(legend.position = "bottom") +``` + +Per state: when each state peaked relative to the national peak, and how +large the peak was per 100k. + +```{r state-peaks, fig.height=4} +state_pk <- truth %>% + filter(season %in% LIVE) %>% + group_by(season, location) %>% + slice_max(truth, n = 1, with_ties = FALSE) %>% + ungroup() %>% + left_join(peaks, by = "season") %>% + left_join(pop, by = "location") %>% + mutate(weeks_from_national = as.integer(target_end_date - peak_week) %/% 7L, peak_per_100k = truth / scale) +knitr::kable( + state_pk %>% + group_by(season) %>% + summarize( + `peak offset, weeks (10th / 50th / 90th pct)` = sprintf("%+d / %+d / %+d", quantile(weeks_from_national, 0.1) %/% 1, median(weeks_from_national) %/% 1, quantile(weeks_from_national, 0.9) %/% 1), + `states within ±1 week of national` = sum(abs(weeks_from_national) <= 1), + `peak per 100k (median)` = round(median(peak_per_100k, na.rm = TRUE), 1), + .groups = "drop" + ), + caption = "State peaks relative to the national peak week" +) +ggplot(state_pk, aes(weeks_from_national, fill = season)) + + geom_bar(position = "dodge") + + scale_fill_manual(values = SEASON_COLS) + + labs(x = "state peak week minus national peak week", y = "states", fill = NULL) + + theme(legend.position = "bottom") +``` + +# Revisions + +Every vintage of each (state, week) in the season, summarized by +`epiprocess::revision_analysis()`: how many times the value changed, the +spread between its smallest and largest published value relative to the +largest, and how long it took to settle within 10% of the final value. Weeks +before the first archive version (2024-11-19) have no revision history and +are dropped. + +```{r revisions} +archive_of <- function(d) { + d %>% + transmute(geo_value = location, time_value = target_end_date, version, value = truth) %>% + epiprocess::as_epi_archive(compactify = TRUE) +} +first_version <- min(vint$version) +rev <- purrr::map(LIVE, function(s) { + a <- vint %>% filter(season == s, target_end_date >= first_version) %>% archive_of() + a$time_type <- "day" + epiprocess::revision_analysis(a, value, min_waiting_period = NULL, return_only_tibble = TRUE) %>% mutate(season = s) +}) %>% bind_rows() +knitr::kable( + rev %>% + group_by(season) %>% + summarize( + keys = n(), + `no revision` = sprintf("%.0f%%", 100 * mean(n_revisions == 0)), + `revisions (median / 90th pct)` = sprintf("%d / %d", median(n_revisions), quantile(n_revisions, 0.9)), + `rel. spread (median / 90th pct)` = sprintf("%.3f / %.3f", median(rel_spread, na.rm = TRUE), quantile(rel_spread, 0.9, na.rm = TRUE)), + `rel. spread > 10%` = sprintf("%.0f%%", 100 * mean(rel_spread > 0.1, na.rm = TRUE)), + `days to within 10% of final (median / 90th pct)` = sprintf("%s / %s", median(as.numeric(lag_near_latest), na.rm = TRUE), quantile(as.numeric(lag_near_latest), 0.9, na.rm = TRUE)), + .groups = "drop" + ), + caption = "Revision behaviour of (state, week) values, per season" +) +``` + +```{r revisions-plot, fig.height=4} +ggplot(rev %>% filter(!is.na(rel_spread)), aes(rel_spread, colour = season)) + + stat_ecdf() + + coord_cartesian(xlim = c(0, 0.5)) + + scale_colour_manual(values = SEASON_COLS) + + labs(x = "relative spread of published values", y = "share of (state, week) keys", colour = NULL) + + theme(legend.position = "bottom") +``` + +# The value the tracker learns from + +The tracker learns from the value as published, with one week of extra +delay (`settle_days = 14`), so a week's outcome enters about two Wednesdays +after the week ends. Here, for each (state, week), the value as of the +Wednesday L weeks after the week end against the final value: the signed +relative gap (negative = the early value was below the final one), per +season and lag, and by month at L = 2. + +```{r learn-gap} +as_of_lag <- function(L) { + vint %>% + filter(season %in% LIVE, version <= target_end_date + 7L * L - 3L) %>% + group_by(season, location, target_end_date) %>% + slice_max(version, n = 1, with_ties = FALSE) %>% + ungroup() %>% + select(season, location, target_end_date, early = truth) %>% + mutate(lag = L) +} +gap <- purrr::map(1:4, as_of_lag) %>% + bind_rows() %>% + inner_join(truth %>% select(location, target_end_date, final = truth), by = c("location", "target_end_date")) %>% + filter(target_end_date >= first_version, final > 0) %>% + mutate(rel = (early - final) / final, month = factor(format(target_end_date, "%b"), levels = month.abb[c(8:12, 1:7)])) +knitr::kable( + gap %>% + group_by(season, lag) %>% + summarize( + keys = n(), + `not yet published` = sprintf("%.0f%%", 100 * mean(is.na(early))), + `signed gap (10th / 50th / 90th pct)` = sprintf("%+.3f / %+.3f / %+.3f", quantile(rel, 0.1, na.rm = TRUE), median(rel, na.rm = TRUE), quantile(rel, 0.9, na.rm = TRUE)), + `|gap| > 10%` = sprintf("%.0f%%", 100 * mean(abs(rel) > 0.1, na.rm = TRUE)), + .groups = "drop" + ), + caption = "Value as of the Wednesday L weeks after the week end, relative to the final value" +) +``` + +```{r learn-gap-month, fig.height=4} +gm <- gap %>% + filter(lag == 2, !is.na(rel)) %>% + group_by(season, month) %>% + summarize(p10 = quantile(rel, 0.1), p50 = median(rel), p90 = quantile(rel, 0.9), n = n(), .groups = "drop") +ggplot(gm, aes(month, p50, colour = season, group = season)) + + geom_hline(yintercept = 0, colour = "grey60") + + geom_ribbon(aes(ymin = p10, ymax = p90, fill = season), alpha = 0.15, colour = NA) + + geom_line() + + geom_point() + + scale_colour_manual(values = SEASON_COLS) + + scale_fill_manual(values = SEASON_COLS) + + coord_cartesian(ylim = c(-0.3, 0.3)) + + labs(x = NULL, y = "(value at lag 2 − final) / final", colour = NULL, fill = NULL, subtitle = "Median with the 10th–90th percentile band over (state, week)") + + theme(legend.position = "bottom") +``` + +# The 2023-24 burn-in + +Warm configs read the 2023-24 season from the HHS archive stitched in before +NHSN's first version (E13). NHSN's final values also cover 2023-24, so the +two sources can be compared week by week: the latest HHS value against the +NHSN final, by month (states summed) and per state. + +```{r burn-in} +hhs <- readRDS(file.path(CH_CACHE_DIR, "hhs_weekly_archive_2020_2024.rds"))[[params$disease]] %>% + as_tibble() %>% + filter(!is.na(value)) %>% + group_by(geo_value, time_value) %>% + slice_max(version, n = 1, with_ties = FALSE) %>% + ungroup() %>% + # HHS weeks are dated by the Wednesday; NHSN by the Saturday week end. + transmute(geo_value, target_end_date = time_value + 3L, hhs = value) %>% + nhsn_to_hub_locations() %>% + states() %>% + filter(season_of(target_end_date) == "2023-2024") +cmp <- hhs %>% + inner_join(truth %>% select(location, target_end_date, nhsn = truth), by = c("location", "target_end_date")) %>% + mutate(month = factor(format(target_end_date, "%b"), levels = month.abb[c(8:12, 1:7)])) +knitr::kable( + cmp %>% + group_by(month) %>% + summarize(weeks = n_distinct(target_end_date), states = n_distinct(location), hhs = sum(hhs), nhsn = sum(nhsn), .groups = "drop") %>% + mutate(`hhs / nhsn` = sprintf("%.3f", hhs / nhsn), across(c(hhs, nhsn), \(x) round(x))), + caption = "2023-24, states summed: latest HHS value against the NHSN final" +) +# States with no NHSN admissions over the season have no ratio. +state_ratio <- cmp %>% + group_by(location) %>% + summarize(ratio = sum(hhs) / sum(nhsn), .groups = "drop") %>% + filter(is.finite(ratio)) +knitr::kable( + state_ratio %>% + summarize( + `ratio (10th / 50th / 90th pct)` = sprintf("%.3f / %.3f / %.3f", quantile(ratio, 0.1), median(ratio), quantile(ratio, 0.9)), + `states below 0.9` = sum(ratio < 0.9), + `states above 1.1` = sum(ratio > 1.1) + ), + caption = "Per-state ratio of HHS to NHSN over the 2023-24 season" +) +``` + +```{r burn-in-plot, fig.height=4} +ggplot(cmp %>% group_by(target_end_date) %>% summarize(hhs = sum(hhs), nhsn = sum(nhsn), .groups = "drop") %>% tidyr::pivot_longer(-target_end_date), + aes(target_end_date, value, colour = name)) + + geom_line() + + scale_colour_manual(values = c(hhs = "#E69F00", nhsn = "#000000")) + + labs(x = NULL, y = "admissions, states summed", colour = NULL) + + theme(legend.position = "bottom") +``` From 092ee9bd332782c89ceca18216c6e393a94e8e1b Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:39:08 -0700 Subject: [PATCH 101/107] =?UTF-8?q?docs(calibration):=20index=20and=20ledg?= =?UTF-8?q?er=20add=20E19=E2=80=93E21,=20the=20Data=20section=20and=20the?= =?UTF-8?q?=20reorganization=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notes/calibration-ledger.md | 25 +++++++++++++------ .../calibration_experiments/index.Rmd | 16 ++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 6bd4ee41..a0288cd7 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -122,8 +122,12 @@ Finalists (E10 part 1, E18), states only, h1/h2/h3 pooled over both seasons: REF-op has the lower WIS in 44–45 of 52 states). At h−1 sqrt 0.018 gains the most of any config, +14.9 vs REF-op +10.2 (E10 part 2). No config helps covid at h1–h3 on either base. -- **The season flip is broad.** REF-op beats sqrt 0.018 cold in 41–45 of - 52 flu states in 2024-25 and 14–18 in 2025-26 (E18). +- **The season flip is broad, but the pair is unmatched.** REF-op warm + beats sqrt 0.018 cold in 41–45 of 52 flu states in 2024-25 and 14–18 in + 2025-26 (E18). Cold configs play the base through Nov–Dec 2024, and on the + same replay sqrt 0.018 warm beats REF-op warm at every horizon in both + seasons, so the per-state flip is partly the start, not the tracker. E18's + matched pairs (warm vs warm, cold vs cold) are written, not yet rendered. Operating point: for covid nothing tested beats REF-op cold (prod) at h0–h3. For flu, sqrt 0.018 warm is ahead of REF-op warm (prod) by about 1 @@ -182,7 +186,9 @@ gating), exact learning truth, spoiled submissions excluded. | E16 | `e16_sqrt_constant_lr` | replay, cold | constant rate on sqrt scale | sqrt 0.018 about +1 over rate constants at h0; at h1–h3 within 0.2 pooled, up to +0.7 in 2025-26. vs REF-op cold: behind in 2024-25 h0–h1, ahead elsewhere; month-avg coverage tie | 0.018 picked in-sample | | E17 | `e17_late_decay` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | | E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008. Branch sqrt 0.018 warm: +3.4/+1.6/+1.6/+2.5 pooled, best of all, but month-avg coverage 0.011–0.018 worse than base at h2–h3 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance) | -| E19 | none (`calibration_tracker_internals.R`) | replay, cold, final truth | why do the leak and constant 0.018 score alike? | REF-op cold and no-leak eta identical; adaptive eta / width peaks Apr–May (h3 0.10) vs constant ≤ 0.03. h3 median offset / width, Feb/Mar/Apr/May 2025: REF-op 0.045/0.078/0.114/0.009, no leak 0.048/0.109/0.237/0.199, constant 0.013/0.026/0.057/0.074; Apr/May 2026: 0.093/0.013, 0.165/0.149, 0.101/0.143. Share of truths above the base median at h3: Mar 0.27–0.43, Apr–May 2026 0.62–0.71 | final truth, not exact; h1–h3; median-level offset only; no WIS by month in the script | +| E19 | `e19_tracker_internals` (was a script; the numbers here are the script's, on final truth, until the notebook is rendered) | replay | why do the leak and constant 0.018 score alike? Step size, offset and base bias by month on the E20 grid; carry vs reset on both anchors | REF-op cold and no-leak eta identical; adaptive eta / width peaks Apr–May (h3 0.10) vs constant ≤ 0.03. h3 median offset / width, Feb/Mar/Apr/May 2025: REF-op 0.045/0.078/0.114/0.009, no leak 0.048/0.109/0.237/0.199, constant 0.013/0.026/0.057/0.074; Apr/May 2026: 0.093/0.013, 0.165/0.149, 0.101/0.143. Share of truths above the base median at h3: Mar 0.27–0.43, Apr–May 2026 0.62–0.71 | script numbers: final truth, h1–h3, median-level offset only, no WIS by month. The notebook uses exact truth and adds WIS by month and the resets; not yet rendered | +| E20 | `e20_structure_grid` | replay | warm start and leak on both anchors: {adaptive 0.03, constant 0.018} × {cold, warm} × {no leak, leak 0.1}, plus REF-op warm and cold | not yet run | replaces E03–E05 and E11's questions on the replay; those stay until read against it. The slow term is left out (E18 step 2) | +| E21 | `e21_data_by_season` | replay data | how the two live seasons differ as data: shape, revisions, learning-truth gap at lags 1–4, HHS-stitched burn-in vs NHSN | 2024-25: 27 rounds from 2024-11-23, peak 2025-02-08 at 55.6k (states summed), total 559k; 2025-26: 28 rounds from 2025-10-18, peak 2026-01-03 at 42.5k, total 336k. State peaks within ±1 week of the national one: 37 vs 47 of 52. Revisions: 26% vs 40% of (state, week) keys never revised; relative spread > 10% for 48% vs 39%. Value at lag 2 (what the tracker learns from) vs final: median gap −1.1% vs 0.0%, \|gap\| > 10% for 19% vs 14% of keys; at lag 1, 44% vs 41%. HHS vs NHSN final over 2023-24: 0.957–0.978 in Nov–Apr (states summed), per-state median 1.004 | describes the data only; no tracker runs. Revision keys start at the first archive version (2024-11-19) | E06 (ILI+ burn-in) and E08 (old covid) were run on code or data later found broken and were not rebuilt. @@ -218,7 +224,9 @@ Checked and resolved on 2026-10-02: |---|---| | E00, E01 | verified | | E02, E07, E09 | commented; changes made, not re-verified | -| E10 part 1, E18 | changes made (finalists runs, state counts), not reviewed | +| E10 part 1, E18 | changes made (finalists runs, state counts), not reviewed; E18's matched pairs not yet rendered | +| E19, E20 | written 2026-10-06, not run | +| E21 | rendered 2026-10-06, not reviewed | | E03, E04, E05, E10, E11, E14, E15, E16, E17 | not reviewed | On 2026-10-02 every notebook's introduction and Findings were rewritten to @@ -240,8 +248,10 @@ reviewed. - `R/calibration/views.R`: reference configs, cached runs, standard views. - `scripts/calibration/calibration_ws_replay.R`: E12/E13. Scores in `cache/calibration/ws_replay_scores_*.csv`. -- `scripts/calibration/calibration_tracker_internals.R`: E19. Per-round - eta and offsets in `cache/calibration/tracker_internals/`. +- `scripts/calibration/calibration_ws_replay.R` also holds the notebook + helpers: `ws_grid_configs()` (the E19/E20 structure grid), + `ws_prod_specs()`, `ws_run_spec()`, `ws_wis_by_season()` and + `ws_internals()` (per-round eta and offsets from an exact run). - `scripts/calibration/calibration_ili_backfill.R`: replays `windowed_seasonal` over the ILI+ state history (2010–2024) into hub-schema parquets in `cache/calibration/`, for an ILI+ burn-in (thread closed 2026-10-02, unused). @@ -314,10 +324,11 @@ operating point. | item | status | |---|---| +| **Reorganization (2026-10-06).** The constant-rate line was never given the structure analysis (E03–E05, E11 ran the adaptive rate on the ensemble; E16 inherited "cold" from E11's framing and E13's constant 0.1 warm covid loss). Plan: (1) E18 matched-pair scatters, warm vs warm and cold vs cold; (2) E20, the {anchor} × {start} × {leak} grid on the replay; (3) E19 as a notebook on that grid with WIS by month and carry vs reset; (4) E21, the seasons as data. Then re-read E03–E05 and E11 against E20 and retire or keep them | written, not run: E18's new pair uses cached runs; E19 and E20 need about 7 new exact runs (the four leak variants, adaptive warm without leak, the two resets; the rest are cached from E01, E13, E16 and E18). E21 rendered 2026-10-06. Render with `just calibration-experiments e18_ref_op_bridge e19_tracker_internals e20_structure_grid` inside the rocker container | | Uncertainty for WIS and coverage differences (bootstrap over rounds or locations) | open; needed before any ranking under ~2 points means anything | | Shrink only the played offset after the peak, keep the stored state | open; `late_decay` shrinks the stored state | | Disease-specific operating point | provisional (2026-10-02): covid REF-op cold; flu REF-op warm or sqrt 0.018 warm, to confirm after the notebook review. Not implemented | -| HHS burn-in level (3–9% below NHSN) | used as-is (E13, E18); settle it with item 1f if a warm flu config is chosen | +| HHS burn-in level | E21: 2–4% below the NHSN final in Nov–Apr 2023-24 (states summed), per-state median ratio 1.004. Used as-is (E13, E18); settle it with item 1f if a warm flu config is chosen | | `windowed_seasonal_extra_sources` retrospective | parked (ROADMAP) | | Method ideas: cross-horizon gradients for staleness, data-driven phase gating, scale proxy | open; no code | | Collaborator email | unknown | diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index f929633a..ef9bb3b1 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -64,11 +64,21 @@ on the submitted ensemble. | `r link("E01", "e01_eta_settings")` | replay, cold | How do the adaptive learning-rate settings (multiplier, window, carry or reset) change WIS and coverage? | | `r link("E09", "e09_lr_delay")` | ensemble | With a constant learning rate on the rate scale, how do WIS and coverage change with the rate and with extra learning delay? | | `r link("E16", "e16_sqrt_constant_lr")` | replay, cold | How do WIS and coverage change with a constant learning rate, season by season, on the square-root and rate scales? | +| `r link("E19", "e19_tracker_internals")` | replay | Month by month, how large a step does each tracker take, how large an offset does it play, and does the month's WIS change follow from those and the base's bias? How much of 2025-26 is the offset carried in from 2024-25? | + +The two learning-rate anchors used by the structure notebooks are E01's +adaptive multiplier 0.03 and E16's constant 0.018 on the sqrt per-100k +scale. ## Tracker structure +E20 asks the warm-start and leak questions on both anchors, on the replay. +E03, E04, E05 and E11 asked them on the ensemble with the adaptive rate +only; they stay until E20 has been read against them. + | | base | question | |---|---|---| +| `r link("E20", "e20_structure_grid")` | replay | Do the warm start and the leak change WIS and coverage in the same way on the adaptive and the constant learning rate? State by state, which anchor wins at the same start and leak? | | `r link("E03", "e03_scale")` | ensemble | Which scale should the offsets live on: counts, square root, or rate per 100k? | | `r link("E04", "e04_offset_structure")` | ensemble | Does a leak or a second, slow offset term change WIS and coverage? | | `r link("E05", "e05_warm_start")` | ensemble | Does a warm start from the burn-in season change WIS and coverage for each tracker? | @@ -87,6 +97,12 @@ on the submitted ensemble. |---|---|---| | `r link("E00", "e00_vintage_backtest")` | ensemble | Should the tracker learn from finalized data or from data as published? | +## Data + +| | base | question | +|---|---|---| +| `r link("E21", "e21_data_by_season")` | replay data | How do 2024-25 and 2025-26 differ as data: season shape, revisions, the gap between the value the tracker learns from and the final one, and the HHS-stitched 2023-24 burn-in against NHSN? | + E12 and E13 (the clean replay without and with a 2023-24 burn-in) have no notebook; they are `scripts/calibration/calibration_ws_replay.R`. E14 is part of E16, and E07b's ranking is part of E10. From 04583904ae85f68a792a71a9800196cb2377a138 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 7 Oct 2026 02:04:25 -0700 Subject: [PATCH 102/107] =?UTF-8?q?docs(calibration):=20read=20the=20E18?= =?UTF-8?q?=E2=80=93E20=20renders=20into=20the=20ledger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notes/calibration-ledger.md | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index a0288cd7..85595b57 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -126,13 +126,27 @@ Finalists (E10 part 1, E18), states only, h1/h2/h3 pooled over both seasons: beats sqrt 0.018 cold in 41–45 of 52 flu states in 2024-25 and 14–18 in 2025-26 (E18). Cold configs play the base through Nov–Dec 2024, and on the same replay sqrt 0.018 warm beats REF-op warm at every horizon in both - seasons, so the per-state flip is partly the start, not the tracker. E18's - matched pairs (warm vs warm, cold vs cold) are written, not yet rendered. + seasons, so the per-state flip is the start, not the tracker: matched, + REF-op wins 23 of 52 states in 2024-25 and 15 in 2025-26 warm vs warm, + 17 and 15 cold vs cold (E18). + +- **On the replay, matched on the start, the constant rate leads (E20).** + Constant 0.018 beats adaptive 0.03 in 29–47 of 52 states in every cell + and season. Warm plus leak is the best cell for either anchor; for the + constant rate +3.3/+1.8/+1.9/+2.9 pooled vs REF-op warm +2.8/+1.1/+0.9/ + +1.1, with month-avg coverage 0.012–0.017 above base at h2–h3 in + 2025-26. +- **The constant rate's 2025-26 lead is mostly inherited (E19).** Resetting + its offset at the season boundary takes 2025-26 from +3.0/+1.0/+1.1/+1.5 + to +1.0/+0.1/+0.2/+0.3. Cold configs enter 2024-25 with nothing, so the + two seasons are not the same test of a tracker. Operating point: for covid nothing tested beats REF-op cold (prod) at h0–h3. For flu, sqrt 0.018 warm is ahead of REF-op warm (prod) by about 1 point at h1–h2 and 2 at h3 on the ensemble, all of it in 2024-25, with -month-avg coverage closer to the base than REF-op's. +month-avg coverage closer to the base than REF-op's. E20 adds the leak to +it on the replay (+0.2 to +0.7 at h1–h3 pooled, coverage 0.005–0.012 +better); not yet run on the ensemble. ## Reference configs @@ -185,9 +199,9 @@ gating), exact learning truth, spoiled submissions excluded. | E15 | `e15_wis_sources` | replay, cold | where the WIS gain comes from | offsets a few % of interval width; REF-op gain from the lower half and the median shift | width-only variant's coverage not measured; split is not additive | | E16 | `e16_sqrt_constant_lr` | replay, cold | constant rate on sqrt scale | sqrt 0.018 about +1 over rate constants at h0; at h1–h3 within 0.2 pooled, up to +0.7 in 2025-26. vs REF-op cold: behind in 2024-25 h0–h1, ahead elsewhere; month-avg coverage tie | 0.018 picked in-sample | | E17 | `e17_late_decay` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | -| E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008. Branch sqrt 0.018 warm: +3.4/+1.6/+1.6/+2.5 pooled, best of all, but month-avg coverage 0.011–0.018 worse than base at h2–h3 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance) | -| E19 | `e19_tracker_internals` (was a script; the numbers here are the script's, on final truth, until the notebook is rendered) | replay | why do the leak and constant 0.018 score alike? Step size, offset and base bias by month on the E20 grid; carry vs reset on both anchors | REF-op cold and no-leak eta identical; adaptive eta / width peaks Apr–May (h3 0.10) vs constant ≤ 0.03. h3 median offset / width, Feb/Mar/Apr/May 2025: REF-op 0.045/0.078/0.114/0.009, no leak 0.048/0.109/0.237/0.199, constant 0.013/0.026/0.057/0.074; Apr/May 2026: 0.093/0.013, 0.165/0.149, 0.101/0.143. Share of truths above the base median at h3: Mar 0.27–0.43, Apr–May 2026 0.62–0.71 | script numbers: final truth, h1–h3, median-level offset only, no WIS by month. The notebook uses exact truth and adds WIS by month and the resets; not yet rendered | -| E20 | `e20_structure_grid` | replay | warm start and leak on both anchors: {adaptive 0.03, constant 0.018} × {cold, warm} × {no leak, leak 0.1}, plus REF-op warm and cold | not yet run | replaces E03–E05 and E11's questions on the replay; those stay until read against it. The slow term is left out (E18 step 2) | +| E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008. Branch sqrt 0.018 warm: +3.4/+1.6/+1.6/+2.5 pooled, best of all, but month-avg coverage 0.011–0.018 worse than base at h2–h3. States where REF-op wins, 2024-25 / 2025-26: chain ends (warm vs cold) 45 / 13 of 52; warm vs warm 23 / 15; cold vs cold 17 / 15 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance). The chain-ends pair is unmatched on the start | +| E19 | `e19_tracker_internals` | replay | why do the leak and constant 0.018 score alike? Step size, offset, base bias and WIS by month on the E20 grid; carry vs reset on both anchors | Step / width at h3 rises from 0.015–0.02 (Dec–Feb) to 0.06–0.09 (Apr–May) on the adaptive rate, 0.005–0.007 to 0.013–0.023 on the constant, in both seasons. Offset / width at h3, Mar/Apr/May 2025: adaptive cold no leak 0.105/0.218/0.186, with leak 0.064/0.078/−0.012, constant cold 0.026/0.055/0.072; Apr/May 2026: 0.160/0.150, 0.057/−0.011, 0.097/0.143. Base bias (share above the base median, h3): Mar 0.43 and 0.27, Apr–May 0.54–0.81 and 0.62–0.71. March WIS at h3: adaptive no leak −48 and −81 (2025, 2026), with leak −27 and −22, constant cold −9 and −35, with leak −5 and −8; Apr–May 2026 constant no leak +4.8/+11.0. **Carry vs reset:** resetting at the season boundary takes the constant tracker's 2025-26 from +3.0/+1.0/+1.1/+1.5 to +1.0/+0.1/+0.2/+0.3, the adaptive one's from +2.2/−0.6/−0.7/−0.2 to +0.7/−0.9/−1.3/−1.6; the carried offset is 0.01–0.025 of width. Carry's gain is Oct–Feb (h3 Nov–Dec 2025 +5.7/+3.8 vs +0.6/+0.4); reset is better after the peak | offsets from the exact runs, step sizes from one pass on final truth per config. `lr_level` for the two-term REF-op configs reports the slow term's rate (about a tenth of the fast one), so their step-size rows are not comparable; REF-op's fast step equals the adaptive anchor's by construction | +| E20 | `e20_structure_grid` | replay | warm start and leak on both anchors: {adaptive 0.03, constant 0.018} × {cold, warm} × {no leak, leak 0.1}, plus REF-op warm and cold | Warm start: +1.0 to +2.6 (adaptive) and +1.5 to +3.8 (constant) in 2024-25, −0.5 to +0.6 in 2025-26, on both anchors at either leak. Leak: on the adaptive rate +0.7 to +2.0 at h1–h3 in both seasons; on the constant rate −0.2 to +0.7 warm and −0.6 to −1.7 in 2025-26 cold. Best cell constant warm leak: +3.3/+1.8/+1.9/+2.9 pooled vs REF-op warm +2.8/+1.1/+0.9/+1.1, but month-avg coverage 0.012–0.017 above base at h2–h3 in 2025-26 (the warm constant without leak is 0.024–0.037 above). Constant beats adaptive in 29–47 of 52 states in every cell and season | replaces E03–E05 and E11's questions on the replay; those stay until read against it. The slow term is left out (E18 step 2). All constant-rate configs tuned in-sample (E16) | | E21 | `e21_data_by_season` | replay data | how the two live seasons differ as data: shape, revisions, learning-truth gap at lags 1–4, HHS-stitched burn-in vs NHSN | 2024-25: 27 rounds from 2024-11-23, peak 2025-02-08 at 55.6k (states summed), total 559k; 2025-26: 28 rounds from 2025-10-18, peak 2026-01-03 at 42.5k, total 336k. State peaks within ±1 week of the national one: 37 vs 47 of 52. Revisions: 26% vs 40% of (state, week) keys never revised; relative spread > 10% for 48% vs 39%. Value at lag 2 (what the tracker learns from) vs final: median gap −1.1% vs 0.0%, \|gap\| > 10% for 19% vs 14% of keys; at lag 1, 44% vs 41%. HHS vs NHSN final over 2023-24: 0.957–0.978 in Nov–Apr (states summed), per-state median 1.004 | describes the data only; no tracker runs. Revision keys start at the first archive version (2024-11-19) | E06 (ILI+ burn-in) and E08 (old covid) were run on code or data later found @@ -225,7 +239,8 @@ Checked and resolved on 2026-10-02: | E00, E01 | verified | | E02, E07, E09 | commented; changes made, not re-verified | | E10 part 1, E18 | changes made (finalists runs, state counts), not reviewed; E18's matched pairs not yet rendered | -| E19, E20 | written 2026-10-06, not run | +| E20 | rendered 2026-10-07, not reviewed | +| E19 | rendered 2026-10-07, not reviewed | | E21 | rendered 2026-10-06, not reviewed | | E03, E04, E05, E10, E11, E14, E15, E16, E17 | not reviewed | @@ -324,7 +339,7 @@ operating point. | item | status | |---|---| -| **Reorganization (2026-10-06).** The constant-rate line was never given the structure analysis (E03–E05, E11 ran the adaptive rate on the ensemble; E16 inherited "cold" from E11's framing and E13's constant 0.1 warm covid loss). Plan: (1) E18 matched-pair scatters, warm vs warm and cold vs cold; (2) E20, the {anchor} × {start} × {leak} grid on the replay; (3) E19 as a notebook on that grid with WIS by month and carry vs reset; (4) E21, the seasons as data. Then re-read E03–E05 and E11 against E20 and retire or keep them | written, not run: E18's new pair uses cached runs; E19 and E20 need about 7 new exact runs (the four leak variants, adaptive warm without leak, the two resets; the rest are cached from E01, E13, E16 and E18). E21 rendered 2026-10-06. Render with `just calibration-experiments e18_ref_op_bridge e19_tracker_internals e20_structure_grid` inside the rocker container | +| **Reorganization (2026-10-06).** The constant-rate line was never given the structure analysis (E03–E05, E11 ran the adaptive rate on the ensemble; E16 inherited "cold" from E11's framing and E13's constant 0.1 warm covid loss). Plan: (1) E18 matched-pair scatters, warm vs warm and cold vs cold; (2) E20, the {anchor} × {start} × {leak} grid on the replay; (3) E19 as a notebook on that grid with WIS by month and carry vs reset; (4) E21, the seasons as data. Then re-read E03–E05 and E11 against E20 and retire or keep them | all four rendered 2026-10-07 (E21 on 2026-10-06) and read into this file; E03–E05 and E11 not yet re-read against E20. Flu-only so far; covid needs the same grid. Render with `just calibration-experiments e18_ref_op_bridge e19_tracker_internals e20_structure_grid` inside the rocker container | | Uncertainty for WIS and coverage differences (bootstrap over rounds or locations) | open; needed before any ranking under ~2 points means anything | | Shrink only the played offset after the peak, keep the stored state | open; `late_decay` shrinks the stored state | | Disease-specific operating point | provisional (2026-10-02): covid REF-op cold; flu REF-op warm or sqrt 0.018 warm, to confirm after the notebook review. Not implemented | From abad0f6e32703af4b9c5d3bf8e41836f85107646 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 7 Oct 2026 02:17:36 -0700 Subject: [PATCH 103/107] feat(calibration): E20 runs the structure grid on covid too; sqrt 0.018 warm + leak joins the finalists --- notes/calibration-ledger.md | 35 ++-- .../e20_structure_grid.Rmd | 169 ++++++++++-------- .../calibration_experiments/index.Rmd | 5 +- scripts/calibration/calibration_ws_replay.R | 4 +- 4 files changed, 118 insertions(+), 95 deletions(-) diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 85595b57..1b574423 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -110,13 +110,17 @@ Coverage: Finalists (E10 part 1, E18), states only, h1/h2/h3 pooled over both seasons: -- **Flu: sqrt 0.018 with a warm start leads on both bases.** Ensemble - +2.5/+2.7/+4.3 vs REF-op warm +1.9/+1.5/+2.3; replay +1.6/+1.5/+2.7 vs - +1.1/+0.8/+1.2. On the ensemble all of its lead is 2024-25; in 2025-26 it - is −0.2/−0.9/−1.7, slightly below REF-op warm. Month-avg coverage: on the - ensemble REF-op warm is the only config below the base at every horizon; - sqrt warm is level with the base there and 0.009–0.017 above it at h2–h3 - on the replay. +- **Flu: sqrt 0.018 warm + leak leads on both bases (E10, E20).** + Ensemble +2.8/+3.0/+4.8 vs sqrt 0.018 warm +2.5/+2.7/+4.3 and REF-op warm + +1.9/+1.5/+2.3; replay +1.8/+1.9/+2.9 vs +1.6/+1.5/+2.7 and +1.1/+0.8/+1.2. + The leak repairs the warm constant's 2025-26 on the ensemble: +0.7/−0.0/ + −0.8 vs −0.2/−0.9/−1.7 (REF-op warm +0.1/−0.7/−0.9). Month-avg coverage on + the ensemble, pooled: 0.108/0.126/0.140 at h1–h3 vs REF-op warm + 0.110/0.124/0.138 and base 0.123/0.137/0.151; in 2025-26 it is 0.004–0.007 + below REF-op warm at h1–h2. Ranks first pooled on both bases, with REF-op + warm first on ensemble coverage by 0.001. +- **Covid: no grid cell helps at h1–h3, and the warm start hurts (E20).** + See the E20 row; REF-op cold stays the least bad config overall. - **Covid: sqrt 0.018 is worse than REF-op cold on the ensemble.** −2.7/−3.7/−4.2 vs −0.1/−0.3/−0.8, from 2025-26 (−6.9 to −8.3 at h1–h3; REF-op has the lower WIS in 44–45 of 52 states). At h−1 sqrt 0.018 gains @@ -142,11 +146,10 @@ Finalists (E10 part 1, E18), states only, h1/h2/h3 pooled over both seasons: two seasons are not the same test of a tracker. Operating point: for covid nothing tested beats REF-op cold (prod) at -h0–h3. For flu, sqrt 0.018 warm is ahead of REF-op warm (prod) by about 1 -point at h1–h2 and 2 at h3 on the ensemble, all of it in 2024-25, with -month-avg coverage closer to the base than REF-op's. E20 adds the leak to -it on the replay (+0.2 to +0.7 at h1–h3 pooled, coverage 0.005–0.012 -better); not yet run on the ensemble. +h0–h3. For flu, sqrt 0.018 warm + leak is ahead of REF-op warm (prod) by +about 1 point at h1, 1.5 at h2 and 2.5 at h3 on the ensemble, no longer +behind it in 2025-26, with month-avg coverage within 0.002 of REF-op +warm's. Its rate and leak were picked in-sample. ## Reference configs @@ -191,7 +194,7 @@ gating), exact learning truth, spoiled submissions excluded. | E05 | `e05_warm_start` | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | | E07 | `e07_across_eras`; the candidate ranking (formerly E07b) is in `e10_covid` | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | | | E09 | `e09_lr_delay` | ensemble | constant rate × revision delay | knee 0.056: +3 to +5.5 over REF-paper at h1–h3; REF-op +1.2 to +2.4 over knee. Extra delay costs about 1 h−1 point per week. Above 0.1 loses steeply | month-avg: no constant rate beats base at h1–h3 | -| E10 | `e10_covid` | flu + covid, ensemble + replay | the finalists on both diseases and bases (part 1); REF-paper, REF-op and sqrt 0.018 on the covid ensemble at h−1–h3 (part 2) | see "Finalists" above. Part 2: h−1 +10 to +15 (sqrt 0.018 largest). REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 and sqrt 0.018 −7.7 to −9.5 at h1–h3 in 2025-26 | part 1 is h0–h3 on rows both bases have; part 2 tables run to 2026-09-19, including the 2026 summer wave | +| E10 | `e10_covid` | flu + covid, ensemble + replay | the finalists (now including sqrt 0.018 warm + leak, E20's best flu cell) on both diseases and bases (part 1); REF-paper, REF-op and sqrt 0.018 on the covid ensemble at h−1–h3 (part 2) | see "Finalists" above. Part 2: h−1 +10 to +15 (sqrt 0.018 largest). REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 and sqrt 0.018 −7.7 to −9.5 at h1–h3 in 2025-26 | part 1 is h0–h3 on rows both bases have; part 2 tables run to 2026-09-19, including the 2026 summer wave | | E11 | `e11_constant_lr` | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | | E12 | none (`calibration_ws_replay.R`) | replay, cold, flu + covid | first look at the clean replay | flu: REF-op cold leads constant 0.1 by 0–2.5. Covid: REF-op cold h0 +1.3 states, losses at h1–h3 in 2025-26 | **all locations** (US is 47% of WIS); covid h0 gains are mostly US | | E13 | none (`calibration_ws_replay.R burn_in`) | replay, flu + covid | 2023-24 HHS burn-in | flu warm +1 to +2.6 in 2024-25, ≤ 0 in 2025-26. Covid: warm worse at h0–h1, better at h2–h3 (+1.3, +4.4); constant 0.1 warm −8 to −27 | HHS used as-is (3–9% below NHSN); coverage is season-pooled only | @@ -201,7 +204,7 @@ gating), exact learning truth, spoiled submissions excluded. | E17 | `e17_late_decay` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | | E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008. Branch sqrt 0.018 warm: +3.4/+1.6/+1.6/+2.5 pooled, best of all, but month-avg coverage 0.011–0.018 worse than base at h2–h3. States where REF-op wins, 2024-25 / 2025-26: chain ends (warm vs cold) 45 / 13 of 52; warm vs warm 23 / 15; cold vs cold 17 / 15 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance). The chain-ends pair is unmatched on the start | | E19 | `e19_tracker_internals` | replay | why do the leak and constant 0.018 score alike? Step size, offset, base bias and WIS by month on the E20 grid; carry vs reset on both anchors | Step / width at h3 rises from 0.015–0.02 (Dec–Feb) to 0.06–0.09 (Apr–May) on the adaptive rate, 0.005–0.007 to 0.013–0.023 on the constant, in both seasons. Offset / width at h3, Mar/Apr/May 2025: adaptive cold no leak 0.105/0.218/0.186, with leak 0.064/0.078/−0.012, constant cold 0.026/0.055/0.072; Apr/May 2026: 0.160/0.150, 0.057/−0.011, 0.097/0.143. Base bias (share above the base median, h3): Mar 0.43 and 0.27, Apr–May 0.54–0.81 and 0.62–0.71. March WIS at h3: adaptive no leak −48 and −81 (2025, 2026), with leak −27 and −22, constant cold −9 and −35, with leak −5 and −8; Apr–May 2026 constant no leak +4.8/+11.0. **Carry vs reset:** resetting at the season boundary takes the constant tracker's 2025-26 from +3.0/+1.0/+1.1/+1.5 to +1.0/+0.1/+0.2/+0.3, the adaptive one's from +2.2/−0.6/−0.7/−0.2 to +0.7/−0.9/−1.3/−1.6; the carried offset is 0.01–0.025 of width. Carry's gain is Oct–Feb (h3 Nov–Dec 2025 +5.7/+3.8 vs +0.6/+0.4); reset is better after the peak | offsets from the exact runs, step sizes from one pass on final truth per config. `lr_level` for the two-term REF-op configs reports the slow term's rate (about a tenth of the fast one), so their step-size rows are not comparable; REF-op's fast step equals the adaptive anchor's by construction | -| E20 | `e20_structure_grid` | replay | warm start and leak on both anchors: {adaptive 0.03, constant 0.018} × {cold, warm} × {no leak, leak 0.1}, plus REF-op warm and cold | Warm start: +1.0 to +2.6 (adaptive) and +1.5 to +3.8 (constant) in 2024-25, −0.5 to +0.6 in 2025-26, on both anchors at either leak. Leak: on the adaptive rate +0.7 to +2.0 at h1–h3 in both seasons; on the constant rate −0.2 to +0.7 warm and −0.6 to −1.7 in 2025-26 cold. Best cell constant warm leak: +3.3/+1.8/+1.9/+2.9 pooled vs REF-op warm +2.8/+1.1/+0.9/+1.1, but month-avg coverage 0.012–0.017 above base at h2–h3 in 2025-26 (the warm constant without leak is 0.024–0.037 above). Constant beats adaptive in 29–47 of 52 states in every cell and season | replaces E03–E05 and E11's questions on the replay; those stay until read against it. The slow term is left out (E18 step 2). All constant-rate configs tuned in-sample (E16) | +| E20 | `e20_structure_grid` | replay | warm start and leak on both anchors: {adaptive 0.03, constant 0.018} × {cold, warm} × {no leak, leak 0.1}, plus REF-op warm and cold | Warm start: +1.0 to +2.6 (adaptive) and +1.5 to +3.8 (constant) in 2024-25, −0.5 to +0.6 in 2025-26, on both anchors at either leak. Leak: on the adaptive rate +0.7 to +2.0 at h1–h3 in both seasons; on the constant rate −0.2 to +0.7 warm and −0.6 to −1.7 in 2025-26 cold. Best cell constant warm leak: +3.3/+1.8/+1.9/+2.9 pooled vs REF-op warm +2.8/+1.1/+0.9/+1.1, but month-avg coverage 0.012–0.017 above base at h2–h3 in 2025-26 (the warm constant without leak is 0.024–0.037 above). Constant beats adaptive in 29–47 of 52 states in every cell and season. **Covid:** the warm start costs 2–7 points at h0–h2 in 2024-25 (constant) and 1–3 (adaptive), and 0.01–0.04 of month-avg coverage; the leak adds 1–3 on the adaptive anchor and on the cold constant in 2024-25, costs 1–3 on the warm constant. No cell beats base at h1–h3 pooled; least bad is constant cold leak −0.5/−0.8/−0.5 vs REF-op cold −1.1/−1.7/−1.6, adaptive warm leak +3.4 at h3 but −1.9 at h1 | replaces E03–E05 and E11's questions on the replay; those stay until read against it. The slow term is left out (E18 step 2). All constant-rate configs tuned in-sample (E16) | | E21 | `e21_data_by_season` | replay data | how the two live seasons differ as data: shape, revisions, learning-truth gap at lags 1–4, HHS-stitched burn-in vs NHSN | 2024-25: 27 rounds from 2024-11-23, peak 2025-02-08 at 55.6k (states summed), total 559k; 2025-26: 28 rounds from 2025-10-18, peak 2026-01-03 at 42.5k, total 336k. State peaks within ±1 week of the national one: 37 vs 47 of 52. Revisions: 26% vs 40% of (state, week) keys never revised; relative spread > 10% for 48% vs 39%. Value at lag 2 (what the tracker learns from) vs final: median gap −1.1% vs 0.0%, \|gap\| > 10% for 19% vs 14% of keys; at lag 1, 44% vs 41%. HHS vs NHSN final over 2023-24: 0.957–0.978 in Nov–Apr (states summed), per-state median 1.004 | describes the data only; no tracker runs. Revision keys start at the first archive version (2024-11-19) | E06 (ILI+ burn-in) and E08 (old covid) were run on code or data later found @@ -239,7 +242,7 @@ Checked and resolved on 2026-10-02: | E00, E01 | verified | | E02, E07, E09 | commented; changes made, not re-verified | | E10 part 1, E18 | changes made (finalists runs, state counts), not reviewed; E18's matched pairs not yet rendered | -| E20 | rendered 2026-10-07, not reviewed | +| E20 (flu and covid), E10, E15 | rendered 2026-10-07 with the covid grid and the new finalist, not reviewed | | E19 | rendered 2026-10-07, not reviewed | | E21 | rendered 2026-10-06, not reviewed | | E03, E04, E05, E10, E11, E14, E15, E16, E17 | not reviewed | @@ -339,7 +342,7 @@ operating point. | item | status | |---|---| -| **Reorganization (2026-10-06).** The constant-rate line was never given the structure analysis (E03–E05, E11 ran the adaptive rate on the ensemble; E16 inherited "cold" from E11's framing and E13's constant 0.1 warm covid loss). Plan: (1) E18 matched-pair scatters, warm vs warm and cold vs cold; (2) E20, the {anchor} × {start} × {leak} grid on the replay; (3) E19 as a notebook on that grid with WIS by month and carry vs reset; (4) E21, the seasons as data. Then re-read E03–E05 and E11 against E20 and retire or keep them | all four rendered 2026-10-07 (E21 on 2026-10-06) and read into this file; E03–E05 and E11 not yet re-read against E20. Flu-only so far; covid needs the same grid. Render with `just calibration-experiments e18_ref_op_bridge e19_tracker_internals e20_structure_grid` inside the rocker container | +| **Reorganization (2026-10-06).** The constant-rate line was never given the structure analysis (E03–E05, E11 ran the adaptive rate on the ensemble; E16 inherited "cold" from E11's framing and E13's constant 0.1 warm covid loss). Plan: (1) E18 matched-pair scatters, warm vs warm and cold vs cold; (2) E20, the {anchor} × {start} × {leak} grid on the replay; (3) E19 as a notebook on that grid with WIS by month and carry vs reset; (4) E21, the seasons as data. Then re-read E03–E05 and E11 against E20 and retire or keep them | all rendered 2026-10-07 (E21 on 2026-10-06) and read into this file; E20 covers covid too, and sqrt 0.018 warm + leak is a finalist in E10 and E15. E03–E05 and E11 not yet re-read against E20 | Render with `just calibration-experiments e18_ref_op_bridge e19_tracker_internals e20_structure_grid` inside the rocker container | | Uncertainty for WIS and coverage differences (bootstrap over rounds or locations) | open; needed before any ranking under ~2 points means anything | | Shrink only the played offset after the peak, keep the stored state | open; `late_decay` shrinks the stored state | | Disease-specific operating point | provisional (2026-10-02): covid REF-op cold; flu REF-op warm or sqrt 0.018 warm, to confirm after the notebook review. Not implemented | diff --git a/reports/writeups/calibration_experiments/e20_structure_grid.Rmd b/reports/writeups/calibration_experiments/e20_structure_grid.Rmd index 91b925a9..e8cf78e6 100644 --- a/reports/writeups/calibration_experiments/e20_structure_grid.Rmd +++ b/reports/writeups/calibration_experiments/e20_structure_grid.Rmd @@ -1,41 +1,42 @@ --- -title: "E20: warm start and leak on both learning-rate anchors" +title: "E20: warm start and leak on both learning-rate anchors, flu and covid" date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" output: html_document: code_folding: hide toc: true toc_float: true - toc_depth: 2 + toc_depth: 3 params: workers: 14 - disease: flu editor_options: chunk_output_type: console --- ```{r setup, echo=FALSE} knitr::opts_chunk$set(fig.align = "center", fig.width = 11, fig.height = 5, message = FALSE, warning = FALSE) +# The month view is knit once per disease, so its chunk labels repeat. +options(knitr.duplicate.label = "allow") knitr::opts_knit$set(root.dir = here::here()) ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) ``` Question: do the warm start and the leak change WIS and coverage in the same -way on the adaptive learning rate and on the constant one? E04 and E05 asked -this on the ensemble with the adaptive rate only; E18 found that the leak's -effect depends on how large the adaptive step gets late in the season, so the -two anchors may answer differently. +way on the adaptive learning rate and on the constant one, and on both +diseases? E04 and E05 asked this on the flu ensemble with the adaptive rate +only; E18 found that the leak's effect depends on how large the adaptive +step gets late in the season, so the two anchors may answer differently. Setup: -- Disease: `r params$disease`. Base forecaster: the clean `windowed_seasonal` +- Diseases: flu and covid. Base forecaster: the clean `windowed_seasonal` replay (53 locations, the hub rounds). Horizons h0–h3. - Learning truth: exact (re-run at every round on the data published by then; E00), one week of extra delay. - Warm configs run with the 2023-24 season present (the HHS-stitched backfill from E13) and start the offset from that season's error quantiles; cold configs start at zero with no burn-in rounds, so they play the base through - Nov–Dec 2024. + the first live rounds. - Scores over the two live seasons, each on its own rounds; states only (US excluded). @@ -44,7 +45,7 @@ left out), plus prod's two configs: | factor | levels | |---|---| -| anchor | `adaptive 0.03`: sqrt scale, eta = 0.03 × the 90th percentile of the last 20 rounds' absolute errors, floor 0.001 (E01's elbow). `constant 0.018`: sqrt of admissions per 100k, eta = 0.018 (E16's elbow) | +| anchor | `adaptive 0.03`: sqrt scale, eta = 0.03 × the 90th percentile of the last 20 rounds' absolute errors, floor 0.001 (E01's elbow). `constant 0.018`: sqrt of admissions per 100k, eta = 0.018 (E16's elbow, picked on flu) | | start | `cold`: offsets start at zero. `warm`: `slow_init = "burn_in_quantile"` from the 2023-24 season | | leak | `no leak`. `leak 0.1`: the offset shrinks by 10% each round | | references | `REF-op warm` (flu prod: adaptive, warm, leak, plus a slow term); `REF-op cold` (covid prod) | @@ -55,34 +56,43 @@ bias is computed within each calendar month and then averaged. ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) -wi <- ws_inputs(params$disease) -fc_burn_in <- bind_rows(ws_burn_in_forecasts(params$disease), wi$fc) -SPECS <- c(ws_grid_configs(wi$rate_scales), ws_prod_specs(wi$rate_scales)) -cals <- purrr::map(SPECS, ws_run_spec, wi = wi, fc_burn_in = fc_burn_in, workers = params$workers) -grid_names <- names(ws_grid_configs(wi$rate_scales)) +DISEASES <- c(flu = "flu", covid = "covid") +RUNS <- purrr::map(DISEASES, function(d) { + wi <- ws_inputs(d) + fc_burn_in <- bind_rows(ws_burn_in_forecasts(d), wi$fc) + specs <- c(ws_grid_configs(wi$rate_scales), ws_prod_specs(wi$rate_scales)) + list(wi = wi, specs = specs, cals = purrr::map(specs, ws_run_spec, wi = wi, fc_burn_in = fc_burn_in, workers = params$workers)) +}) +grid_names <- names(ws_grid_configs(RUNS$flu$wi$rate_scales)) split_name <- function(d) { d %>% mutate(config = as.character(config)) %>% tidyr::separate_wider_delim(config, " | ", names = c("anchor", "start", "leak"), cols_remove = FALSE, too_few = "align_start") } +ANCHOR_COLS <- c(`adaptive 0.03` = "#E69F00", `constant 0.018` = "#0072B2") +h_levels <- paste0("h", 0:3) ``` ```{r metrics} -m <- ws_wis_by_season(cals) -mv <- cal_month_view(cals, seasons = LIVE) -cov <- mv$cov %>% - transmute(config, season, h = factor(paste0("h", horizon), levels = paste0("h", 0:3)), pooled = pooled_cal, mavg = mavg_cal) +M <- purrr::map(RUNS, function(r) { + m <- ws_wis_by_season(r$cals) + mv <- cal_month_view(r$cals, seasons = LIVE) + cov <- mv$cov %>% transmute(config, season, h = factor(paste0("h", horizon), levels = h_levels), pooled = pooled_cal, mavg = mavg_cal) + list(m = m, mv = mv, cov = cov) +}) ``` # WIS by season and horizon -```{r wis-table} -knitr::kable( - m %>% +```{r wis-table, results="asis"} +for (d in names(M)) { + cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") + M[[d]]$m %>% transmute(config, col = paste(season, h), cell = sprintf("%+.1f", wis_pct)) %>% - tidyr::pivot_wider(names_from = col, values_from = cell), - caption = "WIS reduction % (positive = calibrated WIS lower than base), states only; `both` pools the seasons" -) + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + knitr::kable(caption = paste0(d, ": WIS reduction % (positive = calibrated WIS lower than base), states only; `both` pools the seasons")) %>% + print() +} ``` # What each factor changes, per anchor @@ -92,12 +102,7 @@ from adding the leak (leak − no leak) at each start, per anchor. WIS: change in WIS reduction % (positive = the factor lowers WIS). Coverage: change in month-avg L1 coverage bias (negative = the factor improves coverage). -```{r factor-deltas} -g <- m %>% - filter(config %in% grid_names) %>% - select(config, season, h, wis_pct) %>% - left_join(cov, by = c("config", "season", "h")) %>% - split_name() +```{r factor-deltas, results="asis"} contrast <- function(d, factor, hi, lo, held) { d %>% select(anchor, start, leak, season, h, wis_pct, mavg) %>% @@ -109,26 +114,33 @@ contrast <- function(d, factor, hi, lo, held) { d_mavg = .data[[paste0("mavg_", hi)]] - .data[[paste0("mavg_", lo)]] ) } -deltas <- bind_rows( - contrast(g, "start", "warm", "cold", "leak"), - contrast(g, "leak", "leak 0.1", "no leak", "start") -) -knitr::kable( - deltas %>% +DELTAS <- purrr::imap(M, function(x, d) { + g <- x$m %>% + filter(config %in% grid_names) %>% + select(config, season, h, wis_pct) %>% + left_join(x$cov, by = c("config", "season", "h")) %>% + split_name() + bind_rows(contrast(g, "start", "warm", "cold", "leak"), contrast(g, "leak", "leak 0.1", "no leak", "start")) %>% + mutate(disease = d) +}) +for (d in names(DELTAS)) { + cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") + DELTAS[[d]] %>% transmute(anchor, factor, col = paste(season, h), cell = sprintf("%+.1f / %+.3f", d_wis, d_mavg)) %>% tidyr::pivot_wider(names_from = col, values_from = cell) %>% - arrange(anchor, factor), - caption = "Change in WIS reduction % / month-avg L1 coverage bias from adding the factor, per anchor and season" -) + arrange(anchor, factor) %>% + knitr::kable(caption = paste0(d, ": change in WIS reduction % / month-avg L1 coverage bias from adding the factor, per anchor and season")) %>% + print() +} ``` -```{r factor-plot, fig.height=5} -ggplot(deltas %>% filter(season != "both"), aes(h, d_wis, colour = anchor, group = anchor)) + +```{r factor-plot, fig.height=8} +ggplot(bind_rows(DELTAS) %>% filter(season != "both"), aes(h, d_wis, colour = anchor, group = anchor)) + geom_hline(yintercept = 0, colour = "grey60") + geom_line() + geom_point() + - facet_grid(season ~ factor) + - scale_colour_manual(values = c(`adaptive 0.03` = "#E69F00", `constant 0.018` = "#0072B2")) + + facet_grid(disease + season ~ factor) + + scale_colour_manual(values = ANCHOR_COLS) + labs(x = NULL, y = "change in WIS reduction %", colour = NULL) + theme(legend.position = "bottom") ``` @@ -139,42 +151,47 @@ Per state, pooled over h0–h3: WIS reduction % of the adaptive anchor against the constant one at the same start and leak (above the line, adaptive is better). -```{r state-scatter, fig.height=5.5} -state_pct <- cal_state_wis(cals[grid_names]) %>% - group_by(config, season, location) %>% - summarize(pct = 100 * (sum(wis_base) - sum(wis_cal)) / sum(wis_base), .groups = "drop") %>% - split_name() %>% - select(-config) %>% - tidyr::pivot_wider(names_from = anchor, values_from = pct) %>% - mutate(cell = paste(start, leak, sep = ", ")) -ggplot(state_pct, aes(`constant 0.018`, `adaptive 0.03`)) + - geom_abline(slope = 1, intercept = 0, colour = "grey50") + - geom_hline(yintercept = 0, colour = "grey85") + - geom_vline(xintercept = 0, colour = "grey85") + - geom_point(alpha = 0.8) + - facet_grid(season ~ cell) + - labs(x = "constant 0.018: WIS reduction %, h0–h3", y = "adaptive 0.03: WIS reduction %, h0–h3") -``` - -```{r state-counts} -knitr::kable( - state_pct %>% +```{r state-scatter, results="asis", fig.height=5.5} +STATE <- purrr::map(RUNS, function(r) { + cal_state_wis(r$cals[grid_names]) %>% + group_by(config, season, location) %>% + summarize(pct = 100 * (sum(wis_base) - sum(wis_cal)) / sum(wis_base), .groups = "drop") %>% + split_name() %>% + select(-config) %>% + tidyr::pivot_wider(names_from = anchor, values_from = pct) %>% + mutate(cell = paste(start, leak, sep = ", ")) +}) +for (d in names(STATE)) { + cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") + print( + ggplot(STATE[[d]], aes(`constant 0.018`, `adaptive 0.03`)) + + geom_abline(slope = 1, intercept = 0, colour = "grey50") + + geom_hline(yintercept = 0, colour = "grey85") + + geom_vline(xintercept = 0, colour = "grey85") + + geom_point(alpha = 0.8) + + facet_grid(season ~ cell) + + labs(title = d, x = "constant 0.018: WIS reduction %, h0–h3", y = "adaptive 0.03: WIS reduction %, h0–h3") + ) + STATE[[d]] %>% group_by(cell, season) %>% summarize(adaptive_better = sum(`adaptive 0.03` > `constant 0.018`), states = n(), .groups = "drop") %>% transmute(cell, season, v = sprintf("%d / %d", adaptive_better, states)) %>% - tidyr::pivot_wider(names_from = season, values_from = v), - caption = "States (of those scored) where the adaptive anchor beats the constant one, h0–h3 pooled" -) + tidyr::pivot_wider(names_from = season, values_from = v) %>% + knitr::kable(caption = paste0(d, ": states (of those scored) where the adaptive anchor beats the constant one, h0–h3 pooled")) %>% + print() +} ``` -```{r month-view-cfgs} -# The month plots show the no-leak grid and flu prod; the tables keep every config. -mv_plot_cfgs <- c(grid_names[c(1, 3, 5, 7)], "REF-op warm") -mv_cols <- setNames( - c("#E69F00", "#D55E00", "#0072B2", "#56B4E9", "#000000"), - mv_plot_cfgs -) -``` +# Month views + +The month plots show the no-leak grid and flu prod; the tables keep every +config. -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} +```{r month-views, results="asis"} +mv_plot_cfgs <- c(grid_names[c(1, 3, 5, 7)], "REF-op warm") +mv_cols <- setNames(c("#E69F00", "#D55E00", "#0072B2", "#56B4E9", "#000000"), mv_plot_cfgs) +for (d in names(M)) { + mv <- M[[d]]$mv + cal_month_section(d, level = 2L) +} ``` diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index ef9bb3b1..663d1572 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -72,13 +72,14 @@ scale. ## Tracker structure -E20 asks the warm-start and leak questions on both anchors, on the replay. +E20 asks the warm-start and leak questions on both anchors and both +diseases, on the replay. E03, E04, E05 and E11 asked them on the ensemble with the adaptive rate only; they stay until E20 has been read against them. | | base | question | |---|---|---| -| `r link("E20", "e20_structure_grid")` | replay | Do the warm start and the leak change WIS and coverage in the same way on the adaptive and the constant learning rate? State by state, which anchor wins at the same start and leak? | +| `r link("E20", "e20_structure_grid")` | replay, both diseases | Do the warm start and the leak change WIS and coverage in the same way on the adaptive and the constant learning rate, for flu and for covid? State by state, which anchor wins at the same start and leak? | | `r link("E03", "e03_scale")` | ensemble | Which scale should the offsets live on: counts, square root, or rate per 100k? | | `r link("E04", "e04_offset_structure")` | ensemble | Does a leak or a second, slow offset term change WIS and coverage? | | `r link("E05", "e05_warm_start")` | ensemble | Does a warm start from the burn-in season change WIS and coverage for each tracker? | diff --git a/scripts/calibration/calibration_ws_replay.R b/scripts/calibration/calibration_ws_replay.R index dd9f8efa..e27487f4 100644 --- a/scripts/calibration/calibration_ws_replay.R +++ b/scripts/calibration/calibration_ws_replay.R @@ -298,7 +298,9 @@ fin_candidates <- function(rate_scales) { `REF-op warm` = list(cfg = list(ref = "op"), warm = TRUE), `REF-op cold` = list(cfg = list(ref = "op", burn_in_seasons = character(0), slow_init = NULL), warm = FALSE), `sqrt 0.018` = list(cfg = sqrt_018, warm = FALSE), - `sqrt 0.018 warm` = list(cfg = utils::modifyList(sqrt_018, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")), warm = TRUE) + `sqrt 0.018 warm` = list(cfg = utils::modifyList(sqrt_018, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile")), warm = TRUE), + # E20's best cell: the warm start plus the leak (same construction as ws_grid_configs()). + `sqrt 0.018 warm + leak` = list(cfg = utils::modifyList(sqrt_018, list(burn_in_seasons = "2023-2024", slow_init = "burn_in_quantile", fast_decay = 0.1)), warm = TRUE) ) } From 1c69bfce2e75175041cb13a5f4fc793ffeac47c3 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 7 Oct 2026 02:38:44 -0700 Subject: [PATCH 104/107] docs(calibration): retire E04, E05 and E11 (superseded by E20), park ROADMAP 1c/1d, ledger gains an Ideas section Co-Authored-By: Claude Fable 5.1 --- notes/ROADMAP.md | 54 ++++++------- notes/calibration-ledger.md | 36 ++++++--- .../e04_offset_structure.Rmd | 67 ---------------- .../e05_warm_start.Rmd | 78 ------------------- .../e11_constant_lr.Rmd | 69 ---------------- .../calibration_experiments/index.Rmd | 11 ++- 6 files changed, 57 insertions(+), 258 deletions(-) delete mode 100644 reports/writeups/calibration_experiments/e04_offset_structure.Rmd delete mode 100644 reports/writeups/calibration_experiments/e05_warm_start.Rmd delete mode 100644 reports/writeups/calibration_experiments/e11_constant_lr.Rmd diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index dd6065b8..5bcbb970 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -10,10 +10,11 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. 1. **Use the evaluation project as the calibration testbed.** The operating-point work (hub harness, `notes/calibration-ledger.md`, "Finalists") is down to a disease-specific choice. Provisional - (2026-10-02, to confirm after the notebook review, not implemented): - covid REF-op cold (sqrt 0.018 loses 3–4 points at h1–h3), flu REF-op warm - or sqrt 0.018 warm (ahead by 1–2 points, all in 2024-25). Done so far: - a1–a4, b1–b4 (E12, E13). Next: item 1f, in its own thread. Calibrating needs full quantile forecasts over many rounds, + (2026-10-07, not implemented): covid REF-op cold (no tracker tested helps + at h1–h3, E20), flu sqrt 0.018 warm + leak (about 1/1.5/2.5 points over + REF-op warm at h1–h3 on the ensemble, coverage within 0.002; E10, E20). + Done so far: a1–a4, b1–b4 (E12, E13), the structure grid on both + anchors and diseases (E18–E21). Next: item 1f, in its own thread. Calibrating needs full quantile forecasts over many rounds, not just scores. The evaluation project (`flu_hosp_evaluation` / `covid_hosp_evaluation`) replays the current prod components and ensemble weekly since 2024-11-20, which is what calibration would see in @@ -66,9 +67,9 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. later. b. *2023-24 as a burn-in season.* REF-op warm-starts from a burn-in season, and the replay has none. The warm start helps only in the - first live season (E05, E18), and sqrt 0.018 needs no burn-in, so if - it is chosen the open level and stitching questions below drop off the - critical path. Delphi's `hhs` source + first live season (E05, E18, E20), but the flu finalist uses it, so the + stitch stays on the critical path for 1f. Its level is settled (E21: + HHS 2–4% below the NHSN final over 2023-24). Delphi's `hhs` source (`confirmed_admissions_{influenza,covid}_1d`) has real issue history for 2023-24 (CA 2023-12-01: issues 12-06, 12-08, 12-20, 12-22), until HHS reporting ended 2024-04-30. Steps, each checked before the next: @@ -124,27 +125,12 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. an ensemble component. To evaluate calibration there: calibrate the replayed forecasts over all dates, learn from per-round snapshots (`calibrate_hub_forecasts_exact()`), take the config as a parameter so - both finalists run, and score states only. A burn-in for REF-op needs + prod's config and the finalist (flu: sqrt 0.018 warm + leak; covid: + REF-op cold is prod) both run, and score states only. A burn-in for REF-op needs the HHS stitching (b), which today exists only in the harness. - **Later (not needed to evaluate calibration):** + **Later (not needed to evaluate calibration; c and d moved to Parked):** - c. *The ensemble with fixed weights.* The geo-exclusions CSVs carry the - ensemble weights as well as exclusions, so "no hand edits" for - `ensemble_mix` means one fixed weight block for every date. Decide - which block (the default one at the top of the file, or the current - one), check the resolved weights on one date, then replay. Not needed - to choose the operating point: E10 already scores the finalists on - the submitted ensemble, and E07 shows calibration acts the same way - on the ensemble and the single-component replay. Revisit if 1f needs - a clean ensemble history. - d. *Hand edits on vs off as its own question.* A separate evaluation - project (like the `_regr` ones) replaying with the edits, to measure - whether the weekly substitutions and weight changes help. The only - past check found is the 2025 revision report - (`revision_summary_report_2025.Rmd`), which scored substitutions by - whether they moved values toward the final value. Calibration needs - only the clean replay. e. *ILINet as a target*, to check our methods on a dataset we have not tuned on. `fluview` has real issue history (CA wILI for week 2023-50 is revised weekly through May 2024) back to about 2010. Follows the @@ -254,9 +240,25 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. no burn-in. - CMU-climate_baseline US forecast is too low (item 2b). - `us_method` spec column and removing the `usa` alias (item 2b). -- Ensemble-base calibration headline tables (E00, E03–E05, E09–E11) pool +- Ensemble-base calibration headline tables (E00, E03, E09, E10 part 2) pool raw counts over all locations, so US is about half of their "WIS change %". The replay experiments report states only (except E12). +- *The ensemble with fixed weights* (was item 1c, parked 2026-10-07). The + geo-exclusions CSVs carry the ensemble weights as well as exclusions, so "no hand edits" for + `ensemble_mix` means one fixed weight block for every date. Decide + which block (the default one at the top of the file, or the current + one), check the resolved weights on one date, then replay. Not needed + to choose the operating point: E10 already scores the finalists on + the submitted ensemble, and E07 shows calibration acts the same way + on the ensemble and the single-component replay. Revisit if 1f needs + a clean ensemble history. +- *Hand edits on vs off as its own question* (was item 1d, parked + 2026-10-07). A separate evaluation project (like the `_regr` ones) replaying with the edits, to measure + whether the weekly substitutions and weight changes help. The only + past check found is the 2025 revision report + (`revision_summary_report_2025.Rmd`), which scored substitutions by + whether they moved values toward the final value. Calibration needs + only the clean replay. ## Tech debt diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 1b574423..8a0dba52 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -190,12 +190,12 @@ gating), exact learning truth, spoiled submissions excluded. | E01 | `e01_eta_settings` | replay, cold | adaptive eta settings | mult ≥ 0.1 costs 7–81 at h1–h3. 0.01 beats 0.03 by 0.5–2.4 at h1–h3 in both seasons; 0.03 better at h0 in 2024-25 (0.8). Window and carry vs reset: ≤ 2.4, mostly < 1. Count is 2–5 worse than sqrt | month-avg coverage of 0.01 vs 0.03 within noise | | E02 | `e02_eta_variants` | replay, cold | `off_after`, per-level eta, seasonal window | off 1 Mar / 15 Feb +2 to +4 at h1–h3 over the 0.03 reference, both seasons, no month-avg coverage cost. Per-level +1. Seasonal window, burn-in, off 1 Apr: < 1 | dates in-sample; half the 2025-26 gain is December carry-over; "seasonal window" also changes `lr_window` 20 → 10 | | E03 | `e03_scale` | ensemble | count vs sqrt vs rate | sqrt +1.4 to +3.9 over count pooled, mostly 2025-26. Coverage: neither better (sqrt worse per season pooled, better month-avg only in 2024-25) | rate identical to count in every cell (adaptive rate is scale invariant) | -| E04 | `e04_offset_structure` | ensemble | leak, slow term | leak +2.5 to +5.6 at h1–h3 in 2025-26, −0.4 to −1.3 in 2024-25; month-avg coverage flips the same way (±0.015). Slow term without warm start: ≤ 0.5 | | -| E05 | `e05_warm_start` | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | +| E04 | retired 2026-10-07 (superseded by E20) | ensemble | leak, slow term | leak +2.5 to +5.6 at h1–h3 in 2025-26, −0.4 to −1.3 in 2024-25; month-avg coverage flips the same way (±0.015). Slow term without warm start: ≤ 0.5 | | +| E05 | retired 2026-10-07 (superseded by E20) | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | | E07 | `e07_across_eras`; the candidate ranking (formerly E07b) is in `e10_covid` | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | | | E09 | `e09_lr_delay` | ensemble | constant rate × revision delay | knee 0.056: +3 to +5.5 over REF-paper at h1–h3; REF-op +1.2 to +2.4 over knee. Extra delay costs about 1 h−1 point per week. Above 0.1 loses steeply | month-avg: no constant rate beats base at h1–h3 | | E10 | `e10_covid` | flu + covid, ensemble + replay | the finalists (now including sqrt 0.018 warm + leak, E20's best flu cell) on both diseases and bases (part 1); REF-paper, REF-op and sqrt 0.018 on the covid ensemble at h−1–h3 (part 2) | see "Finalists" above. Part 2: h−1 +10 to +15 (sqrt 0.018 largest). REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 and sqrt 0.018 −7.7 to −9.5 at h1–h3 in 2025-26 | part 1 is h0–h3 on rows both bases have; part 2 tables run to 2026-09-19, including the 2026 summer wave | -| E11 | `e11_constant_lr` | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | +| E11 | retired 2026-10-07 (superseded by E20) | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | | E12 | none (`calibration_ws_replay.R`) | replay, cold, flu + covid | first look at the clean replay | flu: REF-op cold leads constant 0.1 by 0–2.5. Covid: REF-op cold h0 +1.3 states, losses at h1–h3 in 2025-26 | **all locations** (US is 47% of WIS); covid h0 gains are mostly US | | E13 | none (`calibration_ws_replay.R burn_in`) | replay, flu + covid | 2023-24 HHS burn-in | flu warm +1 to +2.6 in 2024-25, ≤ 0 in 2025-26. Covid: warm worse at h0–h1, better at h2–h3 (+1.3, +4.4); constant 0.1 warm −8 to −27 | HHS used as-is (3–9% below NHSN); coverage is season-pooled only | | E14 | folded into `e16_sqrt_constant_lr` | replay, cold | constant rate grid by season | every rate 0.0032–0.056 is ≥ 0 in both seasons and within 1.2 of each other; 0.018 vs 0.032 ≤ 0.3. Above 0.1 loses | month-avg: no rate ≤ 0.032 beats base beyond 0.01 | @@ -204,7 +204,7 @@ gating), exact learning truth, spoiled submissions excluded. | E17 | `e17_late_decay` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | | E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008. Branch sqrt 0.018 warm: +3.4/+1.6/+1.6/+2.5 pooled, best of all, but month-avg coverage 0.011–0.018 worse than base at h2–h3. States where REF-op wins, 2024-25 / 2025-26: chain ends (warm vs cold) 45 / 13 of 52; warm vs warm 23 / 15; cold vs cold 17 / 15 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance). The chain-ends pair is unmatched on the start | | E19 | `e19_tracker_internals` | replay | why do the leak and constant 0.018 score alike? Step size, offset, base bias and WIS by month on the E20 grid; carry vs reset on both anchors | Step / width at h3 rises from 0.015–0.02 (Dec–Feb) to 0.06–0.09 (Apr–May) on the adaptive rate, 0.005–0.007 to 0.013–0.023 on the constant, in both seasons. Offset / width at h3, Mar/Apr/May 2025: adaptive cold no leak 0.105/0.218/0.186, with leak 0.064/0.078/−0.012, constant cold 0.026/0.055/0.072; Apr/May 2026: 0.160/0.150, 0.057/−0.011, 0.097/0.143. Base bias (share above the base median, h3): Mar 0.43 and 0.27, Apr–May 0.54–0.81 and 0.62–0.71. March WIS at h3: adaptive no leak −48 and −81 (2025, 2026), with leak −27 and −22, constant cold −9 and −35, with leak −5 and −8; Apr–May 2026 constant no leak +4.8/+11.0. **Carry vs reset:** resetting at the season boundary takes the constant tracker's 2025-26 from +3.0/+1.0/+1.1/+1.5 to +1.0/+0.1/+0.2/+0.3, the adaptive one's from +2.2/−0.6/−0.7/−0.2 to +0.7/−0.9/−1.3/−1.6; the carried offset is 0.01–0.025 of width. Carry's gain is Oct–Feb (h3 Nov–Dec 2025 +5.7/+3.8 vs +0.6/+0.4); reset is better after the peak | offsets from the exact runs, step sizes from one pass on final truth per config. `lr_level` for the two-term REF-op configs reports the slow term's rate (about a tenth of the fast one), so their step-size rows are not comparable; REF-op's fast step equals the adaptive anchor's by construction | -| E20 | `e20_structure_grid` | replay | warm start and leak on both anchors: {adaptive 0.03, constant 0.018} × {cold, warm} × {no leak, leak 0.1}, plus REF-op warm and cold | Warm start: +1.0 to +2.6 (adaptive) and +1.5 to +3.8 (constant) in 2024-25, −0.5 to +0.6 in 2025-26, on both anchors at either leak. Leak: on the adaptive rate +0.7 to +2.0 at h1–h3 in both seasons; on the constant rate −0.2 to +0.7 warm and −0.6 to −1.7 in 2025-26 cold. Best cell constant warm leak: +3.3/+1.8/+1.9/+2.9 pooled vs REF-op warm +2.8/+1.1/+0.9/+1.1, but month-avg coverage 0.012–0.017 above base at h2–h3 in 2025-26 (the warm constant without leak is 0.024–0.037 above). Constant beats adaptive in 29–47 of 52 states in every cell and season. **Covid:** the warm start costs 2–7 points at h0–h2 in 2024-25 (constant) and 1–3 (adaptive), and 0.01–0.04 of month-avg coverage; the leak adds 1–3 on the adaptive anchor and on the cold constant in 2024-25, costs 1–3 on the warm constant. No cell beats base at h1–h3 pooled; least bad is constant cold leak −0.5/−0.8/−0.5 vs REF-op cold −1.1/−1.7/−1.6, adaptive warm leak +3.4 at h3 but −1.9 at h1 | replaces E03–E05 and E11's questions on the replay; those stay until read against it. The slow term is left out (E18 step 2). All constant-rate configs tuned in-sample (E16) | +| E20 | `e20_structure_grid` | replay | warm start and leak on both anchors: {adaptive 0.03, constant 0.018} × {cold, warm} × {no leak, leak 0.1}, plus REF-op warm and cold | Warm start: +1.0 to +2.6 (adaptive) and +1.5 to +3.8 (constant) in 2024-25, −0.5 to +0.6 in 2025-26, on both anchors at either leak. Leak: on the adaptive rate +0.7 to +2.0 at h1–h3 in both seasons; on the constant rate −0.2 to +0.7 warm and −0.6 to −1.7 in 2025-26 cold. Best cell constant warm leak: +3.3/+1.8/+1.9/+2.9 pooled vs REF-op warm +2.8/+1.1/+0.9/+1.1, but month-avg coverage 0.012–0.017 above base at h2–h3 in 2025-26 (the warm constant without leak is 0.024–0.037 above). Constant beats adaptive in 29–47 of 52 states in every cell and season. **Covid:** the warm start costs 2–7 points at h0–h2 in 2024-25 (constant) and 1–3 (adaptive), and 0.01–0.04 of month-avg coverage; the leak adds 1–3 on the adaptive anchor and on the cold constant in 2024-25, costs 1–3 on the warm constant. No cell beats base at h1–h3 pooled; least bad is constant cold leak −0.5/−0.8/−0.5 vs REF-op cold −1.1/−1.7/−1.6, adaptive warm leak +3.4 at h3 but −1.9 at h1 | replaces E04, E05 and E11 on the replay (retired). Not a scale comparison: every cell runs on sqrt, so E03 stays. The slow term is left out (E18 step 2). All constant-rate configs tuned in-sample (E16) | | E21 | `e21_data_by_season` | replay data | how the two live seasons differ as data: shape, revisions, learning-truth gap at lags 1–4, HHS-stitched burn-in vs NHSN | 2024-25: 27 rounds from 2024-11-23, peak 2025-02-08 at 55.6k (states summed), total 559k; 2025-26: 28 rounds from 2025-10-18, peak 2026-01-03 at 42.5k, total 336k. State peaks within ±1 week of the national one: 37 vs 47 of 52. Revisions: 26% vs 40% of (state, week) keys never revised; relative spread > 10% for 48% vs 39%. Value at lag 2 (what the tracker learns from) vs final: median gap −1.1% vs 0.0%, \|gap\| > 10% for 19% vs 14% of keys; at lag 1, 44% vs 41%. HHS vs NHSN final over 2023-24: 0.957–0.978 in Nov–Apr (states summed), per-state median 1.004 | describes the data only; no tracker runs. Revision keys start at the first archive version (2024-11-19) | E06 (ILI+ burn-in) and E08 (old covid) were run on code or data later found @@ -241,11 +241,12 @@ Checked and resolved on 2026-10-02: |---|---| | E00, E01 | verified | | E02, E07, E09 | commented; changes made, not re-verified | -| E10 part 1, E18 | changes made (finalists runs, state counts), not reviewed; E18's matched pairs not yet rendered | +| E10 part 1, E18 | changes made (finalists runs, matched pairs), rendered 2026-10-07, not reviewed | | E20 (flu and covid), E10, E15 | rendered 2026-10-07 with the covid grid and the new finalist, not reviewed | | E19 | rendered 2026-10-07, not reviewed | | E21 | rendered 2026-10-06, not reviewed | -| E03, E04, E05, E10, E11, E14, E15, E16, E17 | not reviewed | +| E03, E10, E14, E15, E16, E17 | not reviewed | +| E04, E05, E11 | retired 2026-10-07, never reviewed | On 2026-10-02 every notebook's introduction and Findings were rewritten to state setup and measured sizes only; the rewritten text has not been @@ -342,16 +343,27 @@ operating point. | item | status | |---|---| -| **Reorganization (2026-10-06).** The constant-rate line was never given the structure analysis (E03–E05, E11 ran the adaptive rate on the ensemble; E16 inherited "cold" from E11's framing and E13's constant 0.1 warm covid loss). Plan: (1) E18 matched-pair scatters, warm vs warm and cold vs cold; (2) E20, the {anchor} × {start} × {leak} grid on the replay; (3) E19 as a notebook on that grid with WIS by month and carry vs reset; (4) E21, the seasons as data. Then re-read E03–E05 and E11 against E20 and retire or keep them | all rendered 2026-10-07 (E21 on 2026-10-06) and read into this file; E20 covers covid too, and sqrt 0.018 warm + leak is a finalist in E10 and E15. E03–E05 and E11 not yet re-read against E20 | Render with `just calibration-experiments e18_ref_op_bridge e19_tracker_internals e20_structure_grid` inside the rocker container | -| Uncertainty for WIS and coverage differences (bootstrap over rounds or locations) | open; needed before any ranking under ~2 points means anything | +| **Reorganization (2026-10-06).** The constant-rate line was never given the structure analysis (E03–E05, E11 ran the adaptive rate on the ensemble; E16 inherited "cold" from E11's framing and E13's constant 0.1 warm covid loss). Plan: (1) E18 matched-pair scatters, warm vs warm and cold vs cold; (2) E20, the {anchor} × {start} × {leak} grid on the replay; (3) E19 as a notebook on that grid with WIS by month and carry vs reset; (4) E21, the seasons as data. Then re-read E03–E05 and E11 against E20 and retire or keep them | all rendered 2026-10-07 (E21 on 2026-10-06) and read into this file; E20 covers covid too, and sqrt 0.018 warm + leak is a finalist in E10 and E15. E03–E05 and E11 re-read against E20 on 2026-10-07: E20 reproduces E04, E05 and E11's directions on the replay (warm start first-season only, leak season-dependent on the adaptive rate, slow term nothing, constant beats adaptive), so those three were deleted; their rows stay in "Experiments". E03 stays: E20 runs every cell on sqrt and never compares scales | Render with `just calibration-experiments e18_ref_op_bridge e19_tracker_internals e20_structure_grid` inside the rocker container | +| Uncertainty for WIS differences (bootstrap over rounds) | next (2026-10-07): the two warm constant cells in E20 differ by 0.3 pooled, under the 1-point convention. A round bootstrap in the season-table helper gives every E20 and E10 table an interval | | Shrink only the played offset after the peak, keep the stored state | open; `late_decay` shrinks the stored state | -| Disease-specific operating point | provisional (2026-10-02): covid REF-op cold; flu REF-op warm or sqrt 0.018 warm, to confirm after the notebook review. Not implemented | -| HHS burn-in level | E21: 2–4% below the NHSN final in Nov–Apr 2023-24 (states summed), per-state median ratio 1.004. Used as-is (E13, E18); settle it with item 1f if a warm flu config is chosen | +| Disease-specific operating point | provisional (2026-10-07): covid REF-op cold (no grid cell helps at h1–h3, E20); flu sqrt 0.018 warm + leak (E10, E20), about 1/1.5/2.5 points over REF-op warm at h1–h3 on the ensemble, coverage within 0.002. Rate and leak picked in-sample; two seasons, no uncertainty. Not implemented (ROADMAP 1f) | +| HHS burn-in level | closed 2026-10-07: E21 measures it at 2–4% below the NHSN final in Nov–Apr 2023-24 (states summed), per-state median ratio 1.004, within the warm start's own noise. Used as-is; the flu finalist is warm, so 1f needs the stitch in the pipeline | | `windowed_seasonal_extra_sources` retrospective | parked (ROADMAP) | -| Method ideas: cross-horizon gradients for staleness, data-driven phase gating, scale proxy | open; no code | -| Collaborator email | unknown | Closed 2026-10-02 without further work: ramp vs post-peak split (the month view shows it), h−1 calibration (left to the revision-aware methods, ROADMAP 3), ILI+ burn-in (the HHS burn-in covers it), gallery review (review uses the month view), speeding up exact runs. + +## Ideas + +Method ideas with no code and no notebook that would test them. None of +E18–E21 points at any of them; they are kept so they are not re-invented. + +- Cross-horizon gradients as a staleness signal: the h0 offset is fresher + than the h3 one by three rounds, so their gap could steer the h3 step. +- Data-driven phase gating: switch the offset off (or shrink what is + played) when the season's phase changes, instead of a fixed date + (E02, E17) or a leak. +- A scale proxy: a per-location scale from the base's own interval width + rather than population, so the sqrt-per-100k transform is not needed. diff --git a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd b/reports/writeups/calibration_experiments/e04_offset_structure.Rmd deleted file mode 100644 index 3bb031b1..00000000 --- a/reports/writeups/calibration_experiments/e04_offset_structure.Rmd +++ /dev/null @@ -1,67 +0,0 @@ ---- -title: "E04: offset structure (leak and slow term)" -date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" -output: - html_document: - code_folding: hide - toc: true - toc_float: true - toc_depth: 2 -params: - hub_dir: "../FluSight-forecast-hub" - workers: 14 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} -``` - -Question: does adding a leak or a second, slow offset term to the tracker -change WIS and coverage? - -Setup: - -- Disease: flu. Base forecaster: the submitted hub ensemble (spoiled - submissions dropped). -- Reference: REF-paper on the sqrt scale (E03's `sqrt`): adaptive learning - rate of 0.03 × the 90th percentile of the last 20 rounds' absolute errors, - floor 0.001; offsets carried across seasons; one week of extra delay - (`settle_days = 14`). -- Burn-in: 2023-24, used only to warm up the learning rate. Offsets start at - zero at the first 2024-25 round (cold start; no warm start for the slow - term). -- Learning truth: exact (re-run at every round on the data published by then, - as prod does; E00). Scores against finalized truth. -- Seasons scored: 2024-25 and 2025-26. Horizons h−1 to h3. States only - (US excluded). - -| config | change from the sqrt reference | -|---|---| -| `single` | none (one offset per level) | -| `leak 0.1` | the offset shrinks by 10% each round (half-life about 6.6 rounds) | -| `slow 0.003` | adds a second offset term with learning-rate multiplier 0.003 (vs 0.03 for the main term), no leak | -| `slow 0.003 + leak 0.1` | both; this is REF-op without its warm start (E05) | - -```{r runs} -sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) -cals <- list( - single = do.call(run, sqrt_args), - `leak 0.1` = do.call(run, c(sqrt_args, fast_decay = 0.1)), - `slow 0.003` = do.call(run, c(sqrt_args, list(lr_slow = list(mult = 0.003)))), - `slow 0.003 + leak 0.1` = do.call(run, c(sqrt_args, list(lr_slow = list(mult = 0.003)), fast_decay = 0.1)) -) -``` - -```{r month-view-data} -mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` diff --git a/reports/writeups/calibration_experiments/e05_warm_start.Rmd b/reports/writeups/calibration_experiments/e05_warm_start.Rmd deleted file mode 100644 index c7a9ad6a..00000000 --- a/reports/writeups/calibration_experiments/e05_warm_start.Rmd +++ /dev/null @@ -1,78 +0,0 @@ ---- -title: "E05: warm start from the burn-in season" -date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" -output: - html_document: - code_folding: hide - toc: true - toc_float: true - toc_depth: 2 -params: - hub_dir: "../FluSight-forecast-hub" - workers: 14 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} -``` - -Question: does starting the persistent offset from the burn-in season's -errors, rather than from zero, change WIS and coverage for each tracker? - -**Setup.** - -- Flu, the submitted hub ensemble (`CMU-TimeSeries`), horizons h−1 to h3. -- 2023-24 is the burn-in season; 2024-25 and 2025-26 are scored. Burn-in - outcomes never move the offsets, so a cold run plays the base forecast - until its first live outcomes arrive. -- Square-root scale, REF-paper otherwise (adaptive learning rate, mult - 0.03, floor 0.001, window 20, offsets carried across seasons). -- Learning truth: NHSN data as published each week, re-run from scratch - each week (prod's design, E00). Scoring: finalized truth. -- Scores cover states only (US excluded). -- The warm start sets the slow offset, per horizon and level, to the - level-quantile of the burn-in season's errors pooled over locations, on - the sqrt scale. - -| config | warm start | tracking after the start | -|---|---|---| -| `warm only` | yes | none (learning rate ≈ 0) | -| `single` | no | REF-paper tracker | -| `warm + single` | yes | REF-paper tracker | -| `leak 0.1` | no | fast offset shrinks 10% per round | -| `warm + leak 0.1` | yes | as `leak 0.1` | -| `slow + leak` | no | leak 0.1 plus a slow offset (mult 0.003) | -| `warm + slow + leak (REF-op)` | yes | as `slow + leak`; prod's config | - -WIS reduction % is `100 × (WIS_base − WIS_cal) / WIS_base`, so positive = -calibrated WIS lower than base. L1 coverage bias: lower is better. - -```{r runs} -sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) -warm <- list(slow_init = "burn_in_quantile") -cals <- list( - `warm only` = do.call(run, c(sqrt_args[1:2], warm, list(lr_args = list(mult = 1e-6, floor = 0)))), - single = do.call(run, sqrt_args), - `warm + single` = do.call(run, c(sqrt_args, warm)), - `leak 0.1` = do.call(run, c(sqrt_args, fast_decay = 0.1)), - `warm + leak 0.1` = do.call(run, c(sqrt_args, warm, fast_decay = 0.1)), - `slow + leak` = do.call(run, c(sqrt_args, list(lr_slow = list(mult = 0.003)), fast_decay = 0.1)), - `warm + slow + leak (REF-op)` = run("op") -) -# The month view drops the two leak-only runs to keep it readable. -mv_plot_cfgs <- c("warm only", "single", "warm + single", "slow + leak", "warm + slow + leak (REF-op)") -``` - -```{r month-view-data} -mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` diff --git a/reports/writeups/calibration_experiments/e11_constant_lr.Rmd b/reports/writeups/calibration_experiments/e11_constant_lr.Rmd deleted file mode 100644 index 76196c56..00000000 --- a/reports/writeups/calibration_experiments/e11_constant_lr.Rmd +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: "E11: constant learning rate vs adaptive" -date: "`r format(Sys.time(), '%d %B %Y, %H:%M %Z', tz = 'America/Los_Angeles')`" -output: - html_document: - code_folding: hide - toc: true - toc_float: true - toc_depth: 2 -params: - hub_dir: "../FluSight-forecast-hub" - workers: 14 -editor_options: - chunk_output_type: console ---- - -```{r setup, echo=FALSE} -knitr::opts_chunk$set(fig.align = "center", fig.width = 10, fig.height = 5, message = FALSE, warning = FALSE) -knitr::opts_knit$set(root.dir = here::here()) -ggplot2::theme_set(ggplot2::theme_bw() + ggplot2::theme(panel.grid.minor = ggplot2::element_blank())) -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_load.Rmd")} -``` - -Question: does a constant learning rate on the rate scale do better than the -adaptive learning rate? - -Setup: - -- Disease: flu. Base forecaster: the submitted hub ensemble (spoiled - submissions dropped). -- Burn-in: 2023-24, used only to warm up the adaptive learning rate. Offsets - start at zero at the first 2024-25 round unless a config has a warm start. - The constant-rate config has no learning rate to warm up, so it is - effectively cold. -- Learning truth: exact (re-run at every round on the data published by then; - E00), one week of extra delay. Scores against finalized truth. -- Seasons scored: 2024-25 and 2025-26. Horizons h−1 to h3. States only - (US excluded). - -| config | change from REF-paper (counts, adaptive 0.03 × p90 of the last 20 errors) | -|---|---| -| `REF-paper` | none | -| `sqrt, adaptive` | sqrt scale, learning-rate floor 0.001 (E03 `sqrt`) | -| `warm + sqrt, adaptive` | as above, plus a warm start from the burn-in season (E05 `warm + single`) | -| `rate, constant 0.1` | rate scale (per 100k), constant learning rate 0.1 (E09, +1 week, re-run) | -| `REF-op` | the prod config; differs in several settings | - -WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated -WIS lower than base. L1 coverage bias: lower is better. - -```{r runs} -sqrt_args <- list("paper", transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3)) -cals <- list( - `REF-paper` = run("paper"), - `sqrt, adaptive` = do.call(run, sqrt_args), - `warm + sqrt, adaptive` = do.call(run, c(sqrt_args, slow_init = "burn_in_quantile")), - `rate, constant 0.1` = run("paper", scales = rate_scales, lr = 0.1), - `REF-op` = run("op") -) -``` - -```{r month-view-data} -mv <- cal_month_view(cals, seasons = LIVE, horizons = NULL) -``` - -```{r child=here::here("reports/writeups/calibration_experiments/_month_view.Rmd")} -``` diff --git a/reports/writeups/calibration_experiments/index.Rmd b/reports/writeups/calibration_experiments/index.Rmd index 663d1572..4e52cec8 100644 --- a/reports/writeups/calibration_experiments/index.Rmd +++ b/reports/writeups/calibration_experiments/index.Rmd @@ -73,17 +73,16 @@ scale. ## Tracker structure E20 asks the warm-start and leak questions on both anchors and both -diseases, on the replay. -E03, E04, E05 and E11 asked them on the ensemble with the adaptive rate -only; they stay until E20 has been read against them. +diseases, on the replay. E03 is the only scale comparison: every E20 cell +runs on the square-root scale. (E04, E05 and E11 asked the leak, warm-start +and constant-rate questions on the ensemble with the adaptive rate; E20 +reproduces their directions on the replay, so they were retired on +2026-10-07. Their numbers stay in the ledger.) | | base | question | |---|---|---| | `r link("E20", "e20_structure_grid")` | replay, both diseases | Do the warm start and the leak change WIS and coverage in the same way on the adaptive and the constant learning rate, for flu and for covid? State by state, which anchor wins at the same start and leak? | | `r link("E03", "e03_scale")` | ensemble | Which scale should the offsets live on: counts, square root, or rate per 100k? | -| `r link("E04", "e04_offset_structure")` | ensemble | Does a leak or a second, slow offset term change WIS and coverage? | -| `r link("E05", "e05_warm_start")` | ensemble | Does a warm start from the burn-in season change WIS and coverage for each tracker? | -| `r link("E11", "e11_constant_lr")` | ensemble | Does a constant learning rate do better than the adaptive rate? | ## Late season From a0887fbde02e0715571599f6aa544caef0d74ede Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 7 Oct 2026 02:50:46 -0700 Subject: [PATCH 105/107] feat(calibration): round bootstrap intervals for WIS reduction in E20 and E10 Co-Authored-By: Claude Fable 5.1 --- R/calibration/views.R | 122 ++++++++++++++++++ notes/ROADMAP.md | 3 +- notes/calibration-ledger.md | 49 +++++-- .../calibration_experiments/e10_covid.Rmd | 39 ++++++ .../e20_structure_grid.Rmd | 50 ++++++- 5 files changed, 247 insertions(+), 16 deletions(-) diff --git a/R/calibration/views.R b/R/calibration/views.R index 2719bf8c..4e01dfa1 100644 --- a/R/calibration/views.R +++ b/R/calibration/views.R @@ -472,6 +472,128 @@ cal_state_wis <- function(cals, seasons = c("2024-2025", "2025-2026")) { }) %>% bind_rows() } +#' Round bootstrap for WIS reduction %, states only, live seasons, h0–h3. +#' +#' Rounds (reference dates) are resampled with replacement within each +#' season; `"both"` pools the two seasons' resampled rounds. Every config is +#' scored on the same draws, so differences between configs taken draw by +#' draw are paired. Each draw's WIS reduction % is `100 × (WIS_base − +#' WIS_cal) / WIS_base` over the resampled rounds. The interval is the +#' central `level` percentile band of the draws; see +#' notes/calibration-ledger.md, "How to read the numbers". +#' +#' @param cals named list of runs (each with `$forecasts`) or forecast frames. +#' @return list of `ci` (one row per config, season, h, with `wis_pct`, `lo`, +#' `hi`) and `draws` (one row per config, season, h, draw, with `wis_pct`). +#' @export +cal_wis_boot <- function(cals, seasons = c("2024-2025", "2025-2026"), horizons = 0:3, n_draws = 1000, level = 0.9, seed = 1, states_only = TRUE) { + pinball <- function(y, q, tau) ifelse(y >= q, tau * (y - q), (1 - tau) * (q - y)) + per_round <- purrr::imap(cals, function(x, nm) { + fc <- if (is.data.frame(x)) x else x$forecasts + if (!"is_burn_in" %in% names(fc)) fc$is_burn_in <- FALSE + if (states_only) fc <- fc %>% filter(.data$location != "US") + fc %>% + mutate(season = cal_season_of(.data$reference_date)) %>% + filter(!.data$is_burn_in, .data$horizon %in% horizons, .data$season %in% seasons, !is.na(.data$truth), !is.na(.data$value_base)) %>% + group_by(.data$season, .data$reference_date, .data$horizon) %>% + summarize( + wis_base = sum(pinball(.data$truth, .data$value_base, .data$level)), + wis_cal = sum(pinball(.data$truth, .data$value_cal, .data$level)), .groups = "drop" + ) %>% + mutate(config = nm, .before = 1) + }) %>% bind_rows() + # One weight matrix per season (draw × round), shared by every config. + rounds <- per_round %>% distinct(.data$season, .data$reference_date) %>% arrange(.data$season, .data$reference_date) + set.seed(seed) + weights <- purrr::map(split(rounds$reference_date, rounds$season), function(r) { + n <- length(r) + w <- t(replicate(n_draws, tabulate(sample.int(n, n, replace = TRUE), nbins = n))) + colnames(w) <- as.character(r) + w + }) + # A config or horizon missing a round (e.g. h3 at the season's end) just + # contributes nothing for it. + score <- function(d, s) { + w <- weights[[s]][, as.character(d$reference_date), drop = FALSE] + list(base = as.vector(w %*% d$wis_base), cal = as.vector(w %*% d$wis_cal)) + } + draws <- per_round %>% + group_by(.data$config, .data$horizon) %>% + group_modify(function(d, key) { + by_season <- purrr::map(split(d, d$season), function(ds) score(ds, ds$season[1])) + by_season$both <- list( + base = Reduce(`+`, purrr::map(by_season, "base")), + cal = Reduce(`+`, purrr::map(by_season, "cal")) + ) + purrr::imap(by_season, function(x, s) { + tibble::tibble(season = s, draw = seq_len(n_draws), wis_pct = 100 * (x$base - x$cal) / x$base) + }) %>% bind_rows() + }) %>% + ungroup() %>% + mutate( + config = factor(.data$config, levels = names(cals)), + season = factor(.data$season, levels = c(seasons, "both")), + h = factor(paste0("h", .data$horizon), levels = paste0("h", sort(horizons))) + ) %>% + select("config", "season", "h", "draw", "wis_pct") + point <- per_round %>% + bind_rows(per_round %>% mutate(season = "both")) %>% + group_by(.data$config, .data$season, .data$horizon) %>% + summarize(wis_pct = 100 * (sum(.data$wis_base) - sum(.data$wis_cal)) / sum(.data$wis_base), .groups = "drop") %>% + mutate( + config = factor(.data$config, levels = names(cals)), + season = factor(.data$season, levels = c(seasons, "both")), + h = factor(paste0("h", .data$horizon), levels = paste0("h", sort(horizons))) + ) %>% + select("config", "season", "h", "wis_pct") + a <- (1 - level) / 2 + ci <- draws %>% + group_by(.data$config, .data$season, .data$h) %>% + summarize(lo = quantile(.data$wis_pct, a), hi = quantile(.data$wis_pct, 1 - a), .groups = "drop") %>% + inner_join(point, by = c("config", "season", "h")) %>% + select("config", "season", "h", "wis_pct", "lo", "hi") + list(ci = ci, draws = draws, level = level, n_draws = n_draws) +} + +#' Paired bootstrap interval for the WIS reduction % gap between two configs +#' of a [cal_wis_boot()] result, `a − b`, draw by draw. +#' @return one row per season and h with `diff`, `lo`, `hi`. +#' @export +cal_wis_boot_diff <- function(boot, a, b) { + al <- (1 - boot$level) / 2 + d <- boot$draws %>% + filter(.data$config %in% c(a, b)) %>% + mutate(config = ifelse(.data$config == a, "a", "b")) %>% + tidyr::pivot_wider(names_from = "config", values_from = "wis_pct") %>% + group_by(.data$season, .data$h) %>% + summarize(lo = quantile(.data$a - .data$b, al), hi = quantile(.data$a - .data$b, 1 - al), .groups = "drop") + p <- boot$ci %>% + filter(.data$config %in% c(a, b)) %>% + mutate(config = ifelse(.data$config == a, "a", "b")) %>% + select("config", "season", "h", "wis_pct") %>% + tidyr::pivot_wider(names_from = "config", values_from = "wis_pct") %>% + transmute(.data$season, .data$h, diff = .data$a - .data$b) + inner_join(p, d, by = c("season", "h")) +} + +#' Wide table of a [cal_wis_boot()] result: configs as rows, `season hX` as +#' columns, cells `point [lo, hi]`. +#' @export +cal_wis_boot_table <- function(boot) { + boot$ci %>% + transmute(.data$config, col = paste(.data$season, .data$h), cell = sprintf("%+.1f [%+.1f, %+.1f]", .data$wis_pct, .data$lo, .data$hi)) %>% + tidyr::pivot_wider(names_from = "col", values_from = "cell") +} + +#' Caption for [cal_wis_boot_table()]. +#' @export +cal_wis_boot_caption <- function(boot) { + sprintf( + "WIS reduction %% (positive = calibrated WIS lower than base), states only, with a %d%% interval from %d round bootstrap draws (rounds resampled within each season; `both` pools the seasons' draws).", + round(100 * boot$level), boot$n_draws + ) +} + # August-July season label, as `season_of()` in the replay scripts. cal_season_of <- function(d) { y <- as.integer(format(d, "%Y")) - (as.integer(format(d, "%m")) < 8) diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index 5bcbb970..67e8dbef 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -12,7 +12,8 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. "Finalists") is down to a disease-specific choice. Provisional (2026-10-07, not implemented): covid REF-op cold (no tracker tested helps at h1–h3, E20), flu sqrt 0.018 warm + leak (about 1/1.5/2.5 points over - REF-op warm at h1–h3 on the ensemble, coverage within 0.002; E10, E20). + REF-op warm at h1–h3 on the ensemble, coverage within 0.002, the 90% + round-bootstrap interval on that gap excluding zero at h2–h3; E10, E20). Done so far: a1–a4, b1–b4 (E12, E13), the structure grid on both anchors and diseases (E18–E21). Next: item 1f, in its own thread. Calibrating needs full quantile forecasts over many rounds, not just scores. The evaluation project (`flu_hosp_evaluation` / diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 8a0dba52..3a239525 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -19,9 +19,17 @@ removed; they are in VCS history at change `uxposooz` - **WIS change %** is against the uncalibrated base; positive is better. **Coverage bias** is mean |coverage − nominal| over levels; lower is better. -- **There are no uncertainty estimates.** Two live seasons, 27 and 28 - rounds. This file calls a gap under 1 WIS point "no clear difference". - That is a reading convention, not a test. +- **Uncertainty is a round bootstrap (E10, E20; `cal_wis_boot()`).** Two + live seasons, 27 and 28 rounds; rounds are resampled within a season, + 1000 draws, 90% intervals, the same draws for every config so gaps + between configs are paired. A single cell's interval is wide: about ±1 to + ±4 points per season on the adaptive anchor, ±0.5 to ±2 on the constant + one, and a 2025-26 cell is wider than a 2024-25 one. A paired gap between + two similar configs is much tighter, about ±0.5 to ±1 pooled. The old + convention, a gap under 1 WIS point is "no clear difference", holds for + pooled paired gaps and is too optimistic for single cells. The round is the + resampling unit, which ignores dependence between neighbouring rounds, so + the intervals are if anything too narrow. - **Use month-avg coverage, not season-pooled.** Pooling over a season (or two) lets ramp under-prediction cancel post-peak over-prediction, and lets one season's error cancel the other's. Season-pooled gains of 2–5× mostly @@ -118,9 +126,26 @@ Finalists (E10 part 1, E18), states only, h1/h2/h3 pooled over both seasons: the ensemble, pooled: 0.108/0.126/0.140 at h1–h3 vs REF-op warm 0.110/0.124/0.138 and base 0.123/0.137/0.151; in 2025-26 it is 0.004–0.007 below REF-op warm at h1–h2. Ranks first pooled on both bases, with REF-op - warm first on ensemble coverage by 0.001. + warm first on ensemble coverage by 0.001. **Bootstrap (2026-10-07):** the + paired gap to REF-op warm, pooled h1/h2/h3, is +0.9 [+0.3, +1.3] / +1.6 + [+0.9, +2.1] / +2.4 [+1.4, +3.2] on the ensemble and +0.6 [−0.1, +1.7] / + +0.9 [+0.1, +2.1] / +1.9 [+1.2, +2.8] on the replay, so the interval + excludes zero at h2–h3 on both bases; per season it excludes zero at + h2–h3 in 2024-25 and only at h3 on the replay in 2025-26. The leak's own + gap over sqrt 0.018 warm is +0.3 [−0.4, +1.1] / +0.4 [−0.3, +1.2] / +0.5 + [−0.1, +1.2] on the ensemble and +0.2 / +0.3 / +0.4 with the same spread + on the replay: for WIS the leak is a coin flip, and it is in the finalist + for its 2025-26 coverage (E20), not its WIS. Against the warm adaptive + cell at the same leak, the constant anchor's gap is +0.6 [+0.0, +1.5] / + +0.8 [+0.2, +1.7] / +1.6 [+1.0, +2.3] pooled (E20). - **Covid: no grid cell helps at h1–h3, and the warm start hurts (E20).** - See the E20 row; REF-op cold stays the least bad config overall. + See the E20 row; REF-op cold stays the least bad config overall. The + bootstrap puts constant cold leak above REF-op cold on the replay by + +0.5 [+0.1, +0.9] / +0.9 [+0.3, +1.4] / +1.1 [+0.2, +1.9] pooled at + h1–h3, all from 2025-26, and both are below the base there (−1.2 to −1.7 + vs −2.4 to −3.2). That cell has not been run on the covid ensemble, where + the constant anchor without a leak loses to REF-op cold by 2.6–3.4 (next + bullet), so it does not change the operating point. - **Covid: sqrt 0.018 is worse than REF-op cold on the ensemble.** −2.7/−3.7/−4.2 vs −0.1/−0.3/−0.8, from 2025-26 (−6.9 to −8.3 at h1–h3; REF-op has the lower WIS in 44–45 of 52 states). At h−1 sqrt 0.018 gains @@ -194,7 +219,7 @@ gating), exact learning truth, spoiled submissions excluded. | E05 | retired 2026-10-07 (superseded by E20) | ensemble | warm start from burn-in | warm +1.7 to +3.2 at h2–h3 in 2024-25, ±0.4 in 2025-26, for every tracker. REF-op ≈ warm + leak (≤ 0.4). Month-avg coverage: all within 0.01 of each other | | | E07 | `e07_across_eras`; the candidate ranking (formerly E07b) is in `e10_covid` | both, same rows | do results transfer between ensemble and replay? | Same direction on both; ensemble gains about 1 point more with REF-op warm. Replay base 5–7% worse pooled, 10–33% worse at h0–h1 in 2025-26. Candidate order flips by season on both; rank correlation of coverage in 2025-26 is 0.31 | | | E09 | `e09_lr_delay` | ensemble | constant rate × revision delay | knee 0.056: +3 to +5.5 over REF-paper at h1–h3; REF-op +1.2 to +2.4 over knee. Extra delay costs about 1 h−1 point per week. Above 0.1 loses steeply | month-avg: no constant rate beats base at h1–h3 | -| E10 | `e10_covid` | flu + covid, ensemble + replay | the finalists (now including sqrt 0.018 warm + leak, E20's best flu cell) on both diseases and bases (part 1); REF-paper, REF-op and sqrt 0.018 on the covid ensemble at h−1–h3 (part 2) | see "Finalists" above. Part 2: h−1 +10 to +15 (sqrt 0.018 largest). REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 and sqrt 0.018 −7.7 to −9.5 at h1–h3 in 2025-26 | part 1 is h0–h3 on rows both bases have; part 2 tables run to 2026-09-19, including the 2026 summer wave | +| E10 | `e10_covid` | flu + covid, ensemble + replay | the finalists (now including sqrt 0.018 warm + leak, E20's best flu cell) on both diseases and bases, with round-bootstrap intervals and paired gaps (part 1); REF-paper, REF-op and sqrt 0.018 on the covid ensemble at h−1–h3 (part 2) | see "Finalists" above, including the bootstrap gaps. Part 2: h−1 +10 to +15 (sqrt 0.018 largest). REF-op h1–h3: +1.9 to +2.8 in 2024-25, −3.0 to −4.3 in 2025-26. REF-paper −10 to −19 and sqrt 0.018 −7.7 to −9.5 at h1–h3 in 2025-26 | part 1 is h0–h3 on rows both bases have; part 2 tables run to 2026-09-19, including the 2026 summer wave | | E11 | retired 2026-10-07 (superseded by E20) | ensemble | constant 0.1 vs adaptive | REF-op best pooled WIS at every horizon (up to 2 over warm + sqrt). Constant 0.1 gives up 1.5–2.3 at h−1/h0 vs adaptive sqrt | constant 0.1 is effectively cold; its coverage edge is season-pooled only | | E12 | none (`calibration_ws_replay.R`) | replay, cold, flu + covid | first look at the clean replay | flu: REF-op cold leads constant 0.1 by 0–2.5. Covid: REF-op cold h0 +1.3 states, losses at h1–h3 in 2025-26 | **all locations** (US is 47% of WIS); covid h0 gains are mostly US | | E13 | none (`calibration_ws_replay.R burn_in`) | replay, flu + covid | 2023-24 HHS burn-in | flu warm +1 to +2.6 in 2024-25, ≤ 0 in 2025-26. Covid: warm worse at h0–h1, better at h2–h3 (+1.3, +4.4); constant 0.1 warm −8 to −27 | HHS used as-is (3–9% below NHSN); coverage is season-pooled only | @@ -204,7 +229,7 @@ gating), exact learning truth, spoiled submissions excluded. | E17 | `e17_late_decay` | replay, cold | shrink or stop offsets from 1 Mar | `late_decay` fixes March but costs about 0.5 at h0, otherwise ±0.2. Off 1 Mar +0.4 to +0.7 at h1–h3 for sqrt 0.018 (mostly 2025-26), ≤ 0 for rate 0.018 | `late_decay` shrinks the stored offset, so it carries into the next season | | E18 | `e18_ref_op_bridge` | replay | which piece of REF-op matters | REF-op vs sqrt 0.018 within 0.6 pooled; REF-op ahead in 2024-25 by 1.3–1.5, behind in 2025-26 by 0.8–1.3. Warm start +0.9 to +2.0 in 2024-25, −0.2 to −0.7 in 2025-26. No step moves month-avg coverage beyond 0.008. Branch sqrt 0.018 warm: +3.4/+1.6/+1.6/+2.5 pooled, best of all, but month-avg coverage 0.011–0.018 worse than base at h2–h3. States where REF-op wins, 2024-25 / 2025-26: chain ends (warm vs cold) 45 / 13 of 52; warm vs warm 23 / 15; cold vs cold 17 / 15 | steps 0 and 4 are the same cached runs as E13; steps 4 and 5 identical (scale invariance). The chain-ends pair is unmatched on the start | | E19 | `e19_tracker_internals` | replay | why do the leak and constant 0.018 score alike? Step size, offset, base bias and WIS by month on the E20 grid; carry vs reset on both anchors | Step / width at h3 rises from 0.015–0.02 (Dec–Feb) to 0.06–0.09 (Apr–May) on the adaptive rate, 0.005–0.007 to 0.013–0.023 on the constant, in both seasons. Offset / width at h3, Mar/Apr/May 2025: adaptive cold no leak 0.105/0.218/0.186, with leak 0.064/0.078/−0.012, constant cold 0.026/0.055/0.072; Apr/May 2026: 0.160/0.150, 0.057/−0.011, 0.097/0.143. Base bias (share above the base median, h3): Mar 0.43 and 0.27, Apr–May 0.54–0.81 and 0.62–0.71. March WIS at h3: adaptive no leak −48 and −81 (2025, 2026), with leak −27 and −22, constant cold −9 and −35, with leak −5 and −8; Apr–May 2026 constant no leak +4.8/+11.0. **Carry vs reset:** resetting at the season boundary takes the constant tracker's 2025-26 from +3.0/+1.0/+1.1/+1.5 to +1.0/+0.1/+0.2/+0.3, the adaptive one's from +2.2/−0.6/−0.7/−0.2 to +0.7/−0.9/−1.3/−1.6; the carried offset is 0.01–0.025 of width. Carry's gain is Oct–Feb (h3 Nov–Dec 2025 +5.7/+3.8 vs +0.6/+0.4); reset is better after the peak | offsets from the exact runs, step sizes from one pass on final truth per config. `lr_level` for the two-term REF-op configs reports the slow term's rate (about a tenth of the fast one), so their step-size rows are not comparable; REF-op's fast step equals the adaptive anchor's by construction | -| E20 | `e20_structure_grid` | replay | warm start and leak on both anchors: {adaptive 0.03, constant 0.018} × {cold, warm} × {no leak, leak 0.1}, plus REF-op warm and cold | Warm start: +1.0 to +2.6 (adaptive) and +1.5 to +3.8 (constant) in 2024-25, −0.5 to +0.6 in 2025-26, on both anchors at either leak. Leak: on the adaptive rate +0.7 to +2.0 at h1–h3 in both seasons; on the constant rate −0.2 to +0.7 warm and −0.6 to −1.7 in 2025-26 cold. Best cell constant warm leak: +3.3/+1.8/+1.9/+2.9 pooled vs REF-op warm +2.8/+1.1/+0.9/+1.1, but month-avg coverage 0.012–0.017 above base at h2–h3 in 2025-26 (the warm constant without leak is 0.024–0.037 above). Constant beats adaptive in 29–47 of 52 states in every cell and season. **Covid:** the warm start costs 2–7 points at h0–h2 in 2024-25 (constant) and 1–3 (adaptive), and 0.01–0.04 of month-avg coverage; the leak adds 1–3 on the adaptive anchor and on the cold constant in 2024-25, costs 1–3 on the warm constant. No cell beats base at h1–h3 pooled; least bad is constant cold leak −0.5/−0.8/−0.5 vs REF-op cold −1.1/−1.7/−1.6, adaptive warm leak +3.4 at h3 but −1.9 at h1 | replaces E04, E05 and E11 on the replay (retired). Not a scale comparison: every cell runs on sqrt, so E03 stays. The slow term is left out (E18 step 2). All constant-rate configs tuned in-sample (E16) | +| E20 | `e20_structure_grid` | replay | warm start and leak on both anchors: {adaptive 0.03, constant 0.018} × {cold, warm} × {no leak, leak 0.1}, plus REF-op warm and cold | Warm start: +1.0 to +2.6 (adaptive) and +1.5 to +3.8 (constant) in 2024-25, −0.5 to +0.6 in 2025-26, on both anchors at either leak. Leak: on the adaptive rate +0.7 to +2.0 at h1–h3 in both seasons; on the constant rate −0.2 to +0.7 warm and −0.6 to −1.7 in 2025-26 cold. Best cell constant warm leak: +3.3/+1.8/+1.9/+2.9 pooled vs REF-op warm +2.8/+1.1/+0.9/+1.1, but month-avg coverage 0.012–0.017 above base at h2–h3 in 2025-26 (the warm constant without leak is 0.024–0.037 above). Constant beats adaptive in 29–47 of 52 states in every cell and season. **Bootstrap (90%, rounds):** flu single cells are ±1.5 to ±4 per season on the adaptive anchor and ±0.5 to ±2 on the constant; paired gaps: best cell − REF-op warm +0.5/+0.7/+1.0/+1.8 pooled at h0–h3 with the interval excluding zero at h2–h3; best cell − warm constant without leak +0.2/+0.3/+0.4 at h1–h3, intervals include zero (2024-25 alone excludes it, +0.4 to +0.7); warm − cold on the constant with leak +1.8 to +3.8 in 2024-25 excluding zero, +0.4 to +1.6 in 2025-26 including it. **Covid:** the warm start costs 2–7 points at h0–h2 in 2024-25 (constant) and 1–3 (adaptive), and 0.01–0.04 of month-avg coverage; the leak adds 1–3 on the adaptive anchor and on the cold constant in 2024-25, costs 1–3 on the warm constant. No cell beats base at h1–h3 pooled; least bad is constant cold leak −0.5/−0.8/−0.5 vs REF-op cold −1.1/−1.7/−1.6, adaptive warm leak +3.4 at h3 but −1.9 at h1 | replaces E04, E05 and E11 on the replay (retired). Not a scale comparison: every cell runs on sqrt, so E03 stays. The slow term is left out (E18 step 2). All constant-rate configs tuned in-sample (E16) | | E21 | `e21_data_by_season` | replay data | how the two live seasons differ as data: shape, revisions, learning-truth gap at lags 1–4, HHS-stitched burn-in vs NHSN | 2024-25: 27 rounds from 2024-11-23, peak 2025-02-08 at 55.6k (states summed), total 559k; 2025-26: 28 rounds from 2025-10-18, peak 2026-01-03 at 42.5k, total 336k. State peaks within ±1 week of the national one: 37 vs 47 of 52. Revisions: 26% vs 40% of (state, week) keys never revised; relative spread > 10% for 48% vs 39%. Value at lag 2 (what the tracker learns from) vs final: median gap −1.1% vs 0.0%, \|gap\| > 10% for 19% vs 14% of keys; at lag 1, 44% vs 41%. HHS vs NHSN final over 2023-24: 0.957–0.978 in Nov–Apr (states summed), per-state median 1.004 | describes the data only; no tracker runs. Revision keys start at the first archive version (2024-11-19) | E06 (ILI+ burn-in) and E08 (old covid) were run on code or data later found @@ -242,7 +267,7 @@ Checked and resolved on 2026-10-02: | E00, E01 | verified | | E02, E07, E09 | commented; changes made, not re-verified | | E10 part 1, E18 | changes made (finalists runs, matched pairs), rendered 2026-10-07, not reviewed | -| E20 (flu and covid), E10, E15 | rendered 2026-10-07 with the covid grid and the new finalist, not reviewed | +| E20 (flu and covid), E10, E15 | rendered 2026-10-07 with the covid grid, the new finalist and (E10, E20) the bootstrap tables, not reviewed | | E19 | rendered 2026-10-07, not reviewed | | E21 | rendered 2026-10-06, not reviewed | | E03, E10, E14, E15, E16, E17 | not reviewed | @@ -264,7 +289,9 @@ reviewed. `off_after`, `late_decay`, `lr_seasonal`. - `R/calibration/hub_data.R`: hub readers (`hub_read_forecasts()` drops spoiled submissions), `nhsn_read_truth()`. -- `R/calibration/views.R`: reference configs, cached runs, standard views. +- `R/calibration/views.R`: reference configs, cached runs, standard views, + and `cal_wis_boot()` (round bootstrap of WIS reduction %, shared draws + across configs) with `cal_wis_boot_diff()` for paired gaps. - `scripts/calibration/calibration_ws_replay.R`: E12/E13. Scores in `cache/calibration/ws_replay_scores_*.csv`. - `scripts/calibration/calibration_ws_replay.R` also holds the notebook @@ -344,9 +371,9 @@ operating point. | item | status | |---|---| | **Reorganization (2026-10-06).** The constant-rate line was never given the structure analysis (E03–E05, E11 ran the adaptive rate on the ensemble; E16 inherited "cold" from E11's framing and E13's constant 0.1 warm covid loss). Plan: (1) E18 matched-pair scatters, warm vs warm and cold vs cold; (2) E20, the {anchor} × {start} × {leak} grid on the replay; (3) E19 as a notebook on that grid with WIS by month and carry vs reset; (4) E21, the seasons as data. Then re-read E03–E05 and E11 against E20 and retire or keep them | all rendered 2026-10-07 (E21 on 2026-10-06) and read into this file; E20 covers covid too, and sqrt 0.018 warm + leak is a finalist in E10 and E15. E03–E05 and E11 re-read against E20 on 2026-10-07: E20 reproduces E04, E05 and E11's directions on the replay (warm start first-season only, leak season-dependent on the adaptive rate, slow term nothing, constant beats adaptive), so those three were deleted; their rows stay in "Experiments". E03 stays: E20 runs every cell on sqrt and never compares scales | Render with `just calibration-experiments e18_ref_op_bridge e19_tracker_internals e20_structure_grid` inside the rocker container | -| Uncertainty for WIS differences (bootstrap over rounds) | next (2026-10-07): the two warm constant cells in E20 differ by 0.3 pooled, under the 1-point convention. A round bootstrap in the season-table helper gives every E20 and E10 table an interval | +| Uncertainty for WIS differences (bootstrap over rounds) | closed 2026-10-07: `cal_wis_boot()` in every E10 and E20 season table, with paired gaps. The flu finalist's lead over REF-op warm survives at h2–h3 on both bases; the leak's WIS gain does not (its case is coverage); the constant anchor's lead over the adaptive one survives at h2–h3; the warm start's gain is 2024-25 only. See "How to read the numbers" and "Finalists" | | Shrink only the played offset after the peak, keep the stored state | open; `late_decay` shrinks the stored state | -| Disease-specific operating point | provisional (2026-10-07): covid REF-op cold (no grid cell helps at h1–h3, E20); flu sqrt 0.018 warm + leak (E10, E20), about 1/1.5/2.5 points over REF-op warm at h1–h3 on the ensemble, coverage within 0.002. Rate and leak picked in-sample; two seasons, no uncertainty. Not implemented (ROADMAP 1f) | +| Disease-specific operating point | provisional (2026-10-07): covid REF-op cold (no grid cell helps at h1–h3, E20); flu sqrt 0.018 warm + leak (E10, E20), about 1/1.5/2.5 points over REF-op warm at h1–h3 on the ensemble, coverage within 0.002. Rate and leak picked in-sample; two seasons. The bootstrap backs the flu lead at h2–h3 on both bases but not the leak's WIS share of it. Not implemented (ROADMAP 1f) | | HHS burn-in level | closed 2026-10-07: E21 measures it at 2–4% below the NHSN final in Nov–Apr 2023-24 (states summed), per-state median ratio 1.004, within the warm start's own noise. Used as-is; the flu finalist is warm, so 1f needs the stitch in the pipeline | | `windowed_seasonal_extra_sources` retrospective | parked (ROADMAP) | diff --git a/reports/writeups/calibration_experiments/e10_covid.Rmd b/reports/writeups/calibration_experiments/e10_covid.Rmd index 8498f6b1..614756f8 100644 --- a/reports/writeups/calibration_experiments/e10_covid.Rmd +++ b/reports/writeups/calibration_experiments/e10_covid.Rmd @@ -69,6 +69,45 @@ for (d in names(FIN)) { } ``` +## WIS by season with bootstrap intervals + +Every cell carries a 90% interval from a round bootstrap: the season's +rounds are resampled with replacement 1000 times and the WIS reduction +recomputed on each draw (`cal_wis_boot()`), the same draws for every +config. The paired-gap tables give the finalist's gap to prod's config on +each base with its paired interval. The round is the resampling unit, which +ignores dependence between neighbouring rounds, so the intervals are if +anything too narrow. + +```{r fin-boot, results="asis"} +FIN_PAIRS <- list( + c("sqrt 0.018 warm + leak", "REF-op warm"), + c("sqrt 0.018 warm + leak", "sqrt 0.018 warm"), + c("sqrt 0.018", "REF-op cold") +) +for (d in names(FIN)) { + cat("\n\n", cal_h(3), " ", d, "\n\n", sep = "") + bt <- cal_wis_boot(FIN[[d]]$cals, seasons = LIVE) + cal_wis_boot_table(bt) %>% + knitr::kable(caption = paste0(d, ": ", cal_wis_boot_caption(bt))) %>% + print() + gaps <- purrr::map(c("ensemble", "replay"), function(b) { + purrr::map(FIN_PAIRS, function(pr) { + a <- paste(b, pr[1], sep = ", "); bb <- paste(b, pr[2], sep = ", ") + if (!all(c(a, bb) %in% names(FIN[[d]]$cals))) return(NULL) + cal_wis_boot_diff(bt, a, bb) %>% mutate(base = b, pair = paste(pr[1], "−", pr[2]), .before = 1) + }) %>% bind_rows() + }) %>% bind_rows() + if (nrow(gaps) > 0) { + gaps %>% + transmute(base, pair, col = paste(season, h), cell = sprintf("%+.1f [%+.1f, %+.1f]", diff, lo, hi)) %>% + tidyr::pivot_wider(names_from = col, values_from = cell) %>% + knitr::kable(caption = paste0(d, ": gap in WIS reduction % (first − second) with its 90% paired bootstrap interval")) %>% + print() + } +} +``` + ## Do the finalists rank the same way on both bases? Per disease, the candidates run on both bases (covid has no warm runs on the diff --git a/reports/writeups/calibration_experiments/e20_structure_grid.Rmd b/reports/writeups/calibration_experiments/e20_structure_grid.Rmd index e8cf78e6..7bb1be9d 100644 --- a/reports/writeups/calibration_experiments/e20_structure_grid.Rmd +++ b/reports/writeups/calibration_experiments/e20_structure_grid.Rmd @@ -54,6 +54,15 @@ WIS reduction % = 100 × (base − calibrated) / base, so positive = calibrated WIS lower than base. L1 coverage bias: lower is better. "Month-avg" coverage bias is computed within each calendar month and then averaged. +Every WIS cell carries a 90% interval from a round bootstrap: the season's +rounds are resampled with replacement 1000 times and the WIS reduction +recomputed on each draw (`cal_wis_boot()`). The same draws score every +config, so a gap between two configs has a paired interval; the "Paired +gaps" tables show it for the cells the ledger compares. Outcomes within a +round move together, so the round is the resampling unit; this ignores +dependence between neighbouring rounds, so the intervals are if anything +too narrow. + ```{r load} source(here::here("scripts/calibration/calibration_ws_replay.R")) DISEASES <- c(flu = "flu", covid = "covid") @@ -78,7 +87,8 @@ M <- purrr::map(RUNS, function(r) { m <- ws_wis_by_season(r$cals) mv <- cal_month_view(r$cals, seasons = LIVE) cov <- mv$cov %>% transmute(config, season, h = factor(paste0("h", horizon), levels = h_levels), pooled = pooled_cal, mavg = mavg_cal) - list(m = m, mv = mv, cov = cov) + bt <- cal_wis_boot(r$cals, seasons = LIVE) + list(m = m, mv = mv, cov = cov, bt = bt) }) ``` @@ -87,10 +97,42 @@ M <- purrr::map(RUNS, function(r) { ```{r wis-table, results="asis"} for (d in names(M)) { cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") - M[[d]]$m %>% - transmute(config, col = paste(season, h), cell = sprintf("%+.1f", wis_pct)) %>% + cal_wis_boot_table(M[[d]]$bt) %>% + knitr::kable(caption = paste0(d, ": ", cal_wis_boot_caption(M[[d]]$bt))) %>% + print() +} +``` + +# Paired gaps + +The gap in WIS reduction % between two cells, with its paired bootstrap +interval (same draws on both sides). An interval that excludes zero in a +season means the ranking of the pair survived resampling that season's +rounds. + +```{r paired-gaps, results="asis"} +PAIRS <- list( + flu = list( + c("constant 0.018 | warm | leak 0.1", "constant 0.018 | warm | no leak"), + c("constant 0.018 | warm | leak 0.1", "REF-op warm"), + c("constant 0.018 | warm | leak 0.1", "adaptive 0.03 | warm | leak 0.1"), + c("constant 0.018 | warm | leak 0.1", "constant 0.018 | cold | leak 0.1") + ), + covid = list( + c("constant 0.018 | cold | leak 0.1", "REF-op cold"), + c("constant 0.018 | cold | leak 0.1", "constant 0.018 | cold | no leak"), + c("constant 0.018 | cold | leak 0.1", "constant 0.018 | warm | leak 0.1") + ) +) +for (d in names(M)) { + cat("\n\n", cal_h(2), " ", d, "\n\n", sep = "") + purrr::map(PAIRS[[d]], function(pr) { + cal_wis_boot_diff(M[[d]]$bt, pr[1], pr[2]) %>% mutate(pair = paste(pr[1], "−", pr[2]), .before = 1) + }) %>% + bind_rows() %>% + transmute(pair, col = paste(season, h), cell = sprintf("%+.1f [%+.1f, %+.1f]", diff, lo, hi)) %>% tidyr::pivot_wider(names_from = col, values_from = cell) %>% - knitr::kable(caption = paste0(d, ": WIS reduction % (positive = calibrated WIS lower than base), states only; `both` pools the seasons")) %>% + knitr::kable(caption = paste0(d, ": gap in WIS reduction % (first − second) with its 90% paired bootstrap interval")) %>% print() } ``` From 94752e9173d31c83a480b671685958f52a9c2eff Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:00:15 -0700 Subject: [PATCH 106/107] docs(calibration): ROADMAP item 1 points at the ledger; the pipeline plan and the ILINet idea move there. Ledger: coverage-first rule, preliminary operating point (flu sqrt 0.018 cold), priorities, rewrite TODO --- notes/ROADMAP.md | 149 +++--------------------------------- notes/calibration-ledger.md | 133 ++++++++++++++++++++++++++++++-- 2 files changed, 137 insertions(+), 145 deletions(-) diff --git a/notes/ROADMAP.md b/notes/ROADMAP.md index 67e8dbef..561bb7b3 100644 --- a/notes/ROADMAP.md +++ b/notes/ROADMAP.md @@ -7,144 +7,17 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. ## Forecast evaluation -1. **Use the evaluation project as the calibration testbed.** The - operating-point work (hub harness, `notes/calibration-ledger.md`, - "Finalists") is down to a disease-specific choice. Provisional - (2026-10-07, not implemented): covid REF-op cold (no tracker tested helps - at h1–h3, E20), flu sqrt 0.018 warm + leak (about 1/1.5/2.5 points over - REF-op warm at h1–h3 on the ensemble, coverage within 0.002, the 90% - round-bootstrap interval on that gap excluding zero at h2–h3; E10, E20). - Done so far: a1–a4, b1–b4 (E12, E13), the structure grid on both - anchors and diseases (E18–E21). Next: item 1f, in its own thread. Calibrating needs full quantile forecasts over many rounds, - not just scores. The evaluation project (`flu_hosp_evaluation` / - `covid_hosp_evaluation`) replays the current prod components and - ensemble weekly since 2024-11-20, which is what calibration would see in - prod from now on. The hub-submission harness stays the record of what was - actually submitted; the explore stores (per-forecaster quantile forecasts, - NHSN only, on S3) can answer "does calibrating a candidate beat picking a - different one" without a new project. Three gaps, below. Work them in - small steps (AGENTS.md, "Working incrementally"); the S3 evaluation - stores are empty, so even the baseline needs a run. +1. **Calibration in the evaluation pipeline.** The operating point is a + provisional disease-specific choice (covid REF-op cold, flu sqrt 0.018 + warm + leak), not yet implemented. The plan, the pipeline's current + state and the open decisions are in `notes/calibration-ledger.md`, + "TODO: calibration in the evaluation pipeline". The clean replay + (`EVALUATION_FORECASTERS`, `EVALUATION_SUBSTITUTIONS=false`, AGENTS.md) + and the harness-side 2023-24 burn-in are done; see the ledger's "Code, + data and prod". The fixed-weight ensemble and hand-edits-on-vs-off + questions (old 1c, 1d) are in Parked; ILINet as a held-out target (old + 1e) is in the ledger's "Ideas". - **Critical path for evaluating calibration:** - - a. *Clean replay, one component.* Start with `windowed_seasonal` alone - (the harness proxy). A single component has no ensemble weights, so - the only hand edit to switch off is the data substitutions - (`make_forecast_snapshot(substitutions = NULL)`). Steps: a way to run - one forecaster (e.g. an env var that filters the grid), 2–3 dates, - diff against the edited run (changes should appear only at the - substituted geo-dates), then all dates, timed. Feed it to the - calibration harness. - - a1–a2 done 2026-10-01: `EVALUATION_FORECASTERS`, `EVALUATION_DATES`, - `EVALUATION_SUBSTITUTIONS=false` (AGENTS.md "Key env vars"); prod - manifests unchanged. Filtering out any ensemble component skips all - ensemble, submission, report and calibration targets. On vs off, - the snapshot differs exactly at the CSV cells (flu 2025-01-08, - 2025-02-12, 2026-01-28; covid 2025-02-19; control date - bit-identical). The forecasts move a lot at the substituted geos - and slightly (median ~0.1%, max ~6%) at every other geo on the - same date, because `windowed_seasonal` fits one model pooled - across geos. So the expected diff is "only on substituted dates", - not "only at substituted geos". Single-forecaster cost: ~40 - CPU-s per date plus ~4 min fixed overhead per run. - - a3 done 2026-10-01: `EVALUATION_FORECASTERS=windowed_seasonal - EVALUATION_SUBSTITUTIONS=false make eval-{flu,covid}` into the - local stores (not pushed). Flu 11.5 min, covid 9 min, 0 errors, 98 - dates (2024-11-20 to 2026-09-30), all 53 geos, no NAs. Forecasts - are in `forecast_nhsn_full`. Gaps: `local_scores_nhsn` never - scores `us` (52 of 53 geos, likely a geo-name mismatch with - `nhsn_latest_data`); NSSP inputs drop `mo` on 81 dates, `wy` on 28 - and `nh` on 5, for both diseases. - - a4 done 2026-10-01: `ch_use("{flu,covid}_windowed_seasonal")` in - `scripts/calibration/calibration_harness.R` reads forecasts from - the clean store (decision: read the store, backfill only for spec - changes; the two paths match exactly on 5 dates per disease). - Results are E12 in `notes/calibration-ledger.md`. The two diseases' - replays differ on 2024-11-20: flu generates on 11-21, covid on - 11-20 (NHSN's bad release; the hub-round filter drops that round). - Decision (2026-10-01): calibrate h0 and up only. h−1 belongs to - the revision-aware methods (item 3) and gets integrated with them - later. - b. *2023-24 as a burn-in season.* REF-op warm-starts from a burn-in - season, and the replay has none. The warm start helps only in the - first live season (E05, E18, E20), but the flu finalist uses it, so the - stitch stays on the critical path for 1f. Its level is settled (E21: - HHS 2–4% below the NHSN final over 2023-24). Delphi's `hhs` source - (`confirmed_admissions_{influenza,covid}_1d`) has real issue history - for 2023-24 (CA 2023-12-01: issues 12-06, 12-08, 12-20, 12-22), until - HHS reporting ended 2024-04-30. Steps, each checked before the next: - - HHS weekly archive (done 2026-10-01, - `scripts/one_offs/hhs_2023_24_archive.R`). Decisions: NHSN's week - ending Saturday S is the sum of hhs daily `time_value` S−7..S−1 - (exact match in 71% of flu geo-weeks; other alignments 17–24%), - labelled `time_value = S − 3` (Wednesday). Every daily issue is a - version; a week's value as of v sums its 7 days at their latest - issue ≤ v, and a week appears only once all 7 days exist (first - report lag is a median of 4 days, as with NHSN). US is hhs - `nation`, which equals the sum of states plus territories, as NHSN - `us` does. The `hhs` source matches the cached healthdata.gov - snapshots exactly, so the gap below is between sources, not a - fetch problem. **Open: level.** Finalized HHS runs below finalized - NHSN (in-season sum ratio flu 0.968, covid 0.941, falling to ~0.91 - by April; tx/in/ks/ar/pr 15–20% low). The gap is already there in - NHSN's first 2023-24 version, so it isn't an NHSN revision. Choose: - use as-is with a caveat, rescale per geo by the finalized ratio - (uses finalized data), or drop the worst geos. Also decide how to - stitch: NHSN's archive holds 2023-24 from version 2024-11-19, so - dates after that see NHSN history, a level jump from HHS; - - ILI+/flusurv training extras (checked 2026-10-01). They are folded - into `nhsn_prod_archive` (`pipelines/flu_hosp_prod.R`) with - `version = time_value`, and ILI+ runs to 2024-07-24. A 2023-12-06 - snapshot would train on finalized ILI+ up to the forecast week - itself. The snapshot's version-faithfulness abort can't see this - (no row is newer than its version); only the archive-build - `stopifnot` catches it. Decision: version each extras row at its - season's end (`group_by(source, season)`, - `version = max(time_value)`) and replace that `stopifnot` with a - check on versions. Dates from 2024-11-21 on see every extras row - either way, so the golden diff should be empty. Covid has no - extras; - - NSSP vintages (checked 2026-10-01): none before 2024-04-18 from any - source (v5 API, epidatr `nssp`, the S3 Socrata snapshots, the hub - mirror). Decision: `windowed_seasonal_extra_sources` sits out - 2023-24. Gotcha: an NSSP snapshot before the first vintage - silently returns 0 rows, so a 2023-24 replay needs an explicit skip - or abort for every component that reads NSSP. NHSN versions start - 2024-11-19, so the HHS archive must supply every 2023-24 target - row; - - Done 2026-10-01 in the harness, not the pipeline: HHS back to - 2020-08 (issues before 2023-07 collapse to one version), stitched - before NHSN's first version; a 2024-12-04 control matches the - store exactly. Results: E13 in `notes/calibration-ledger.md`. - f. *Calibration in the evaluation pipeline.* The calibration targets in - `pipelines/{flu,covid}_hosp_prod.R` run in evaluation mode too, but - there they calibrate the submitted `CMU-TimeSeries` history from the - hub checkout plus only the current replayed round; learn from - `hhs_evaluation_data` (the latest version, so final truth, not exact); - hard-code REF-op; and are skipped when `EVALUATION_FORECASTERS` drops - an ensemble component. To evaluate calibration there: calibrate the - replayed forecasts over all dates, learn from per-round snapshots - (`calibrate_hub_forecasts_exact()`), take the config as a parameter so - prod's config and the finalist (flu: sqrt 0.018 warm + leak; covid: - REF-op cold is prod) both run, and score states only. A burn-in for REF-op needs - the HHS stitching (b), which today exists only in the harness. - - **Later (not needed to evaluate calibration; c and d moved to Parked):** - - e. *ILINet as a target*, to check our methods on a dataset we have not - tuned on. `fluview` has real issue history (CA wILI for week 2023-50 - is revised weekly through May 2024) back to about 2010. Follows the - `nssp_target_archive` + `primary_source` pattern; the ILI+ training - rows must be dropped in this mode, and wILI is a percentage, so it - shares the scale question with NSSP-target scoring (item 2). Prototype - on one forecaster and a few seasons; `calibration_ili_backfill.R` - (ILI+, no vintages) is a reference, not the base to extend. - Source (checked 2026-10-02, `notes/ilinet-sources.md`): v5 - `fluview_ilinet` matches epidatr `fluview` at every issue outside NY - (99.99% of state rows), with version date = CDC release Friday. State - version history starts 2017w40 in both, so a vintage backtest has - 2017-18 on; use state `ili` (v5 has no state `wili`). 2. **NSSP-target backtesting is second-class and needs dedicated attention.** - Explore forecasts NHSN only: every family sets `outcome = "hhs"`. NSSP @@ -251,7 +124,7 @@ validator, simplification inventory) live in `notes/refactor-ideas.md`. one), check the resolved weights on one date, then replay. Not needed to choose the operating point: E10 already scores the finalists on the submitted ensemble, and E07 shows calibration acts the same way - on the ensemble and the single-component replay. Revisit if 1f needs + on the ensemble and the single-component replay. Revisit if the calibration pipeline plan (ledger TODO) needs a clean ensemble history. - *Hand edits on vs off as its own question* (was item 1d, parked 2026-10-07). A separate evaluation project (like the `_regr` ones) replaying with the edits, to measure diff --git a/notes/calibration-ledger.md b/notes/calibration-ledger.md index 3a239525..3a4be6b8 100644 --- a/notes/calibration-ledger.md +++ b/notes/calibration-ledger.md @@ -30,6 +30,13 @@ removed; they are in VCS history at change `uxposooz` pooled paired gaps and is too optimistic for single cells. The round is the resampling unit, which ignores dependence between neighbouring rounds, so the intervals are if anything too narrow. +- **Coverage bias is the selection criterion; WIS is the tie-breaker + (2026-10-07).** Pick the config on the Pareto front where both improve, + and never trade month-avg coverage for WIS: WIS rewards a one-directional + shift when the base runs low (see "The correction is a small upward + shift") and is the less trusted metric. A config whose month-avg coverage + is worse than base in any season at h1–h3 is not a finalist, whatever its + WIS. - **Use month-avg coverage, not season-pooled.** Pooling over a season (or two) lets ramp under-prediction cancel post-peak over-prediction, and lets one season's error cancel the other's. Season-pooled gains of 2–5× mostly @@ -170,11 +177,23 @@ Finalists (E10 part 1, E18), states only, h1/h2/h3 pooled over both seasons: to +1.0/+0.1/+0.2/+0.3. Cold configs enter 2024-25 with nothing, so the two seasons are not the same test of a tracker. -Operating point: for covid nothing tested beats REF-op cold (prod) at -h0–h3. For flu, sqrt 0.018 warm + leak is ahead of REF-op warm (prod) by -about 1 point at h1, 1.5 at h2 and 2.5 at h3 on the ensemble, no longer -behind it in 2025-26, with month-avg coverage within 0.002 of REF-op -warm's. Its rate and leak were picked in-sample. +Operating point (preliminary, chosen 2026-10-07 for that week's +submission; the open threads below can still move it): + +- **Flu: sqrt 0.018 cold** (in `pipelines/flu_hosp_prod.R`). Chosen on + coverage, not WIS: it is the only flu config whose month-avg coverage is + better than base in both seasons on both bases (by 0.005–0.014 at h1–h3) + and whose WIS is never below base. It gives up the warm start's 2024-25 + WIS (sqrt 0.018 warm + leak is +2.8/+3.0/+4.8 pooled on the ensemble vs + +0.6/+0.1/−0.0), because the warm configs are worse than base on + month-avg coverage in 2025-26 on the replay (+0.008/+0.014 at h2–h3 with + the leak, +0.022/+0.027 without). The rate was picked in-sample (E16). +- **Covid: REF-op cold** (prod, unchanged). No config improves covid + coverage at h1–h3 on the ensemble (every one is 0.000–0.008 worse than + base) and every one loses WIS there; REF-op cold loses least (−2.0 to + −2.9 in 2025-26 vs −6.9 to −8.3 for sqrt 0.018 cold, which is only + competitive on the replay). Its value is h0 (+0.8) and h−1 (+10 to +15, + E10 part 2). ## Reference configs @@ -318,6 +337,51 @@ reviewed. `windowed_seasonal_extra_sources` at h1–h3 and `revision_aware` at h−1 (weights in `pipelines/covid_geo_exclusions.csv`, edited weekly). +## TODO: calibration in the evaluation pipeline + +The evidence for the operating point comes from the harness +(`scripts/calibration/`), which reads the evaluation store's forecasts, +learns from per-round NHSN snapshots and scores states only. The pipeline's +calibration targets (`pipelines/{flu,covid}_hosp_prod.R`) run in evaluation +mode but unchanged from prod: they calibrate the hub checkout's submitted +history plus only the current replayed round, learn from +`hhs_evaluation_data` (final truth), hard-code REF-op, and are skipped when +`EVALUATION_FORECASTERS` drops an ensemble component. The goal is a +calibrated row in the store beside the other forecasters, so the score +notebooks pick it up. + +Three steps, each a run and a row in "Experiments": + +1. **Covid, REF-op cold, on the store's ensemble.** In evaluation mode the + target takes `ensemble_mix` over all replayed dates from the store, + calls `calibrate_hub_forecasts_exact()` with truth and vintages from + `nhsn_archive_data` (as `ch_hub_truth()` does), and takes the config + from a per-disease list. No burn-in. Score it like any other forecaster; + states-only is a scoring filter, so keep the US row. Check against E10's + covid ensemble REF-op cold row, the same computation in the harness. +2. **Flu, sqrt 0.018 warm + leak, on the store's ensemble.** Same target, + plus the hub checkout's 2023-24 rounds as the burn-in season (the prod + target already reads them). The burn-in learns from the NHSN archive's + first version, which carries 2023-24 as final, as E10's ensemble warm + runs do (`fin_inputs()`); no HHS stitch on this base. Check against + E10's ensemble sqrt 0.018 warm + leak row. +3. **Prod's config beside the finalist.** REF-op warm (flu) and REF-op cold + (covid) through the same target as a second row, so the evaluation + reports show both. Then decide whether prod switches. + +Not in these steps: a calibrated row on the clean `windowed_seasonal` base. +The store has nothing before 2024-11-20; the harness backfills 2023-24 by +rerunning the forecaster on the HHS-stitched archive +(`ws_burn_in_forecasts()`). Doing that in the pipeline needs two decisions +already made but not implemented: ILI+/flusurv extras are folded into +`nhsn_prod_archive` with `version = time_value`, so a 2023-24 snapshot +would train on finalized ILI+ (version each extras row at its season's end +and replace the archive-build `stopifnot` with a version check; dates from +2024-11-21 on are unaffected); and NSSP has no vintages before 2024-04-18, +so every NSSP-reading component needs an explicit skip for 2023-24 (a +snapshot before the first vintage silently returns 0 rows). Parked unless +the evaluation reports need the replay base. + ## Running things Commands run from the repo root (inside the rocker container, @@ -373,9 +437,40 @@ operating point. | **Reorganization (2026-10-06).** The constant-rate line was never given the structure analysis (E03–E05, E11 ran the adaptive rate on the ensemble; E16 inherited "cold" from E11's framing and E13's constant 0.1 warm covid loss). Plan: (1) E18 matched-pair scatters, warm vs warm and cold vs cold; (2) E20, the {anchor} × {start} × {leak} grid on the replay; (3) E19 as a notebook on that grid with WIS by month and carry vs reset; (4) E21, the seasons as data. Then re-read E03–E05 and E11 against E20 and retire or keep them | all rendered 2026-10-07 (E21 on 2026-10-06) and read into this file; E20 covers covid too, and sqrt 0.018 warm + leak is a finalist in E10 and E15. E03–E05 and E11 re-read against E20 on 2026-10-07: E20 reproduces E04, E05 and E11's directions on the replay (warm start first-season only, leak season-dependent on the adaptive rate, slow term nothing, constant beats adaptive), so those three were deleted; their rows stay in "Experiments". E03 stays: E20 runs every cell on sqrt and never compares scales | Render with `just calibration-experiments e18_ref_op_bridge e19_tracker_internals e20_structure_grid` inside the rocker container | | Uncertainty for WIS differences (bootstrap over rounds) | closed 2026-10-07: `cal_wis_boot()` in every E10 and E20 season table, with paired gaps. The flu finalist's lead over REF-op warm survives at h2–h3 on both bases; the leak's WIS gain does not (its case is coverage); the constant anchor's lead over the adaptive one survives at h2–h3; the warm start's gain is 2024-25 only. See "How to read the numbers" and "Finalists" | | Shrink only the played offset after the peak, keep the stored state | open; `late_decay` shrinks the stored state | -| Disease-specific operating point | provisional (2026-10-07): covid REF-op cold (no grid cell helps at h1–h3, E20); flu sqrt 0.018 warm + leak (E10, E20), about 1/1.5/2.5 points over REF-op warm at h1–h3 on the ensemble, coverage within 0.002. Rate and leak picked in-sample; two seasons. The bootstrap backs the flu lead at h2–h3 on both bases but not the leak's WIS share of it. Not implemented (ROADMAP 1f) | -| HHS burn-in level | closed 2026-10-07: E21 measures it at 2–4% below the NHSN final in Nov–Apr 2023-24 (states summed), per-state median ratio 1.004, within the warm start's own noise. Used as-is; the flu finalist is warm, so 1f needs the stitch in the pipeline | +| Disease-specific operating point | preliminary choice made 2026-10-07 (see "Operating point"): flu sqrt 0.018 cold on coverage grounds, covid REF-op cold. Earlier provisional reading, WIS-led: covid REF-op cold (no grid cell helps at h1–h3, E20); flu sqrt 0.018 warm + leak (E10, E20), about 1/1.5/2.5 points over REF-op warm at h1–h3 on the ensemble, coverage within 0.002. Rate and leak picked in-sample; two seasons. The bootstrap backs the flu lead at h2–h3 on both bases but not the leak's WIS share of it. Not implemented; see "TODO: calibration in the evaluation pipeline" | +| HHS burn-in level | closed 2026-10-07: E21 measures it at 2–4% below the NHSN final in Nov–Apr 2023-24 (states summed), per-state median ratio 1.004, within the warm start's own noise. Used as-is. The pipeline's flu step (the TODO section) calibrates the ensemble base, whose 2023-24 rounds are hub submissions and whose burn-in truth is the NHSN archive's first version, so the stitch is only needed for a replay-base burn-in | | `windowed_seasonal_extra_sources` retrospective | parked (ROADMAP) | +| Re-rank the finalists coverage-first (2026-10-07) | acted on the same day: flu moved to sqrt 0.018 cold (see "Operating point"). Still open as a question, since the replay 2025-26 excess is near the noise floor. Under the coverage rule the flu finalist (sqrt 0.018 warm + leak) fails on the replay in 2025-26: month-avg coverage +0.008/+0.014 above base at h2–h3 (warm without leak +0.022/+0.027), while on the ensemble it is within 0.004 of base there and 0.024–0.028 better in 2024-25. The cold configs (sqrt 0.018, REF-op cold) improve coverage in both seasons on both bases by 0.005–0.019 but give up 2–4 WIS points at h2–h3 in 2024-25, partly because cold = base through Nov–Dec 2024. Month-level noise is about 0.01, so the replay 2025-26 excess is at the edge. To settle: a coverage bootstrap like `cal_wis_boot()`, and the E20 grid read by coverage first | + +**Priorities (2026-10-07).** The configs tested are within about a point +of each other and differ only in how they handle the rise and the fall, so +further config-vs-config distinctions are deprioritized. Order of work: + +1. **Mechanistic view of one tracker on one season.** The per-round view + (offset relative to width, base bias, truth vs base median, WIS by + round) for the operating config, with the real-time offsets against + what the finalized data later showed. The 2025-26 zigzag reads cleanly + this way: a one-directional offset against a base whose error changes + sign at the peak, the plateau and the final decline. Build this as a + standing notebook (E19 is the start) and run it on every new season + before any config comparison. +2. **Coverage as the criterion.** A coverage bootstrap beside + `cal_wis_boot()`, so the replay 2025-26 excess (0.008–0.014, noise about + 0.01) can be called. Then re-read E20 by coverage. +3. **Covid at h1–h3.** Nothing helps; decide whether to stop calibrating + those horizons and keep h0/h−1. +4. Everything else in this table and in "Ideas" waits. Do not add a config + unless the mechanistic view predicts where it would help; do not tune + for a single season's anomaly. + +**TODO: rewrite this ledger and the notebooks around the mechanistic view +(noted 2026-10-07, not started).** The experiment summaries above are +detailed, expensive to read, and mostly record that the scores are +indifferent to the config; every session starts by triaging them. Slim the +ledger to the data facts, the operating point and its reasoning, the +coverage rule, and the per-season mechanistic reads; drop the E-by-E result +prose (it stays recoverable from the commit log and the rendered +notebooks). Decide what to keep in a dedicated session, not in passing. Closed 2026-10-02 without further work: ramp vs post-peak split (the month view shows it), h−1 calibration (left to the revision-aware methods, @@ -394,3 +489,27 @@ E18–E21 points at any of them; they are kept so they are not re-invented. (E02, E17) or a leak. - A scale proxy: a per-location scale from the base's own interval width rather than population, so the sqrt-per-100k transform is not needed. + +Housekeeping, low priority: + +- Rename the tracker's `slow` / `fast` terms to `init_offset` / + `tracked_offset` (`lr_slow`, `fast_decay`, `slow_init`, `init_slow`, + `burn_in_learns_slow`). The slow term's online learning never registered + (E04, E18); what it does is hold the warm start where the leak cannot + drain it, so the real split is a fixed burn-in offset plus a learned, + leaky one. "Constant" / "adaptive" are taken by the learning rates. + +Held-out data, no code yet: + +- **ILINet as a target**, to check the trackers on a dataset they were not + tuned on. `fluview` has real issue history back to about 2010 (CA wILI + for week 2023-50 is revised weekly through May 2024). Source checked + 2026-10-02 (`notes/ilinet-sources.md`): v5 `fluview_ilinet` matches + epidatr `fluview` at every issue outside NY, version date = CDC release + Friday; state version history starts 2017w40 in both, so a vintage + backtest has 2017-18 on; use state `ili` (v5 has no state `wili`). + Would follow the `nssp_target_archive` + `primary_source` pattern, drop + the ILI+ training rows in this mode, and face the percentage-scale + question NSSP-target scoring has (ROADMAP 2). Prototype on one forecaster + and a few seasons; `calibration_ili_backfill.R` (ILI+, no vintages) is a + reference, not the base to extend. From e7c9623ef6af9aab5857d7c7db0983f2acff1967 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov <1810426+dshemetov@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:59:38 -0700 Subject: [PATCH 107/107] feat(calibration): prod flu switches to sqrt 0.018 cold; settle_days explicit in both pipelines --- pipelines/covid_hosp_prod.R | 2 ++ pipelines/flu_hosp_prod.R | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pipelines/covid_hosp_prod.R b/pipelines/covid_hosp_prod.R index 3c603939..d5f0a865 100644 --- a/pipelines/covid_hosp_prod.R +++ b/pipelines/covid_hosp_prod.R @@ -614,8 +614,10 @@ calibration_targets <- list( by = c("geo_value" = "state_id") ) %>% select("target_end_date", "location", truth = "true_value") + # REF-op cold: see "Operating point" in notes/calibration-ledger.md. calibrate_hub_forecasts( historical_fc, truth, + settle_days = 14L, burn_in_seasons = character(0), transform = "sqrt", lr_args = list(mult = 0.03, floor = 1e-3), diff --git a/pipelines/flu_hosp_prod.R b/pipelines/flu_hosp_prod.R index bf8aa570..611101ef 100644 --- a/pipelines/flu_hosp_prod.R +++ b/pipelines/flu_hosp_prod.R @@ -614,16 +614,19 @@ calibration_targets <- list( by = c("geo_value" = "state_id") ) %>% select("target_end_date", "location", truth = "true_value") + # sqrt constant 0.018 per 100k, cold: see "Operating point" in + # notes/calibration-ledger.md (chosen 2026-10-07 on month-avg coverage). + rate_scales <- get_population_data() %>% + distinct(.data$state_code, .keep_all = TRUE) %>% + transmute(location = .data$state_code, from = as.Date("2000-01-01"), scale = .data$population / 1e5) calibrate_hub_forecasts( historical_fc, truth, - burn_in_seasons = "2023-2024", + settle_days = 14L, + burn_in_seasons = character(0), transform = "sqrt", - lr_args = list(mult = 0.03, floor = 1e-3), - lr_window = 20, - season_policy = "carry", - slow_init = "burn_in_quantile", - lr_slow = list(mult = 0.003), - fast_decay = 0.1 + scales = rate_scales, + lr = 0.018, + season_policy = "carry" ) }, cue = tar_cue("always")