From a6eab4608c33c4a24959e696d11f7a692f1177fc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 01:15:54 +0000 Subject: [PATCH 1/3] fix(neural): make the ESN forecaster actually forecast MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trained ESN predicted no better than the series mean. Three defects in the readout: - Train/predict mismatch. solve_ridge/2 fits a linear map, but compute_output/3 passed the result through a sigmoid the fit never saw, so a correct estimate of 0.5 came out as 0.62 and every forecast was squeezed towards the middle. Apply the linear readout and clamp to [0, 1]. - No bias. The "+1 for bias" slot held the input value; there was no constant feature. Features are now [state..., input, 1.0]. - The "ridge regression" was 200 epochs of SGD at lr 5e-4, far short of the minimum. Replace it with the closed-form solve of (XᵀX + λI')w = Xᵀy (bias unpenalised) by Gaussian elimination with partial pivoting. One-step MSE on held-out data (last 200 of 600 steps, 80-node reservoir, seeds 1-3; baselines: series mean, and persistence "next = current"): series mean persistence old ESN new ESN smooth 3.6e-2 9.9e-4 3.6e-2 1.4e-4 - 1.9e-4 memory 2.2e-2 1.4e-3 3.3e-2 6.8e-4 confidence 3.3e-2 1.4e-6 4.5e-3 1.6e-6 - 1.7e-6 Tests: a constant series is forecast as itself; held-out forecasts beat persistence by 2x; forecasts stay in [0, 1]. The first two fail on the old readout (it lost to persistence by 42x). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0165rGu3tsKmJtevmrdMVyfJ --- lib/neural/echo_state_network.ex | 101 +++++++++++++++++++++++++------ test/neural_test.exs | 52 ++++++++++++++++ 2 files changed, 133 insertions(+), 20 deletions(-) diff --git a/lib/neural/echo_state_network.ex b/lib/neural/echo_state_network.ex index 7d4e33ca..324ead9e 100644 --- a/lib/neural/echo_state_network.ex +++ b/lib/neural/echo_state_network.ex @@ -67,8 +67,8 @@ defmodule Hypatia.Neural.EchoStateNetwork do reservoir: generate_reservoir(size), input_weights: generate_weights(size, @input_scaling), feedback_weights: generate_weights(size, @teacher_scaling), - # +1 for bias - output_weights: List.duplicate(0.0, size + 1), + # Readout over [state..., input, bias] + output_weights: List.duplicate(0.0, size + 2), state: List.duplicate(0.0, size) } end @@ -97,9 +97,7 @@ defmodule Hypatia.Neural.EchoStateNetwork do {states, _final_esn} = Enum.reduce(time_series, {[], esn}, fn value, {states_acc, current_esn} -> new_state = update_state(current_esn, value) - # Append input as extra feature - extended = new_state ++ [value] - {[extended | states_acc], %{current_esn | state: new_state}} + {[features(new_state, value) | states_acc], %{current_esn | state: new_state}} end) all_states = Enum.reverse(states) @@ -250,14 +248,17 @@ defmodule Hypatia.Neural.EchoStateNetwork do end end + # The readout is fitted as a linear map (solve_ridge/2), so it is applied as + # one. It used to be passed through a sigmoid here, which the fit never saw: + # a correct linear estimate of 0.5 came out as 0.62, and every forecast was + # squeezed towards the middle. Clamp instead, since the series are + # confidences in [0, 1]. defp compute_output(%__MODULE__{output_weights: ow, trained: true}, state, input) do - extended = state ++ [input] - raw = - Enum.zip(extended, ow) + Enum.zip(features(state, input), ow) |> Enum.reduce(0.0, fn {s, w}, acc -> acc + s * w end) - sigmoid(raw) + raw |> max(0.0) |> min(1.0) end defp compute_output(_esn, state, _input) do @@ -266,23 +267,83 @@ defmodule Hypatia.Neural.EchoStateNetwork do sigmoid(mean) end + # Readout features: reservoir state, the current input, and a constant 1.0 + # for the bias. The input lets the readout fall back to persistence + # ("next ≈ current"), which is a strong baseline for smooth confidence series. + defp features(state, input), do: state ++ [input, 1.0] + + # Closed-form ridge regression: solve (XᵀX + λI') w = Xᵀy, where I' leaves the + # bias unpenalised. This replaces 200 epochs of SGD at lr 5e-4, which stopped + # far short of the minimum (the trained ESN forecast no better than the series + # mean). The system is (reservoir size + 2) square, so Gaussian elimination is + # cheap next to collecting the states. defp solve_ridge(states, targets) do - # Gradient descent approximation of ridge regression - size = length(List.first(states, [])) - weights = List.duplicate(0.0, size) - lr = 0.0005 - epochs = 200 + dim = length(List.first(states, [])) + bias = dim - 1 - Enum.reduce(1..epochs, weights, fn _epoch, w -> + gram = + Enum.reduce(states, List.duplicate(List.duplicate(0.0, dim), dim), fn x, acc -> + Enum.zip_with(acc, x, fn row, xi -> + Enum.zip_with(row, x, fn a, xj -> a + xi * xj end) + end) + end) + + rhs = Enum.zip(states, targets) - |> Enum.reduce(w, fn {state, target}, w_acc -> - pred = Enum.zip(state, w_acc) |> Enum.reduce(0.0, fn {s, wi}, acc -> acc + s * wi end) - error = target - pred + |> Enum.reduce(List.duplicate(0.0, dim), fn {x, y}, acc -> + Enum.zip_with(acc, x, fn a, xi -> a + xi * y end) + end) + + gram + |> Enum.with_index() + |> Enum.map(fn {row, i} -> + if i == bias, do: row, else: List.update_at(row, i, &(&1 + @ridge_lambda)) + end) + |> gaussian_solve(rhs) + end - Enum.zip(w_acc, state) - |> Enum.map(fn {wi, si} -> wi + lr * error * si - lr * @ridge_lambda * wi end) + # Gaussian elimination with partial pivoting. A pivot below 1e-12 (only + # possible with no training rows) contributes a zero weight. + defp gaussian_solve(a, b) do + rows = Enum.zip_with(a, b, fn row, bi -> row ++ [bi] end) + n = length(rows) + + upper = + Enum.reduce(0..(n - 1), rows, fn col, m -> + {done, rest} = Enum.split(m, col) + pivot_row = Enum.max_by(rest, fn r -> abs(Enum.at(r, col)) end) + rest = List.delete(rest, pivot_row) + p = Enum.at(pivot_row, col) + + if abs(p) < 1.0e-12 do + done ++ [pivot_row | rest] + else + reduced = + Enum.map(rest, fn r -> + f = Enum.at(r, col) / p + Enum.zip_with(r, pivot_row, fn x, y -> x - f * y end) + end) + + done ++ [pivot_row | reduced] + end end) + + upper + |> Enum.with_index() + |> Enum.reverse() + |> Enum.reduce(%{}, fn {row, i}, solved -> + p = Enum.at(row, i) + + if abs(p) < 1.0e-12 do + Map.put(solved, i, 0.0) + else + known = + Enum.reduce((i + 1)..(n - 1)//1, 0.0, fn j, acc -> acc + Enum.at(row, j) * solved[j] end) + + Map.put(solved, i, (Enum.at(row, n) - known) / p) + end end) + |> then(fn solved -> Enum.map(0..(n - 1), &solved[&1]) end) end defp sigmoid(x), do: 1.0 / (1.0 + :math.exp(-max(min(x, 500), -500))) diff --git a/test/neural_test.exs b/test/neural_test.exs index bc165efb..ecf89a7f 100644 --- a/test/neural_test.exs +++ b/test/neural_test.exs @@ -86,6 +86,58 @@ defmodule Hypatia.NeuralTest do assert EchoStateNetwork.detect_drift(esn) == :insufficient_data end + # A constant series must be forecast as itself. The old readout had no bias + # and squashed its linear fit through a sigmoid, so it forecast ~0.6 here. + test "trained ESN forecasts a constant series as that constant" do + esn = EchoStateNetwork.init(reservoir_size: 20, seed: 1) + trained = EchoStateNetwork.train(esn, List.duplicate(0.3, 100)) + + {_, ran} = + Enum.reduce(1..20, {nil, trained}, fn _, {_, e} -> EchoStateNetwork.step(e, 0.3) end) + + {prediction, _} = EchoStateNetwork.step(ran, 0.3) + assert_in_delta prediction, 0.3, 1.0e-3 + end + + # One-step forecasts on held-out data must beat "next = current", the + # baseline any forecaster has to clear. The old SGD readout lost to it by + # about 40x on this series and did no better than the series mean. + test "trained ESN beats the persistence baseline on held-out data" do + series = for t <- 1..300, do: 0.5 + 0.25 * :math.sin(t / 9) + 0.1 * :math.sin(t / 3.1) + {train, _} = Enum.split(series, 200) + + trained = + EchoStateNetwork.init(reservoir_size: 20, seed: 1) |> EchoStateNetwork.train(train) + + {_, ran} = + Enum.reduce(series, {nil, trained}, fn u, {_, e} -> EchoStateNetwork.step(e, u) end) + + %{mse: esn_mse, samples: n} = EchoStateNetwork.accuracy_report(ran) + held_out = Enum.take(series, -(n + 1)) + + persistence_mse = + held_out + |> Enum.chunk_every(2, 1, :discard) + |> Enum.map(fn [a, b] -> (a - b) * (a - b) end) + |> Enum.sum() + |> Kernel./(n) + + assert esn_mse < persistence_mse / 2, + "esn_mse=#{esn_mse} persistence_mse=#{persistence_mse}" + end + + test "trained ESN forecasts stay inside [0, 1]" do + series = for t <- 1..120, do: if(rem(t, 10) < 5, do: 0.0, else: 1.0) + + trained = + EchoStateNetwork.init(reservoir_size: 20, seed: 2) |> EchoStateNetwork.train(series) + + {_, ran} = + Enum.reduce(series, {nil, trained}, fn u, {_, e} -> EchoStateNetwork.step(e, u) end) + + assert Enum.all?(ran.predictions, fn p -> p.prediction >= 0.0 and p.prediction <= 1.0 end) + end + test "detect_drift/1 with trained network returns drift status" do esn = EchoStateNetwork.init(reservoir_size: 20) time_series = for i <- 1..100, do: :math.sin(i * 0.1) * 0.5 + 0.5 From 0fad06cdad8dcd1675299f1ca79a957d00b05df7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 01:22:49 +0000 Subject: [PATCH 2/3] test(neural): score the ESN persistence test on held-out points only The test stepped through all 300 points, so accuracy_report/1 (last 200 predictions) included training points 101-200. Warm the reservoir on the training series, clear the prediction history, and score only the 100 held-out points. Still fails against the old readout (47x worse than persistence there). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0165rGu3tsKmJtevmrdMVyfJ --- test/neural_test.exs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/neural_test.exs b/test/neural_test.exs index ecf89a7f..0c5561c5 100644 --- a/test/neural_test.exs +++ b/test/neural_test.exs @@ -101,19 +101,27 @@ defmodule Hypatia.NeuralTest do # One-step forecasts on held-out data must beat "next = current", the # baseline any forecaster has to clear. The old SGD readout lost to it by - # about 40x on this series and did no better than the series mean. + # about 47x on these held-out points. test "trained ESN beats the persistence baseline on held-out data" do series = for t <- 1..300, do: 0.5 + 0.25 * :math.sin(t / 9) + 0.1 * :math.sin(t / 3.1) - {train, _} = Enum.split(series, 200) + {train, held_out} = Enum.split(series, 200) trained = EchoStateNetwork.init(reservoir_size: 20, seed: 1) |> EchoStateNetwork.train(train) + # Warm the reservoir on the training series, then score only forecasts + # made on held-out inputs: accuracy_report/1 keeps the last 200 + # predictions, which would otherwise include training points. + {_, warmed} = + Enum.reduce(train, {nil, trained}, fn u, {_, e} -> EchoStateNetwork.step(e, u) end) + {_, ran} = - Enum.reduce(series, {nil, trained}, fn u, {_, e} -> EchoStateNetwork.step(e, u) end) + Enum.reduce(held_out, {nil, %{warmed | predictions: []}}, fn u, {_, e} -> + EchoStateNetwork.step(e, u) + end) %{mse: esn_mse, samples: n} = EchoStateNetwork.accuracy_report(ran) - held_out = Enum.take(series, -(n + 1)) + assert n == length(held_out) - 1 persistence_mse = held_out From a15c372397531377c8ee28e7a8b4ccdc4a6ceebe Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:38:33 +0000 Subject: [PATCH 3/3] docs(neural): clarify ESN initialization and training behavior --- lib/neural/echo_state_network.ex | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/neural/echo_state_network.ex b/lib/neural/echo_state_network.ex index 324ead9e..1fd93196 100644 --- a/lib/neural/echo_state_network.ex +++ b/lib/neural/echo_state_network.ex @@ -56,7 +56,13 @@ defmodule Hypatia.Neural.EchoStateNetwork do # --- Public API --- - @doc "Initialize a new Echo State Network" + @doc """ + Initialise an untrained network with zero reservoir state and no predictions. + + Options are `:reservoir_size` (default: 80 nodes) and `:seed` (default: + current system time in microseconds). Seeds the calling process's random + number generator to initialise the weights. + """ def init(opts \\ []) do size = Keyword.get(opts, :reservoir_size, @reservoir_size) seed = Keyword.get(opts, :seed, :erlang.system_time(:microsecond)) @@ -89,7 +95,18 @@ defmodule Hypatia.Neural.EchoStateNetwork do {prediction, %{esn | state: new_state, predictions: updated_preds}} end - @doc "Train ESN on a time series (list of floats)" + @doc """ + Fit the readout to predict the next value in a chronological confidence series. + + Requires more than 60 numeric values and discards the first 50 reservoir + states as washout. Fits the reservoir features, current input and an + unpenalised bias using ridge regression. + + Returns the network with fitted output weights and `trained: true`, preserving + its original reservoir state and prediction history. With 60 or fewer values, + returns the network unchanged. Non-numeric values raise `ArithmeticError` + when training proceeds. + """ def train(%__MODULE__{} = esn, time_series) when length(time_series) > @washout + 10 do _size = length(esn.state)