diff --git a/aieng-forecasting/aieng/forecasting/langfuse_tracing.py b/aieng-forecasting/aieng/forecasting/langfuse_tracing.py index 0a94e5db..1936a7a8 100644 --- a/aieng-forecasting/aieng/forecasting/langfuse_tracing.py +++ b/aieng-forecasting/aieng/forecasting/langfuse_tracing.py @@ -24,12 +24,11 @@ def _langfuse_credentials_present() -> bool: class _LangfuseTracingBootstrap: - """Registers LiteLLM + ADK exporters at most once per process.""" + """Registers the Langfuse client and ADK instrumentation once per process.""" - __slots__ = ("_google_adk_instrumented", "_langfuse_client_initialized", "_litellm_instrumented") + __slots__ = ("_google_adk_instrumented", "_langfuse_client_initialized") def __init__(self) -> None: - self._litellm_instrumented = False self._google_adk_instrumented = False self._langfuse_client_initialized = False @@ -46,7 +45,6 @@ def init(self) -> None: # this, ADK spans are emitted into a no-op provider and never reach Langfuse. self._ensure_langfuse_client() - self._register_litellm_langfuse_otel() self._instrument_google_adk() def _ensure_langfuse_client(self) -> None: @@ -64,21 +62,6 @@ def _ensure_langfuse_client(self) -> None: return self._langfuse_client_initialized = True - def _register_litellm_langfuse_otel(self) -> None: - """Register LiteLLM Langfuse callback.""" - if self._litellm_instrumented: - return - try: - import litellm # noqa: PLC0415 - except ImportError: - logger.debug("litellm not installed; skipping LiteLLM Langfuse callback.") - return - - existing = list(getattr(litellm, "callbacks", None) or []) - if "langfuse_otel" not in existing: - litellm.callbacks = [*existing, "langfuse_otel"] - self._litellm_instrumented = True - def _instrument_google_adk(self) -> None: """Instrument Google ADK.""" if self._google_adk_instrumented: @@ -120,11 +103,13 @@ def init_langfuse_tracing() -> None: ``TracerProvider`` receives Langfuse's span processor. This is required for ADK spans emitted via ``openinference-instrumentation-google-adk`` to reach Langfuse. - 2. Appends ``"langfuse_otel"`` to ``litellm.callbacks`` once (if - ``litellm`` is importable). - 3. Runs ``GoogleADKInstrumentor().instrument()`` once (if + 2. Runs ``GoogleADKInstrumentor().instrument()`` once (if ``openinference-instrumentation-google-adk`` is importable). + LiteLLM's ``langfuse_otel`` callback is deliberately not registered: it is + unusable against the Langfuse v4 SDK and stamps a zero ``llm.cost.total`` + on the active span, which suppresses Langfuse's own cost calculation. + Set ``LANGFUSE_HOST`` or ``LANGFUSE_BASE_URL`` for non-default regions. For short-lived processes, call ``langfuse.get_client().flush()`` before exit so pending spans are exported. diff --git a/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py b/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py index 5866997d..e5dd7daf 100644 --- a/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py +++ b/aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py @@ -669,12 +669,16 @@ def build_adk_agent( if isinstance(model, str) and config.openai_base_url: from google.adk.models.lite_llm import LiteLlm # noqa: PLC0415 - # Prefix with "openai/" so LiteLLM uses the OpenAI-compatible path. - # LiteLLM strips the prefix before sending, so the proxy receives the - # bare model name. - litellm_model = model if model.startswith("openai/") else f"openai/{model}" + # Route via LiteLLM's OpenAI-compatible path with ``custom_llm_provider`` + # rather than an ``openai/`` model prefix. ADK stamps ``LlmRequest.model`` + # from this name and OpenInference reports it to Langfuse, which matches + # its per-model price table on the bare name. A prefixed name matches + # nothing, so the generation is logged at zero cost. Both forms route + # identically. + bare_model = model[len("openai/") :] if model.startswith("openai/") else model model = LiteLlm( - model=litellm_model, + model=bare_model, + custom_llm_provider="openai", api_base=config.openai_base_url, api_key=config.openai_api_key, ) diff --git a/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py b/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py index 24b91d72..fcbaa336 100644 --- a/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py +++ b/aieng-forecasting/aieng/forecasting/methods/llm_processes/_client.py @@ -20,13 +20,13 @@ from __future__ import annotations import asyncio +import contextlib import contextvars import json import logging -import os import warnings from concurrent.futures import ThreadPoolExecutor -from typing import Any, Callable, TypeVar +from typing import Any, Callable, Iterator, TypeVar from pydantic import BaseModel, ValidationError @@ -39,21 +39,22 @@ def bootstrap_litellm() -> None: - """One-time wiring of LiteLLM callbacks. + """Suppress LiteLLM and OpenTelemetry logging noise, once per process. Lazy and idempotent so non-LLM predictors do not require Langfuse env vars. - The Langfuse OTEL callback is registered only when ``LANGFUSE_PUBLIC_KEY`` - is set in the environment. + + LiteLLM's ``langfuse_otel`` callback is deliberately not registered. It is + unusable against the Langfuse v4 SDK this repo depends on, and it stamps + ``llm.cost.total`` on the active span from LiteLLM's own ``response_cost``, + which is ``0`` for every proxy-routed model. Langfuse honours a supplied + cost instead of deriving one from usage, so the callback forced agent-path + generations to $0. Instead, :func:`langfuse_generation` creates LLM-process + generations directly and OpenInference covers the agent path, so both price + correctly from ``usage_details``. """ global _BOOTSTRAP_DONE # noqa: PLW0603 if _BOOTSTRAP_DONE: return - import litellm # noqa: PLC0415 - - if os.environ.get("LANGFUSE_PUBLIC_KEY"): - existing = list(getattr(litellm, "callbacks", []) or []) - if "langfuse_otel" not in existing: - litellm.callbacks = [*existing, "langfuse_otel"] # Suppress LiteLLM startup and OTEL noise (mirrors agent_factory.py filter). # Bedrock/SageMaker "no botocore" and OTEL proxy-server notices are harmless. @@ -91,6 +92,61 @@ def _noop(fn: Any) -> Any: return _noop +class _NoopGeneration: + """Stand-in used when Langfuse is unavailable, so callers need no branching.""" + + def update(self, **kwargs: Any) -> None: + """Discard the update.""" + return + + +@contextlib.contextmanager +def langfuse_generation(*, name: str, model: str, input_messages: Any) -> Iterator[Any]: + """Create a Langfuse ``generation`` around one LLM call. + + LiteLLM's ``langfuse_otel`` callback emits no generation when the call runs + inside an already-active Langfuse span, which is every LLM-process + ``predict`` because they are wrapped in :func:`langfuse_observe`. Token + usage, and so cost, never reached Langfuse for those runs. Creating the + generation here works when nested and keeps the model, usage, and payload + under this module's control. + + Cost is deliberately not set. Langfuse derives it from ``usage_details`` + against its own per-model prices, which match the Vector proxy's published + rates. + + ``start_as_current_observation(as_type="generation")`` is a first-class + Langfuse v4 instrumentation API. LiteLLM's Langfuse bridge targets the v2 + SDK, pinning ``langfuse = ^2.45.0``, while this repo requires + ``langfuse>=4.5.1``. Support for v4 is BerriAI/litellm#24123, open and + unanswered since 2026-03-19. Retire this helper in favour of the callback + once that issue is closed and ``langfuse_otel`` is confirmed to emit a + generation under an active Langfuse span. + + Yields a handle exposing ``update(**kwargs)``. That handle is a no-op + stand-in when Langfuse is not installed or a generation cannot be started, + so predictors remain usable without the ``agentic`` and ``llm`` extras. + """ + manager = None + try: + from langfuse import get_client # noqa: PLC0415 + + manager = get_client().start_as_current_observation( + as_type="generation", + name=name, + model=model, + input=input_messages, + ) + except Exception: # pragma: no cover - depends on optional dependency + logger.debug("Langfuse generation unavailable; usage will not be traced.", exc_info=True) + + if manager is None: + yield _NoopGeneration() + return + with manager as generation: + yield generation + + def current_trace_info() -> tuple[str | None, str | None]: """Return ``(trace_id, trace_url)`` from the active Langfuse client, if any.""" try: @@ -299,17 +355,21 @@ async def _one_completion_async( # models that don't support them (e.g. temperature on some o-series). kwargs["drop_params"] = True - resp = await litellm.acompletion(**kwargs) - cost = float(getattr(resp, "_hidden_params", {}).get("response_cost") or 0.0) - usage = getattr(resp, "usage", None) - in_tok = int(getattr(usage, "prompt_tokens", 0) or 0) if usage is not None else 0 - out_tok = int(getattr(usage, "completion_tokens", 0) or 0) if usage is not None else 0 - # Log full usage so we can see thinking-token breakdown when available. - # The proxy may populate completion_tokens_details.reasoning_tokens. - if usage is not None: - logger.debug("LLM usage: %s", vars(usage) if hasattr(usage, "__dict__") else usage) - raw = resp.choices[0].message.content - content = strip_markdown_fence(raw) if raw else raw + # ``model`` is the bare name as configured, before any "openai/" prefixing + # above; that is what Langfuse's price table matches on. + with langfuse_generation(name="llm_completion", model=model, input_messages=messages) as generation: + resp = await litellm.acompletion(**kwargs) + cost = float(getattr(resp, "_hidden_params", {}).get("response_cost") or 0.0) + usage = getattr(resp, "usage", None) + in_tok = int(getattr(usage, "prompt_tokens", 0) or 0) if usage is not None else 0 + out_tok = int(getattr(usage, "completion_tokens", 0) or 0) if usage is not None else 0 + # Full usage exposes the thinking-token breakdown when the proxy + # populates completion_tokens_details.reasoning_tokens. + if usage is not None: + logger.debug("LLM usage: %s", vars(usage) if hasattr(usage, "__dict__") else usage) + raw = resp.choices[0].message.content + content = strip_markdown_fence(raw) if raw else raw + generation.update(output=content, usage_details={"input": in_tok, "output": out_tok}) return content, cost, in_tok, out_tok diff --git a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py index c49b098f..9bee828f 100644 --- a/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py +++ b/aieng-forecasting/tests/aieng/forecasting/methods/agentic/test_agent_factory.py @@ -108,10 +108,12 @@ def test_string_model_wrapped_in_litellm_when_proxy_set(self) -> None: agent = build_adk_agent(config) assert isinstance(agent.model, LiteLlm) - # LiteLlm receives the "openai/" prefix so LiteLLM routes via the - # OpenAI-compatible proxy path; the prefix is stripped before the - # proxy sees the model name. - assert agent.model.model == "openai/gemini-3.1-flash-lite-preview" + # The bare model name is kept and the OpenAI-compatible proxy route is + # selected via custom_llm_provider instead of an "openai/" prefix: + # OpenInference reports this name to Langfuse, which matches its price + # table on the bare name (a prefixed name logs zero cost). + assert agent.model.model == "gemini-3.1-flash-lite-preview" + assert agent.model._additional_args["custom_llm_provider"] == "openai" def test_string_model_kept_as_string_without_proxy(self, monkeypatch: pytest.MonkeyPatch) -> None: """Without a proxy URL the model is passed as a plain string to LlmAgent.""" diff --git a/aieng-forecasting/tests/aieng/forecasting/methods/llm_processes/test__client.py b/aieng-forecasting/tests/aieng/forecasting/methods/llm_processes/test__client.py index d7154cc6..a7d7fc36 100644 --- a/aieng-forecasting/tests/aieng/forecasting/methods/llm_processes/test__client.py +++ b/aieng-forecasting/tests/aieng/forecasting/methods/llm_processes/test__client.py @@ -7,6 +7,8 @@ from __future__ import annotations +import contextlib +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -121,6 +123,7 @@ def _mock_litellm_response(content: str) -> MagicMock: return resp +_CLIENT = "aieng.forecasting.methods.llm_processes._client" _DUMMY_MESSAGES = [{"role": "user", "content": "forecast"}] _DUMMY_FORMAT = {"type": "json_schema", "json_schema": {"name": "x", "schema": {}, "strict": True}} @@ -257,3 +260,75 @@ async def fake_acompletion(**kwargs): # type: ignore[override] assert kw["reasoning_effort"] == "low" assert "extra_body" not in kw or "reasoning_effort" not in kw.get("extra_body", {}) assert kw.get("drop_params") is True + + +# --------------------------------------------------------------------------- +# langfuse_generation: usage and cost reporting for the Langfuse trace +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_generation_receives_bare_model_usage_and_output() -> None: + """The Langfuse generation gets the bare model name, token usage, and output. + + The bare (un-prefixed) name matters: Langfuse matches its per-model price + table on it, so sending ``openai/`` would yield no cost. + """ + captured: dict = {} + + @contextlib.contextmanager + def _fake_generation(*, name: str, model: str, input_messages: object): + captured.update(name=name, model=model, input_messages=input_messages) + + class _Gen: + def update(self, **kwargs: object) -> None: + captured["update"] = kwargs + + yield _Gen() + + resp = _mock_litellm_response('{"ok": 1}') + resp.usage = SimpleNamespace(prompt_tokens=11, completion_tokens=7) + + with ( + patch(f"{_CLIENT}.langfuse_generation", _fake_generation), + patch("litellm.acompletion", new=AsyncMock(return_value=resp)), + ): + await _one_completion_async( + model="gemini-3.5-flash", + messages=_DUMMY_MESSAGES, + response_format=_DUMMY_FORMAT, + temperature=1.0, + max_tokens=512, + timeout_s=30.0, + reasoning_effort=None, + api_base="https://proxy.example.com/v1", + ) + + assert captured["model"] == "gemini-3.5-flash" # not "openai/gemini-3.5-flash" + assert captured["input_messages"] == _DUMMY_MESSAGES + assert captured["update"]["usage_details"] == {"input": 11, "output": 7} + assert captured["update"]["output"] == '{"ok": 1}' + + +@pytest.mark.asyncio +async def test_completion_succeeds_when_langfuse_is_unavailable() -> None: + """A failing Langfuse client degrades to a no-op and the completion returns.""" + resp = _mock_litellm_response('{"ok": 1}') + resp.usage = SimpleNamespace(prompt_tokens=3, completion_tokens=4) + + with ( + patch(f"{_CLIENT}.get_client", side_effect=RuntimeError("no langfuse"), create=True), + patch("litellm.acompletion", new=AsyncMock(return_value=resp)), + ): + content, _cost, in_tok, out_tok = await _one_completion_async( + model="gemini-3.5-flash", + messages=_DUMMY_MESSAGES, + response_format=_DUMMY_FORMAT, + temperature=1.0, + max_tokens=512, + timeout_s=30.0, + reasoning_effort=None, + ) + + assert content == '{"ok": 1}' + assert (in_tok, out_tok) == (3, 4) diff --git a/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py b/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py index 7d2c2d0b..17f148ad 100644 --- a/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py +++ b/aieng-forecasting/tests/aieng/forecasting/test_langfuse_tracing.py @@ -84,13 +84,6 @@ def test_get_client_exception_does_not_propagate(self) -> None: bootstrap._ensure_langfuse_client() assert bootstrap._langfuse_client_initialized is False - def test_missing_litellm_package_does_not_raise(self) -> None: - """Absent ``litellm`` skips callback registration without raising.""" - with patch.dict(sys.modules, {"litellm": None}): - bootstrap = _LangfuseTracingBootstrap() - bootstrap._register_litellm_langfuse_otel() - assert bootstrap._litellm_instrumented is False - def test_missing_openinference_package_does_not_raise(self) -> None: """Missing OpenInference ADK shim skips instrumentation silently.""" with patch.dict( @@ -130,25 +123,23 @@ def test_instrumentor_exception_does_not_propagate(self) -> None: class TestBootstrapLiteLLMCallbackContract: - """LiteLLM global ``callbacks`` list is updated idempotently.""" + """LiteLLM's ``langfuse_otel`` callback must never be registered.""" - def test_langfuse_otel_not_appended_when_already_present(self) -> None: - """Existing langfuse_otel entry must not be duplicated.""" - litellm_mod = MagicMock() - litellm_mod.callbacks = ["langfuse_otel", "other_hook"] - with patch.dict(sys.modules, {"litellm": litellm_mod}): - bootstrap = _LangfuseTracingBootstrap() - bootstrap._register_litellm_langfuse_otel() - assert litellm_mod.callbacks.count("langfuse_otel") == 1 - - def test_existing_callbacks_are_preserved(self) -> None: - """Appending ``langfuse_otel`` keeps prior callback entries.""" - litellm_mod = MagicMock() - litellm_mod.callbacks = ["other_hook"] - with patch.dict(sys.modules, {"litellm": litellm_mod}): + def test_langfuse_otel_is_not_registered(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Init must not add ``langfuse_otel`` to ``litellm.callbacks``. + + The callback is unusable against the Langfuse v4 SDK and stamps a zero + ``llm.cost.total`` on the active span, which suppresses Langfuse's own + cost calculation and forced agent-path generations to $0. + """ + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") + deps = _all_deps_present(litellm_callbacks=["other_hook"]) + with patch.dict(sys.modules, deps): bootstrap = _LangfuseTracingBootstrap() - bootstrap._register_litellm_langfuse_otel() - assert "other_hook" in litellm_mod.callbacks + bootstrap.init() + assert "langfuse_otel" not in deps["litellm"].callbacks + assert deps["litellm"].callbacks == ["other_hook"], "unrelated callbacks must be left alone" # --------------------------------------------------------------------------- @@ -168,11 +159,10 @@ def test_no_op_without_credentials(self, monkeypatch: pytest.MonkeyPatch) -> Non bootstrap = _LangfuseTracingBootstrap() bootstrap.init() assert not bootstrap._langfuse_client_initialized - assert not bootstrap._litellm_instrumented assert not bootstrap._google_adk_instrumented - def test_repeated_init_does_not_duplicate_litellm_callback(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Calling ``init`` twice adds ``langfuse_otel`` at most once.""" + def test_repeated_init_instruments_adk_only_once(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Calling ``init`` twice instruments Google ADK at most once.""" monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") deps = _all_deps_present() @@ -180,7 +170,8 @@ def test_repeated_init_does_not_duplicate_litellm_callback(self, monkeypatch: py bootstrap = _LangfuseTracingBootstrap() bootstrap.init() bootstrap.init() - assert deps["litellm"].callbacks.count("langfuse_otel") == 1 + instrumentor = deps["openinference.instrumentation.google_adk"].GoogleADKInstrumentor + assert instrumentor.return_value.instrument.call_count == 1 # --------------------------------------------------------------------------- diff --git a/aieng-forecasting/tests/conftest.py b/aieng-forecasting/tests/conftest.py new file mode 100644 index 00000000..b43316f5 --- /dev/null +++ b/aieng-forecasting/tests/conftest.py @@ -0,0 +1,19 @@ +"""Suite-wide test configuration. + +Disables Langfuse tracing for the whole test suite. The LLM-call seam emits a +Langfuse generation per completion (see +``aieng.forecasting.methods.llm_processes._client.langfuse_generation``), and +several suites load real credentials from the repo-root ``.env``. Without this +guard, unit tests with a mocked ``litellm`` would ship real, zero-usage +observations to the live Langfuse project. + +Set at import time rather than in a fixture so it lands before any test module +constructs a Langfuse client. +""" + +from __future__ import annotations + +import os + + +os.environ["LANGFUSE_TRACING_ENABLED"] = "false" diff --git a/implementations/getting_started/concierge_agent/context/artifacts/AGENTS.md.md b/implementations/getting_started/concierge_agent/context/artifacts/AGENTS.md.md index a7d37df5..3e353114 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/AGENTS.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/AGENTS.md.md @@ -20,7 +20,7 @@ So "done" always includes a documentation reconciliation step. Before considerin 1. **Grep for what you touched** across docs — the feature name, the module/class/function, the dataset, the spec, the notebook. The fast version: `grep -rn "" --include="*.md" .` (and check notebook markdown cells). Don't rely on memory for where something is mentioned. 2. **Reconcile every hit.** If a doc calls something "planned", "deferred", "not yet wired in", a "seam", or "out of scope" and you just made it real, update that wording. If a doc lists files, notebooks, predictors, specs, or data sources and you added or removed one, fix the list. If you changed a default, a metric, or a command, fix it everywhere it appears. -3. **Update the layered docs together**, not just the nearest one: the use-case README (most detail), the reference-implementations table in the root `README.md`, the method catalog (`aieng-forecasting/aieng/forecasting/methods/README.md`) when you touch a reusable predictor, and `planning-docs/roadmap.md` when something moves from "extension idea" to "shipped". +3. **Update the layered docs together**, not just the nearest one: the use-case README (most detail), the contents table in the root `README.md`, the method catalog (`aieng-forecasting/aieng/forecasting/methods/README.md`) when you touch a reusable predictor, and `planning-docs/roadmap.md` when something moves from "extension idea" to "shipped". Concrete example: integrating Canada's Food Price Report PDFs into the food-price LLM-Process prompt is **not done when the code runs** — it is done when `implementations/food_price_forecasting/README.md` (which currently frames report→prompt wiring as a deferred extension) and the "Reports as predictor context" entry in `planning-docs/roadmap.md` no longer describe it as future work. Shipping the code while those still say "deferred" is the regression the reviewer should catch. @@ -35,6 +35,7 @@ The older planning log, backlog, project charter, and technical-design files und Project shape to keep in mind: - The core library `aieng.forecasting` owns stable infrastructure; reusable predictors live in `aieng.forecasting.methods`; use-case material lives in `implementations//`. +- Strategy guides live under `guides/` (onboard a dataset, create an experiment, customize an agent, audit a result). - YAML specs are co-located under `implementations//specs/`. - Reference implementations: Getting Started, Food Price Forecasting, Energy/Oil (stateless capability track plus an adaptive learning agent), BoC Rate Decisions (quantitative path, cutoff-aware press-release ingestion, and a reasoning-alignment evaluator), and S&P 500 (in active development). - Energy/oil's older information-session notebooks are archived under `playground/energy_case_study/`. @@ -42,7 +43,7 @@ Project shape to keep in mind: ### README files -Search the repo for `README.md` files (excluding `.venv/`) to find every README — there is one at the root, one per package (`aieng-forecasting/`, `implementations/`), the method catalog under `aieng-forecasting/aieng/forecasting/methods/`, and one per use case under `implementations//`. These are the primary user surface and the first thing a new contributor reads; the reconciliation rule above applies to all of them. Keep them accurate and production-quality: describe what the code does and what you can build from it, with no internal program, scheduling, or ownership framing. +Search the repo for `README.md` files (excluding `.venv/`) to find every README — there is one at the root, one per package (`aieng-forecasting/`, `implementations/`), the method catalog under `aieng-forecasting/aieng/forecasting/methods/`, one per use case under `implementations//`, and the strategy-guide index under `guides/`. These are the primary user surface and the first thing a new contributor reads; the reconciliation rule above applies to all of them. Keep them accurate and production-quality: describe what the code does and what you can build from it, with no internal program, scheduling, or ownership framing. --- diff --git a/implementations/getting_started/concierge_agent/context/artifacts/README.md.md b/implementations/getting_started/concierge_agent/context/artifacts/README.md.md index 4279d40e..213fb6e7 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/README.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/README.md.md @@ -6,7 +6,19 @@ kind: markdown A foundation for building, evaluating, and comparing forecasting systems — conventional numerical models, LLM Processes, and agentic forecasters — on real economic, financial, and event-prediction tasks. -The repository pairs a small, stable core library with a set of self-contained reference implementations. The library gives you cutoff-safe data handling, a single `Predictor` interface, and a backtest/evaluation harness. Each reference implementation is a worked example of a different forecasting problem and the techniques that suit it. Start from whichever one is closest to what you want to build. +## Contents + +The repo has two layers. A small core library (`aieng.forecasting`) owns cutoff-safe data handling, a shared `Predictor` interface, reusable methods, and the backtest/evaluation harness. Self-contained **reference implementations** under [`implementations/`](implementations/) apply those methods to real forecasting problems — pick the one closest to what you want to build; each directory has its own README. + +| # | Implementation | Use case | Methods | +| --- | --- | --- | --- | +| 0 | [Getting started](implementations/getting_started/) | Canada CPI gasoline, one month ahead — the smallest end-to-end loop | Naive last-value, AutoARIMA; CRPS via `backtest()` / `evaluate()` | +| 1 | [S&P 500](implementations/sp500_forecasting/) | Daily index returns under a leak-safe macro/market covariate panel (1 / 5 / 21 business-day horizons) | Naive, ETS, Kalman, AutoARIMA, linear regression, LightGBM; covariate-aware LLM-Process | +| 2 | [Food price forecasting](implementations/food_price_forecasting/) | Multivariate Canadian food CPI in the style of Canada's Food Price Report (nine sub-indices, 12-month trajectory, avg/avg YoY) | Naive last-value, AutoARIMA; report-grounded LLM-Process (quantile grid and sampled trajectory) | +| 3 | [Energy / oil](implementations/energy_oil_forecasting/) | Daily WTI crude under regime-breaking news (continuous trajectory, binary up-shock, scenario analysis) | Prophet, LLM-Process, news-grounded agent, code-executing agent, adaptive (curriculum-trained) agent | +| 4 | [BoC rate decisions](implementations/boc_rate_decisions/) | Will the Bank of Canada cut, hold, or hike at its next meeting? (ordered categorical; binary cut-vs-not special case) | Climatological frequency, multinomial logistic, categorical LLM-Process, analyst agent; LLM-as-judge reasoning alignment | + +Also in this README: [Setup](#setup) · [Core concepts](#core-concepts) · [Repository layout](#repository-layout) · [Documentation](#documentation) > **👉 First time here? Run the environment check.** After `uv sync` (see [Setup](#setup)), open [`implementations/getting_started/00_environment_check.ipynb`](implementations/getting_started/00_environment_check.ipynb) and run it top to bottom. It's a self-guided preflight that verifies every capability — proxy LLM inference, Langfuse, E2B code execution, StatCan/FRED data access, and an end-to-end mini backtest — and tells you exactly what to fix when something isn't set up. **Do this before anything else.** @@ -27,18 +39,11 @@ Every method can be used in one of two modes, and the distinction runs through t ## Reference implementations -Each is independent and self-contained — pick the one that matches the problem you care about, and read that directory's `README.md` for the full walkthrough. They are numbered in a recommended order that mirrors the bootcamp progression — conventional numerical methods → LLM Processes → agents → agentic evaluation — but any one stands on its own, so jump straight to the problem you care about. - -**Start here → #0 [`getting_started/`](implementations/getting_started/)** — one CPI series, one month ahead. The smallest end-to-end loop: a `Predictor`, a `BacktestSpec` and `EvalSpec`, naive + AutoARIMA baselines, CRPS scoring. The place to learn the evaluation framework before picking a domain below. Also includes [`99_repo_concierge.ipynb`](implementations/getting_started/99_repo_concierge.ipynb) — a lite-model repo guide for “how does this codebase work?” questions (`uv run adk run implementations/getting_started/concierge_agent` from the repo root). +Use cases, methods, and links are in the [contents](#contents) table above. Each implementation is independent — pick the problem you care about and read that directory's `README.md` for the full walkthrough. They are numbered in a recommended order that mirrors the bootcamp progression — conventional numerical methods → LLM Processes → agents → agentic evaluation — but any one stands on its own. -| # | Implementation | The problem | Concepts & techniques it demonstrates | -| --- | -------------------------------------------------------------------- | ------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | [`sp500_forecasting/`](implementations/sp500_forecasting/) | S&P 500 returns under a macro/market covariate panel. | A head-to-head of conventional numerical methods (naive, ETS, Kalman, AutoARIMA, linear regression, LightGBM) plus a covariate-aware LLM-Process, all reading the same leak-safe covariate panel. Cumulative-return targets at 1/5/21-business-day horizons, CRPS + direction metrics, config-driven specs. | -| 2 | [`food_price_forecasting/`](implementations/food_price_forecasting/) | A multivariate food-CPI trajectory, in the style of Canada's Food Price Report. | Nine correlated sub-indices, a 12-step trajectory, a domain metric (avg/avg YoY), baselines vs LLM-Process predictors, leakage-aware backtests, and cached artifacts for fast iteration. | -| 3 | [`energy_oil_forecasting/`](implementations/energy_oil_forecasting/) | Daily WTI crude-oil price under regime-breaking news. | A capability progression — Prophet → LLM-Process → news-grounded agent → code-executing agent — plus an adaptive agent that learns a strategy from data and is scored before vs after. Continuous trajectories, a binary up-shock task, and interactive scenario analysis. | -| 4 | [`boc_rate_decisions/`](implementations/boc_rate_decisions/) | Will the Bank of Canada cut, hold, or hike at its next meeting? | Discrete-event forecasting: ordered-categorical outcomes on an irregular calendar, RPS scoring and one-vs-rest calibration (instead of CRPS), a binary (Brier) special case, cutoff-aware document ingestion, and an LLM-as-judge that scores an agent's reasoning against the official rationale. | +**Start here → #0 [`getting_started/`](implementations/getting_started/)** if the evaluation loop is new to you. That directory also includes [`99_repo_concierge.ipynb`](implementations/getting_started/99_repo_concierge.ipynb) — a lite-model repo guide for “how does this codebase work?” questions (`uv run adk run implementations/getting_started/concierge_agent` from the repo root). -**Not sure where to start building?** Each of the four domain implementations above ends with a `99_starter_agent.ipynb` — a fresh, hackable **starter agent** (a `starter_agent/` module) with toggleable news search and code execution, two lightweight tool-usage skills, an interactive cell, and one scored forecast. It's the consistent "continue from here" entry point for taking any reference use case in an agentic direction, and a quick end-to-end test of that use case's agent stack. +**Not sure where to start building?** Each of the four domain implementations (#1–#4) ends with a `99_starter_agent.ipynb` — a fresh, hackable **starter agent** (a `starter_agent/` module) with toggleable news search and code execution, two lightweight tool-usage skills, an interactive cell, and one scored forecast. It's the consistent "continue from here" entry point for taking any reference use case in an agentic direction, and a quick end-to-end test of that use case's agent stack. ## Time Series Data sources @@ -71,6 +76,7 @@ On Coder workspaces, bootcamp keys (`OPENAI_*`, `E2B_*`, `LANGFUSE_*`) live in y ```text aieng-forecasting/ # Installable library: import as aieng.forecasting implementations/ # Self-contained reference implementations + co-located specs +guides/ # Step-by-step strategy guides for common build-phase tasks scripts/ # Data-fetch scripts + E2B template builder tests/ # Onboarding integration tests (not run in CI) planning-docs/ # Architecture notes and the extension/roadmap catalog @@ -201,6 +207,8 @@ uv run pre-commit run --all-files ## Documentation - Per-implementation READMEs under [`implementations/`](implementations/) — the primary user surface. +- [`guides/`](guides/) — self-contained, step-by-step strategy guides for the most common build-phase tasks: onboarding a dataset, creating an experiment, customizing an agent's strategy, and auditing a result before you believe it. +- [Architecture atlas](https://vectorinstitute.github.io/agentic-forecasting/architecture-atlas.html) ([source](docs/architecture-atlas.html)) — a self-contained visual atlas of the system architecture: the loop, the temporal fence, predictor families, the harness, agent anatomy, and how each reference implementation instantiates them. - [`aieng-forecasting/README.md`](aieng-forecasting/README.md) and [`aieng-forecasting/aieng/forecasting/methods/README.md`](aieng-forecasting/aieng/forecasting/methods/README.md) — the library and the method catalog. - [`planning-docs/roadmap.md`](planning-docs/roadmap.md) — architecture principles and extension ideas. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md index 105042e6..3aa109fd 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md @@ -29,12 +29,11 @@ def _langfuse_credentials_present() -> bool: class _LangfuseTracingBootstrap: - """Registers LiteLLM + ADK exporters at most once per process.""" + """Registers the Langfuse client and ADK instrumentation once per process.""" - __slots__ = ("_google_adk_instrumented", "_langfuse_client_initialized", "_litellm_instrumented") + __slots__ = ("_google_adk_instrumented", "_langfuse_client_initialized") def __init__(self) -> None: - self._litellm_instrumented = False self._google_adk_instrumented = False self._langfuse_client_initialized = False @@ -51,7 +50,6 @@ class _LangfuseTracingBootstrap: # this, ADK spans are emitted into a no-op provider and never reach Langfuse. self._ensure_langfuse_client() - self._register_litellm_langfuse_otel() self._instrument_google_adk() def _ensure_langfuse_client(self) -> None: @@ -69,21 +67,6 @@ class _LangfuseTracingBootstrap: return self._langfuse_client_initialized = True - def _register_litellm_langfuse_otel(self) -> None: - """Register LiteLLM Langfuse callback.""" - if self._litellm_instrumented: - return - try: - import litellm # noqa: PLC0415 - except ImportError: - logger.debug("litellm not installed; skipping LiteLLM Langfuse callback.") - return - - existing = list(getattr(litellm, "callbacks", None) or []) - if "langfuse_otel" not in existing: - litellm.callbacks = [*existing, "langfuse_otel"] - self._litellm_instrumented = True - def _instrument_google_adk(self) -> None: """Instrument Google ADK.""" if self._google_adk_instrumented: @@ -125,11 +108,13 @@ def init_langfuse_tracing() -> None: ``TracerProvider`` receives Langfuse's span processor. This is required for ADK spans emitted via ``openinference-instrumentation-google-adk`` to reach Langfuse. - 2. Appends ``"langfuse_otel"`` to ``litellm.callbacks`` once (if - ``litellm`` is importable). - 3. Runs ``GoogleADKInstrumentor().instrument()`` once (if + 2. Runs ``GoogleADKInstrumentor().instrument()`` once (if ``openinference-instrumentation-google-adk`` is importable). + LiteLLM's ``langfuse_otel`` callback is deliberately not registered: it is + unusable against the Langfuse v4 SDK and stamps a zero ``llm.cost.total`` + on the active span, which suppresses Langfuse's own cost calculation. + Set ``LANGFUSE_HOST`` or ``LANGFUSE_BASE_URL`` for non-default regions. For short-lived processes, call ``langfuse.get_client().flush()`` before exit so pending spans are exported. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md index 6337eb98..f1a37153 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md @@ -187,7 +187,13 @@ class AdkTextRunner: """Underlying ADK runner (session, artifact, memory services).""" return self._runner - async def _resolve_session_id(self, user_id: str | None, session_id: str | None) -> str: + async def _resolve_session_id( + self, + user_id: str | None, + session_id: str | None, + *, + initial_state: dict[str, Any] | None = None, + ) -> str: """Return the ADK session id to use for a single turn. Parameters @@ -197,6 +203,11 @@ class AdkTextRunner: session_id : str or None Explicit session id from the caller. ``None`` triggers sticky-session lookup or new-session creation depending on ``fresh_session_per_message``. + initial_state : dict[str, Any] or None + Seeded into a newly-created session's state. Only takes effect when + this call actually creates a session — has no effect when an + existing sticky session (``fresh_session_per_message=False``) is + reused, since session state can only be seeded at creation. Returns ------- @@ -210,6 +221,7 @@ class AdkTextRunner: new_session = await self._runner.session_service.create_session( app_name=self.config.app_name, user_id=user_id, + state=initial_state, ) sid = new_session.id elif session_id is not None: @@ -221,6 +233,7 @@ class AdkTextRunner: new_session = await self._runner.session_service.create_session( app_name=self.config.app_name, user_id=user_id, + state=initial_state, ) sid = new_session.id self._conversation_session_by_user[user_id] = sid @@ -234,6 +247,7 @@ class AdkTextRunner: user_id: str | None = None, session_id: str | None = None, run_config: RunConfig | None = None, + initial_state: dict[str, Any] | None = None, ) -> str: """Run one user turn; return the first final model text or an empty string. @@ -252,6 +266,12 @@ class AdkTextRunner: run_config : RunConfig | None, optional The run configuration to use for the run. If not provided, the default run configuration is used. + initial_state : dict[str, Any] | None, optional + Seeded into the session's state when this call creates a new + session (see :meth:`_resolve_session_id`). Use this to pass + harness-controlled values (e.g. a forecast's ``as_of`` date) that + tools can read via ``ToolContext.state`` without the LLM being + able to see or influence them. Returns ------- @@ -275,7 +295,7 @@ class AdkTextRunner: user_id = user_id or self.config.default_user_id - session_id = await self._resolve_session_id(user_id, session_id) + session_id = await self._resolve_session_id(user_id, session_id, initial_state=initial_state) content = genai_types.Content(role="user", parts=[genai_types.Part(text=prompt)]) diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md index 494c0ed4..be9c2a65 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md @@ -17,6 +17,7 @@ raises :class:`ImportError` with installation guidance. from __future__ import annotations +import json import logging import os import warnings @@ -24,9 +25,12 @@ from pathlib import Path from typing import Any, Callable, Sequence from aieng.forecasting.methods.agentic.outputs import AgentForecastOutput -from aieng.forecasting.models import LITE_MODEL +from aieng.forecasting.models import ADVANCED_MODEL, LITE_MODEL from google.adk.models.base_llm import BaseLlm -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator + + +logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -80,6 +84,13 @@ except ModuleNotFoundError as exc: # final text, giving the predictor the structured JSON it expects. SMR_STATE_KEY = "__smr_output__" +# Session-state key AgentPredictor seeds with the current prediction's as_of +# date before each run (see AdkTextRunner.run_text_async's initial_state). +# search_web reads this via its ADK-injected ToolContext as the authoritative +# cutoff — unlike the LLM-supplied cutoff_date argument, the model can never +# see or influence this key, so it can't be silently omitted or spoofed. +AS_OF_STATE_KEY = "__as_of__" + def _build_set_model_response_tool() -> FunctionTool: """Return a proxy-compatible ``set_model_response`` shim. @@ -118,12 +129,18 @@ class ContextRetrievalConfig(BaseModel): the calling agent can retrieve grounded, sourced web context without a direct Gemini API key. - Temporal cutoff enforcement is soft (LLM-judgment-based): when - ``enforce_cutoff`` is ``True`` and the calling agent passes a - ``cutoff_date`` to the tool, the inner proxy prompt explicitly asks the - model to exclude post-cutoff sources. This is the same trust model used - by the prior Google Search sub-agent — backtest leakage is a - pedagogically useful discussion point, not a hard guarantee. + Temporal cutoff enforcement has two layers. The first is soft + (LLM-judgment-based): when ``enforce_cutoff`` is ``True`` and the calling + agent passes a ``cutoff_date`` to the tool, the inner proxy prompt + explicitly asks the model to exclude post-cutoff sources. This alone is + not a hard guarantee — backtests have shown it leak real post-cutoff + information despite the instruction. The second, hard layer is an + independent verifier call (see ``verifier_model`` etc. below): a separate + LLM call extracts and judges each factual claim in the search result + against the cutoff, strips violations, and retries the search with + feedback when it cannot produce a sufficiently confident result — + returning an explicit failure sentinel rather than silently risky + content if verification never succeeds within the attempt budget. Attributes ---------- @@ -139,12 +156,27 @@ class ContextRetrievalConfig(BaseModel): enforce_cutoff : bool, default=True When ``True``, the ``search_web`` tool appends a cutoff-date constraint to the user prompt whenever ``cutoff_date`` is supplied by - the calling agent. Set to ``False`` for live (non-backtest) agents - where no temporal fence is needed. + the calling agent, and runs the independent leakage verifier + described above. Set to ``False`` for live (non-backtest) agents + where no temporal fence is needed — the verifier is skipped entirely + in that case, at zero extra cost. temperature : float | None, default=None Sampling temperature for the inner search call. max_output_tokens : int | None, default=None Maximum output tokens for the inner search call. + verifier_model : str, default=ADVANCED_MODEL (``"gemini-3.5-flash"``) + Model used for the independent leakage-verification call. Defaults + to a different model than ``search_model`` so the verifier does not + share the same blind spot as the call it's checking. + verifier_max_attempts : int, default=3 + Maximum number of search-then-verify attempts before giving up and + returning the ``[SEARCH_VERIFICATION_FAILED]`` sentinel. + verifier_confidence_threshold : int, default=8 + Minimum self-reported confidence (1-10) the verifier must report, + alongside a clean verdict, for a result to be accepted. Kept + configurable rather than hardcoded because LLM self-reported + confidence is not well-calibrated: too strict a bar (e.g. a literal + 10) risks exhausting retries on results that were actually fine. """ model_config = {"extra": "forbid"} @@ -159,6 +191,9 @@ class ContextRetrievalConfig(BaseModel): enforce_cutoff: bool = True temperature: float | None = Field(default=None, ge=0.0, le=2.0) max_output_tokens: int | None = Field(default=None, ge=1) + verifier_model: str = ADVANCED_MODEL + verifier_max_attempts: int = Field(default=3, ge=1) + verifier_confidence_threshold: int = Field(default=8, ge=1, le=10) class CodeExecutionConfig(BaseModel): @@ -212,6 +247,106 @@ def _build_automatic_function_calling_config( return AutomaticFunctionCallingConfig(disable=True) +class _LeakageVerification(BaseModel): + """Structured verdict from the independent temporal-leakage verifier.""" + + flagged_claims: list[str] = Field(default_factory=list) + filtered_text: str = "" + confidence: int = Field(ge=1, le=10) + clean: bool + + +def _build_leakage_verification_schema() -> dict[str, Any]: + """Strict JSON schema for the verifier's structured output. + + Field order matters: the model must extract/flag claims and produce + ``filtered_text`` before declaring ``confidence``/``clean``, so the + verdict follows the claim-level reasoning instead of preceding it. + """ + return { + "type": "object", + "properties": { + "flagged_claims": {"type": "array", "items": {"type": "string"}}, + "filtered_text": {"type": "string"}, + "confidence": {"type": "integer", "minimum": 1, "maximum": 10}, + "clean": {"type": "boolean"}, + }, + "required": ["flagged_claims", "filtered_text", "confidence", "clean"], + } + + +_LEAKAGE_VERIFIER_INSTRUCTION = """\ +You are an independent fact-checker verifying that a web search result contains \ +no information published on or after a given cutoff date. + +Extract every discrete factual claim relevant to the query. For each claim, \ +judge whether its *content* (the event itself, prices/figures referenced, \ +described developments) could only be known on or after the cutoff date. \ +Do NOT trust a source's claimed publish date, byline timestamp, or URL — \ +page metadata and timestamps are frequently updated after original \ +publication and are not reliable evidence of when the underlying facts \ +became known. Reason from the substance of the claim itself. + +Remove every claim that fails this test and produce `filtered_text`: the \ +original text with only the surviving, pre-cutoff claims. Report the removed \ +claims in `flagged_claims`. Set `confidence` (1-10) to how confident you are \ +that `filtered_text` now contains zero post-cutoff leakage. Set `clean` to \ +true only if you removed all identifiable violations.""" + + +async def _verify_no_leakage( + *, + text: str, + query: str, + cutoff_date: str, + verifier_model: str, + openai_base_url: str, + openai_api_key: str | None, +) -> _LeakageVerification: + """Judge a search result for post-cutoff claims via an independent verifier call. + + Uses a different model (by default) than the search call it's checking, + so the verifier does not share the same knowledge-attribution blind spot + that caused the leak in the first place. Never raises on a malformed + verifier response — a parse failure is treated as a non-clean verdict so + it consumes a retry attempt like any other rejection. + """ + import litellm # noqa: PLC0415 + from aieng.forecasting.methods.llm_processes._client import ( # noqa: PLC0415 + make_json_schema_response_format, + strip_markdown_fence, + ) + + model = verifier_model if verifier_model.startswith("openai/") else f"openai/{verifier_model}" + resp = await litellm.acompletion( + model=model, + api_base=openai_base_url, + api_key=openai_api_key, + messages=[ + {"role": "system", "content": _LEAKAGE_VERIFIER_INSTRUCTION}, + { + "role": "user", + "content": f"Original query: {query}\nCutoff date: {cutoff_date}\n\nSearch result to verify:\n{text}", + }, + ], + response_format=make_json_schema_response_format("LeakageVerification", _build_leakage_verification_schema()), + temperature=0.0, + max_tokens=2048, + timeout=60.0, + ) + raw = resp.choices[0].message.content or "{}" + try: + return _LeakageVerification.model_validate(json.loads(strip_markdown_fence(raw))) + except (json.JSONDecodeError, ValidationError): + logger.warning("Leakage verifier returned unparseable output; treating as non-clean: %r", raw[:200]) + return _LeakageVerification( + flagged_claims=["verifier response could not be parsed"], + filtered_text=text, + confidence=1, + clean=False, + ) + + def _build_search_tool( config: ContextRetrievalConfig, *, @@ -224,25 +359,25 @@ def _build_search_tool( the proxy with ``"tools": [{"googleSearch": {}}]`` so the model does server-side grounding and returns a synthesised answer plus source URLs extracted from ``choices[0].provider_specific_fields["grounding_metadata"]``. - """ - async def search_web(query: str, cutoff_date: str | None = None) -> str: - """Search the web and return a grounded summary with source URLs. + When a ``cutoff_date`` is supplied and ``config.enforce_cutoff`` is + ``True``, the raw result is passed through an independent leakage + verifier (:func:`_verify_no_leakage`) before being returned. On a flagged + result, the search is retried (up to ``config.verifier_max_attempts`` + times) with the previously flagged claims injected as explicit negative + feedback. If no attempt is verified clean, an explicit + ``[SEARCH_VERIFICATION_FAILED]`` sentinel is returned instead of + potentially-leaky content. + """ - Args: - query: What to search for. - cutoff_date: ISO date (YYYY-MM-DD). When provided, only include - information published strictly before this date. + def _format_result(content: str, sources: list[str]) -> str: + if sources: + content += "\n\nSources:\n" + "\n".join(sources[:5]) + return content - Returns - ------- - A grounded summary of search results, with source URLs appended. - """ + async def _do_search(user_content: str) -> tuple[str, list[str]]: import litellm # noqa: PLC0415 - user_content = query - if cutoff_date and config.enforce_cutoff: - user_content += f"\n\nOnly include and cite information published strictly before {cutoff_date}." search_model = config.search_model if not search_model.startswith("openai/"): search_model = f"openai/{search_model}" @@ -265,9 +400,88 @@ def _build_search_tool( sources: list[str] = [ uri for c in gm.get("groundingChunks", []) if (uri := (c.get("web") or {}).get("uri")) is not None ] - if sources: - content += "\n\nSources:\n" + "\n".join(sources[:5]) - return content + return content, sources + + async def search_web(query: str, cutoff_date: str | None = None, tool_context: ToolContext | None = None) -> str: + """Search the web and return a grounded summary with source URLs. + + Args: + query: What to search for. + cutoff_date: ISO date (YYYY-MM-DD). When provided, only include + information published strictly before this date. + + Returns + ------- + A grounded summary of search results, with source URLs appended. + When cutoff verification is enabled and cannot be satisfied + within the attempt budget, returns a ``[SEARCH_VERIFICATION_FAILED]`` + sentinel instead of unverified content. + + Notes + ----- + ``tool_context`` is not part of the LLM-visible tool schema — ADK + injects it automatically because of its type annotation. When + :class:`AgentPredictor` runs this tool, it has already seeded the + session with the current prediction's ``as_of`` date under + :data:`AS_OF_STATE_KEY`; that harness-controlled value is the + authoritative cutoff whenever present, since (unlike ``cutoff_date``) + the calling LLM cannot see, omit, or alter it. This closes a bypass + where the model simply didn't pass ``cutoff_date`` and both the soft + cutoff instruction and the verifier below were silently skipped. + """ + harness_as_of = tool_context.state.get(AS_OF_STATE_KEY) if tool_context is not None else None + if harness_as_of and cutoff_date and harness_as_of != cutoff_date: + logger.warning( + "search_web: cutoff_date=%r disagrees with harness as_of=%r; using the harness value.", + cutoff_date, + harness_as_of, + ) + effective_cutoff = harness_as_of or cutoff_date + + needs_verification = bool(effective_cutoff and config.enforce_cutoff) + if not needs_verification: + content, sources = await _do_search(query) + return _format_result(content, sources) + + negative_guidance = "" + for attempt in range(1, config.verifier_max_attempts + 1): + user_content = ( + query + f"\n\nOnly include and cite information published strictly before {effective_cutoff}." + ) + if negative_guidance: + user_content += f"\n\n{negative_guidance}" + content, sources = await _do_search(user_content) + verdict = await _verify_no_leakage( + text=content, + query=query, + cutoff_date=effective_cutoff, # type: ignore[arg-type] + verifier_model=config.verifier_model, + openai_base_url=openai_base_url, + openai_api_key=openai_api_key, + ) + logger.info( + "search_web verification attempt %d/%d: clean=%s confidence=%d flagged=%d", + attempt, + config.verifier_max_attempts, + verdict.clean, + verdict.confidence, + len(verdict.flagged_claims), + ) + if verdict.clean and verdict.confidence >= config.verifier_confidence_threshold: + return _format_result(verdict.filtered_text, sources) + logger.warning("search_web attempt %d flagged %d claim(s); retrying.", attempt, len(verdict.flagged_claims)) + negative_guidance = ( + f"Your previous search result may have included information published on or after " + f"{effective_cutoff}. Do not repeat or rely on these claims:\n- " + + "\n- ".join(verdict.flagged_claims or ["(unspecified — be more conservative)"]) + ) + + logger.error("search_web exhausted %d attempts without a verified clean result.", config.verifier_max_attempts) + return ( + f"[SEARCH_VERIFICATION_FAILED] Could not verify search results as free of information " + f"published on or after {effective_cutoff} after {config.verifier_max_attempts} attempts. " + "Treat this as no verified news context being available for this query." + ) return search_web @@ -460,12 +674,16 @@ def build_adk_agent( if isinstance(model, str) and config.openai_base_url: from google.adk.models.lite_llm import LiteLlm # noqa: PLC0415 - # Prefix with "openai/" so LiteLLM uses the OpenAI-compatible path. - # LiteLLM strips the prefix before sending, so the proxy receives the - # bare model name. - litellm_model = model if model.startswith("openai/") else f"openai/{model}" + # Route via LiteLLM's OpenAI-compatible path with ``custom_llm_provider`` + # rather than an ``openai/`` model prefix. ADK stamps ``LlmRequest.model`` + # from this name and OpenInference reports it to Langfuse, which matches + # its per-model price table on the bare name. A prefixed name matches + # nothing, so the generation is logged at zero cost. Both forms route + # identically. + bare_model = model[len("openai/") :] if model.startswith("openai/") else model model = LiteLlm( - model=litellm_model, + model=bare_model, + custom_llm_provider="openai", api_base=config.openai_base_url, api_key=config.openai_api_key, ) diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md index 6e613431..10baed0c 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md @@ -34,7 +34,7 @@ from aieng.forecasting.evaluation.prediction import Prediction from aieng.forecasting.evaluation.predictor import Predictor from aieng.forecasting.evaluation.task import ForecastingTask from aieng.forecasting.methods.agentic.adk_runner import AdkTextRunner, AdkTextRunnerConfig -from aieng.forecasting.methods.agentic.agent_factory import AgentConfig, build_adk_agent +from aieng.forecasting.methods.agentic.agent_factory import AS_OF_STATE_KEY, AgentConfig, build_adk_agent from aieng.forecasting.methods.agentic.outputs import AgentForecastOutput from aieng.forecasting.methods.llm_processes._client import strip_markdown_fence, trace_url_for from google.adk.agents.base_agent import BaseAgent @@ -282,7 +282,11 @@ class AgentPredictor(Predictor): validation errors on the agent's JSON are not swallowed. """ prompt = self.prompt_builder(task=task, context=context) - output_str = _run_coroutine_sync(self._runner.run_text_async(prompt)) + # Seed the harness-controlled as_of into the ADK session before the run, + # so search_web can enforce it via ToolContext.state regardless of + # whether the LLM remembers to pass a matching cutoff_date argument. + initial_state = {AS_OF_STATE_KEY: str(context.as_of)[:10]} + output_str = _run_coroutine_sync(self._runner.run_text_async(prompt, initial_state=initial_state)) # Normalise: strip markdown fences before validation so any model can # be swapped in without breaking the parse layer. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md index 877bd51c..63511409 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md @@ -25,13 +25,13 @@ into a single response, which would defeat sample-based forecasting. from __future__ import annotations import asyncio +import contextlib import contextvars import json import logging -import os import warnings from concurrent.futures import ThreadPoolExecutor -from typing import Any, Callable, TypeVar +from typing import Any, Callable, Iterator, TypeVar from pydantic import BaseModel, ValidationError @@ -44,21 +44,22 @@ _BOOTSTRAP_DONE = False def bootstrap_litellm() -> None: - """One-time wiring of LiteLLM callbacks. + """Suppress LiteLLM and OpenTelemetry logging noise, once per process. Lazy and idempotent so non-LLM predictors do not require Langfuse env vars. - The Langfuse OTEL callback is registered only when ``LANGFUSE_PUBLIC_KEY`` - is set in the environment. + + LiteLLM's ``langfuse_otel`` callback is deliberately not registered. It is + unusable against the Langfuse v4 SDK this repo depends on, and it stamps + ``llm.cost.total`` on the active span from LiteLLM's own ``response_cost``, + which is ``0`` for every proxy-routed model. Langfuse honours a supplied + cost instead of deriving one from usage, so the callback forced agent-path + generations to $0. Instead, :func:`langfuse_generation` creates LLM-process + generations directly and OpenInference covers the agent path, so both price + correctly from ``usage_details``. """ global _BOOTSTRAP_DONE # noqa: PLW0603 if _BOOTSTRAP_DONE: return - import litellm # noqa: PLC0415 - - if os.environ.get("LANGFUSE_PUBLIC_KEY"): - existing = list(getattr(litellm, "callbacks", []) or []) - if "langfuse_otel" not in existing: - litellm.callbacks = [*existing, "langfuse_otel"] # Suppress LiteLLM startup and OTEL noise (mirrors agent_factory.py filter). # Bedrock/SageMaker "no botocore" and OTEL proxy-server notices are harmless. @@ -96,6 +97,61 @@ def langfuse_observe(name: str) -> Callable[..., Any]: return _noop +class _NoopGeneration: + """Stand-in used when Langfuse is unavailable, so callers need no branching.""" + + def update(self, **kwargs: Any) -> None: + """Discard the update.""" + return + + +@contextlib.contextmanager +def langfuse_generation(*, name: str, model: str, input_messages: Any) -> Iterator[Any]: + """Create a Langfuse ``generation`` around one LLM call. + + LiteLLM's ``langfuse_otel`` callback emits no generation when the call runs + inside an already-active Langfuse span, which is every LLM-process + ``predict`` because they are wrapped in :func:`langfuse_observe`. Token + usage, and so cost, never reached Langfuse for those runs. Creating the + generation here works when nested and keeps the model, usage, and payload + under this module's control. + + Cost is deliberately not set. Langfuse derives it from ``usage_details`` + against its own per-model prices, which match the Vector proxy's published + rates. + + ``start_as_current_observation(as_type="generation")`` is a first-class + Langfuse v4 instrumentation API. LiteLLM's Langfuse bridge targets the v2 + SDK, pinning ``langfuse = ^2.45.0``, while this repo requires + ``langfuse>=4.5.1``. Support for v4 is BerriAI/litellm#24123, open and + unanswered since 2026-03-19. Retire this helper in favour of the callback + once that issue is closed and ``langfuse_otel`` is confirmed to emit a + generation under an active Langfuse span. + + Yields a handle exposing ``update(**kwargs)``. That handle is a no-op + stand-in when Langfuse is not installed or a generation cannot be started, + so predictors remain usable without the ``agentic`` and ``llm`` extras. + """ + manager = None + try: + from langfuse import get_client # noqa: PLC0415 + + manager = get_client().start_as_current_observation( + as_type="generation", + name=name, + model=model, + input=input_messages, + ) + except Exception: # pragma: no cover - depends on optional dependency + logger.debug("Langfuse generation unavailable; usage will not be traced.", exc_info=True) + + if manager is None: + yield _NoopGeneration() + return + with manager as generation: + yield generation + + def current_trace_info() -> tuple[str | None, str | None]: """Return ``(trace_id, trace_url)`` from the active Langfuse client, if any.""" try: @@ -304,17 +360,21 @@ async def _one_completion_async( # models that don't support them (e.g. temperature on some o-series). kwargs["drop_params"] = True - resp = await litellm.acompletion(**kwargs) - cost = float(getattr(resp, "_hidden_params", {}).get("response_cost") or 0.0) - usage = getattr(resp, "usage", None) - in_tok = int(getattr(usage, "prompt_tokens", 0) or 0) if usage is not None else 0 - out_tok = int(getattr(usage, "completion_tokens", 0) or 0) if usage is not None else 0 - # Log full usage so we can see thinking-token breakdown when available. - # The proxy may populate completion_tokens_details.reasoning_tokens. - if usage is not None: - logger.debug("LLM usage: %s", vars(usage) if hasattr(usage, "__dict__") else usage) - raw = resp.choices[0].message.content - content = strip_markdown_fence(raw) if raw else raw + # ``model`` is the bare name as configured, before any "openai/" prefixing + # above; that is what Langfuse's price table matches on. + with langfuse_generation(name="llm_completion", model=model, input_messages=messages) as generation: + resp = await litellm.acompletion(**kwargs) + cost = float(getattr(resp, "_hidden_params", {}).get("response_cost") or 0.0) + usage = getattr(resp, "usage", None) + in_tok = int(getattr(usage, "prompt_tokens", 0) or 0) if usage is not None else 0 + out_tok = int(getattr(usage, "completion_tokens", 0) or 0) if usage is not None else 0 + # Full usage exposes the thinking-token breakdown when the proxy + # populates completion_tokens_details.reasoning_tokens. + if usage is not None: + logger.debug("LLM usage: %s", vars(usage) if hasattr(usage, "__dict__") else usage) + raw = resp.choices[0].message.content + content = strip_markdown_fence(raw) if raw else raw + generation.update(output=content, usage_details={"input": in_tok, "output": out_tok}) return content, cost, in_tok, out_tok diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__README.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__README.md.md index c8b6e6a4..52ecf07e 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__README.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__README.md.md @@ -32,7 +32,7 @@ implementations/ `-- pyproject.toml # local workspace packaging ``` -YAML backtest and eval specs live under each use case in `specs/`. Each directory is independent; see its `README.md` for the walkthrough. +YAML backtest and eval specs live under each use case in `specs/`. Each directory is independent; see its `README.md` for the walkthrough. For the build-phase moves — onboarding data, standing up an experiment, customizing an agent, auditing a result — see [`guides/`](../guides/). Every domain use case (all except `getting_started`) also ships a `starter_agent/` module and a `99_starter_agent.ipynb` — a fresh, hackable **starter agent** that is the consistent "build your own" entry point for that use case (toggleable news search + code execution, two lightweight tool-usage skills, an interactive cell, and one scored forecast). diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md index 500357a7..0ed0cf45 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md @@ -105,7 +105,16 @@ labour market; market pricing of the upcoming decision (OIS, economist surveys); and macro shocks relevant to Canada (oil, exchange rate, US policy, trade). Ground every claim in the search results you actually retrieve. When a cutoff -date is specified, never report or speculate about events after it.\ +date is specified, never report or speculate about events after it. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ """ diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md index 96ad5034..d824d247 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md @@ -21,6 +21,12 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. +`search_web` runs an independent verifier on every result and returns +`[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as +pre-cutoff. Treat that as no verified news for the query — proceed on your +other signals and say so, never filling the gap from your own background +knowledge. + ## How to search - **Search before you forecast, not after.** Gather context first, then reason. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md index 003940ad..a3989b15 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md @@ -14,8 +14,10 @@ This notebook simulates a rigorous production forecasting workflow: 3. Select the **top contender configurations** based solely on 2025 historical performance (no peeking at 2026). 4. Let the contenders compete in the **2026 Protected Arena** - (`energy_oil_eval.yaml`) during the geopolitical price shock — - measuring adaptive real-time responsiveness and calibration. + (`energy_oil_eval.yaml`) across the geopolitical price shock and its + aftermath — measuring adaptive real-time responsiveness and calibration. + The eval window runs through the most recent origin whose 21-business-day + horizon still resolves against cached data (see `scripts/fetch_wti.py`). The line-up spans three families behind one `Predictor` interface: **baselines** (Naive, AutoARIMA), **numerical ML** (LightGBM ± a leak-safe covariate panel), @@ -56,7 +58,7 @@ warnings.filterwarnings("ignore") # Set SMOKE_TEST = True to run a 2-origin, 1-sample version of the notebook # for fast local development and end-to-end CI testing. The full specs run # 51 backtest + 8 eval origins; smoke runs 2 + 2. -SMOKE_TEST = True +SMOKE_TEST = False # ── Models ──────────────────────────────────────────────────────────────────── # The project standardises on two Vector-proxy models. Every LLM and agent @@ -324,9 +326,11 @@ print(f"Saved {sum(n in _BASELINE_PREDICTORS for n in backtest_results)} backtes --- ## 5. 2026 Evaluation — Held-Out Test Period -We run every active predictor on **8 weekly origins in early 2026** -(`energy_oil_eval.yaml`) — a period of major geopolitical volatility not seen -during the 2025 backtest. +We run every active predictor on **18 weekly origins spanning Feb–Jun 2026** +(`energy_oil_eval.yaml`) — the major geopolitical volatility spike not seen +during the 2025 backtest, plus its aftermath. The window runs through the +most recent origin that still fully resolves against cached WTI data (the +21-business-day horizon needs data 21 business days past the origin). This evaluation serves two purposes: 1. **Measure out-of-sample robustness** — do the 2025 edges (statistical, @@ -402,55 +406,161 @@ print(df_scorecard.to_string()) ## Cell 16 (markdown) --- -## 7. Core Takeaways +## 7. Diagnostics — reading past the leaderboard -1. **Numerical methods beat the naive baseline** by extracting structure from the - price history — AutoARIMA via local autocorrelation, LightGBM via lagged - gradient-boosted quantiles. In stable regimes this translates to better CRPS. +The scorecard above is a single number per method. That hides *where* the score +comes from and *whether the ranking is even real*. The next cells decompose it +straight from the eval predictions — so they recompute on any rerun, smoke or +full: -2. **Covariates can sharpen LightGBM.** The `LightGBM + cov` variant adds a - leak-safe panel (Brent, natural gas, gasoline, gold, the USD index, the - USL/USO futures-curve contango proxy, and VIX). Comparing it to plain - `LightGBM` isolates how much the cross-market context is worth — the same - lesson that made covariates decisive in the S&P 500 study. +- **CRPS by horizon** — does a method win everywhere, or is its mean dominated by + one horizon? (For a short forecast, the 5-day calls are easy and nearly tied; + the ranking is usually decided by the longest horizon.) +- **Mean CRPS ± standard error** — with only a handful of origins, are the gaps + between methods bigger than the noise, or is the "winner" a coin flip? -3. **Tree models extrapolate poorly through regime shifts.** LightGBM forecasts - the price *level*, and gradient-boosted trees cannot predict outside the range - seen in training. When the 2026 shock pushes WTI to new levels, expect the - tree methods — like AutoARIMA — to lag and produce biased, under-confident - intervals. This is a structural limitation, not a tuning problem. +With the **smoke spec (2 origins → a few scored points)** expect wide error bars +and an unstable ranking. That is exactly why a surprising leaderboard here is not +yet evidence of anything — it is a pipeline check. -4. **LLM/agent methods bring a different prior.** The LLM-process forecasters and - the news-reading agent are run on both `gemini-3.1-flash-lite-preview` and - `gemini-3.5-flash`, so the scorecard shows both *method* and *model* effects — - and whether reading the news helps when the numerical methods are blindsided. +## Cell 17 (code) -5. **The `Predictor` abstraction makes the comparison clean.** The same harness, - scoring functions, covariate panel, and eval spec serve every family, and the - registry lets you switch any predictor on or off without touching the pipeline. +```python +from energy_oil_forecasting import viz +from energy_oil_forecasting.analysis import ( + build_price_frame, + eval_narrative_md, + extract_agent_rationales, + leaderboard_with_uncertainty, + per_horizon_crps, + predictions_to_frame, +) +from IPython.display import HTML, Markdown, display # noqa: A004 + + +# Explode every scored 2026 eval prediction into one tidy row per +# (predictor, origin, horizon): point, 80% interval, realised price, and CRPS. +# Everything in Sections 7–10 reads from this frame, so it all recomputes when +# you flip SMOKE_TEST off and rerun. +price_df = build_price_frame(data_service) +eval_frame = predictions_to_frame(eval_results, data_service) +eval_board = leaderboard_with_uncertainty(eval_frame) +ph_crps = per_horizon_crps(eval_frame) + +print("━" * 72) +print("MEAN CRPS BY PREDICTOR × HORIZON (lower = better; 'All' = overall mean):") +print("━" * 72) +print(ph_crps.round(2).to_string()) +``` + +## Cell 18 (code) + +```python +# Heatmap of the table above. Read it left-to-right: the short-horizon columns +# are usually a near-uniform green (everyone is right), and one long-horizon +# column carries the colour spread that sets the 'All' ranking. +viz.make_crps_heatmap(ph_crps) +``` + +## Cell 19 (code) + +```python +# Same leaderboard, now with a standard-error bar on each mean. If the bars of +# the top methods overlap, their ordering is not statistically distinguishable — +# the honest verdict when only a few origins have been scored. +viz.make_leaderboard_interval_chart(eval_board) +``` + +## Cell 20 (markdown) + +--- +## 8. What are the top methods actually forecasting? + +A CRPS number doesn't show *behaviour*. Below, each leading method's **median +forecast and 80% interval** are drawn against the realised WTI path at every +eval origin. This is where the leaderboard becomes legible — watch for who +tracks the move, who simply anchors to the last price, and whose intervals are +too narrow to cover the outcome when the market jumps. + +## Cell 21 (code) + +```python +# Plot the leaderboard's top methods, and always include the best LLM/agent +# method for contrast (so the chart compares families even when a baseline leads). +_leaders = list(eval_board.index[:3]) +_best_llm = next((p for p in eval_board.index if eval_board.loc[p, "family"] == "LLM / Agent"), None) +if _best_llm and _best_llm not in _leaders: + _leaders.append(_best_llm) +print(f"Showing: {', '.join(_leaders)}") +viz.make_eval_forecast_chart(eval_frame, price_df, _leaders) +``` + +## Cell 22 (markdown) + +--- +## 9. Reading the agent's reasoning + +The news-reading agent attaches a free-text **rationale** to every forecast, and +a link to the full **Langfuse trace**. These are pulled straight from the +prediction metadata. This is where a surprising score becomes interpretable: you +can read whether the agent actually saw the geopolitical risk, and *how* it +turned that into a price and an interval — including, often, an interval far too +narrow for a regime shift. + +## Cell 23 (code) + +```python +# One card per (agent, origin): the rationale, the per-horizon note, and a link +# to the full reasoning trace. Empty only if no LLM/agent predictor is enabled. +eval_rationales = extract_agent_rationales(eval_results) +display(HTML(viz.render_rationales_html(eval_rationales))) +``` + +## Cell 24 (markdown) + +--- +## 10. Takeaways — computed from this run + +The summary below is **generated from the eval results in memory, not +hard-coded**, so it always matches what actually ran: the real winner, whether +its lead clears the noise floor, the horizon that decided the ranking, the +best-performing family, and a calibration line. Flip `SMOKE_TEST` off, rerun, +and these takeaways update themselves with the full leaderboard. + +## Cell 25 (code) + +```python +display(Markdown(eval_narrative_md(eval_frame, smoke=SMOKE_TEST))) +``` + +## Cell 26 (markdown) --- -## 8. What stateless methods can't do +## 11. What stateless methods can't do -Every method here is calibrated (or prompted) once and never updated between -rounds. This is intentional — it creates a clean baseline — but it leaves a -systematic gap: +Sections 7–10 score and dissect this run on its own terms. But every method here +shares one structural limit, independent of who topped the leaderboard: it is +calibrated (or prompted) **once and never updated between rounds**. That is +intentional — it creates a clean baseline — but it leaves a systematic gap: -- **No error feedback.** If a method consistently produces intervals that are too - narrow in elevated-vol regimes, it keeps making the same mistake. There is no - mechanism to update calibration between rounds. +- **No error feedback.** If a method's intervals are consistently too narrow in + an elevated-vol regime (read the coverage line in Section 10, and the squashed + error bars in Section 8), it keeps making the same mistake. Nothing updates its + calibration between origins. -- **No strategy evolution.** Each prediction starts from the same prior (the same - fitted model, or the same prompt). Resolved outcomes disappear without +- **No strategy evolution.** Each prediction starts from the same prior — the + same fitted model, or the same prompt. Resolved outcomes disappear without influencing future forecasts. - **Context without memory.** Even the news agent re-reads the world each origin; - it does not accumulate what worked. + it does not accumulate what worked. The rationales in Section 9 are written + fresh every time, with no record of how the last one resolved. -→ **Notebook 5** introduces adaptive agents that study the 2025 backtest, -record systematic observations, and calibrate their strategies accordingly. At -inference time, each agent receives the live stateless estimate and decides how -to adjust it — applying what it learned from training. +→ **Notebook 5** introduces adaptive agents that study the 2025 backtest, record +systematic observations, and calibrate their strategies accordingly. At inference +time, each agent receives the live stateless estimate and decides how to adjust +it — applying what it learned from training. → **Notebook 6** evaluates whether any training approach actually improved -out-of-sample performance on the held-out 2026 data. +out-of-sample performance on the held-out 2026 data — measured against the +stateless baseline this notebook just established. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__06_protected_eval.ipynb.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__06_protected_eval.ipynb.md index 672e8a9f..fa126a84 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__06_protected_eval.ipynb.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__06_protected_eval.ipynb.md @@ -103,7 +103,7 @@ This has a concrete implication for this evaluation: fetch data via yfinance and reason from what it computed — not from memorized facts about 2025 WTI prices. -- The **evaluation period** (Feb–Mar 2026) is definitively post-cutoff. +- The **evaluation period** (Feb–Jun 2026) is definitively post-cutoff. During eval, the agent must rely entirely on: 1. Live Google Search (with `cutoff_date` enforcement per origin) 2. Code execution (for statistical analysis of fetched data) diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md index 33691c76..ceb8021c 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md @@ -8,13 +8,14 @@ kind: notebook **If you're not sure what to do next, continue from here.** -This notebook is a fresh, hackable agent for the WTI crude-oil use case — deliberately *not* wired into the numbered curriculum. It gives you our common building blocks behind simple toggles, so you can start building something of your own: +This notebook is a fresh, hackable agent for the WTI crude-oil use case — deliberately *not* wired into the numbered curriculum. An agent is a **persona** plus a **toolbelt**, and you assemble that toolbelt right here in the notebook from a menu of one-line tool factories: -- **optional news search** — bounded, cutoff-aware Google Search (proxy-only) -- **optional code execution** — an E2B Python sandbox -- **two lightweight skills** — *tool-usage playbooks* in `starter_agent/skills/` +- **`news_search()`** — bounded, cutoff-aware Google Search (proxy-only) +- **`arima_forecast()`** — an AutoARIMA statistical anchor the agent can call directly (no code-gen) +- **`code_sandbox()`** — an E2B Python sandbox for the agent to compute its own diagnostics +- each tool pulls in its own *playbook* skill from `starter_agent/skills/` -It does two things: lets you **talk to the agent** (open-ended, Track 2) and **score one real forecast** (Track 1). The live cells are gated by `RUN_AGENT` so a fresh `Run All` is safe and free; flip it to `True` to actually call the model. +The factories live in `starter_agent/tools.py` — open it to see how a tool is built, or add your own. It does two things: lets you **talk to the agent** (open-ended, Track 2) and **score one real forecast** (Track 1). The live cells are gated by `RUN_AGENT` so a fresh `Run All` is safe and free; flip it to `True` to actually call the model. ## Cell 2 (code) @@ -47,6 +48,7 @@ RUN_AGENT = False from energy_oil_forecasting.starter_agent import ( build_starter_agent_config, build_starter_agent_predictor, + tools, ) @@ -56,24 +58,31 @@ print("RUN_AGENT =", RUN_AGENT, "| model =", AGENT_MODEL) ## Cell 3 (markdown) --- -## 1. Meet your agent +## 1. Build your agent's toolbelt -`build_starter_agent_config` returns an `AgentConfig` with two toggles. The default turns **news search on** (proxy-only, no extra key) and **code execution off** (it needs `E2B_API_KEY` and is slower). Flip them and re-run — the loaded skills follow the enabled tools. +This is where you compose the agent. `build_starter_agent_config` takes a `tools=[...]` list — the toolbelt — and folds each tool onto the agent (its config, its skill, its instructions). **Comment a line to drop a tool; uncomment to add one**, then re-run. That's the whole model: an agent is a persona plus the tools you hand it. ## Cell 4 (code) ```python -config = build_starter_agent_config( - model=AGENT_MODEL, - enable_search=True, # ← cutoff-aware Google Search (proxy-only) - enable_code_exec=False, # ← E2B Python sandbox (needs E2B_API_KEY); try True! -) - -print("Agent:", config.name) -print("Search enabled: ", config.context_retrieval.enabled) -print("Code-exec enabled: ", config.code_execution.enabled) -print("Skills loaded: ", [p.name for p in config.skills_dirs]) -print("\n── System instruction (edit this in starter_agent/agent.py) ──\n") +# ── Your agent's toolbelt ────────────────────────────── +# Each factory returns one tool. Comment a line to drop it, uncomment to add it. +# See starter_agent/tools.py for how each is built — and to write your own. +toolbelt = [ + tools.news_search(), # cutoff-aware Google Search (proxy-only, no extra key) + tools.arima_forecast(), # AutoARIMA anchor — the agent calls a forecast directly, no code-gen + # tools.code_sandbox(), # E2B Python sandbox (needs E2B_API_KEY, slower) — uncomment to add +] + +config = build_starter_agent_config(model=AGENT_MODEL, tools=toolbelt) + +print("Agent: ", config.name) +print("Toolbelt:", [t.label for t in toolbelt]) +print(" search enabled: ", config.context_retrieval.enabled) +print(" forecast tool: ", bool(config.function_tools)) +print(" code-exec enabled:", config.code_execution.enabled) +print("Skills loaded: ", [p.name for p in config.skills_dirs]) +print("\n── System instruction (edit the persona in starter_agent/agent.py) ──\n") print(config.instruction[:1200], "...") ``` @@ -164,11 +173,11 @@ else: This agent is a starting point. Here are concrete next steps, easiest first — each is a small edit, then re-run the cells above. -1. **Flip code execution on.** Set `enable_code_exec=True` in §1 (needs `E2B_API_KEY`). The agent loads the `code-analysis-playbook` skill and can compute its own diagnostics before forecasting. Compare the rationale. +1. **Change the toolbelt.** In §1, uncomment `tools.code_sandbox()` (needs `E2B_API_KEY`) to let the agent compute its own diagnostics, or drop `arima_forecast()` and compare the rationale with and without a statistical anchor. Adding a tool automatically loads its playbook skill and its instructions. 2. **Edit the agent's personality.** Open `starter_agent/agent.py` and change `_build_starter_instruction()` — make it more cautious, more contrarian, focused on one driver. Re-run §1 to see the new instruction. -3. **Sharpen the skills.** The two files in `starter_agent/skills/` are short on purpose. Add your best queries to `research-playbook`, or a new diagnostic to `code-analysis-playbook`. The agent picks them up automatically. +3. **Sharpen the skills.** The files in `starter_agent/skills/` are short on purpose. Add your best queries to `research-playbook`, or a new diagnostic to `code-analysis-playbook`. The agent picks them up automatically. 4. **Change the question and the origin.** Try a different `QUESTION` in §2 and a different origin in §3. -5. **Add a tool.** Give the agent a conventional forecast tool as a statistical anchor — see `analyst_agent.build_wti_tool_config` for the `ForecastTool` pattern. +5. **Write your own tool.** Open `starter_agent/tools.py` and add a factory that returns a `ToolSpec` — point `arima_forecast()` at a different series, swap AutoARIMA for another predictor, or wrap a brand-new function tool. Then add it to the toolbelt in §1. 6. **Score it properly.** Run it across several origins with `backtest()` (see `04_systematic_backtest_eval.ipynb`) and compare CRPS against the baselines. Bigger ideas — an agent that *learns* a strategy (notebooks 05–06), news vs. no-news lift, live prospective forecasting — are in the use-case `README.md` and `planning-docs/roadmap.md`. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md index 1041fba0..f9ffc196 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md @@ -183,6 +183,10 @@ def _build_adaptive_analyst_instruction() -> str: "## Temporal discipline\n\n" "Every forecast is anchored to an `as_of` date. Never use information beyond " "that date — in web search, code analysis, or reasoning.\n\n" + "If `search_web` returns a result beginning with `[SEARCH_VERIFICATION_FAILED]`, " + "treat it as no verified news context for that query. Do not use your own " + "background knowledge to fill the gap — proceed on price history and other " + "available signals only, and note the gap in your reasoning.\n\n" "When fetching data inside `run_code`, always pass `end=as_of_date` to " "yfinance to enforce the temporal cutoff — for example:\n\n" "```python\nraw = ticker.history(start='2004-01-01', end='2026-02-16', " @@ -221,7 +225,16 @@ markdown summary (3-5 paragraphs) covering relevant aspects of: Ground your summary in the search results you actually retrieve. \ When a cutoff date is specified, do not report or speculate about events \ -that occurred after that date.\ +that occurred after that date. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ """ diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analysis.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analysis.py.md index 0d07636d..6058a6d3 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analysis.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__analysis.py.md @@ -166,9 +166,292 @@ def select_top_predictors( return [str(x) for x in leaderboard.head(n)["predictor_id"].tolist()] +# ── Per-prediction diagnostics ──────────────────────────────────────────────── +# The leaderboard collapses every forecast into a single mean CRPS. To understand +# *why* a method wins or loses we need the predictions un-aggregated: one row per +# (predictor, origin, horizon) carrying the point estimate, the 80% interval, the +# realised outcome, and the CRPS the harness assigned. Everything below builds on +# this tidy frame so the notebook charts and the written narrative read from the +# same numbers. + +# Map of predictor-name fragments → family label, checked in order. Used to group +# the leaderboard into baselines / numerical-ML / LLM-agent without hard-coding a +# per-predictor table (the registry can grow without touching this). +_FAMILY_RULES: list[tuple[tuple[str, ...], str]] = [ + (("naive", "last value"), "Baseline"), + (("arima", "lightgbm", "prophet"), "Numerical ML"), + (("llmp", "agent", "news", "llm"), "LLM / Agent"), +] + + +def predictor_family(name: str) -> str: + """Classify a predictor display name into a forecasting family.""" + low = name.lower() + for fragments, label in _FAMILY_RULES: + if any(frag in low for frag in fragments): + return label + return "Other" + + +def _qval(quantiles: dict[Any, float], q: float) -> float: + """Read a quantile value tolerating both float and string dict keys.""" + for key in (q, str(q), f"{q:.2f}"): + if key in quantiles: + return float(quantiles[key]) + return float("nan") + + +def _business_horizon(as_of: pd.Timestamp, forecast_date: pd.Timestamp) -> int: + """Trading-day distance between an information cutoff and a forecast date.""" + return max(len(pd.bdate_range(as_of.normalize(), forecast_date.normalize())) - 1, 0) + + +def predictions_to_frame( + results_by_predictor: dict[str, dict[str, BacktestResult]], + data_service: DataService, + *, + actuals_as_of: datetime | None = None, +) -> pd.DataFrame: + """Explode backtest results into one tidy row per scored prediction. + + Parameters + ---------- + results_by_predictor + ``{display_name: {task_id: BacktestResult}}`` — exactly the shape the + notebook holds in ``eval_results`` / ``backtest_results``. + data_service + Used to look up realised target values for error/coverage columns. + actuals_as_of + Cutoff for realised-value lookup; defaults to *now* so every horizon that + has already resolved is scored (see :func:`score_backtest_results`). + + Returns + ------- + pd.DataFrame + Columns: ``predictor``, ``family``, ``as_of``, ``forecast_date``, + ``horizon`` (trading days), ``point``, ``q10``/``q20``/``q50``/``q80``/ + ``q90``, ``actual``, ``crps``, ``abs_error``, ``signed_error``, + ``width80`` (80% interval width), and ``inside80`` (1.0/0.0/NaN). + """ + resolved_as_of = actuals_as_of or datetime.now(tz=timezone.utc).replace(tzinfo=None) + actual_cache: dict[str, dict[pd.Timestamp, float]] = {} + + def _actuals(series_id: str) -> dict[pd.Timestamp, float]: + if series_id not in actual_cache: + df = data_service.get_series(series_id, as_of=resolved_as_of) + actual_cache[series_id] = { + pd.Timestamp(row["timestamp"]).normalize(): float(row["value"]) for _, row in df.iterrows() + } + return actual_cache[series_id] + + rows: list[dict[str, Any]] = [] + for predictor_name, task_results in results_by_predictor.items(): + for result in task_results.values(): + actual_by_date = _actuals(result.spec.task.target_series_id) + for pred, score in zip(result.predictions, result.scores, strict=False): + if not isinstance(pred.payload, ContinuousForecast): + continue + as_of = pd.Timestamp(pred.as_of) + fdate = pd.Timestamp(pred.forecast_date).normalize() + q = pred.payload.quantiles + lo80, hi80 = _qval(q, 0.2), _qval(q, 0.8) + point = float(pred.payload.point_forecast) + actual = actual_by_date.get(fdate) + rows.append( + { + "predictor": predictor_name, + "family": predictor_family(predictor_name), + "as_of": as_of, + "forecast_date": fdate, + "horizon": _business_horizon(as_of, fdate), + "point": point, + "q10": _qval(q, 0.1), + "q20": lo80, + "q50": _qval(q, 0.5), + "q80": hi80, + "q90": _qval(q, 0.9), + "actual": actual, + "crps": float(score), + "abs_error": abs(point - actual) if actual is not None else float("nan"), + "signed_error": (actual - point) if actual is not None else float("nan"), + "width80": hi80 - lo80, + "inside80": float(lo80 <= actual <= hi80) if actual is not None else float("nan"), + } + ) + return pd.DataFrame(rows) + + +def per_horizon_crps(pred_frame: pd.DataFrame) -> pd.DataFrame: + """Pivot mean CRPS to a predictor × horizon matrix with an ``All`` column. + + Rows are sorted by overall mean CRPS (best first) so the table doubles as the + leaderboard and reveals which horizon decides the ranking. + """ + if pred_frame.empty: + return pd.DataFrame() + pivot = pred_frame.pivot_table(index="predictor", columns="horizon", values="crps", aggfunc="mean") + pivot.columns = [f"h={int(h)}d" for h in pivot.columns] + pivot["All"] = pred_frame.groupby("predictor")["crps"].mean() + return pivot.sort_values("All") + + +def leaderboard_with_uncertainty(pred_frame: pd.DataFrame) -> pd.DataFrame: + """Mean CRPS per predictor with a standard error, sorted best-first. + + The ``se`` column (sample standard deviation / √n) is the lens for the + "is this lead real or noise?" question: when the gap between two predictors + is small relative to their SEs — common with only a handful of scored + origins — the ranking is not statistically meaningful. + """ + if pred_frame.empty: + return pd.DataFrame() + grp = pred_frame.groupby("predictor")["crps"] + out = pd.DataFrame( + { + "mean_crps": grp.mean(), + "se": grp.std(ddof=1) / np.sqrt(grp.count()), + "n": grp.count().astype(int), + "family": pred_frame.groupby("predictor")["family"].first(), + } + ) + return out.sort_values("mean_crps") + + +def extract_agent_rationales(results_by_predictor: dict[str, dict[str, BacktestResult]]) -> pd.DataFrame: + """Pull free-text rationale and trace links from agent/LLM prediction metadata. + + Only predictions whose ``metadata`` carries a ``rationale`` (the analyst agent + and any LLM method that returns one) produce rows. The result is the raw + material for inspecting *what the model was thinking* origin by origin. + """ + rows: list[dict[str, Any]] = [] + for predictor_name, task_results in results_by_predictor.items(): + for result in task_results.values(): + for pred in result.predictions: + meta = pred.metadata or {} + if "rationale" not in meta and "horizon_rationale" not in meta: + continue + rows.append( + { + "predictor": predictor_name, + "as_of": pd.Timestamp(pred.as_of), + "horizon": _business_horizon(pd.Timestamp(pred.as_of), pd.Timestamp(pred.forecast_date)), + "point": float(pred.payload.point_forecast) + if isinstance(pred.payload, ContinuousForecast) + else float("nan"), + "rationale": str(meta.get("rationale", "")).strip(), + "horizon_rationale": str(meta.get("horizon_rationale", "")).strip(), + "trace_url": meta.get("langfuse_trace_url", ""), + } + ) + return pd.DataFrame(rows) + + +def eval_narrative_md( + pred_frame: pd.DataFrame, + *, + smoke: bool = False, + period_label: str = "2026 evaluation", +) -> str: + """Generate the eval takeaways as Markdown computed from the results. + + Replaces hard-coded prose so the narrative always matches the run — including + after switching from smoke to the full suite. Reports the actual winner, the + gap to the runner-up relative to the noise floor, the decisive horizon, the + best family, and a calibration line, plus an explicit small-sample caveat. + """ + if pred_frame.empty: + return "_No scored predictions available to summarise._" + + board = leaderboard_with_uncertainty(pred_frame) + horizons = sorted(pred_frame["horizon"].unique()) + n_origins = pred_frame["as_of"].nunique() + n_points = len(pred_frame) + + winner = board.index[0] + win_crps, win_se = board.loc[winner, "mean_crps"], board.loc[winner, "se"] + lines: list[str] = [] + + # 1. Winner + significance vs runner-up. + if len(board) > 1: + runner = board.index[1] + gap = board.loc[runner, "mean_crps"] - win_crps + noise = float(np.nan_to_num(win_se) + np.nan_to_num(board.loc[runner, "se"])) + significant = noise > 0 and gap > noise + verdict = ( + "a gap larger than the combined standard error — a real edge over this window" + if significant + else "**well within the combined standard error**, so the ranking here is not statistically distinguishable from noise" + ) + lines.append( + f"1. **{winner}** has the best mean CRPS ({win_crps:.2f}) on the {period_label}, " + f"ahead of **{runner}** ({board.loc[runner, 'mean_crps']:.2f}) by {gap:.2f} — {verdict}." + ) + else: + lines.append(f"1. **{winner}** scored {win_crps:.2f} mean CRPS on the {period_label}.") + + # 2. Where the ranking is decided (per-horizon spread). + if len(horizons) > 1: + by_h = pred_frame.groupby("horizon")["crps"] + spread = (by_h.max() - by_h.min()).sort_values(ascending=False) + decisive_h = int(spread.index[0]) + easy_h = int(spread.index[-1]) + lines.append( + f"2. The leaderboard is **decided at h={decisive_h}d**, where CRPS ranges {by_h.min()[decisive_h]:.1f}–" + f"{by_h.max()[decisive_h]:.1f} across methods; at the short h={easy_h}d horizon the methods are nearly " + f"tied (range {by_h.min()[easy_h]:.1f}–{by_h.max()[easy_h]:.1f}). A handful of long-horizon points " + f"drives the whole ranking." + ) + + # 3. Best family — does the agentic/LLM bet pay off here? + fam = pred_frame.groupby("family")["crps"].mean().sort_values() + best_fam = fam.index[0] + fam_str = ", ".join(f"{f} {v:.2f}" for f, v in fam.items()) + lines.append(f"3. **By family** (mean CRPS): {fam_str}. Best family this window: **{best_fam}**.") + + # 4. Calibration — is the winner's 80% interval honest? + cov = pred_frame.dropna(subset=["inside80"]).groupby("predictor")["inside80"].mean() * 100 + if winner in cov.index: + n_win = int(board.loc[winner, "n"]) + lines.append( + f"4. **Calibration:** {winner}'s 80% interval covered {cov[winner]:.0f}% of outcomes " + f"(target 80%) over its {n_win} scored point(s). With this few, coverage this far from " + f"target is itself a small-sample artefact, not necessarily mis-calibration." + ) + + # 5. Sample-size caveat — the honest health warning. + caveat = ( + f"⚠️ **Smoke run:** only {n_origins} origin(s) / {n_points} scored points. Treat the ranking as a " + f"pipeline check, not evidence — rerun the full suite before drawing conclusions." + if smoke or n_origins <= 2 + else f"Based on {n_origins} origins / {n_points} scored points." + ) + lines.append(f"5. {caveat}") + return "\n".join(lines) + + +def build_price_frame(data_service: DataService, *, as_of: datetime | None = None) -> pd.DataFrame: + """Return the target price series as a ``price``-column DataFrame for plotting.""" + resolved_as_of = as_of or datetime.now(tz=timezone.utc).replace(tzinfo=None) + series = data_service.get_series("wti_crude_oil_price", as_of=resolved_as_of) + frame = pd.DataFrame( + {"price": series["value"].astype(float).to_numpy()}, + index=pd.to_datetime(series["timestamp"]), + ) + frame.index.name = "date" + return frame.sort_index() + + __all__ = [ "backtest_results_to_frame", + "build_price_frame", "compute_brier_score", + "eval_narrative_md", + "extract_agent_rationales", + "leaderboard_with_uncertainty", + "per_horizon_crps", + "predictions_to_frame", + "predictor_family", "rolling_coverage_pct", "score_backtest_results", "select_top_predictors", diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md index 0c0a4371..80e1c480 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md @@ -5,8 +5,12 @@ kind: yaml ```yaml # Energy Oil Eval Spec — 2026 Prospective Competition # -# Runs on 8 weekly origins from Feb 2, 2026 to Mar 23, 2026. -# Covers the high-volatility Persian Gulf geopolitical price shock period. +# Runs on 18 weekly origins from Feb 2, 2026 to Jun 1, 2026. +# Covers the high-volatility Persian Gulf geopolitical price shock and its +# aftermath. The end date is set to the latest origin whose longest horizon +# (21 business days) still resolves against available data — keep it at +# most 21 business days behind the most recent cached WTI price (see +# scripts/fetch_wti.py) so every origin fully resolves. # Target is WTI Crude Oil price (yfinance ticker: CL=F). # Horizons: 5, 10, 21 business days. @@ -14,8 +18,9 @@ spec_id: energy_oil_eval description: >- Prospective/out-of-sample evaluation period in 2026 for daily WTI crude oil. - Evaluates selected contender models on 8 weekly origins during the early 2026 - geopolitical price shock to measure adaptive real-time forecasting performance. + Evaluates selected contender models on 18 weekly origins from the early 2026 + geopolitical price shock through its aftermath, to measure adaptive + real-time forecasting performance. tasks: - task_id: wti_oil_price_forecast @@ -27,7 +32,7 @@ tasks: projected 5, 10, and 21 trading days ahead. start: "2026-02-02" -end: "2026-03-23" +end: "2026-06-01" stride: 5 warmup: 250 ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md index 80597220..f4fdfe8c 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md @@ -9,7 +9,7 @@ kind: yaml # arena cheaply during development and end-to-end testing. # Use by setting SMOKE_TEST = True in the notebook setup cell. # -# Origin count : 2 (vs. 8 in the full eval) +# Origin count : 2 (vs. 18 in the full eval) # Warmup : 250 trading days (~1 year) of historical prices spec_id: energy_oil_eval_smoke diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md index 2e5a2e84..080dd814 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md @@ -5,10 +5,12 @@ kind: python ```python """WTI starter agent — a fresh, hackable template for your own exploration. -Exports the toggle-driven :class:`AgentConfig` factory and the predictor -convenience factory. See ``99_starter_agent.ipynb`` and ``agent.py``. +Exports the toolbelt-driven :class:`AgentConfig` factory, the predictor +convenience factory, and the :mod:`tools` module of per-tool factories you +compose in the notebook. See ``99_starter_agent.ipynb`` and ``agent.py``. """ +from energy_oil_forecasting.starter_agent import tools from energy_oil_forecasting.starter_agent.agent import ( build_starter_agent_config, build_starter_agent_predictor, @@ -18,5 +20,6 @@ from energy_oil_forecasting.starter_agent.agent import ( __all__ = [ "build_starter_agent_config", "build_starter_agent_predictor", + "tools", ] ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md index c7076021..c625c4d9 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md @@ -32,7 +32,7 @@ from __future__ import annotations import json from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Sequence from aieng.forecasting.data.context import ForecastContext from aieng.forecasting.evaluation.task import ForecastingTask @@ -51,13 +51,13 @@ from aieng.forecasting.models import LITE_MODEL # Reuse the existing WTI prompt builder + history compression — these serialise # the task/context into the agent's JSON payload and are not worth duplicating. from energy_oil_forecasting.analyst_agent import WtiPriceForecastPromptBuilder +from energy_oil_forecasting.starter_agent.tools import ToolSpec, news_search -# Skills live next to this module. +# Skills live next to this module. The forecasting contract is always loaded; +# each tool loads its own playbook via its ToolSpec (see tools.py). _SKILLS_ROOT = Path(__file__).parent / "skills" _FORECASTING_SKILL = _SKILLS_ROOT / "forecasting" -_RESEARCH_SKILL = _SKILLS_ROOT / "research-playbook" -_CODE_ANALYSIS_SKILL = _SKILLS_ROOT / "code-analysis-playbook" # --------------------------------------------------------------------------- @@ -93,19 +93,6 @@ def _build_starter_instruction() -> str: _STARTER_INSTRUCTION = _build_starter_instruction() -_CONTEXT_RETRIEVAL_INSTRUCTION = """\ -You are an oil-market intelligence specialist with web search. - -Return a concise structured markdown summary (3-5 paragraphs) covering, as the -query warrants: WTI/Brent price level and trend; OPEC+ supply decisions; -geopolitical risk in the Persian Gulf and key shipping lanes; US SPR / energy -policy; notable supply-disruption signals; and published analyst price targets. - -Ground every claim in the search results you actually retrieve. When a cutoff -date is specified, never report or speculate about events after it.\ -""" - - # --------------------------------------------------------------------------- # Config factory # --------------------------------------------------------------------------- @@ -113,59 +100,75 @@ date is specified, never report or speculate about events after it.\ def build_starter_agent_config( model: str = LITE_MODEL, - search_model: str = LITE_MODEL, *, - enable_search: bool = True, - enable_code_exec: bool = False, + tools: Sequence[ToolSpec] = (), ) -> AgentConfig: - """Build the WTI starter :class:`AgentConfig`. + """Build the WTI starter :class:`AgentConfig` from a toolbelt. + + An agent is a persona plus a list of tools. The persona is fixed here (edit + ``_build_starter_instruction``); the *toolbelt* is what you compose in the + notebook — a list of :class:`~energy_oil_forecasting.starter_agent.tools.ToolSpec` + from the factories in :mod:`energy_oil_forecasting.starter_agent.tools`:: + + from energy_oil_forecasting.starter_agent import tools, build_starter_agent_config + + config = build_starter_agent_config( + model=AGENT_MODEL, + tools=[tools.news_search(), tools.arima_forecast()], + ) + + Each spec lands in a different ``AgentConfig`` field; this function folds the + list, routing every fragment to the right place — search sub-agent, code + sandbox, function tools — and loading each tool's playbook skill and prompt + supplement. Adding or removing a tool is one line in the notebook. Parameters ---------- model : str Model for the analyst agent (default: lite). Pass the advanced model (``"gemini-3.5-flash"``) for higher-quality runs. - search_model : str - Model for the bounded web-search sub-tool. - enable_search : bool, default=True - Wire a cutoff-aware ``search_web`` tool and load the - ``research-playbook`` skill. Proxy-only — no extra API key. - enable_code_exec : bool, default=False - Wire an E2B Python sandbox and load the ``code-analysis-playbook`` - skill. Needs ``E2B_API_KEY`` and is slower, so it is off by default — - flip it on to let the agent compute its own diagnostics. + tools : Sequence[ToolSpec], default=() + The agent's toolbelt. Build entries with the factories in ``tools.py`` + (``news_search()``, ``code_sandbox()``, ``arima_forecast()``), or write + your own factory that returns a ``ToolSpec``. Returns ------- AgentConfig """ - # Every attached skill is loaded on demand: ADK injects each skill's name + - # description into the system prompt, and the agent reads the full SKILL.md - # only when relevant — so toggling a tool just adds its skill, no persona edits. + # The forecasting contract is always loaded. Each tool's own playbook is + # loaded on demand: ADK injects each skill's name + description into the + # system prompt, and the agent reads the full SKILL.md only when relevant — + # so adding a tool just adds its skill, no persona edits. skills_dirs: list[Path] = [_FORECASTING_SKILL] - if enable_search: - skills_dirs.append(_RESEARCH_SKILL) - if enable_code_exec: - skills_dirs.append(_CODE_ANALYSIS_SKILL) - - context_retrieval = ( - ContextRetrievalConfig( - enabled=True, - instruction=_CONTEXT_RETRIEVAL_INSTRUCTION, - search_model=search_model, - ) - if enable_search - else ContextRetrievalConfig() - ) + instruction = _STARTER_INSTRUCTION + context_retrieval = ContextRetrievalConfig() + code_execution = CodeExecutionConfig() + function_tools: list[Any] = [] + max_output_tokens: int | None = None + + for spec in tools: + if spec.skill_dir is not None: + skills_dirs.append(spec.skill_dir) + if spec.instruction_supplement: + instruction += spec.instruction_supplement + if spec.context_retrieval is not None: + context_retrieval = spec.context_retrieval + if spec.code_execution is not None: + code_execution = spec.code_execution + if spec.function_tool is not None: + function_tools.append(spec.function_tool) + if spec.max_output_tokens is not None: + max_output_tokens = max(max_output_tokens or 0, spec.max_output_tokens) return AgentConfig( name="wti_starter_agent", model=model, - instruction=_STARTER_INSTRUCTION, - # 16k headroom: enough for a complete run_code script + structured output. - max_output_tokens=16_384 if enable_code_exec else None, + instruction=instruction, + max_output_tokens=max_output_tokens, context_retrieval=context_retrieval, - code_execution=CodeExecutionConfig(enabled=enable_code_exec), + code_execution=code_execution, + function_tools=function_tools, skills_dirs=skills_dirs, ) @@ -228,6 +231,7 @@ def build_starter_agent_predictor(config: AgentConfig) -> AgentPredictor: def __getattr__(name: str) -> Any: """Expose ``root_agent`` lazily for schema-free interactive use via ``adk web``.""" if name == "root_agent": - return build_adk_agent(build_starter_agent_config()) + # Interactive default: news search on (proxy-only, no extra key). + return build_adk_agent(build_starter_agent_config(tools=[news_search()])) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md index a6969936..f43e96ae 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md @@ -21,6 +21,12 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. +`search_web` runs an independent verifier on every result and returns +`[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as +pre-cutoff. Treat that as no verified news for the query — proceed on your +other signals and say so, never filling the gap from your own background +knowledge. + ## How to search - **Search before you forecast, not after.** Gather context first, then reason. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md new file mode 100644 index 00000000..1cd0911d --- /dev/null +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md @@ -0,0 +1,237 @@ +# Source: implementations/energy_oil_forecasting/starter_agent/tools.py + +kind: python + +```python +"""The starter agent's toolbelt — one factory per tool, composed in the notebook. + +An agent is a *persona* plus a *list of tools*. This module makes that list the +thing you edit: each function here returns a :class:`ToolSpec` describing one +capability, and you assemble an agent by handing a list of them to +:func:`~energy_oil_forecasting.starter_agent.agent.build_starter_agent_config`:: + + from energy_oil_forecasting.starter_agent import tools, build_starter_agent_config + + config = build_starter_agent_config( + model=AGENT_MODEL, + tools=[ + tools.news_search(), # cutoff-aware Google Search (proxy-only) + tools.arima_forecast(), # AutoARIMA statistical anchor — no code-gen + # tools.code_sandbox(), # E2B Python sandbox (needs E2B_API_KEY) + ], + ) + +Each tool lands in a *different* field of the underlying ``AgentConfig`` (search +is a sub-agent, code execution is a sandbox capability, the forecast is a +plain function tool). A :class:`ToolSpec` carries everything a tool needs — the +config fragment it fills, its playbook skill, and any prompt supplement — so the +config factory can route it without the notebook ever touching ADK plumbing. + +To add your own tool, write a factory that returns a ``ToolSpec``. Point it at a +different series, swap AutoARIMA for another predictor, or wrap a brand-new +function tool — the notebook composition and the config fold both keep working. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from aieng.forecasting.data import DataService +from aieng.forecasting.methods.agentic import ForecastTool +from aieng.forecasting.methods.agentic.agent_factory import ( + CodeExecutionConfig, + ContextRetrievalConfig, +) +from aieng.forecasting.methods.numerical.darts_arima import DartsAutoARIMAPredictor +from aieng.forecasting.models import LITE_MODEL +from energy_oil_forecasting.data import WTI_SERIES_ID, build_wti_service + + +# Skills live next to this module; each tool loads its own playbook. +_SKILLS_ROOT = Path(__file__).parent / "skills" +_RESEARCH_SKILL = _SKILLS_ROOT / "research-playbook" +_CODE_ANALYSIS_SKILL = _SKILLS_ROOT / "code-analysis-playbook" + + +# --------------------------------------------------------------------------- +# ToolSpec — the seam between the notebook's toolbelt and AgentConfig +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ToolSpec: + """One item on the agent's toolbelt. + + A tool is more than a callable: it may need a config fragment, a skill that + teaches the agent how to use it, and a line of instruction. This descriptor + bundles all of that so + :func:`~energy_oil_forecasting.starter_agent.agent.build_starter_agent_config` + can fold a list of specs onto a single ``AgentConfig`` — routing each field + to the right place — without the caller knowing the internals. + + Attributes + ---------- + label : str + Short human-readable name, shown when the config is printed. + context_retrieval : ContextRetrievalConfig or None + Web-search sub-agent config, if this tool provides search. + code_execution : CodeExecutionConfig or None + E2B sandbox config, if this tool provides code execution. + function_tool : Any or None + A ready-to-register ADK function tool (e.g. from + ``ForecastTool.as_function_tool()``). + skill_dir : Path or None + A playbook skill directory to load alongside the tool. + instruction_supplement : str + Text appended to the agent's system instruction when this tool is on. + max_output_tokens : int or None + A per-tool floor on the response budget (e.g. code execution needs + headroom for a full script). The config takes the max across all tools. + """ + + label: str + context_retrieval: ContextRetrievalConfig | None = None + code_execution: CodeExecutionConfig | None = None + function_tool: Any | None = None + skill_dir: Path | None = None + instruction_supplement: str = "" + max_output_tokens: int | None = None + + +# --------------------------------------------------------------------------- +# Tool-specific prompt text +# --------------------------------------------------------------------------- + + +_CONTEXT_RETRIEVAL_INSTRUCTION = """\ +You are an oil-market intelligence specialist with web search. + +Return a concise structured markdown summary (3-5 paragraphs) covering, as the +query warrants: WTI/Brent price level and trend; OPEC+ supply decisions; +geopolitical risk in the Persian Gulf and key shipping lanes; US SPR / energy +policy; notable supply-disruption signals; and published analyst price targets. + +Ground every claim in the search results you actually retrieve. When a cutoff +date is specified, never report or speculate about events after it. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ +""" + + +def _forecast_tool_supplement(series_id: str, frequency: str) -> str: + """Instruction appended when the statistical forecast tool is attached. + + Kept in the factory (not hard-coded) so a tool built for a different series + or frequency describes itself correctly to the agent. + """ + return f""" + +## Statistical forecast tool + +You have access to `run_forecast`, a conventional statistical baseline +(AutoARIMA) you can call directly. Unlike open-ended code, this tool has a fixed, +auditable interface and returns a structured forecast you can reason from. + +Call it ONCE before producing your forecast, with: +- `series_id`: "{series_id}" +- `cutoff_date`: the `as_of` date from the payload (YYYY-MM-DD). This is the + information cutoff — the model uses only data on or before it. +- `horizons`: the `horizons` list from the payload. +- `frequency`: "{frequency}" (the business calendar the series trades on). + +The tool returns JSON with point forecasts and 80%/90% prediction intervals per +horizon. Treat it as a disciplined statistical anchor: combine it with any +market context you have. You may adjust away from the baseline when fundamentals +or geopolitical risk justify it — document your reasoning in the `rationale` +fields.\ +""" + + +# --------------------------------------------------------------------------- +# Tool factories — each returns one ToolSpec +# --------------------------------------------------------------------------- + + +def news_search(*, search_model: str = LITE_MODEL) -> ToolSpec: + """Build a cutoff-aware Google Search tool, run by a bounded sub-agent (proxy-only). + + Wires a ``search_web`` tool and loads the ``research-playbook`` skill. No + extra API key — everything routes through the Vector proxy. + + Parameters + ---------- + search_model : str + Model for the web-search sub-agent. Defaults to the lite model. + """ + return ToolSpec( + label="news_search", + context_retrieval=ContextRetrievalConfig( + enabled=True, + instruction=_CONTEXT_RETRIEVAL_INSTRUCTION, + search_model=search_model, + ), + skill_dir=_RESEARCH_SKILL, + ) + + +def code_sandbox() -> ToolSpec: + """Build an E2B Python sandbox for the agent to compute its own diagnostics. + + Wires the code-execution capability and loads the ``code-analysis-playbook`` + skill. Needs ``E2B_API_KEY`` and is slower than the other tools, so it is + off by default — add it to the toolbelt to turn it on. + """ + return ToolSpec( + label="code_sandbox", + code_execution=CodeExecutionConfig(enabled=True), + skill_dir=_CODE_ANALYSIS_SKILL, + # 16k headroom: enough for a complete run_code script + structured output. + max_output_tokens=16_384, + ) + + +def arima_forecast( + *, + series_id: str = WTI_SERIES_ID, + frequency: str = "B", + num_samples: int = 200, + data_service: DataService | None = None, +) -> ToolSpec: + """Build a conventional statistical anchor: AutoARIMA behind a `run_forecast` tool. + + Lets the agent invoke a statistical forecast *directly* — a rigid, auditable + interface — instead of writing forecasting code. In contrast to + :func:`code_sandbox` (open-ended), this trades flexibility for control and + reproducibility. The tool reads series data server-side; it never enters the + LLM context. + + Parameters + ---------- + series_id : str + Series the agent should forecast. Defaults to the WTI target. + frequency : str + Business calendar passed to the predictor (``"B"`` for WTI). + num_samples : int + Monte Carlo sample count for AutoARIMA. Kept modest to bound latency. + data_service : DataService or None + Pre-populated data service. When ``None``, a cache-backed WTI service is + built. Pass one to point the tool at your own series (or to avoid a data + fetch in tests). + """ + service = data_service if data_service is not None else build_wti_service() + tool = ForecastTool(service, predictor=DartsAutoARIMAPredictor(num_samples=num_samples)) + return ToolSpec( + label="arima_forecast", + function_tool=tool.as_function_tool(), + instruction_supplement=_forecast_tool_supplement(series_id, frequency), + ) +``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__viz.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__viz.py.md index e8693676..ddc06844 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__viz.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__energy_oil_forecasting__viz.py.md @@ -1220,4 +1220,274 @@ def prob_bar(val: float, width: int = 10) -> str: def conf_bar(conf: str) -> str: """Map confidence label to emoji indicator.""" return {"high": "🟢", "medium": "🟡", "low": "🔴"}.get(conf.lower(), "⚪") + + +# ── NB4 eval-diagnostic charts ──────────────────────────────────────────────── +# These read the tidy per-prediction frame from ``analysis.predictions_to_frame`` +# (one row per predictor × origin × horizon) and answer three questions the bare +# leaderboard can't: *where* the ranking is decided (heatmap), whether a lead is +# real or noise (leaderboard with error bars), and *what* the methods actually +# forecast vs reality (trajectory chart). + +# Qualitative palette for an arbitrary, growing predictor set. Stable per call: +# colours are assigned by sorted predictor name so a method keeps its colour +# across the heatmap, leaderboard, and trajectory charts within one notebook run. +_PREDICTOR_PALETTE = [ + "#1f77b4", + "#ff7f0e", + "#2ca02c", + "#d62728", + "#9467bd", + "#8c564b", + "#e377c2", + "#7f7f7f", + "#bcbd22", + "#17becf", + "#393b79", + "#b5651d", +] + + +def predictor_colors(predictors: list[str]) -> dict[str, str]: + """Assign a stable colour to each predictor name.""" + return {name: _PREDICTOR_PALETTE[i % len(_PREDICTOR_PALETTE)] for i, name in enumerate(predictors)} + + +def make_crps_heatmap(per_horizon_df: pd.DataFrame) -> go.Figure: + """Predictor × horizon mean-CRPS heatmap (lower = better, sorted best-first). + + Expects the output of ``analysis.per_horizon_crps`` — horizon columns plus a + final ``All`` column. Reveals which horizon decides the ranking: typically the + short horizons are a wash and one long horizon dominates the mean. + """ + df = per_horizon_df.copy() + # Best predictor on top: reverse so plotly's bottom-up y-axis shows it first. + df = df.iloc[::-1] + z = df.to_numpy(dtype=float) + fig = go.Figure( + go.Heatmap( + z=z, + x=list(df.columns), + y=list(df.index), + colorscale="RdYlGn_r", + colorbar={"title": "CRPS"}, + text=[[f"{v:.2f}" if np.isfinite(v) else "" for v in row] for row in z], + texttemplate="%{text}", + textfont={"size": 12}, + hovertemplate="%{y}
%{x}: %{z:.3f}", + ) + ) + fig.update_layout( + title={"text": "Mean CRPS by Predictor × Horizon (lower = better)", "font": {"size": 16}}, + xaxis={"title": "Horizon", "side": "top"}, + yaxis={"title": ""}, + template="plotly_white", + width=720, + height=40 * len(df) + 160, + margin={"t": 90, "b": 40, "l": 230, "r": 40}, + ) + # Visually separate the "All" summary column. + if "All" in per_horizon_df.columns: + fig.add_vline(x=len(per_horizon_df.columns) - 1.5, line={"color": "#333333", "width": 1.5}) + return fig + + +def make_leaderboard_interval_chart(board_df: pd.DataFrame) -> go.Figure: + """Mean CRPS ± standard error per predictor, exposing whether a lead is noise. + + Expects ``analysis.leaderboard_with_uncertainty``. When the error bars of the + top methods overlap heavily, the ranking is not statistically meaningful — the + honest read on a short eval window. + """ + df = board_df.iloc[::-1] # best at top of the bottom-up axis + fam_colors = {"Baseline": "#7f7f7f", "Numerical ML": "#1f77b4", "LLM / Agent": "#2ca02c", "Other": "#b5651d"} + colors = [fam_colors.get(f, "#b5651d") for f in df["family"]] + fig = go.Figure( + go.Scatter( + x=df["mean_crps"], + y=df.index, + mode="markers", + marker={"size": 11, "color": colors}, + error_x={"type": "data", "array": df["se"].fillna(0.0), "thickness": 1.6, "width": 6, "color": "#888888"}, + hovertemplate="%{y}
CRPS %{x:.3f} ± %{error_x.array:.3f}", + showlegend=False, + ) + ) + best = float(df["mean_crps"].min()) + fig.add_vline( + x=best, + line={"color": "#31a354", "dash": "dot", "width": 1.5}, + annotation_text=" best", + annotation_position="top", + annotation_font={"size": 11, "color": "#31a354"}, + ) + fig.update_layout( + title={"text": "Eval Leaderboard — Mean CRPS ± 1 SE (overlap ⇒ tied)", "font": {"size": 16}}, + xaxis={"title": "Mean CRPS (lower = better)", "showgrid": True, "gridcolor": "#f0f0f0"}, + yaxis={"title": ""}, + template="plotly_white", + width=760, + height=34 * len(df) + 150, + margin={"t": 70, "b": 50, "l": 230, "r": 40}, + ) + return fig + + +def make_eval_forecast_chart( + pred_frame: pd.DataFrame, + price_df: pd.DataFrame, + predictors: list[str], + *, + history_window: int = 25, +) -> go.Figure: + """Per-origin trajectory chart: each method's median + 80% band vs reality. + + One column per forecast origin. Shows the pre-origin price history, the + realised price path, and — for each selected predictor — point forecasts at + each horizon with 80% interval error bars. This is the "what are the top + methods actually doing" view: you can see who tracks the move, who lags, and + whose intervals are too tight. + """ + origins = sorted(pred_frame["as_of"].unique()) + colors = predictor_colors(predictors) + + titles = [] + for o_raw in origins: + o = pd.Timestamp(o_raw) + rows = price_df[price_df.index >= o.normalize()] + spot = f"${float(rows.iloc[0]['price']):.0f}" if not rows.empty else "" + titles.append(f"{o.strftime('%b %d, %Y')} WTI {spot}") + + fig = psp.make_subplots( + rows=1, cols=len(origins), subplot_titles=titles, shared_yaxes=True, horizontal_spacing=0.03 + ) + + for col, origin_raw in enumerate(origins, start=1): + origin = pd.Timestamp(origin_raw) + show_legend = col == 1 + sub = pred_frame[pred_frame["as_of"] == origin] + last_fdate = pd.Timestamp(sub["forecast_date"].max()) + + # Pre-origin history + realised future path. + hist = price_df[price_df.index <= origin.normalize()].iloc[-history_window:] + future = price_df[(price_df.index > origin.normalize()) & (price_df.index <= last_fdate)] + fig.add_trace( + go.Scatter( + x=hist.index.tolist(), + y=hist["price"].tolist(), + mode="lines", + line={"color": CLR_HISTORY, "width": 1.5}, + name="WTI history", + showlegend=show_legend, + legendgroup="hist", + ), + row=1, + col=col, + ) + fig.add_trace( + go.Scatter( + x=future.index.tolist(), + y=future["price"].tolist(), + mode="lines+markers", + line={"color": CLR_ACTUAL, "width": 2.5}, + marker={"size": 5}, + name="Realised price", + showlegend=show_legend, + legendgroup="actual", + ), + row=1, + col=col, + ) + + # Each predictor's median + 80% interval at every horizon. + for name in predictors: + pr = sub[sub["predictor"] == name].sort_values("forecast_date") + if pr.empty: + continue + err_hi = (pr["q80"] - pr["point"]).clip(lower=0).fillna(0.0) + err_lo = (pr["point"] - pr["q20"]).clip(lower=0).fillna(0.0) + fig.add_trace( + go.Scatter( + x=pr["forecast_date"].tolist(), + y=pr["point"].tolist(), + mode="lines+markers", + line={"color": colors[name], "width": 1.4, "dash": "dot"}, + marker={"size": 8, "symbol": "diamond"}, + error_y={ + "type": "data", + "symmetric": False, + "array": err_hi.tolist(), + "arrayminus": err_lo.tolist(), + "color": colors[name], + "thickness": 1.4, + "width": 4, + }, + name=name, + showlegend=show_legend, + legendgroup=name, + ), + row=1, + col=col, + ) + + fig.add_vline( + x=origin.timestamp() * 1000, line={"color": "#aaaaaa", "dash": "dash", "width": 1}, row=1, col=col + ) + + fig.update_layout( + title={"text": "Eval Forecasts vs Reality — Median + 80% Interval by Origin", "font": {"size": 16}}, + template="plotly_white", + width=max(420 * len(origins), 720), + height=480, + margin={"t": 80, "b": 110, "l": 60, "r": 20}, + legend={"orientation": "h", "y": -0.18, "x": 0.0, "xanchor": "left", "font": {"size": 11}}, + ) + fig.update_xaxes(showgrid=True, gridcolor="#f0f0f0", tickfont={"size": 10}) + fig.update_yaxes(showgrid=True, gridcolor="#f0f0f0", tickfont={"size": 11}) + return fig + + +def render_rationales_html(rationale_df: pd.DataFrame, *, max_chars: int = 700) -> str: + """Render agent/LLM rationales as readable HTML cards with trace links. + + Expects ``analysis.extract_agent_rationales``. One card per (predictor, + origin), showing the overall rationale, the per-horizon note, and a link to + the Langfuse trace so the full agent reasoning is one click away. + """ + if rationale_df.empty: + return "

No agent/LLM rationales found in this run's metadata.

" + + def _clip(text: str) -> str: + text = (text or "").strip() + return text if len(text) <= max_chars else text[:max_chars].rsplit(" ", 1)[0] + " …" + + # One representative card per (predictor, origin) — the rationale is shared + # across horizons, so dedupe to the first row of each group. + seen: set[tuple[str, str]] = set() + cards: list[str] = [] + for _, r in rationale_df.sort_values(["predictor", "as_of"]).iterrows(): + key = (r["predictor"], str(pd.Timestamp(r["as_of"]).date())) + if key in seen: + continue + seen.add(key) + link = ( + f"🔗 Langfuse trace" + if r.get("trace_url") + else "" + ) + horizon_note = ( + f"
Horizon note: {_clip(r['horizon_rationale'])}
" + if r.get("horizon_rationale") + else "" + ) + cards.append( + f"
" + f"
" + f"{r['predictor']}" + f"{key[1]}   point ${r['point']:.1f}   {link}
" + f"
{_clip(r['rationale'])}
" + f"{horizon_note}
" + ) + return f"
{''.join(cards)}
" ``` diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md index 139e8d5a..cf88644e 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md @@ -141,7 +141,16 @@ commodity and input costs (grains, energy, fertiliser); supply-chain and weather disruptions; and the CAD exchange rate. Ground every claim in the search results you actually retrieve. When a cutoff -date is specified, never report or speculate about events after it.\ +date is specified, never report or speculate about events after it. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ """ diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md index 467eb3ca..2c9e2463 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md @@ -21,6 +21,12 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. +`search_web` runs an independent verifier on every result and returns +`[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as +pre-cutoff. Treat that as no verified news for the query — proceed on your +other signals and say so, never filling the gap from your own background +knowledge. + ## How to search - **Search before you forecast, not after.** Gather context first, then reason. diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md index 03cc28e0..9819c41d 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md @@ -155,7 +155,16 @@ the VIX and credit spreads; earnings-season tone; and major geopolitical or policy shocks. Ground every claim in the search results you actually retrieve. When a cutoff -date is specified, never report or speculate about events after it.\ +date is specified, never report or speculate about events after it. + +Before finalizing your summary, reason step by step: (1) for each candidate \ +fact, judge its actual recency from the substance of the result itself, \ +never from a source's claimed publish date or byline timestamp — those are \ +frequently stale or updated after original publication; (2) discard \ +anything you cannot confidently place before the cutoff date; (3) only then \ +write your summary. Do not supplement the search results with your own \ +background/training knowledge — if the results are insufficient, say so \ +explicitly rather than filling gaps from memory.\ """ diff --git a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md index dce71dea..359dd966 100644 --- a/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md +++ b/implementations/getting_started/concierge_agent/context/artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md @@ -21,6 +21,12 @@ Always pass `cutoff_date` equal to the `as_of` date in your payload. It is the temporal fence that keeps post-origin information out of a historical forecast. A forecast that "knew" what happened after `as_of` is not a forecast. +`search_web` runs an independent verifier on every result and returns +`[SEARCH_VERIFICATION_FAILED]` instead of content it couldn't confirm as +pre-cutoff. Treat that as no verified news for the query — proceed on your +other signals and say so, never filling the gap from your own background +knowledge. + ## How to search - **Search before you forecast, not after.** Gather context first, then reason. diff --git a/implementations/getting_started/concierge_agent/context/catalog.yaml b/implementations/getting_started/concierge_agent/context/catalog.yaml index 9a6f8110..21831953 100644 --- a/implementations/getting_started/concierge_agent/context/catalog.yaml +++ b/implementations/getting_started/concierge_agent/context/catalog.yaml @@ -1,9 +1,9 @@ source_url: https://github.com/VectorInstitute/agentic-forecasting -git_ref: 0ac6b3098bdb529c08f2445895d82490de2404fd +git_ref: 04507bf406cc5524c3d9523fc4835f29462fe3ea branch: main -built_at: '2026-06-30T15:09:29+00:00' -ingest_source: /home/coder/agentic-forecasting -entry_count: 196 +built_at: '2026-08-24T20:54:40+00:00' +ingest_source: /home/akore/vscodeprojects/agentic-forecasting +entry_count: 197 entries: - path: AGENTS.md kind: markdown @@ -22,7 +22,7 @@ entries: - Model selection - Code quality (not on commit) - Test philosophy - chars: 6702 + chars: 6845 artifact: artifacts/AGENTS.md.md - path: README.md kind: markdown @@ -31,6 +31,7 @@ entries: symbols: [] sections: - Agentic Forecasting + - Contents - What's here - Two ways to use a forecaster - Reference implementations @@ -46,7 +47,7 @@ entries: - Extending the foundation - Code quality - Documentation - chars: 14796 + chars: 14063 artifact: artifacts/README.md.md - path: aieng-forecasting/aieng/forecasting/__init__.py kind: python @@ -428,7 +429,7 @@ entries: - init_langfuse_tracing - print_langfuse_trace_url sections: [] - chars: 6632 + chars: 6087 artifact: artifacts/aieng-forecasting__aieng__forecasting__langfuse_tracing.py.md - path: aieng-forecasting/aieng/forecasting/methods/README.md kind: markdown @@ -497,7 +498,7 @@ entries: - AdkTextRunnerConfig - AdkTextRunner sections: [] - chars: 15228 + chars: 16225 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__adk_runner.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/agent_factory.py kind: python @@ -507,10 +508,11 @@ entries: - _LiteLLMNoiseFilter - ContextRetrievalConfig - CodeExecutionConfig + - _LeakageVerification - AgentConfig - build_adk_agent sections: [] - chars: 24986 + chars: 35772 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__agent_factory.py.md - path: aieng-forecasting/aieng/forecasting/methods/agentic/curriculum.py kind: python @@ -555,7 +557,7 @@ entries: - ForecastPromptBuilder - AgentPredictor sections: [] - chars: 14369 + chars: 14715 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__agentic__predictor.py.md - path: aieng-forecasting/aieng/forecasting/methods/baselines/__init__.py kind: python @@ -623,6 +625,8 @@ entries: symbols: - bootstrap_litellm - langfuse_observe + - _NoopGeneration + - langfuse_generation - current_trace_info - trace_url_for - set_current_trace_name @@ -631,7 +635,7 @@ entries: - sample_n_async - run_async sections: [] - chars: 18293 + chars: 21206 artifact: artifacts/aieng-forecasting__aieng__forecasting__methods__llm_processes___client.py.md - path: aieng-forecasting/aieng/forecasting/methods/llm_processes/base.py kind: python @@ -801,7 +805,7 @@ entries: - Directory layout - Relationship to `aieng-forecasting` - Adding a new use case - chars: 3778 + chars: 3921 artifact: artifacts/implementations__README.md.md - path: implementations/__init__.py kind: python @@ -925,7 +929,7 @@ entries: - build_boc_news_config - build_boc_agent_predictor sections: [] - chars: 16347 + chars: 17929 artifact: artifacts/implementations__boc_rate_decisions__analyst_agent__agent.py.md - path: implementations/boc_rate_decisions/data.py kind: python @@ -1199,7 +1203,7 @@ entries: - _StarterForecastPromptBuilder - build_starter_agent_predictor sections: [] - chars: 9637 + chars: 10211 artifact: artifacts/implementations__boc_rate_decisions__starter_agent__agent.py.md - path: implementations/boc_rate_decisions/starter_agent/skills/code-analysis-playbook/SKILL.md kind: markdown @@ -1238,7 +1242,7 @@ entries: - How to search - Domain focus (edit this for your use case) - Room to grow - chars: 1947 + chars: 2250 artifact: artifacts/implementations__boc_rate_decisions__starter_agent__skills__research-playbook__SKILL.md.md - path: implementations/energy_oil_forecasting/01_wti_case_study.ipynb kind: notebook @@ -1281,7 +1285,7 @@ entries: sections: - "WTI Crude Oil Price Forecasting \u2014 Stateless Methods: Systematic Backtest\ \ (Notebook 4 of 7)" - chars: 18735 + chars: 22648 artifact: artifacts/implementations__energy_oil_forecasting__04_systematic_backtest_eval.ipynb.md - path: implementations/energy_oil_forecasting/05_adaptive_agent_training.ipynb kind: notebook @@ -1321,7 +1325,7 @@ entries: symbols: [] sections: - "WTI Crude Oil \u2014 Your Starter Agent" - chars: 7573 + chars: 8586 artifact: artifacts/implementations__energy_oil_forecasting__99_starter_agent.ipynb.md - path: implementations/energy_oil_forecasting/README.md kind: markdown @@ -1374,7 +1378,7 @@ entries: - build_wti_adaptive_config - build_wti_adaptive_predictor sections: [] - chars: 19884 + chars: 20792 artifact: artifacts/implementations__energy_oil_forecasting__adaptive_agent__agent.py.md - path: implementations/energy_oil_forecasting/adaptive_agent/curriculum/snapshot_utils.py kind: python @@ -1564,14 +1568,28 @@ entries: - backtest_results_to_frame - trajectory_mae_table - select_top_predictors + - predictor_family + - predictions_to_frame + - per_horizon_crps + - leaderboard_with_uncertainty + - extract_agent_rationales + - eval_narrative_md + - build_price_frame - backtest_results_to_frame + - build_price_frame - compute_brier_score + - eval_narrative_md + - extract_agent_rationales + - leaderboard_with_uncertainty + - per_horizon_crps + - predictions_to_frame + - predictor_family - rolling_coverage_pct - score_backtest_results - select_top_predictors - trajectory_mae_table sections: [] - chars: 6984 + chars: 19850 artifact: artifacts/implementations__energy_oil_forecasting__analysis.py.md - path: implementations/energy_oil_forecasting/analyst_agent/__init__.py kind: python @@ -1603,7 +1621,7 @@ entries: - build_wti_tool_config - build_wti_agent_predictor sections: [] - chars: 22599 + chars: 27592 artifact: artifacts/implementations__energy_oil_forecasting__analyst_agent__agent.py.md - path: implementations/energy_oil_forecasting/analyst_agent/skills/statistical-analysis/SKILL.md kind: markdown @@ -1787,11 +1805,15 @@ entries: symbols: [] sections: - "Energy Oil Eval Spec \u2014 2026 Prospective Competition" - - '# Runs on 8 weekly origins from Feb 2, 2026 to Mar 23, 2026.' - - Covers the high-volatility Persian Gulf geopolitical price shock period. + - '# Runs on 18 weekly origins from Feb 2, 2026 to Jun 1, 2026.' + - Covers the high-volatility Persian Gulf geopolitical price shock and its + - aftermath. The end date is set to the latest origin whose longest horizon + - "(21 business days) still resolves against available data \u2014 keep it at" + - most 21 business days behind the most recent cached WTI price (see + - scripts/fetch_wti.py) so every origin fully resolves. - 'Target is WTI Crude Oil price (yfinance ticker: CL=F).' - 'Horizons: 5, 10, 21 business days.' - chars: 1019 + chars: 1316 artifact: artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval.yaml.md - path: implementations/energy_oil_forecasting/specs/energy_oil_eval_smoke.yaml kind: yaml @@ -1803,9 +1825,9 @@ entries: - '# Two-origin subset of energy_oil_eval.yaml for running the 2026 protected' - arena cheaply during development and end-to-end testing. - Use by setting SMOKE_TEST = True in the notebook setup cell. - - '# Origin count : 2 (vs. 8 in the full eval)' + - '# Origin count : 2 (vs. 18 in the full eval)' - 'Warmup : 250 trading days (~1 year) of historical prices' - chars: 1084 + chars: 1085 artifact: artifacts/implementations__energy_oil_forecasting__specs__energy_oil_eval_smoke.yaml.md - path: implementations/energy_oil_forecasting/specs/energy_oil_smoke.yaml kind: yaml @@ -1828,8 +1850,9 @@ entries: symbols: - build_starter_agent_config - build_starter_agent_predictor + - tools sections: [] - chars: 542 + chars: 688 artifact: artifacts/implementations__energy_oil_forecasting__starter_agent____init__.py.md - path: implementations/energy_oil_forecasting/starter_agent/agent.py kind: python @@ -1840,7 +1863,7 @@ entries: - _StarterForecastPromptBuilder - build_starter_agent_predictor sections: [] - chars: 9265 + chars: 9841 artifact: artifacts/implementations__energy_oil_forecasting__starter_agent__agent.py.md - path: implementations/energy_oil_forecasting/starter_agent/skills/code-analysis-playbook/SKILL.md kind: markdown @@ -1880,8 +1903,21 @@ entries: - How to search - Domain focus (edit this for your use case) - Room to grow - chars: 1830 + chars: 2133 artifact: artifacts/implementations__energy_oil_forecasting__starter_agent__skills__research-playbook__SKILL.md.md +- path: implementations/energy_oil_forecasting/starter_agent/tools.py + kind: python + domain: impl.energy_oil_forecasting + summary: "The starter agent's toolbelt \u2014 one factory per tool, composed in\ + \ the notebook." + symbols: + - ToolSpec + - news_search + - code_sandbox + - arima_forecast + sections: [] + chars: 9826 + artifact: artifacts/implementations__energy_oil_forecasting__starter_agent__tools.py.md - path: implementations/energy_oil_forecasting/tasks.py kind: python domain: impl.energy_oil_forecasting @@ -1923,8 +1959,13 @@ entries: - verdict_label - prob_bar - conf_bar + - predictor_colors + - make_crps_heatmap + - make_leaderboard_interval_chart + - make_eval_forecast_chart + - render_rationales_html sections: [] - chars: 43163 + chars: 54100 artifact: artifacts/implementations__energy_oil_forecasting__viz.py.md - path: implementations/food_price_forecasting/01_food_data_exploration.ipynb kind: notebook @@ -2202,7 +2243,7 @@ entries: - _StarterForecastPromptBuilder - build_starter_agent_predictor sections: [] - chars: 10665 + chars: 11239 artifact: artifacts/implementations__food_price_forecasting__starter_agent__agent.py.md - path: implementations/food_price_forecasting/starter_agent/skills/code-analysis-playbook/SKILL.md kind: markdown @@ -2242,7 +2283,7 @@ entries: - How to search - Domain focus (edit this for your use case) - Room to grow - chars: 1871 + chars: 2174 artifact: artifacts/implementations__food_price_forecasting__starter_agent__skills__research-playbook__SKILL.md.md - path: implementations/getting_started/00_environment_check.ipynb kind: notebook @@ -2645,7 +2686,7 @@ entries: - build_sp500_multivariate_service - sp500_logret_series_id sections: [] - chars: 31632 + chars: 32091 artifact: artifacts/implementations__sp500_forecasting__data.py.md - path: implementations/sp500_forecasting/leaderboard.py kind: python @@ -2799,7 +2840,7 @@ entries: - _StarterForecastPromptBuilder - build_starter_agent_predictor sections: [] - chars: 11869 + chars: 12443 artifact: artifacts/implementations__sp500_forecasting__starter_agent__agent.py.md - path: implementations/sp500_forecasting/starter_agent/skills/code-analysis-playbook/SKILL.md kind: markdown @@ -2839,7 +2880,7 @@ entries: - How to search - Domain focus (edit this for your use case) - Room to grow - chars: 1860 + chars: 2163 artifact: artifacts/implementations__sp500_forecasting__starter_agent__skills__research-playbook__SKILL.md.md - path: planning-docs/roadmap.md kind: markdown @@ -2901,12 +2942,13 @@ entries: - path: scripts/fetch_fred.py kind: python domain: scripts - summary: Populate the local FRED cache with series used by the CFPR experiment. + summary: Populate the local FRED cache with series used by the food-price and S&P + 500 experiments. symbols: - build_data_service - main sections: [] - chars: 6020 + chars: 8603 artifact: artifacts/scripts__fetch_fred.py.md - path: scripts/fetch_sp500_market.py kind: python diff --git a/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml b/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml index 14913f1b..a715b400 100644 --- a/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml +++ b/implementations/getting_started/concierge_agent/skills/repo-navigation/references/catalog-summary.yaml @@ -1,9 +1,9 @@ # Concierge catalog summary (regenerated by scripts/build_concierge_context.py) source_url: https://github.com/VectorInstitute/agentic-forecasting branch: main -built_at: '2026-06-30T15:09:29+00:00' -git_ref: 0ac6b3098bdb529c08f2445895d82490de2404fd -entry_count: 196 +built_at: '2026-08-24T20:54:40+00:00' +git_ref: 04507bf406cc5524c3d9523fc4835f29462fe3ea +entry_count: 197 domains: docs: 4 core.root: 3 @@ -14,7 +14,7 @@ domains: impl.README.md: 1 impl.__init__.py: 1 impl.boc_rate_decisions: 27 - impl.energy_oil_forecasting: 45 + impl.energy_oil_forecasting: 46 impl.food_price_forecasting: 21 impl.getting_started: 16 impl.sp500_forecasting: 19