diff --git a/.gitignore b/.gitignore index 15ac48d..6aabe20 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.DS_Store + # Byte-compiled / optimized / DLL files __pycache__/ *.py[codz] diff --git a/2026-04-time-series-forecasting-timesfm/README.md b/2026-04-time-series-forecasting-timesfm/README.md new file mode 100644 index 0000000..f9dee39 --- /dev/null +++ b/2026-04-time-series-forecasting-timesfm/README.md @@ -0,0 +1,115 @@ +# QuestDB → TimesFM: Time-Series Data Loading & Forecasting + +This tutorial demonstrates how to efficiently load time-series data from QuestDB into Python, then run forecasts using Google's TimesFM foundation model. + +**Focus**: Data pipeline patterns. The forecasting is just to show the complete workflow. + +## What's Inside + +| File | Purpose | +|------|---------| +| `steps/method1_direct_query.py` | Load data via REST API and ConnectorX | +| `steps/method2_parquet_export.py` | Export data to Parquet files | +| `steps/method3_parquet_partitions.py` | Read QuestDB partitions directly | +| `steps/forecast.py` | Run TimesFM forecasts on volume & volatility | + +## The Use Case + +We forecast two operationally useful metrics from crypto trade data: + +1. **Volume** – When will liquidity be available? (execution timing) +2. **Volatility** – How much is price moving? (risk management) + +We intentionally avoid price prediction—forecasting "where will BTC be in an hour" isn't credible. Volume and volatility forecasting, however, serve real needs. + +## Quick Start + +> **Requires Python 3.10+** + +```bash +# 1. Clone and setup +git clone https://github.com/questdb/blog-examples.git +cd blog-examples/2025-pytorch-questdb-timesfm + +# 2. Create environment +python3.11 -m venv .venv +source .venv/bin/activate + +# 3. Install base dependencies +pip install pandas pyarrow requests torch + +# 4. Install TimesFM from source (pip package is outdated) +git clone https://github.com/google-research/timesfm.git +cd timesfm +pip install -e ".[torch]" +cd .. + +# 5. Run the pipeline +python steps/method1_direct_query.py # Test QuestDB connection +python steps/method2_parquet_export.py # Export data to Parquet +python steps/forecast.py # Run forecasts +``` + +## Data Source + +Uses the `trades` table from QuestDB's public demo: `https://demo.questdb.io` + +Raw trades are aggregated into 1-minute OHLCV bars: + +```sql +SELECT + timestamp, + first(price) as open, + max(price) as high, + min(price) as low, + last(price) as close, + sum(amount) as volume, + count() as trade_count +FROM trades +WHERE symbol = 'BTC-USDT' +SAMPLE BY 1m +ALIGN TO CALENDAR +``` + +## Three Data Loading Methods + +| Method | Best For | How | +|--------|----------|-----| +| **REST API** | Exploration, small data | `requests.get("/exp?fmt=csv")` | +| **ConnectorX** | Large results (local QuestDB) | PostgreSQL wire protocol | +| **Parquet Export** | Reproducible pipelines | `requests.get("/exp?fmt=parquet")` | +| **Direct Partitions** | Terabyte-scale | Read QuestDB's Parquet files directly | + +> **Note**: ConnectorX requires PostgreSQL port access (8812). The public demo only exposes REST API, so ConnectorX works with local/self-hosted QuestDB only. + +## Project Structure + +``` +├── config/settings.py # Configuration (URLs, symbols, params) +├── src/ +│ ├── data_loader.py # Three data loading methods +│ └── forecaster.py # TimesFM wrapper +├── steps/ +│ ├── method1_direct_query.py +│ ├── method2_parquet_export.py +│ ├── method3_parquet_partitions.py +│ └── forecast.py +├── dataset/ # Exported Parquet files +├── outputs/ # Forecast results (CSV) +└── BLOG_POST.md # Full tutorial writeup +``` + +## Notes + +- **Python 3.10+ required** – TimesFM doesn't support older versions +- **TimesFM must be installed from source** – The PyPI package is outdated +- **Model downloads ~900MB on first run** – Cached in `~/.cache/huggingface/` +- **GPU optional** – CPU inference works fine, just slower + +## Honest Caveats + +1. **Foundation models aren't magic.** TimesFM gives a quick baseline, but tuned domain-specific models may perform better. + +2. **Volume is hard to forecast.** It's driven by news, large orders, and sentiment—all unpredictable. Wide confidence intervals are expected. + +3. **This is a data pipeline tutorial.** Whether these forecasts are useful for your application requires proper backtesting. diff --git a/2026-04-time-series-forecasting-timesfm/config/__init__.py b/2026-04-time-series-forecasting-timesfm/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/2026-04-time-series-forecasting-timesfm/config/settings.py b/2026-04-time-series-forecasting-timesfm/config/settings.py new file mode 100644 index 0000000..a3a435d --- /dev/null +++ b/2026-04-time-series-forecasting-timesfm/config/settings.py @@ -0,0 +1,26 @@ +from pathlib import Path + +# QuestDB connection settings +# REST API - points to public demo instance (works out of the box) +QUESTDB_HTTP = "https://demo.questdb.io" + +# PostgreSQL wire protocol - points to local QuestDB (for ConnectorX) +# The demo instance doesn't expose port 8812 externally, so this requires local QuestDB +QUESTDB_PG = "postgresql://admin:quest@localhost:8812/qdb" + +# For local QuestDB with Parquet partitions (Method 3) +QUESTDB_DATA_DIR = Path("/var/lib/questdb/db") + +# Paths +PROJECT_ROOT = Path(__file__).parent.parent +DATASET_DIR = PROJECT_ROOT / "dataset" +OUTPUT_DIR = PROJECT_ROOT / "outputs" + +# Data settings +TABLE_NAME = "trades" +SYMBOL = "BTC-USDT" + +# TimesFM settings +TIMESFM_MODEL = "google/timesfm-2.5-200m-pytorch" +MAX_CONTEXT = 512 # How much history to use +FORECAST_HORIZON = 60 # How far ahead to forecast (in minutes) diff --git a/2026-04-time-series-forecasting-timesfm/dataset/.gitkeep b/2026-04-time-series-forecasting-timesfm/dataset/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/2026-04-time-series-forecasting-timesfm/outputs/.gitkeep b/2026-04-time-series-forecasting-timesfm/outputs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/2026-04-time-series-forecasting-timesfm/pyproject.toml b/2026-04-time-series-forecasting-timesfm/pyproject.toml new file mode 100644 index 0000000..0cb7dbb --- /dev/null +++ b/2026-04-time-series-forecasting-timesfm/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "pytorch-questdb-timesfm" +version = "0.1.0" +description = "Tutorial: QuestDB time-series data with Google's TimesFM foundation model" +requires-python = ">=3.10" +dependencies = [ + "pandas>=2.0.0", + "pyarrow>=15.0.0", + "requests>=2.31.0", + "torch>=2.0.0", +] + +[project.optional-dependencies] +# NOTE: TimesFM must be installed from source, not pip +# git clone https://github.com/google-research/timesfm.git +# cd timesfm && pip install -e ".[torch]" +fast = [ + "connectorx>=0.3.0", # Faster queries via PostgreSQL protocol +] + +[tool.poe.tasks] +method1 = { cmd = "python steps/method1_direct_query.py", help = "Demo: Direct SQL queries" } +method2 = { cmd = "python steps/method2_parquet_export.py", help = "Demo: Parquet export via API" } +method3 = { cmd = "python steps/method3_parquet_partitions.py", help = "Demo: Direct partition access" } +forecast = { cmd = "python steps/forecast.py", help = "Run TimesFM forecasting" } +all = ["method1", "method2", "forecast"] diff --git a/2026-04-time-series-forecasting-timesfm/requirements.txt b/2026-04-time-series-forecasting-timesfm/requirements.txt new file mode 100644 index 0000000..2169c21 --- /dev/null +++ b/2026-04-time-series-forecasting-timesfm/requirements.txt @@ -0,0 +1,10 @@ +# Core dependencies +pandas>=2.0.0 +pyarrow>=15.0.0 +requests>=2.31.0 +torch>=2.0.0 +connectorx>=0.3.0 + +# TimesFM must be installed separately from source: +# git clone https://github.com/google-research/timesfm.git +# cd timesfm && pip install -e ".[torch]" diff --git a/2026-04-time-series-forecasting-timesfm/src/__init__.py b/2026-04-time-series-forecasting-timesfm/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/2026-04-time-series-forecasting-timesfm/src/data_loader.py b/2026-04-time-series-forecasting-timesfm/src/data_loader.py new file mode 100644 index 0000000..2a5e1fd --- /dev/null +++ b/2026-04-time-series-forecasting-timesfm/src/data_loader.py @@ -0,0 +1,165 @@ +""" +Three methods to load data from QuestDB into Python. +""" +import pandas as pd +import requests +from io import StringIO +from pathlib import Path + +from config.settings import QUESTDB_HTTP, QUESTDB_PG + + +# ============================================================================= +# Method 1: Direct Queries +# ============================================================================= + +def query_rest_api(query: str) -> pd.DataFrame: + """ + Query QuestDB via REST API, return DataFrame. + + Simple and works everywhere. Best for exploration and smaller datasets. + """ + response = requests.get( + f"{QUESTDB_HTTP}/exp", + params={"query": query, "fmt": "csv"} + ) + response.raise_for_status() + return pd.read_csv(StringIO(response.text)) + + +def query_connectorx(query: str) -> pd.DataFrame: + """ + Query QuestDB via PostgreSQL wire protocol using ConnectorX. + + Faster than REST for larger results. Requires: pip install connectorx + + Note: Uses redshift:// driver because QuestDB's PG wire protocol is + compatible with Redshift's simpler dialect. + """ + import connectorx as cx + # Replace postgresql:// with redshift:// for QuestDB compatibility + conn = QUESTDB_PG.replace("postgresql://", "redshift://") + return cx.read_sql(conn, query) + + +# ============================================================================= +# Method 2: Parquet Export via REST API +# ============================================================================= + +def export_to_parquet(query: str, output_path: Path) -> Path: + """ + Export query results to Parquet file via QuestDB REST API. + + Falls back to CSV if Parquet export is disabled on the server. + Best for reproducible datasets. + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Try native Parquet export + response = requests.get( + f"{QUESTDB_HTTP}/exp", + params={"query": query, "fmt": "parquet"}, + stream=True + ) + + content_type = response.headers.get("content-type", "") + + if response.ok and "parquet" in content_type: + with open(output_path, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + print(f"✓ Exported native Parquet: {output_path}") + else: + # Fallback: CSV export + pandas conversion + print(" Parquet export not available, using CSV fallback...") + csv_response = requests.get( + f"{QUESTDB_HTTP}/exp", + params={"query": query, "fmt": "csv"} + ) + csv_response.raise_for_status() + df = pd.read_csv(StringIO(csv_response.text)) + df.to_parquet(output_path, index=False) + print(f"✓ Exported via CSV→Parquet: {output_path}") + + return output_path + + +# ============================================================================= +# Method 3: Direct Parquet Partition Access +# ============================================================================= + +def load_parquet_partitions(table_name: str, data_dir: Path): + """ + Load QuestDB Parquet partitions directly via PyArrow. + + Requires: + - Local/mounted access to QuestDB data directory + - Parquet partition format enabled (o3PartitionFormat = 'PARQUET') + + Best for terabyte-scale data and streaming processing. + """ + import pyarrow.dataset as ds + + table_dir = data_dir / table_name + parquet_files = list(table_dir.glob("**/*.parquet")) + + if not parquet_files: + raise FileNotFoundError( + f"No Parquet files found in {table_dir}. " + "Ensure Parquet partition format is enabled in QuestDB." + ) + + print(f"✓ Found {len(parquet_files)} partition files") + return ds.dataset(table_dir, format="parquet") + + +def stream_from_partitions(dataset, columns: list[str], batch_size: int = 100_000): + """ + Stream data from Parquet partitions in batches. + + Memory-efficient for large datasets. + """ + scanner = dataset.scanner(columns=columns, batch_size=batch_size) + for batch in scanner.to_batches(): + yield batch.to_pandas() + + +# ============================================================================= +# Query builders +# ============================================================================= + +def get_ohlcv_query(symbol: str = "BTC-USDT", days: int = 7, interval: str = "1m") -> str: + """ + Build query for OHLCV data aggregated at specified interval. + + Returns: timestamp, open, high, low, close, volume + """ + return f""" + SELECT + timestamp, + first(price) as open, + max(price) as high, + min(price) as low, + last(price) as close, + sum(amount) as volume, + count() as trade_count + FROM trades + WHERE symbol = '{symbol}' + AND timestamp > dateadd('d', -{days}, now()) + SAMPLE BY {interval} + ALIGN TO CALENDAR + """ + + +def get_volume_series_query(symbol: str = "BTC-USDT", days: int = 7) -> str: + """Build query for volume time series (1-minute intervals).""" + return f""" + SELECT + timestamp, + sum(amount) as volume + FROM trades + WHERE symbol = '{symbol}' + AND timestamp > dateadd('d', -{days}, now()) + SAMPLE BY 1m + ALIGN TO CALENDAR + """ diff --git a/2026-04-time-series-forecasting-timesfm/src/forecaster.py b/2026-04-time-series-forecasting-timesfm/src/forecaster.py new file mode 100644 index 0000000..66371f4 --- /dev/null +++ b/2026-04-time-series-forecasting-timesfm/src/forecaster.py @@ -0,0 +1,222 @@ +""" +TimesFM wrapper for easy forecasting. + +TimesFM is a foundation model for time-series forecasting developed by Google Research. +It provides zero-shot forecasting - no training needed, just pass your data. + +Paper: https://arxiv.org/abs/2310.10688 +Model: https://huggingface.co/google/timesfm-2.5-200m-pytorch +""" +import os +import logging +import warnings +import numpy as np +import pandas as pd +from typing import Optional +from dataclasses import dataclass + +# Suppress noisy HuggingFace Hub warnings before importing anything else +os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1" +os.environ["HF_HUB_VERBOSITY"] = "error" +logging.getLogger("huggingface_hub").setLevel(logging.ERROR) +logging.getLogger("huggingface_hub.utils._http").setLevel(logging.ERROR) +warnings.filterwarnings("ignore", message=".*unauthenticated.*") +warnings.filterwarnings("ignore", message=".*HF Hub.*") + + +@dataclass +class ForecastResult: + """Container for forecast results.""" + timestamps: pd.DatetimeIndex + point_forecast: np.ndarray + quantile_forecast: Optional[np.ndarray] # Shape: (horizon, num_quantiles) + quantile_levels: Optional[list[float]] # e.g., [0.1, 0.2, ..., 0.9] + + def to_dataframe(self) -> pd.DataFrame: + """Convert to DataFrame with point forecast and quantile columns.""" + df = pd.DataFrame({ + "timestamp": self.timestamps, + "forecast": self.point_forecast, + }) + + if self.quantile_forecast is not None and self.quantile_levels is not None: + for i, q in enumerate(self.quantile_levels): + df[f"q{int(q*100):02d}"] = self.quantile_forecast[:, i] + + return df + + +class TimesFMForecaster: + """ + Wrapper around Google's TimesFM model for easy forecasting. + + Usage: + forecaster = TimesFMForecaster() + forecaster.load_model() + + result = forecaster.forecast( + series=df["volume"].values, + timestamps=df["timestamp"], + horizon=60 + ) + """ + + def __init__( + self, + model_name: str = "google/timesfm-2.5-200m-pytorch", + max_context: int = 512, + max_horizon: int = 256, + use_quantiles: bool = True, + ): + self.model_name = model_name + self.max_context = max_context + self.max_horizon = max_horizon + self.use_quantiles = use_quantiles + self.model = None + + def load_model(self): + """Load the TimesFM model from HuggingFace.""" + import warnings + import logging + import torch + + # Suppress HuggingFace warnings aggressively + warnings.filterwarnings("ignore", message=".*unauthenticated.*") + warnings.filterwarnings("ignore", message=".*HF_TOKEN.*") + logging.getLogger("huggingface_hub").setLevel(logging.ERROR) + logging.getLogger("huggingface_hub.utils._http").setLevel(logging.ERROR) + + import timesfm + + print(f"Loading TimesFM model: {self.model_name}") + torch.set_float32_matmul_precision("high") + + # Handle different TimesFM versions + if hasattr(timesfm, 'TimesFM_2p5_200M_torch'): + # Source install (recommended) + self.model = timesfm.TimesFM_2p5_200M_torch.from_pretrained(self.model_name) + elif hasattr(timesfm, 'TimesFm'): + # Older API + self.model = timesfm.TimesFm( + hparams=timesfm.TimesFmHparams( + backend="gpu" if torch.cuda.is_available() else "cpu", + per_core_batch_size=32, + horizon_len=self.max_horizon, + context_len=self.max_context, + ), + checkpoint=timesfm.TimesFmCheckpoint( + huggingface_repo_id=self.model_name + ), + ) + print("✓ Model loaded (legacy API)") + return + else: + raise RuntimeError( + "TimesFM not properly installed. Please install from source:\n" + " git clone https://github.com/google-research/timesfm.git\n" + " cd timesfm && pip install -e '.[torch]'" + ) + + self.model.compile( + timesfm.ForecastConfig( + max_context=self.max_context, + max_horizon=self.max_horizon, + normalize_inputs=True, + use_continuous_quantile_head=self.use_quantiles, + force_flip_invariance=True, + infer_is_positive=True, + fix_quantile_crossing=True, + ) + ) + print("✓ Model loaded and compiled") + + def forecast( + self, + series: np.ndarray, + timestamps: pd.DatetimeIndex, + horizon: int, + freq: str = "1min", + ) -> ForecastResult: + """ + Generate forecasts for a time series. + + Args: + series: 1D numpy array of historical values + timestamps: DatetimeIndex corresponding to series + horizon: Number of steps to forecast + freq: Frequency for generating future timestamps + + Returns: + ForecastResult with point and quantile forecasts + """ + if self.model is None: + self.load_model() + + # Ensure we don't exceed max context + if len(series) > self.max_context: + series = series[-self.max_context:] + timestamps = timestamps[-self.max_context:] + + # Run forecast + point_forecast, quantile_forecast = self.model.forecast( + horizon=horizon, + inputs=[series.astype(np.float32)], + ) + + # Generate future timestamps + last_ts = timestamps[-1] + future_timestamps = pd.date_range( + start=last_ts + pd.Timedelta(freq), + periods=horizon, + freq=freq + ) + + # Extract results (model returns batch dimension) + point = point_forecast[0] # Shape: (horizon,) + + if quantile_forecast is not None: + # Shape: (1, horizon, num_quantiles) -> (horizon, num_quantiles) + quantiles = quantile_forecast[0] + # TimesFM 2.5 returns: mean, then 10th to 90th percentiles + quantile_levels = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] + else: + quantiles = None + quantile_levels = None + + return ForecastResult( + timestamps=future_timestamps, + point_forecast=point, + quantile_forecast=quantiles, + quantile_levels=quantile_levels, + ) + + def forecast_multiple( + self, + series_list: list[np.ndarray], + horizon: int, + ) -> list[np.ndarray]: + """ + Forecast multiple time series efficiently in a single batch. + + Args: + series_list: List of 1D numpy arrays + horizon: Number of steps to forecast + + Returns: + List of point forecasts (one per input series) + """ + if self.model is None: + self.load_model() + + # Truncate each series to max_context + inputs = [ + s[-self.max_context:].astype(np.float32) + for s in series_list + ] + + point_forecasts, _ = self.model.forecast( + horizon=horizon, + inputs=inputs, + ) + + return [point_forecasts[i] for i in range(len(inputs))] diff --git a/2026-04-time-series-forecasting-timesfm/steps/__init__.py b/2026-04-time-series-forecasting-timesfm/steps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/2026-04-time-series-forecasting-timesfm/steps/forecast.py b/2026-04-time-series-forecasting-timesfm/steps/forecast.py new file mode 100644 index 0000000..4f2bc85 --- /dev/null +++ b/2026-04-time-series-forecasting-timesfm/steps/forecast.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +""" +Zero-Shot Forecasting with TimesFM + +This script demonstrates using Google's TimesFM foundation model +for time-series forecasting on QuestDB data. + +We forecast two operationally useful metrics: +1. Volume - When will liquidity be available? Useful for execution timing. +2. Volatility - How much is price moving? Useful for risk management. + +Note: We intentionally avoid price prediction. Forecasting "where will BTC +be in an hour" is essentially claiming to beat the market—not credible. +Volume and volatility forecasting, however, serve real operational needs. +""" +# Suppress HuggingFace warnings before any imports +import os +import logging +import warnings +os.environ["HF_HUB_VERBOSITY"] = "error" +logging.getLogger("huggingface_hub").setLevel(logging.ERROR) +warnings.filterwarnings("ignore", message=".*unauthenticated.*") +warnings.filterwarnings("ignore", message=".*HF_TOKEN.*") + +import sys +sys.path.insert(0, ".") + +import pandas as pd +import numpy as np +from pathlib import Path + +from src.data_loader import query_rest_api, get_ohlcv_query +from src.forecaster import TimesFMForecaster +from config.settings import ( + SYMBOL, DATASET_DIR, OUTPUT_DIR, + MAX_CONTEXT, FORECAST_HORIZON +) + + +def load_data() -> pd.DataFrame: + """ + Load OHLCV data from Parquet (preferred) or live query (fallback). + + Returns DataFrame with columns: timestamp, open, high, low, close, volume, trade_count + """ + parquet_path = DATASET_DIR / "btc_ohlcv_7d.parquet" + + if parquet_path.exists(): + print(f"Loading from Parquet: {parquet_path}") + df = pd.read_parquet(parquet_path) + else: + print(f"Parquet not found, querying live data from QuestDB...") + print(f" (Run method2_parquet_export.py first for reproducible results)") + query = get_ohlcv_query(symbol=SYMBOL, days=7) + df = query_rest_api(query) + + df["timestamp"] = pd.to_datetime(df["timestamp"]) + df = df.sort_values("timestamp").reset_index(drop=True) + + return df + + +def compute_volatility(df: pd.DataFrame) -> np.ndarray: + """ + Compute per-minute volatility as the high-low range. + + This is a simple measure of intra-minute price movement. + Higher values = more volatile = higher risk/reward. + """ + return (df["high"] - df["low"]).values + + +def main(): + print("=" * 70) + print("Zero-Shot Forecasting with TimesFM") + print("=" * 70) + + # --- 1. Load Data --- + print("\n1. LOADING DATA") + print("-" * 70) + df = load_data() + print(f" Rows loaded: {len(df):,}") + print(f" Time range: {df['timestamp'].min()} to {df['timestamp'].max()}") + print(f" Columns: {', '.join(df.columns)}") + + # --- 2. Prepare Series --- + print("\n2. PREPARING TIME SERIES") + print("-" * 70) + + timestamps = pd.DatetimeIndex(df["timestamp"]) + + # Volume: total amount traded per minute + volume_series = df["volume"].values + print(f" Volume series: {len(volume_series)} points") + print(f" Recent values: {volume_series[-5:].round(3)}") + print(f" Mean: {volume_series.mean():.3f}") + + # Volatility: high-low range per minute + volatility_series = compute_volatility(df) + print(f" Volatility series: {len(volatility_series)} points") + print(f" Recent values: {volatility_series[-5:].round(2)}") + print(f" Mean: ${volatility_series.mean():.2f}") + + print(f"\n Using last {MAX_CONTEXT} points as context for forecasting") + print(f" Forecasting {FORECAST_HORIZON} minutes ahead") + + # --- 3. Load Model --- + print("\n3. LOADING TIMESFM MODEL") + print("-" * 70) + print(" TimesFM is a foundation model trained on 100B+ time points.") + print(" It provides zero-shot forecasting—no training on your data needed.") + print() + + forecaster = TimesFMForecaster( + max_context=MAX_CONTEXT, + max_horizon=FORECAST_HORIZON, + use_quantiles=True, + ) + forecaster.load_model() + + # --- 4. Generate Forecasts --- + print("\n4. GENERATING FORECASTS") + print("-" * 70) + + print(" Forecasting volume (liquidity timing)...") + volume_result = forecaster.forecast( + series=volume_series, + timestamps=timestamps, + horizon=FORECAST_HORIZON, + freq="1min", + ) + + print(" Forecasting volatility (risk assessment)...") + volatility_result = forecaster.forecast( + series=volatility_series, + timestamps=timestamps, + horizon=FORECAST_HORIZON, + freq="1min", + ) + + # --- 5. Display Results --- + print("\n" + "=" * 70) + print("FORECAST RESULTS") + print("=" * 70) + + # Volume forecast + print("\n📊 VOLUME FORECAST (next 60 minutes)") + print(" Use case: Execution timing—when will liquidity be available?") + print() + volume_df = volume_result.to_dataframe() + # Show subset of columns for readability + display_cols = ["timestamp", "forecast", "q10", "q50", "q90"] + print(volume_df[display_cols].head(10).to_string(index=False)) + print(" ...") + print(volume_df[display_cols].tail(5).to_string(index=False)) + + # Volatility forecast + print("\n📈 VOLATILITY FORECAST (next 60 minutes)") + print(" Use case: Risk management—how much is price likely to move?") + print() + volatility_df = volatility_result.to_dataframe() + print(volatility_df[display_cols].head(10).to_string(index=False)) + print(" ...") + print(volatility_df[display_cols].tail(5).to_string(index=False)) + + # --- 6. Save Results --- + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + + volume_output = OUTPUT_DIR / "volume_forecast.csv" + volatility_output = OUTPUT_DIR / "volatility_forecast.csv" + + volume_df.to_csv(volume_output, index=False) + volatility_df.to_csv(volatility_output, index=False) + + print(f"\n✓ Saved forecasts to:") + print(f" - {volume_output}") + print(f" - {volatility_output}") + + # --- 7. Summary Statistics --- + print("\n" + "=" * 70) + print("SUMMARY") + print("=" * 70) + + # Volume summary + last_volume = volume_series[-1] + forecast_volume = volume_result.point_forecast + + print(f"\n📊 Volume:") + print(f" Last observed: {last_volume:.3f}") + print(f" Forecast mean: {forecast_volume.mean():.3f}") + print(f" Forecast range: {forecast_volume.min():.3f} – {forecast_volume.max():.3f}") + if volume_result.quantile_forecast is not None: + q10_mean = volume_result.quantile_forecast[:, 0].mean() + q90_mean = volume_result.quantile_forecast[:, -1].mean() + print(f" 80% confidence band: {q10_mean:.3f} – {q90_mean:.3f}") + + # Volatility summary + last_volatility = volatility_series[-1] + forecast_volatility = volatility_result.point_forecast + + print(f"\n📈 Volatility (price range per minute):") + print(f" Last observed: ${last_volatility:.2f}") + print(f" Forecast mean: ${forecast_volatility.mean():.2f}") + print(f" Forecast range: ${forecast_volatility.min():.2f} – ${forecast_volatility.max():.2f}") + if volatility_result.quantile_forecast is not None: + q10_mean = volatility_result.quantile_forecast[:, 0].mean() + q90_mean = volatility_result.quantile_forecast[:, -1].mean() + print(f" 80% confidence band: ${q10_mean:.2f} – ${q90_mean:.2f}") + + # Interpretation + print("\n" + "-" * 70) + print("INTERPRETATION") + print("-" * 70) + print(""" + • Wide confidence intervals are expected—volume and volatility are + inherently noisy and driven by unpredictable factors (news, large + orders, market sentiment). + + • These forecasts provide a baseline. For production use, you'd want + to backtest against historical data and compare with simpler methods + like moving averages or ARIMA. + + • The value of foundation models like TimesFM is speed: you get a + reasonable forecast immediately, without training or tuning. + """) + + print("=" * 70) + print("Done!") + print("=" * 70) + + +if __name__ == "__main__": + main() diff --git a/2026-04-time-series-forecasting-timesfm/steps/method1_direct_query.py b/2026-04-time-series-forecasting-timesfm/steps/method1_direct_query.py new file mode 100644 index 0000000..30ff4f3 --- /dev/null +++ b/2026-04-time-series-forecasting-timesfm/steps/method1_direct_query.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +""" +Method 1: Direct SQL Queries + +Demonstrates querying QuestDB directly via REST API and ConnectorX. +Best for exploration and smaller datasets. +""" +import sys +sys.path.insert(0, ".") + +from src.data_loader import query_rest_api, query_connectorx, get_ohlcv_query +from config.settings import SYMBOL, QUESTDB_HTTP, QUESTDB_PG + + +def main(): + query = get_ohlcv_query(symbol=SYMBOL, days=1) + + print("=" * 60) + print("METHOD 1: Direct SQL Queries") + print("=" * 60) + print("\nConnection settings (from config/settings.py):") + print(f" REST API: {QUESTDB_HTTP}") + print(f" ConnectorX: {QUESTDB_PG}") + + # Method 1a: REST API + print("\n" + "-" * 60) + print("1a. REST API") + print("-" * 60) + print(f"Connecting to: {QUESTDB_HTTP}") + print(f"Query: {SYMBOL} OHLCV, 1-minute bars, last 24 hours") + + df = query_rest_api(query) + print(f"\n✓ Loaded {len(df):,} rows") + print(f" Columns: {list(df.columns)}") + print(f"\nFirst 5 rows:") + print(df.head().to_string(index=False)) + print(f"\nLast 5 rows:") + print(df.tail().to_string(index=False)) + + # Method 1b: ConnectorX + print("\n" + "-" * 60) + print("1b. ConnectorX (PostgreSQL wire protocol)") + print("-" * 60) + print(f"Connecting to: {QUESTDB_PG}") + print(f"Query: {SYMBOL} OHLCV, 1-minute bars, last 24 hours") + print() + + try: + df_cx = query_connectorx(query) + print(f"✓ Loaded {len(df_cx):,} rows") + print(f"\nFirst 5 rows:") + print(df_cx.head().to_string(index=False)) + except ImportError: + print("⚠ ConnectorX not installed. Run: pip install connectorx") + except Exception as e: + print(f"⚠ ConnectorX failed: {e}") + print("\n This is expected if you don't have local QuestDB running.") + print(" ConnectorX requires PostgreSQL wire protocol access (port 8812),") + print(" which the public demo instance doesn't expose.") + + # Summary + print("\n" + "=" * 60) + print("SUMMARY") + print("=" * 60) + print(""" +REST API (demo.questdb.io) + - Works out of the box, no local setup needed + - Good for exploration and moderate datasets + +ConnectorX (localhost:8812) + - Requires local QuestDB with PostgreSQL wire protocol + - ~5x faster for large result sets (100K+ rows) + - Edit config/settings.py to change connection URLs + +Both methods return pandas DataFrames ready for analysis or ML. +""") + + return df + + +if __name__ == "__main__": + main() diff --git a/2026-04-time-series-forecasting-timesfm/steps/method2_parquet_export.py b/2026-04-time-series-forecasting-timesfm/steps/method2_parquet_export.py new file mode 100644 index 0000000..1e2fe56 --- /dev/null +++ b/2026-04-time-series-forecasting-timesfm/steps/method2_parquet_export.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +""" +Method 2: Parquet Export via REST API + +Demonstrates exporting query results to Parquet files. +Best for reproducible datasets and larger exports. +""" +import sys +sys.path.insert(0, ".") + +import pyarrow.parquet as pq + +from src.data_loader import export_to_parquet, get_ohlcv_query +from config.settings import SYMBOL, DATASET_DIR + + +def main(): + print("=" * 60) + print("METHOD 2: Parquet Export via REST API") + print("=" * 60) + + # Export OHLCV data + query = get_ohlcv_query(symbol=SYMBOL, days=7) + output_path = DATASET_DIR / "btc_ohlcv_7d.parquet" + + print(f"\nExporting 7 days of {SYMBOL} OHLCV data...") + print(f"Output: {output_path}") + + export_to_parquet(query, output_path) + + # Verify the export + print("\n" + "-" * 60) + print("Verifying exported file...") + + table = pq.read_table(output_path) + print(f" Rows: {table.num_rows:,}") + print(f" Size: {output_path.stat().st_size / 1024:.1f} KB") + print(f" Schema:") + for field in table.schema: + print(f" - {field.name}: {field.type}") + + # Show sample + df = table.to_pandas() + print(f"\nFirst 5 rows:") + print(df.head().to_string(index=False)) + + print("\n" + "=" * 60) + print("Summary:") + print(" - Parquet preserves types and compresses well") + print(" - File can be versioned and shared") + print(f" - Ready for forecasting: {output_path}") + print("=" * 60) + + return output_path + + +if __name__ == "__main__": + main() diff --git a/2026-04-time-series-forecasting-timesfm/steps/method3_parquet_partitions.py b/2026-04-time-series-forecasting-timesfm/steps/method3_parquet_partitions.py new file mode 100644 index 0000000..9e96d48 --- /dev/null +++ b/2026-04-time-series-forecasting-timesfm/steps/method3_parquet_partitions.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +""" +Method 3: Direct Parquet Partition Access + +Demonstrates reading QuestDB's Parquet partitions directly. +Best for terabyte-scale data and streaming processing. + +NOTE: This requires: + 1. Local or mounted access to QuestDB's data directory + 2. Parquet partition format enabled in QuestDB config + +If you're using the remote demo instance, this won't work—use Method 1 or 2. +""" +import sys +sys.path.insert(0, ".") + +from config.settings import QUESTDB_DATA_DIR, TABLE_NAME + + +def main(): + print("=" * 60) + print("METHOD 3: Direct Parquet Partition Access") + print("=" * 60) + + print(f"\nLooking for partitions in: {QUESTDB_DATA_DIR / TABLE_NAME}") + + table_dir = QUESTDB_DATA_DIR / TABLE_NAME + + if not table_dir.exists(): + print(f""" + ⚠ Directory not found: {table_dir} + + This method requires local QuestDB access with Parquet partitions. + + To enable Parquet partitions in QuestDB: + + 1. In server.conf, set: + cairo.o3.partition.format=PARQUET + + 2. Or convert existing table partitions: + ALTER TABLE {TABLE_NAME} SET PARAM o3PartitionFormat = 'PARQUET'; + + For the remote demo instance, use Method 1 or Method 2 instead. + """) + return + + try: + import pyarrow.dataset as ds + + parquet_files = list(table_dir.glob("**/*.parquet")) + + if not parquet_files: + print(f" ⚠ No .parquet files found in {table_dir}") + return + + print(f" ✓ Found {len(parquet_files)} partition files") + + # Load as dataset + dataset = ds.dataset(table_dir, format="parquet") + print(f" ✓ Schema:") + for field in dataset.schema: + print(f" - {field.name}: {field.type}") + + # Demo streaming + print("\n Streaming first 3 batches...") + scanner = dataset.scanner(batch_size=10000) + + for i, batch in enumerate(scanner.to_batches()): + if i >= 3: + break + print(f" Batch {i}: {batch.num_rows} rows") + + print("\n" + "=" * 60) + print("Summary:") + print(" - Direct partition access bypasses query engine") + print(" - PyArrow dataset API enables efficient streaming") + print(" - Best for large-scale batch processing") + print("=" * 60) + + except Exception as e: + print(f" ⚠ Error: {e}") + + +if __name__ == "__main__": + main()