From 58fab67efed4e60521ad455e438894c6e87ca99b Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sun, 11 Oct 2026 17:49:15 -0700 Subject: [PATCH 1/4] Freeze current-time functions into literals when compiling DuckDB SQL CURRENT_DATE, now() and CURRENT_TIMESTAMP are replaced with literals from a single instant per query, so every reference in one query sees the same time and engines that cannot fold these functions before pruning still skip files. On by default for DuckDB only, since other dialects use a server timezone sidemantic cannot see. The cached SQL stays unfrozen and is frozen per call. --- sidemantic/core/semantic_layer.py | 32 +++- sidemantic/core/transport_security.py | 8 + sidemantic/sql/frozen_time.py | 266 ++++++++++++++++++++++++++ tests/dates/test_frozen_time.py | 222 +++++++++++++++++++++ tests/dates/test_integration.py | 43 ++++- tests/metrics/test_filters.py | 13 +- 6 files changed, 572 insertions(+), 12 deletions(-) create mode 100644 sidemantic/sql/frozen_time.py create mode 100644 tests/dates/test_frozen_time.py diff --git a/sidemantic/core/semantic_layer.py b/sidemantic/core/semantic_layer.py index 68fb9aa80..89c5f9108 100644 --- a/sidemantic/core/semantic_layer.py +++ b/sidemantic/core/semantic_layer.py @@ -5,6 +5,7 @@ import os from collections.abc import Callable from contextvars import ContextVar, Token +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -87,6 +88,7 @@ def __init__( max_limit: int | None = None, allow_non_additive_unsafe: bool = False, enforce_visibility: bool = False, + freeze_current_time: bool | None = None, ): """Initialize semantic layer. @@ -135,6 +137,14 @@ def __init__( enforce_visibility: When True, requesting a dimension/metric whose ``public=False`` raises during compile, and catalog/introspection listings omit non-public fields. Defaults to False so library users are unaffected. + freeze_current_time: Replace ``CURRENT_DATE``/``today()``/``now()``/ + ``CURRENT_TIMESTAMP`` in compiled query SQL with literals taken from one + compile-time instant (``self.clock``) so storage planners such as DuckDB's + Iceberg scan can prune on time filters. ``None`` (default) enables it only + for the DuckDB dialect, where ``CURRENT_DATE`` uses the process's local + timezone like DuckDB's default ``TimeZone`` setting; remote engines evaluate + ``CURRENT_DATE`` in a server/session timezone the client cannot know. + See ``sidemantic.sql.frozen_time`` for the exact rewrite rules. """ from sidemantic.db.base import BaseDatabaseAdapter @@ -170,6 +180,9 @@ def __init__( self.max_limit = max_limit self.allow_non_additive_unsafe = allow_non_additive_unsafe self.enforce_visibility = enforce_visibility + self.freeze_current_time = freeze_current_time + # Source of the instant frozen into compiled SQL; replaceable for deterministic tests. + self.clock: Callable[[], datetime] = lambda: datetime.now(UTC) self._strict_rust_sql_generator_entrypoint = is_strict_for("sql_generator_entrypoint") self._strict_rust_query_validation = is_strict_for("semantic_core_query_validation") if engine == "python": @@ -1349,7 +1362,21 @@ def compile( dialect or self.dialect, aliases=aliases, ) - return self._apply_post_process(inner_sql, post_process) + return self._freeze_current_time_sql(self._apply_post_process(inner_sql, post_process), dialect or self.dialect) + + def _freeze_current_time_sql(self, sql: str, dialect: str | None = None) -> str: + """Freeze current-time functions in compiled query SQL when enabled for ``dialect``. + + Applied after generation (and after the generator/rewrite caches) so cached SQL + never carries a stale instant and both the Python and Rust engines are covered. + """ + dialect = dialect or self.dialect + enabled = self.freeze_current_time if self.freeze_current_time is not None else dialect == "duckdb" + if not enabled: + return sql + from sidemantic.sql.frozen_time import freeze_current_time + + return freeze_current_time(sql, dialect, now=self.clock()) def _validate_query( self, @@ -2254,12 +2281,15 @@ def sql(self, query: str, *, user_attributes: dict | None = None): query, user_attributes=user_attributes, transport="SemanticLayer.sql()", + freeze_current_time=False, ) if use_cache: if len(self._sql_rewrite_cache) >= self._sql_rewrite_cache_limit: self._sql_rewrite_cache.pop(next(iter(self._sql_rewrite_cache))) selection = dict(self.last_engine_selection) if self.last_engine_selection is not None else None self._sql_rewrite_cache[cache_key] = (rewritten_sql, selection) + # The cache holds unfrozen SQL; each execution freezes its own instant. + rewritten_sql = self._freeze_current_time_sql(rewritten_sql) def recompile_raw(): return rewrite_transport_sql( diff --git a/sidemantic/core/transport_security.py b/sidemantic/core/transport_security.py index 6f1daa319..cd08c883b 100644 --- a/sidemantic/core/transport_security.py +++ b/sidemantic/core/transport_security.py @@ -131,6 +131,7 @@ def rewrite_transport_sql( transport: str, strict: bool = True, use_preaggregations: bool | None = None, + freeze_current_time: bool = True, ) -> str: """Rewrite SQL with row/access/column controls and reject unsafe passthrough. @@ -138,6 +139,10 @@ def rewrite_transport_sql( regenerated as semantic SQL. A query that the rewriter leaves untouched is denied before execution because its policies cannot be proven to apply. Projection-only queries such as ``SELECT 1`` remain safe and available. + + ``freeze_current_time`` applies the layer's current-time freezing (see + ``SemanticLayer.freeze_current_time``); callers that cache the rewrite pass + False and freeze each use of the cached SQL themselves. """ from sidemantic.core.semantic_layer import SecurityError from sidemantic.sql.query_rewriter import QueryRewriter @@ -211,6 +216,9 @@ def rewrite_transport_sql( else: rewritten = rewriter.rewrite(query, strict=strict, user_attributes=user_attributes) layer.last_engine_selection = rewriter.last_engine_selection + freeze = getattr(layer, "_freeze_current_time_sql", None) + if freeze_current_time and freeze is not None: + rewritten = freeze(rewritten) return rewritten diff --git a/sidemantic/sql/frozen_time.py b/sidemantic/sql/frozen_time.py new file mode 100644 index 000000000..a6fdb4bc6 --- /dev/null +++ b/sidemantic/sql/frozen_time.py @@ -0,0 +1,266 @@ +"""Replace current-time SQL functions with literals fixed at compile time. + +Storage planners (for example DuckDB's Iceberg scan) prune files only when a +filter compares a column with a constant. ``CURRENT_DATE``/``now()`` are +query-start-stable functions, so substituting one compile-time instant for +every occurrence in a query preserves query semantics while exposing constants +to the planner. + +Rewrites are applied to token spans: everything outside a replaced call (and +any directly folded ``+``/``-`` arithmetic) stays byte-for-byte unchanged, +including comments such as the ``-- sidemantic:`` routing marker. + +Timezone handling mirrors DuckDB: + +- ``now()``/``CURRENT_TIMESTAMP`` (DuckDB type TIMESTAMPTZ) become a TIMESTAMPTZ + literal of the UTC instant. Comparisons against naive TIMESTAMP columns still + cast the column through the session timezone exactly as before. +- ``CURRENT_DATE``/``today()`` (DuckDB type DATE, evaluated in the session + timezone) become a DATE literal of today's date in ``timezone``, which + defaults to the process's local timezone (DuckDB's default ``TimeZone``). +- ``CURRENT_DATE +/- `` folds to a DATE literal and + ``CURRENT_DATE +/- INTERVAL n day|week|hour|minute|second`` folds to a naive + TIMESTAMP literal, matching DuckDB's result types. TIMESTAMPTZ values fold + only exact hour/minute/second intervals because day arithmetic on + TIMESTAMPTZ follows session-timezone calendar days. Calendar units (month, + quarter, year) are never folded; the substituted literal leaves a constant + expression the engine folds itself. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import UTC, date, datetime, timedelta, tzinfo + +from sqlglot import exp +from sqlglot.dialects import Dialect +from sqlglot.errors import SqlglotError +from sqlglot.tokens import Token, TokenType + +# Cheap pre-check so queries without time functions skip tokenization. +_CANDIDATE_PATTERN = re.compile( + r"\b(?:current_date|current_timestamp|now|today|get_current_timestamp|transaction_timestamp)\b", + re.IGNORECASE, +) + +# Function name -> whether the bare keyword form (no parentheses) is a call. +_DATE_FUNCTIONS = {"CURRENT_DATE": True, "TODAY": False} +_TIMESTAMP_FUNCTIONS = { + "CURRENT_TIMESTAMP": True, + "NOW": False, + "GET_CURRENT_TIMESTAMP": False, + "TRANSACTION_TIMESTAMP": False, +} + +_EXACT_UNITS = { + "SECOND": timedelta(seconds=1), + "MINUTE": timedelta(minutes=1), + "HOUR": timedelta(hours=1), + "DAY": timedelta(days=1), + "WEEK": timedelta(weeks=1), +} +_SUB_DAY_UNITS = frozenset({"SECOND", "MINUTE", "HOUR"}) +_INTERVAL_STRING = re.compile(r"^\s*([+-]?\d+)\s*(?:([a-z]+?)s?)?\s*$", re.IGNORECASE) + + +def _token_types(*names: str) -> frozenset[TokenType]: + return frozenset(token_type for name in names if (token_type := getattr(TokenType, name, None)) is not None) + + +_CALL_NAME_TYPES = _token_types("VAR", "CURRENT_DATE", "CURRENT_TIMESTAMP") +_QUERY_START_TYPES = _token_types("SELECT", "WITH", "FROM", "L_PAREN") +# Arithmetic binds tighter than these neighbours, so `` +/- `` +# can be folded without changing how the surrounding expression parses. +_PRECEDING_FOLD_SAFE = _token_types( + "L_PAREN", "COMMA", "AND", "OR", "NOT", "EQ", "NEQ", "GT", "GTE", "LT", "LTE", "NULLSAFE_EQ", + "BETWEEN", "IN", "IS", "WHEN", "THEN", "ELSE", "SELECT", "WHERE", "HAVING", "ON", "QUALIFY", "FROM", +) # fmt: skip +_FOLLOWING_FOLD_SAFE = _token_types( + "R_PAREN", "COMMA", "AND", "OR", "NOT", "EQ", "NEQ", "GT", "GTE", "LT", "LTE", "NULLSAFE_EQ", + "BETWEEN", "IN", "IS", "PLUS", "DASH", "WHEN", "THEN", "ELSE", "END", "ALIAS", "SEMICOLON", + "ORDER_BY", "GROUP_BY", "LIMIT", "HAVING", "QUALIFY", "WINDOW", "UNION", "EXCEPT", "INTERSECT", + "FROM", "WHERE", +) # fmt: skip + + +@dataclass(frozen=True) +class _Value: + """A frozen SQL value and the DuckDB type it represents.""" + + kind: str # "date", "timestamp" (naive), or "timestamptz" + value: date | datetime + + def shifted(self, sign: int, unit: str | None, count: int) -> _Value | None: + """Return ``self +/- count units`` or ``None`` when folding is not exact.""" + if unit is None: + # DATE +/- INTEGER adds days and stays a DATE; other types reject it. + if self.kind != "date": + return None + return _Value("date", self.value + sign * timedelta(days=count)) + step = _EXACT_UNITS.get(unit) + if step is None or (self.kind == "timestamptz" and unit not in _SUB_DAY_UNITS): + return None + if self.kind == "date": + start = datetime.combine(self.value, datetime.min.time()) + return _Value("timestamp", start + sign * count * step) + return _Value(self.kind, self.value + sign * count * step) + + def sql(self, dialect: str) -> str: + if self.kind == "date": + text, data_type = self.value.isoformat(), exp.DataType.Type.DATE + elif self.kind == "timestamp": + text, data_type = _timestamp_text(self.value), exp.DataType.Type.TIMESTAMP + else: + text = self.value.astimezone(UTC).isoformat(sep=" ", timespec="microseconds") + data_type = exp.DataType.Type.TIMESTAMPTZ + return exp.cast(exp.Literal.string(text), data_type).sql(dialect=dialect) + + +def _timestamp_text(value: datetime) -> str: + return value.isoformat(sep=" ", timespec="microseconds" if value.microsecond else "seconds") + + +def freeze_current_time( + sql: str, + dialect: str, + *, + now: datetime | None = None, + timezone: tzinfo | None = None, +) -> str: + """Return ``sql`` with current-time calls replaced by one frozen instant. + + Args: + sql: A single SQL query. Statements that are not queries (DDL/DML, or + multiple statements) are returned unchanged so a frozen literal is + never persisted into a view or table definition. + dialect: SQLGlot dialect used to tokenize the query and render literals. + now: The instant the query is frozen at (defaults to the current time). + A naive value is interpreted as UTC. + timezone: Session timezone used to derive ``CURRENT_DATE`` + (defaults to the process's local timezone). + + Returns: + The rewritten SQL, or the original SQL when nothing can be replaced or + the query cannot be tokenized. + """ + if not _CANDIDATE_PATTERN.search(sql): + return sql + try: + tokens = Dialect.get_or_raise(dialect).tokenize(sql) + except SqlglotError: + return sql + if not _is_single_query(tokens): + return sql + + if now is None: + now = datetime.now(UTC) + elif now.tzinfo is None: + now = now.replace(tzinfo=UTC) + today = now.astimezone(timezone).date() + + replacements: list[tuple[int, int, str]] = [] + index = 0 + while index < len(tokens): + call = _match_call(tokens, index) + if call is None: + index += 1 + continue + kind, call_end = call + value = _Value("date", today) if kind == "date" else _Value("timestamptz", now) + value, span_end = _fold_arithmetic(tokens, index, call_end, value) + replacements.append((tokens[index].start, tokens[span_end].end + 1, value.sql(dialect))) + index = span_end + 1 + + rewritten = sql + for start, end, text in reversed(replacements): + rewritten = rewritten[:start] + text + rewritten[end:] + return rewritten + + +def _is_single_query(tokens: list[Token]) -> bool: + if not tokens or tokens[0].token_type not in _QUERY_START_TYPES: + return False + semicolons = [i for i, token in enumerate(tokens) if token.token_type == TokenType.SEMICOLON] + return not semicolons or semicolons == [len(tokens) - 1] + + +def _match_call(tokens: list[Token], index: int) -> tuple[str, int] | None: + """Match a current-time call at ``index``; return ``(kind, last_token_index)``.""" + token = tokens[index] + if token.token_type not in _CALL_NAME_TYPES: + return None + name = token.text.upper() + if name in _DATE_FUNCTIONS: + kind, bare_ok = "date", _DATE_FUNCTIONS[name] + elif name in _TIMESTAMP_FUNCTIONS: + kind, bare_ok = "timestamptz", _TIMESTAMP_FUNCTIONS[name] + else: + return None + previous = tokens[index - 1].token_type if index > 0 else None + if previous in (TokenType.DOT, TokenType.ALIAS): + return None # qualified column or output alias, not a call + + following = tokens[index + 1].token_type if index + 1 < len(tokens) else None + if following == TokenType.L_PAREN: + # Arguments (for example a CURRENT_TIMESTAMP precision) are left untouched. + if index + 2 < len(tokens) and tokens[index + 2].token_type == TokenType.R_PAREN: + return kind, index + 2 + return None + return (kind, index) if bare_ok else None + + +def _fold_arithmetic(tokens: list[Token], call_start: int, call_end: int, value: _Value) -> tuple[_Value, int]: + """Fold `` +/- `` chains when precedence allows.""" + previous = tokens[call_start - 1].token_type if call_start > 0 else None + if previous is not None and previous not in _PRECEDING_FOLD_SAFE: + return value, call_end + + best_value, best_end = value, call_end + position = call_end + while position + 2 < len(tokens) and tokens[position + 1].token_type in (TokenType.PLUS, TokenType.DASH): + sign = 1 if tokens[position + 1].token_type == TokenType.PLUS else -1 + operand = _parse_operand(tokens, position + 2) + if operand is None: + break + unit, count, operand_end = operand + shifted = value.shifted(sign, unit, count) + if shifted is None: + break + value, position = shifted, operand_end + following = tokens[position + 1].token_type if position + 1 < len(tokens) else None + if following is None or following in _FOLLOWING_FOLD_SAFE: + best_value, best_end = value, position + return best_value, best_end + + +def _parse_operand(tokens: list[Token], index: int) -> tuple[str | None, int, int] | None: + """Parse ```` or ``INTERVAL n [unit]``; return ``(unit, count, last_index)``.""" + token = tokens[index] + if token.token_type == TokenType.NUMBER: + return (None, int(token.text), index) if token.text.isdigit() else None + if token.token_type != TokenType.INTERVAL or index + 1 >= len(tokens): + return None + + amount = tokens[index + 1] + unit_token = tokens[index + 2] if index + 2 < len(tokens) else None + separate_unit = None + if unit_token is not None and unit_token.token_type not in (TokenType.STRING, TokenType.IDENTIFIER): + candidate = unit_token.text.upper().removesuffix("S") + if candidate in _EXACT_UNITS or candidate in {"MONTH", "QUARTER", "YEAR", "MILLISECOND", "MICROSECOND"}: + separate_unit = candidate + + if amount.token_type == TokenType.NUMBER and amount.text.isdigit() and separate_unit: + return separate_unit, int(amount.text), index + 2 + if amount.token_type == TokenType.STRING: + match = _INTERVAL_STRING.match(amount.text) + if match is None: + return None + inline_unit = match.group(2).upper() if match.group(2) else None + if inline_unit and separate_unit: + return None + unit = inline_unit or separate_unit + if unit is None: + return None + return unit, int(match.group(1)), index + 2 if separate_unit else index + 1 + return None diff --git a/tests/dates/test_frozen_time.py b/tests/dates/test_frozen_time.py new file mode 100644 index 000000000..c56255928 --- /dev/null +++ b/tests/dates/test_frozen_time.py @@ -0,0 +1,222 @@ +"""Current-time functions are frozen to compile-time literals for partition pruning.""" + +from datetime import UTC, datetime, timedelta, timezone +from zoneinfo import ZoneInfo + +import duckdb +import pytest + +from sidemantic import Dimension, Metric, Model, SemanticLayer +from sidemantic.sql.frozen_time import freeze_current_time + +NOW = datetime(2026, 10, 11, 18, 30, 15, 123456, tzinfo=UTC) +PACIFIC = timezone(timedelta(hours=-7)) +NOW_TZ = "CAST('2026-10-11 18:30:15.123456+00:00' AS TIMESTAMPTZ)" + + +def freeze(sql: str) -> str: + return freeze_current_time(sql, "duckdb", now=NOW, timezone=PACIFIC) + + +def test_current_date_interval_arithmetic_folds_to_timestamp_literal(): + sql = "SELECT 1 FROM events WHERE timestamp >= CURRENT_DATE - INTERVAL '30' DAY" + + assert freeze(sql) == "SELECT 1 FROM events WHERE timestamp >= CAST('2026-09-11 00:00:00' AS TIMESTAMP)" + + +def test_current_date_integer_arithmetic_folds_to_date_literal(): + assert freeze("SELECT 1 WHERE d >= current_date - 7 AND d <= today()") == ( + "SELECT 1 WHERE d >= CAST('2026-10-04' AS DATE) AND d <= CAST('2026-10-11' AS DATE)" + ) + + +def test_current_date_uses_session_timezone(): + # 02:00 UTC on Oct 12 is still Oct 11 in UTC-7. + late = datetime(2026, 10, 12, 2, 0, tzinfo=UTC) + + assert freeze_current_time("SELECT CURRENT_DATE", "duckdb", now=late, timezone=PACIFIC) == ( + "SELECT CAST('2026-10-11' AS DATE)" + ) + assert freeze_current_time("SELECT CURRENT_DATE", "duckdb", now=late, timezone=UTC) == ( + "SELECT CAST('2026-10-12' AS DATE)" + ) + + +def test_now_becomes_timestamptz_literal_and_folds_exact_intervals(): + assert freeze("SELECT 1 WHERE ts >= now() - INTERVAL 24 HOUR AND x = 1") == ( + "SELECT 1 WHERE ts >= CAST('2026-10-10 18:30:15.123456+00:00' AS TIMESTAMPTZ) AND x = 1" + ) + # Day arithmetic on TIMESTAMPTZ follows session-timezone calendar days, so it stays in SQL. + assert freeze("SELECT 1 WHERE ts >= now() - interval 30 day") == ( + f"SELECT 1 WHERE ts >= {NOW_TZ} - interval 30 day" + ) + + +def test_current_timestamp_becomes_timestamptz_literal(): + assert freeze("SELECT 1 WHERE ts < current_timestamp") == f"SELECT 1 WHERE ts < {NOW_TZ}" + assert freeze("SELECT 1 WHERE ts < CURRENT_TIMESTAMP()") == f"SELECT 1 WHERE ts < {NOW_TZ}" + + +def test_one_instant_is_used_for_every_call(): + frozen = freeze("SELECT now() AS a, CURRENT_TIMESTAMP AS b, get_current_timestamp() AS c") + + assert frozen == f"SELECT {NOW_TZ} AS a, {NOW_TZ} AS b, {NOW_TZ} AS c" + + +def test_sql_without_time_functions_is_unchanged(): + sql = "SELECT status FROM orders WHERE created_at >= '2024-01-01' AND status = 'completed'" + + assert freeze(sql) == sql + + +@pytest.mark.parametrize( + "sql", + [ + "SELECT t.current_date, 'now()' FROM t -- CURRENT_DATE", + 'SELECT "now" FROM t WHERE x = 1', + "SELECT CURRENT_TIMESTAMP(3)", + "CREATE VIEW v AS SELECT * FROM t WHERE d >= CURRENT_DATE", + "SELECT 1; SELECT now()", + ], +) +def test_non_calls_non_queries_and_arguments_are_left_alone(sql): + assert freeze(sql) == sql + + +@pytest.mark.parametrize( + ("sql", "expected"), + [ + # (x - CURRENT_DATE) - 1: folding CURRENT_DATE - 1 would regroup the expression. + ( + "SELECT 1 WHERE x - CURRENT_DATE - 1 > 0", + "SELECT 1 WHERE x - CAST('2026-10-11' AS DATE) - 1 > 0", + ), + # CURRENT_DATE - (INTERVAL 1 DAY * 2): only the leading integer term may fold. + ( + "SELECT 1 WHERE ts > CURRENT_DATE - 1 - INTERVAL 1 DAY * 2", + "SELECT 1 WHERE ts > CAST('2026-10-10' AS DATE) - INTERVAL 1 DAY * 2", + ), + # Calendar units are not folded; the engine folds the constant expression. + ( + "SELECT 1 WHERE d >= DATE_TRUNC('month', CURRENT_DATE) - INTERVAL '1 month'", + "SELECT 1 WHERE d >= DATE_TRUNC('month', CAST('2026-10-11' AS DATE)) - INTERVAL '1 month'", + ), + ], +) +def test_folding_respects_precedence_and_units(sql, expected): + assert freeze(sql) == expected + + +@pytest.mark.parametrize( + "expression", + [ + "current_date - interval '30' day", + "current_date - interval '30 days' - interval 2 hour", + "current_date + 3", + "today()", + "now() - interval 24 hours", + "now() - interval 30 day", + "date_trunc('month', current_date) - interval '1 month'", + ], +) +def test_frozen_sql_matches_duckdb_value_and_type(expression): + zone = "America/Los_Angeles" + con = duckdb.connect() + con.execute(f"SET TimeZone = '{zone}'") + con.execute("BEGIN") + # DuckDB fixes now()/current_date at transaction start; freeze at that same instant. + transaction_start = con.execute("SELECT get_current_timestamp()").fetchone()[0] + frozen = freeze_current_time(f"SELECT {expression}", "duckdb", now=transaction_start, timezone=ZoneInfo(zone)) + frozen_expression = frozen.removeprefix("SELECT ") + + original = con.execute(f"SELECT {expression}, typeof({expression})").fetchone() + replaced = con.execute(f"SELECT {frozen_expression}, typeof({frozen_expression})").fetchone() + + assert "current_date" not in frozen.lower() and "now()" not in frozen.lower() + assert replaced == original + + +def _events_layer(**kwargs) -> SemanticLayer: + layer = SemanticLayer(auto_register=False, **kwargs) + layer.clock = lambda: NOW + layer.add_model( + Model( + name="events", + table="events", + primary_key="event_id", + dimensions=[Dimension(name="timestamp", type="time", granularity="day")], + metrics=[Metric(name="event_count", agg="count")], + ) + ) + return layer + + +def _local_date_literal(days_ago: int) -> str: + return f"CAST('{(NOW.astimezone().date() - timedelta(days=days_ago)).isoformat()} 00:00:00' AS TIMESTAMP)" + + +def test_compile_freezes_pushed_down_filters_on_duckdb(): + layer = _events_layer() + + sql = layer.compile( + metrics=["events.event_count"], + filters=["events.timestamp >= current_date - interval 30 day", "events.timestamp < current_timestamp"], + ) + + assert f"timestamp >= {_local_date_literal(30)}" in sql + assert f"timestamp < {NOW_TZ}" in sql + assert "CURRENT_DATE" not in sql.upper() + assert "-- sidemantic:" in sql + + +def test_compile_freezes_sidemantic_relative_date_range(): + layer = _events_layer() + + sql = layer.compile(metrics=["events.event_count"], filters=["events.timestamp = 'yesterday'"]) + + yesterday = (NOW.astimezone().date() - timedelta(days=1)).isoformat() + assert f"timestamp = CAST('{yesterday}' AS DATE)" in sql + + +def test_freezing_can_be_disabled_and_is_off_for_other_dialects(): + filters = ["events.timestamp >= current_date - interval 30 day"] + + disabled = _events_layer(freeze_current_time=False).compile(metrics=["events.event_count"], filters=filters) + postgres = _events_layer().compile(metrics=["events.event_count"], filters=filters, dialect="postgres") + forced = _events_layer(freeze_current_time=True).compile( + metrics=["events.event_count"], filters=filters, dialect="postgres" + ) + + assert "CURRENT_DATE - INTERVAL '30' DAY" in disabled + assert "CURRENT_DATE" in postgres.upper() + assert "CURRENT_DATE" not in forced.upper() + + +def test_semantic_sql_freezes_each_execution_despite_rewrite_cache(): + layer = _events_layer() + layer.adapter.execute( + "CREATE TABLE events AS SELECT i AS event_id, TIMESTAMP '2026-01-01' + to_days(i) AS timestamp FROM range(60) t(i)" + ) + query = "select events.event_count from events where events.timestamp >= current_date - interval 30 day" + + layer.clock = lambda: datetime(2026, 2, 15, 12, tzinfo=UTC) + first = layer.sql(query).fetchone()[0] + layer.clock = lambda: datetime(2026, 3, 1, 12, tzinfo=UTC) + second = layer.sql(query).fetchone()[0] + + # Cutoffs 2026-01-16 and 2026-01-30 over events dated 2026-01-01 .. 2026-03-01. + assert (first, second) == (45, 31) + + +def test_rust_engine_output_is_frozen(): + pytest.importorskip("sidemantic_rs", reason="Requires the built Rust extension") + layer = _events_layer(engine="rust", fallback=False) + + sql = layer.compile( + metrics=["events.event_count"], + filters=["events.timestamp >= current_date - interval 30 day"], + ) + + assert layer.last_engine_selection == {"engine": "rust", "reason": None} + assert _local_date_literal(30) in sql + assert "CURRENT_DATE" not in sql.upper() diff --git a/tests/dates/test_integration.py b/tests/dates/test_integration.py index cfef158c9..9c1f21a71 100644 --- a/tests/dates/test_integration.py +++ b/tests/dates/test_integration.py @@ -1,6 +1,29 @@ -"""Test relative date integration with queries.""" +"""Test relative date integration with queries. -from sidemantic import Dimension, Metric, Model +Relative dates expand to ``CURRENT_DATE`` expressions, which DuckDB compilation +then freezes to literals of one pinned instant (see ``sidemantic.sql.frozen_time``). +""" + +from datetime import UTC, datetime, timedelta + +import pytest + +from sidemantic import Dimension, Metric, Model, SemanticLayer + +FROZEN_AT = datetime(2026, 10, 11, 12, 0, tzinfo=UTC) +# CURRENT_DATE follows the local timezone, like DuckDB's default session TimeZone. +TODAY = FROZEN_AT.astimezone().date() + + +def _date_literal(days_ago: int = 0) -> str: + return f"CAST('{(TODAY - timedelta(days=days_ago)).isoformat()}' AS DATE)" + + +@pytest.fixture +def layer(): + layer = SemanticLayer(auto_register=False) + layer.clock = lambda: FROZEN_AT + return layer def test_relative_date_in_filter(layer): @@ -30,8 +53,9 @@ def test_relative_date_in_filter(layer): print("SQL with relative date filter:") print(sql) - # Should convert to SQL expression - assert "CURRENT_DATE - 7" in sql + # Should convert to SQL expression, folded to a literal date + assert f"created_at >= {_date_literal(7)}" in sql + assert "CURRENT_DATE" not in sql assert "'last 7 days'" not in sql # Original string should be replaced @@ -57,8 +81,9 @@ def test_multiple_relative_date_filters(layer): ) # Should convert both - assert "CURRENT_DATE - 30" in sql - assert "CURRENT_DATE" in sql + assert f"created_at >= {_date_literal(30)}" in sql + assert f"created_at <= {_date_literal()}" in sql + assert "CURRENT_DATE" not in sql def test_this_month_filter(layer): @@ -81,7 +106,7 @@ def test_this_month_filter(layer): # Should expand to range (case insensitive since SQLGlot may uppercase) sql_upper = sql.upper() - assert "DATE_TRUNC('MONTH', CURRENT_DATE)" in sql_upper or "DATE_TRUNC('month', CURRENT_DATE)" in sql + assert f"DATE_TRUNC('MONTH', {_date_literal()})" in sql_upper assert "INTERVAL" in sql_upper and "MONTH" in sql_upper @@ -147,7 +172,7 @@ def test_not_equal_relative_date_single_day(layer): sql = layer.compile(metrics=["orders.revenue"], filters=["orders_cte.created_at != 'today'"]) - assert "CURRENT_DATE" in sql + assert _date_literal() in sql assert "'today'" not in sql assert "NOT" in sql.upper() or "<>" in sql @@ -221,6 +246,6 @@ def test_mixed_filters(layer): ) # Relative date should be converted - assert "CURRENT_DATE - 7" in sql + assert f"created_at >= {_date_literal(7)}" in sql # Regular filter should remain assert "status = 'completed'" in sql diff --git a/tests/metrics/test_filters.py b/tests/metrics/test_filters.py index b35a62fb2..d759b3f7f 100644 --- a/tests/metrics/test_filters.py +++ b/tests/metrics/test_filters.py @@ -1,5 +1,7 @@ """Test metric-level filters.""" +from datetime import UTC, datetime, timedelta + import duckdb from sqlglot import exp, parse_one @@ -199,12 +201,19 @@ def test_metric_filter_with_time_dimension(layer): ) layer.add_model(orders) + frozen_at = datetime(2026, 10, 11, 12, 0, tzinfo=UTC) + layer.clock = lambda: frozen_at + cutoff = frozen_at.astimezone().date() - timedelta(days=30) sql = layer.compile(metrics=["orders.recent_completed_revenue"], dimensions=["orders.created_at__month"]) - # Both filters should be in CASE WHEN combined with AND + # Both filters should be in CASE WHEN combined with AND; CURRENT_DATE is frozen on DuckDB. _assert_metric_filter( - sql, "orders", "recent_completed_revenue", "status = 'completed' AND created_at >= CURRENT_DATE - 30", "amount" + sql, + "orders", + "recent_completed_revenue", + f"status = 'completed' AND created_at >= CAST('{cutoff.isoformat()}' AS DATE)", + "amount", ) assert "recent_completed_revenue_raw" in sql From 8ec56630bcf8c5d7ace3cad01427990863d1eb21 Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sun, 11 Oct 2026 18:21:01 -0700 Subject: [PATCH 2/4] Freeze CURRENT_DATE in the DuckDB session timezone and skip CTE-prefixed DML CURRENT_DATE depends on the connection's TimeZone setting, which init_sql or SET can change, so read it from the layer's own DuckDB session; the setting is session-scoped, so a cursor would see the default. When the timezone is unknown, leave CURRENT_DATE unchanged unless freezing was explicitly enabled. Classify statements by their parsed root so WITH ... INSERT and other CTE-prefixed DML keep their current-time calls. --- sidemantic/core/semantic_layer.py | 37 +++++++++++++++++++++++++---- sidemantic/sql/frozen_time.py | 39 ++++++++++++++++++++++++++----- tests/dates/test_frozen_time.py | 23 ++++++++++++++++++ 3 files changed, 88 insertions(+), 11 deletions(-) diff --git a/sidemantic/core/semantic_layer.py b/sidemantic/core/semantic_layer.py index 89c5f9108..fa640b85f 100644 --- a/sidemantic/core/semantic_layer.py +++ b/sidemantic/core/semantic_layer.py @@ -5,9 +5,10 @@ import os from collections.abc import Callable from contextvars import ContextVar, Token -from datetime import UTC, datetime +from datetime import UTC, datetime, tzinfo from pathlib import Path from typing import Any +from zoneinfo import ZoneInfo from sidemantic.core.consumption import ( Explore, @@ -141,9 +142,11 @@ def __init__( ``CURRENT_TIMESTAMP`` in compiled query SQL with literals taken from one compile-time instant (``self.clock``) so storage planners such as DuckDB's Iceberg scan can prune on time filters. ``None`` (default) enables it only - for the DuckDB dialect, where ``CURRENT_DATE`` uses the process's local - timezone like DuckDB's default ``TimeZone`` setting; remote engines evaluate + for the DuckDB dialect, and freezes ``CURRENT_DATE`` only when the + connection's ``TimeZone`` setting can be read; remote engines evaluate ``CURRENT_DATE`` in a server/session timezone the client cannot know. + ``True`` also freezes other dialects, using the DuckDB session timezone when + available and the process timezone otherwise. See ``sidemantic.sql.frozen_time`` for the exact rewrite rules. """ from sidemantic.db.base import BaseDatabaseAdapter @@ -1374,9 +1377,33 @@ def _freeze_current_time_sql(self, sql: str, dialect: str | None = None) -> str: enabled = self.freeze_current_time if self.freeze_current_time is not None else dialect == "duckdb" if not enabled: return sql - from sidemantic.sql.frozen_time import freeze_current_time + from sidemantic.sql.frozen_time import freeze_current_time, uses_current_date + + # CURRENT_DATE depends on the connection's TimeZone setting, which init_sql or SET can + # change, so read it from the live DuckDB session. When it is unknown, only an explicit + # opt-in falls back to the process timezone; otherwise CURRENT_DATE stays unchanged. + timezone = self._duckdb_session_timezone() if uses_current_date(sql) else None + return freeze_current_time( + sql, + dialect, + now=self.clock(), + timezone=timezone, + freeze_dates=timezone is not None or self.freeze_current_time is True, + ) - return freeze_current_time(sql, dialect, now=self.clock()) + def _duckdb_session_timezone(self) -> tzinfo | None: + """Return the ``TimeZone`` of this layer's DuckDB connection, or None if it cannot be read. + + The setting is session-scoped, so it must be read on the layer's own connection rather + than a cursor. + """ + if getattr(self.adapter, "dialect", None) != "duckdb": + return None + try: + row = self.adapter.raw_connection.execute("SELECT current_setting('TimeZone')").fetchone() + return ZoneInfo(row[0]) + except Exception: + return None def _validate_query( self, diff --git a/sidemantic/sql/frozen_time.py b/sidemantic/sql/frozen_time.py index a6fdb4bc6..2cc7bc249 100644 --- a/sidemantic/sql/frozen_time.py +++ b/sidemantic/sql/frozen_time.py @@ -16,8 +16,9 @@ literal of the UTC instant. Comparisons against naive TIMESTAMP columns still cast the column through the session timezone exactly as before. - ``CURRENT_DATE``/``today()`` (DuckDB type DATE, evaluated in the session - timezone) become a DATE literal of today's date in ``timezone``, which - defaults to the process's local timezone (DuckDB's default ``TimeZone``). + timezone) become a DATE literal of today's date in ``timezone``. Callers + that cannot determine the session timezone pass ``freeze_dates=False`` and + these calls stay unchanged. - ``CURRENT_DATE +/- `` folds to a DATE literal and ``CURRENT_DATE +/- INTERVAL n day|week|hour|minute|second`` folds to a naive TIMESTAMP literal, matching DuckDB's result types. TIMESTAMPTZ values fold @@ -33,6 +34,7 @@ from dataclasses import dataclass from datetime import UTC, date, datetime, timedelta, tzinfo +import sqlglot from sqlglot import exp from sqlglot.dialects import Dialect from sqlglot.errors import SqlglotError @@ -44,6 +46,8 @@ re.IGNORECASE, ) +_DATE_CANDIDATE_PATTERN = re.compile(r"\b(?:current_date|today)\b", re.IGNORECASE) + # Function name -> whether the bare keyword form (no parentheses) is a call. _DATE_FUNCTIONS = {"CURRENT_DATE": True, "TODAY": False} _TIMESTAMP_FUNCTIONS = { @@ -127,18 +131,24 @@ def freeze_current_time( *, now: datetime | None = None, timezone: tzinfo | None = None, + freeze_dates: bool = True, ) -> str: """Return ``sql`` with current-time calls replaced by one frozen instant. Args: - sql: A single SQL query. Statements that are not queries (DDL/DML, or - multiple statements) are returned unchanged so a frozen literal is - never persisted into a view or table definition. + sql: A single SQL query. Statements that are not queries (DDL/DML, + including CTE-prefixed DML, or multiple statements) are returned + unchanged so a frozen literal is never persisted into a view or + table. dialect: SQLGlot dialect used to tokenize the query and render literals. now: The instant the query is frozen at (defaults to the current time). A naive value is interpreted as UTC. timezone: Session timezone used to derive ``CURRENT_DATE`` (defaults to the process's local timezone). + freeze_dates: Whether to replace ``CURRENT_DATE``/``today()``. Pass + False when the session timezone is unknown; ``now()`` and + ``CURRENT_TIMESTAMP`` are timezone-independent instants and are + still frozen. Returns: The rewritten SQL, or the original SQL when nothing can be replaced or @@ -150,7 +160,7 @@ def freeze_current_time( tokens = Dialect.get_or_raise(dialect).tokenize(sql) except SqlglotError: return sql - if not _is_single_query(tokens): + if not _is_single_query(tokens) or not _is_query_statement(sql, dialect): return sql if now is None: @@ -167,6 +177,9 @@ def freeze_current_time( index += 1 continue kind, call_end = call + if kind == "date" and not freeze_dates: + index = call_end + 1 + continue value = _Value("date", today) if kind == "date" else _Value("timestamptz", now) value, span_end = _fold_arithmetic(tokens, index, call_end, value) replacements.append((tokens[index].start, tokens[span_end].end + 1, value.sql(dialect))) @@ -185,6 +198,20 @@ def _is_single_query(tokens: list[Token]) -> bool: return not semicolons or semicolons == [len(tokens) - 1] +def uses_current_date(sql: str) -> bool: + """Whether ``sql`` may reference ``CURRENT_DATE``/``today()``, which depend on the session timezone.""" + return bool(_DATE_CANDIDATE_PATTERN.search(sql)) + + +def _is_query_statement(sql: str, dialect: str) -> bool: + """Whether the parsed root statement is a query, so CTE-prefixed DML such as ``WITH ... INSERT`` is excluded.""" + try: + statements = sqlglot.parse(sql, read=dialect) + except SqlglotError: + return False + return len(statements) == 1 and isinstance(statements[0], exp.Query) + + def _match_call(tokens: list[Token], index: int) -> tuple[str, int] | None: """Match a current-time call at ``index``; return ``(kind, last_token_index)``.""" token = tokens[index] diff --git a/tests/dates/test_frozen_time.py b/tests/dates/test_frozen_time.py index c56255928..b6371686d 100644 --- a/tests/dates/test_frozen_time.py +++ b/tests/dates/test_frozen_time.py @@ -77,6 +77,7 @@ def test_sql_without_time_functions_is_unchanged(): "SELECT CURRENT_TIMESTAMP(3)", "CREATE VIEW v AS SELECT * FROM t WHERE d >= CURRENT_DATE", "SELECT 1; SELECT now()", + "WITH x AS (SELECT 1 AS a) INSERT INTO history SELECT CURRENT_DATE FROM x", ], ) def test_non_calls_non_queries_and_arguments_are_left_alone(sql): @@ -220,3 +221,25 @@ def test_rust_engine_output_is_frozen(): assert layer.last_engine_selection == {"engine": "rust", "reason": None} assert _local_date_literal(30) in sql assert "CURRENT_DATE" not in sql.upper() + + +def test_current_date_uses_the_duckdb_session_timezone(): + layer = _events_layer() + layer.adapter.execute("SET TimeZone = 'Pacific/Kiritimati'") + # 18:30 UTC on Oct 11 is already Oct 12 in UTC+14. + sql = layer.compile(metrics=["events.event_count"], filters=["events.timestamp >= current_date"]) + + assert "timestamp >= CAST('2026-10-12' AS DATE)" in sql + + +def test_current_date_is_left_alone_when_session_timezone_is_unknown(): + layer = _events_layer() + layer._duckdb_session_timezone = lambda: None + + sql = layer.compile( + metrics=["events.event_count"], + filters=["events.timestamp >= current_date - interval 30 day", "events.timestamp < current_timestamp"], + ) + + assert "CURRENT_DATE - INTERVAL '30' DAY" in sql + assert f"timestamp < {NOW_TZ}" in sql From 226a2e7e04363d4beb594764e4dc8c8177d303d0 Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sun, 11 Oct 2026 18:31:41 -0700 Subject: [PATCH 3/4] Use the pinned clock in pure-Rust parity runs of the date tests The parity harness replaced the dates module's layer fixture with a fresh Rust layer, so its pinned clock was dropped and the frozen dates followed the wall clock, failing whenever the UTC date differed from the pinned one. Build the layer from the module fixture, which already uses the patched Rust SemanticLayer. --- tests/core/test_pure_rust_python_test_parity.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/core/test_pure_rust_python_test_parity.py b/tests/core/test_pure_rust_python_test_parity.py index 6a4f2ac8a..cea81a169 100644 --- a/tests/core/test_pure_rust_python_test_parity.py +++ b/tests/core/test_pure_rust_python_test_parity.py @@ -279,6 +279,9 @@ def _fixture_values(case: PythonTestParityCase) -> dict[str, object]: if case.module is auto_dimension_tests: values["db"] = _call_fixture(case.module, "db") values[fixture_name] = _call_fixture(case.module, "layer", values["db"]) + elif case.module is date_integration_tests: + # The module fixture pins the layer clock; SemanticLayer is patched to Rust above. + values[fixture_name] = _call_fixture(case.module, "layer") else: values[fixture_name] = RustRuntimeSemanticLayer(auto_register=False) elif fixture_name == "test_db": From 396f40a5f29b0279f4642d704ef78e717a0008c8 Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sun, 11 Oct 2026 18:50:50 -0700 Subject: [PATCH 4/4] Accept the Rust engine's parenthesized column in frozen-time tests The Rust engine renders pushed-down filters as (timestamp) >= ...; the frozen literals were correct but the assertions matched only the Python engine's rendering. --- tests/dates/test_frozen_time.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/dates/test_frozen_time.py b/tests/dates/test_frozen_time.py index b6371686d..78fec5df4 100644 --- a/tests/dates/test_frozen_time.py +++ b/tests/dates/test_frozen_time.py @@ -152,6 +152,11 @@ def _events_layer(**kwargs) -> SemanticLayer: return layer +def _compile(layer: SemanticLayer, **kwargs) -> str: + # The Rust engine parenthesizes the filtered column; normalize so assertions cover both engines. + return layer.compile(**kwargs).replace("(timestamp)", "timestamp") + + def _local_date_literal(days_ago: int) -> str: return f"CAST('{(NOW.astimezone().date() - timedelta(days=days_ago)).isoformat()} 00:00:00' AS TIMESTAMP)" @@ -159,7 +164,8 @@ def _local_date_literal(days_ago: int) -> str: def test_compile_freezes_pushed_down_filters_on_duckdb(): layer = _events_layer() - sql = layer.compile( + sql = _compile( + layer, metrics=["events.event_count"], filters=["events.timestamp >= current_date - interval 30 day", "events.timestamp < current_timestamp"], ) @@ -173,7 +179,7 @@ def test_compile_freezes_pushed_down_filters_on_duckdb(): def test_compile_freezes_sidemantic_relative_date_range(): layer = _events_layer() - sql = layer.compile(metrics=["events.event_count"], filters=["events.timestamp = 'yesterday'"]) + sql = _compile(layer, metrics=["events.event_count"], filters=["events.timestamp = 'yesterday'"]) yesterday = (NOW.astimezone().date() - timedelta(days=1)).isoformat() assert f"timestamp = CAST('{yesterday}' AS DATE)" in sql @@ -213,7 +219,8 @@ def test_rust_engine_output_is_frozen(): pytest.importorskip("sidemantic_rs", reason="Requires the built Rust extension") layer = _events_layer(engine="rust", fallback=False) - sql = layer.compile( + sql = _compile( + layer, metrics=["events.event_count"], filters=["events.timestamp >= current_date - interval 30 day"], ) @@ -227,7 +234,7 @@ def test_current_date_uses_the_duckdb_session_timezone(): layer = _events_layer() layer.adapter.execute("SET TimeZone = 'Pacific/Kiritimati'") # 18:30 UTC on Oct 11 is already Oct 12 in UTC+14. - sql = layer.compile(metrics=["events.event_count"], filters=["events.timestamp >= current_date"]) + sql = _compile(layer, metrics=["events.event_count"], filters=["events.timestamp >= current_date"]) assert "timestamp >= CAST('2026-10-12' AS DATE)" in sql @@ -236,7 +243,8 @@ def test_current_date_is_left_alone_when_session_timezone_is_unknown(): layer = _events_layer() layer._duckdb_session_timezone = lambda: None - sql = layer.compile( + sql = _compile( + layer, metrics=["events.event_count"], filters=["events.timestamp >= current_date - interval 30 day", "events.timestamp < current_timestamp"], )