From fc4affd9d46fa555fb8b9166de8bde89ea9527a1 Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Mon, 19 Oct 2026 13:35:28 -0700 Subject: [PATCH 1/3] Enforce model required filters across query clients --- docs/native-format.md | 59 +++++ sidemantic/adapters/sidemantic.py | 7 + sidemantic/core/inheritance.py | 2 + sidemantic/core/model.py | 24 ++- sidemantic/core/required_filters.py | 157 ++++++++++++++ sidemantic/core/semantic_layer.py | 14 ++ sidemantic/core/transport_security.py | 12 +- sidemantic/rust_bridge.py | 45 +++- sidemantic/sql/query_rewriter.py | 6 + tests/core/test_required_filters.py | 298 ++++++++++++++++++++++++++ 10 files changed, 615 insertions(+), 9 deletions(-) create mode 100644 sidemantic/core/required_filters.py create mode 100644 tests/core/test_required_filters.py diff --git a/docs/native-format.md b/docs/native-format.md index 9027125b7..95047b955 100644 --- a/docs/native-format.md +++ b/docs/native-format.md @@ -128,6 +128,8 @@ At least one of `table`, `sql`, or `source_uri` should be present unless the mod | `metrics` | No | List of model-local metrics. | | `relationships` | No | List of relationships from this model to other models. | | `segments` | No | List of model-local named filters. | +| `required_filters` | No | Fields that every query using the model must filter. Enforced by Python-hosted clients before Python or Rust compilation. | +| `required_filters_or_groups` | No | Fields that every query using the model must filter or group by, independently for each listed field. | | `pre_aggregations` | No | List of pre-aggregation definitions. | | `default_time_dimension` | No | Time dimension to add by default when the query needs time grouping. | | `default_grain` | No | Default time grain for the default time dimension. | @@ -140,6 +142,63 @@ also accept compatibility input aliases: model-level `measures` for `metrics`, dimension/metric `expr` for `sql`, and metric `measure` for `sql`. Exports use canonical field names. +### Required filters + +Use model-level caller obligations to reject queries that omit a date range or +combine rows across a required grouping dimension: + +```yaml +models: + - name: daily_activity + table: warehouse.daily_activity + required_filters: [event_date] + required_filters_or_groups: [test_name] + dimensions: + - name: event_date + type: time + granularity: day + - name: test_name + type: categorical + metrics: + - name: activity_count + agg: count +``` + +```sh +sidemantic query --models ./models --metric daily_activity.activity_count \ + --dimension daily_activity.test_name \ + --filter "daily_activity.event_date >= '2026-01-01'" +``` + +Each listed field is mandatory, including on models reached through metric +dependencies, filters, and intermediate joins. Declarations accept local names +or names qualified with the declaring model. Inheritance accumulates parent and +child requirements, rebinding inherited fields to the child. Empty lists are the +default and impose no obligations. + +Matching uses SQL column references, not text or string literals. A time-grained +reference such as `daily_activity.event_date__month` matches `event_date`. +`required_filters` cannot be satisfied by grouping; `required_filters_or_groups` +accepts either a filter or grouping for **each** field. Structured dimensions count +as grouping unless `ungrouped` is set. SQL clients must use an explicit `GROUP BY` +(field or selected alias). Filters on an outer query do not satisfy obligations +inside a CTE or subquery. The check requires a field reference; it does not prove +selectivity or impose a maximum date range. + +Selected segments and resolved Explore filters can satisfy the obligation. +Explore `default_filters` still apply only when filters are omitted; replacing +them with an unrelated filter fails the model requirement. Explore mandatory +filters still always apply. Model `invariant_filters`, security policies, and +metric-internal filters do not substitute for the caller obligation. + +The Python CLI, HTTP `/query`, MCP structured queries, and standard semantic SQL +(including PostgreSQL wire queries) enforce these rules before engine dispatch. +Missing requirements raise `ValueError` and HTTP 400. Raw SQL bypass endpoints are +disabled while requirements are declared. Yardstick SQL is rejected for these +graphs; use structured queries or standard semantic SQL. Standalone Rust and +DuckDB extension runtimes do not yet support these model options; exported +definitions retain them and unsupported native loaders reject them. + Single-column primary key: ```yaml diff --git a/sidemantic/adapters/sidemantic.py b/sidemantic/adapters/sidemantic.py index cc6a7bb44..f370451e4 100644 --- a/sidemantic/adapters/sidemantic.py +++ b/sidemantic/adapters/sidemantic.py @@ -63,6 +63,8 @@ "relationships", "segments", "invariant_filters", + "required_filters", + "required_filters_or_groups", "pre_aggregations", "security", "default_time_dimension", @@ -778,6 +780,8 @@ def _parse_model(self, model_def: dict, *, source_path: Path | None = None) -> M "auto_dimensions", "meta", "invariant_filters", + "required_filters", + "required_filters_or_groups", ]: if field in model_def: model_kwargs[field] = model_def.get(field) @@ -1002,6 +1006,9 @@ def _export_model(self, model: Model) -> dict: result["meta"] = model.meta if model.invariant_filters: result["invariant_filters"] = model.invariant_filters + for policy in ("required_filters", "required_filters_or_groups"): + if getattr(model, policy): + result[policy] = getattr(model, policy) if model.auto_dimensions: result["auto_dimensions"] = True if model.schema_exposure is not None: diff --git a/sidemantic/core/inheritance.py b/sidemantic/core/inheritance.py index 29f7828e7..aa9f75e39 100644 --- a/sidemantic/core/inheritance.py +++ b/sidemantic/core/inheritance.py @@ -41,6 +41,8 @@ def merge_model(child: Model, parent: Model) -> Model: *merged_data.get("invariant_filters", []), *child_data.get("invariant_filters", []), ] + for policy in ("required_filters", "required_filters_or_groups"): + merged_data[policy] = list(dict.fromkeys([*merged_data.get(policy, []), *child_data.get(policy, [])])) # Merge lists (dimensions, metrics, relationships, segments) # Child's items are added to parent's items diff --git a/sidemantic/core/model.py b/sidemantic/core/model.py index 8b8ddd4fa..ba9c573a9 100644 --- a/sidemantic/core/model.py +++ b/sidemantic/core/model.py @@ -2,7 +2,7 @@ from typing import Any, Literal -from pydantic import Field +from pydantic import Field, model_validator from sidemantic.core.dimension import Dimension from sidemantic.core.governance import GovernedObject @@ -51,6 +51,11 @@ class Model(GovernedObject): default_factory=list, description="SQL predicates that always constrain this model before joins and aggregation", ) + required_filters: list[str] = Field(default_factory=list, description="Fields callers must filter") + required_filters_or_groups: list[str] = Field( + default_factory=list, description="Fields callers must filter or group by, independently for each field" + ) + pre_aggregations: list[PreAggregation] = Field( default_factory=list, description="Pre-aggregation definitions for query optimization" ) @@ -81,6 +86,23 @@ class Model(GovernedObject): None, description="Model-level access control and row-level security policy" ) + @model_validator(mode="after") + def normalize_required_fields(self) -> "Model": + """Store local field names so inherited obligations bind to the child model.""" + import re + + for policy in ("required_filters", "required_filters_or_groups"): + normalized = [] + for reference in getattr(self, policy): + field = reference.removeprefix(f"{self.name}.") + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", field): + raise ValueError(f"{policy} must name fields on {self.name}: {reference!r}") + normalized.append(field) + normalized = list(dict.fromkeys(normalized)) + if normalized != getattr(self, policy): + setattr(self, policy, normalized) + return self + def __init__(self, **data): super().__init__(**data) diff --git a/sidemantic/core/required_filters.py b/sidemantic/core/required_filters.py new file mode 100644 index 000000000..76fb61de6 --- /dev/null +++ b/sidemantic/core/required_filters.py @@ -0,0 +1,157 @@ +"""Caller filter obligations shared by structured and SQL query entrypoints.""" + +from sqlglot import exp, parse_one +from sqlglot.errors import SqlglotError +from sqlglot.optimizer.scope import traverse_scope + + +def has_required_filters(graph) -> bool: + return any(model.required_filters or model.required_filters_or_groups for model in graph.models.values()) + + +def _parse(expression: str, dialect: str) -> exp.Expression: + try: + return parse_one(expression, dialect=dialect) + except SqlglotError as error: + raise ValueError(f"Cannot validate required filters: invalid SQL expression {expression!r}") from error + + +def _field(graph, model_name: str, name: str) -> str: + model = graph.models.get(model_name) + # Only strip a grain when it actually identifies a declared time dimension. + if model and "__" in name and not model.get_dimension(name): + base = name.rsplit("__", 1)[0] + dimension = model.get_dimension(base) + if dimension and dimension.type == "time": + name = base + return f"{model_name}.{name}" + + +def _references(graph, expression, models, aliases=None) -> set[str]: + """Bind outer columns only; subquery columns belong to a separate query.""" + references = set() + aliases = aliases or {} + for node in expression.walk(prune=lambda node: isinstance(node, (exp.Select, exp.Subquery))): + if not isinstance(node, exp.Column): + continue + owner = aliases.get(node.table, node.table) + if owner.endswith("_cte") and owner not in graph.models: + owner = owner.removesuffix("_cte") + if not owner: + candidates = [ + name + for name in models + if name in graph.models + and ( + graph.models[name].get_dimension(_field(graph, name, node.name).split(".", 1)[1]) + or graph.models[name].get_metric(node.name) + ) + ] + if len(candidates) != 1: + continue + owner = candidates[0] + if owner in graph.models: + references.add(_field(graph, owner, node.name)) + return references + + +def _enforce(graph, models, filtered, grouped): + for name in sorted(models): + model = graph.models.get(name) + if model is None: + continue + for field in model.required_filters: + reference = _field(graph, name, field) + if reference not in filtered: + raise ValueError(f"a query on {name} must filter {reference}") + for field in model.required_filters_or_groups: + reference = _field(graph, name, field) + if reference not in filtered | grouped: + raise ValueError(f"a query on {name} must filter or group by {reference}") + + +def validate_required_filters( + graph, + *, + metrics=(), + dimensions=(), + filters=(), + segments=(), + ungrouped=False, + base_model=None, + parameters=None, + dialect="duckdb", +): + """Check resolved caller filters before selecting a compiler engine.""" + if not has_required_filters(graph): + return + from sidemantic.core.parameter import ParameterSet + from sidemantic.sql.generator import SQLGenerator + + generator = SQLGenerator(graph, dialect=dialect, base_model=base_model) + parameter_set = ParameterSet(graph.parameters, parameters, dialect=dialect) + predicates = [parameter_set.interpolate(predicate) for predicate in filters or []] + predicates += generator._resolve_segments(segments or [], interpolate=parameter_set.interpolate) + models = generator._participating_models( + generator._find_required_models(list(metrics or []), list(dimensions or []), predicates) + ) + filtered = set() + for predicate in predicates: + filtered.update(_references(graph, _parse(predicate, dialect), models)) + grouped = set() + if not ungrouped: + for reference in dimensions or []: + grouped.update(_references(graph, _parse(reference, dialect), models)) + _enforce(graph, models, filtered, grouped) + + +def validate_required_filters_sql(graph, sql, *, dialect="duckdb"): + """Check each SQL read scope independently, before Rust or Python rewriting. + + WHERE predicates and explicit GROUP BY expressions satisfy obligations. Outer + filters on an already aggregated subquery do not constrain its source read. + """ + if not has_required_filters(graph): + return + from sidemantic.sql.generator import SQLGenerator + + parsed = _parse(sql, dialect) + for table in parsed.find_all(exp.Table): + if isinstance(table.this, exp.Anonymous) and table.this.name.lower() == "yardstick": + raise ValueError("Required model filters require standard semantic SQL; yardstick() is unsupported") + for scope in traverse_scope(parsed): + select = scope.expression + if not isinstance(select, exp.Select): + continue + aliases = { + alias: source.name + for alias, source in scope.sources.items() + if isinstance(source, exp.Table) and source.name in graph.models + } + models = list(dict.fromkeys(aliases.values())) + references = set() + for column in scope.columns: + references.update(_references(graph, column, models, aliases)) + metrics = [ref for ref in references if graph.models[ref.split(".")[0]].get_metric(ref.split(".")[1])] + for column in scope.columns: + reference = f"{column.table}.{column.name}" if column.table else column.name + if reference in graph.metrics: + metrics.append(reference) + dimensions = [ref for ref in references if ref not in metrics] + generator = SQLGenerator(graph, dialect=dialect) + models = generator._participating_models([*models, *generator._find_required_models(metrics, dimensions)]) + where = select.args.get("where") + filtered = _references(graph, where.this, models, aliases) if where else set() + grouped = set() + group = select.args.get("group") + if group: + selected_aliases = {item.alias: item.this for item in select.expressions if isinstance(item, exp.Alias)} + for expression in group.expressions: + if isinstance(expression, exp.Literal) and expression.is_int: + index = int(expression.this) - 1 + if 0 <= index < len(select.expressions): + expression = select.expressions[index] + elif isinstance(expression, exp.Column) and not expression.table: + expression = selected_aliases.get(expression.name, expression) + grouped.update(_references(graph, expression, models, aliases)) + _enforce(graph, models, filtered, grouped) diff --git a/sidemantic/core/semantic_layer.py b/sidemantic/core/semantic_layer.py index fa640b85f..ff77c06aa 100644 --- a/sidemantic/core/semantic_layer.py +++ b/sidemantic/core/semantic_layer.py @@ -1229,6 +1229,20 @@ def compile( metrics = metrics or [] dimensions = dimensions or [] + from sidemantic.core.required_filters import validate_required_filters + + validate_required_filters( + self.graph, + metrics=metrics, + dimensions=dimensions, + filters=filters, + segments=segments, + ungrouped=ungrouped, + base_model=consumption_base_model, + parameters=parameters, + dialect=dialect or self.dialect, + ) + if with_totals and (limit is not None or offset is not None): raise ValueError( "with_totals cannot be combined with limit/offset: the grand-total row shares " diff --git a/sidemantic/core/transport_security.py b/sidemantic/core/transport_security.py index cd08c883b..df9e7e69c 100644 --- a/sidemantic/core/transport_security.py +++ b/sidemantic/core/transport_security.py @@ -27,7 +27,14 @@ def has_invariant_filters(layer: Any) -> bool: def controls_are_active(layer: Any) -> bool: - return has_declared_security(layer) or has_enforced_column_restrictions(layer) or has_invariant_filters(layer) + from sidemantic.core.required_filters import has_required_filters + + return ( + has_declared_security(layer) + or has_enforced_column_restrictions(layer) + or has_invariant_filters(layer) + or has_required_filters(layer.graph) + ) def _reads_from_source(sql: str, dialect: str) -> bool: @@ -224,6 +231,7 @@ def rewrite_transport_sql( def deny_raw_sql(layer: Any, *, transport: str) -> None: """Disable a raw database bypass whenever a declared control needs enforcement.""" + from sidemantic.core.required_filters import has_required_filters from sidemantic.core.semantic_layer import SecurityError controls: list[str] = [] @@ -233,6 +241,8 @@ def deny_raw_sql(layer: Any, *, transport: str) -> None: controls.append("column visibility restrictions") if has_invariant_filters(layer): controls.append("model invariant filters") + if has_required_filters(layer.graph): + controls.append("model required filters") if controls: raise SecurityError( f"{transport} is disabled because {' and '.join(controls)} are active and raw SQL " diff --git a/sidemantic/rust_bridge.py b/sidemantic/rust_bridge.py index 3dc580293..9cc51bf70 100644 --- a/sidemantic/rust_bridge.py +++ b/sidemantic/rust_bridge.py @@ -12,14 +12,16 @@ from sidemantic.semantic_handoff import ( RustBackendUnavailableError, UnsupportedSemanticFeaturesError, - graph_to_semantic_json, + graph_to_semantic_input, ) from sidemantic.yaml_compat import safe_load as _yaml_safe_load -# Lambda-only PreAggregation fields absent from the sidemantic-rs YAML schema -# (which uses deny_unknown_fields). Exclude them when dumping a model so Rust -# deserialization does not reject a model carrying a lambda pre-aggregation. +# Python-managed fields absent from the legacy sidemantic-rs YAML schema +# (which uses deny_unknown_fields). Query entrypoints enforce caller obligations +# before dispatch; public authoring exports retain the policy declarations. _RUST_MODEL_DUMP_EXCLUDE = { + "required_filters": True, + "required_filters_or_groups": True, "invariant_filters": True, "schema_exposure": True, "pre_aggregations": {"__all__": {"rollups", "union_with_source_data"}}, @@ -31,6 +33,19 @@ def _model_dump_for_rust(model_obj) -> dict: return model_obj.model_dump(exclude_none=True, exclude=_RUST_MODEL_DUMP_EXCLUDE) +def _compiler_graph_json(graph, *, input_dialect): + """Omit caller obligations already checked by the Python query boundary. + + Keep them in public semantic snapshots; standalone native consumers must + reject unsupported policies rather than silently forgetting them. + """ + data = graph_to_semantic_input(graph, input_dialect=input_dialect) + for model in data["models"]: + model.pop("required_filters", None) + model.pop("required_filters_or_groups", None) + return json.dumps(data, allow_nan=False) + + def get_rust_module() -> object: """Import and return sidemantic_rs extension module.""" try: @@ -150,6 +165,19 @@ def compile_semantic_input( rust_module=None, ) -> str: """Compile through the versioned graph contract; Rust owns SQL generation.""" + from sidemantic.core.required_filters import validate_required_filters + + validate_required_filters( + graph, + metrics=query.get("metrics"), + dimensions=query.get("dimensions"), + filters=query.get("filters"), + segments=query.get("segments"), + parameters=query.get("parameter_values"), + ungrouped=query.get("ungrouped", False), + base_model=query.get("consumption_base_model"), + dialect=query_dialect or input_dialect, + ) if query_dialect == "postgres": query = dict(query) query["filters"] = [_postgres_query_sql(filter_sql, clause="WHERE") for filter_sql in query.get("filters", [])] @@ -163,7 +191,7 @@ def compile_semantic_input( sql = _call_semantic_entrypoint( module, "compile_with_semantic_input", - graph_to_semantic_json(graph, input_dialect=input_dialect), + _compiler_graph_json(graph, input_dialect=input_dialect), json.dumps(query, allow_nan=False), ) if not isinstance(sql, str): @@ -185,7 +213,7 @@ def validate_semantic_input( errors = _call_semantic_entrypoint( module, "validate_with_semantic_input", - graph_to_semantic_json(graph, input_dialect=input_dialect), + _compiler_graph_json(graph, input_dialect=input_dialect), json.dumps( { "metrics": metrics, @@ -214,6 +242,9 @@ def rewrite_semantic_input( rust_module=None, ) -> str: """Rewrite SQL with caller context through the versioned graph contract.""" + from sidemantic.core.required_filters import validate_required_filters_sql + + validate_required_filters_sql(graph, sql, dialect=sql_dialect or input_dialect) if sql_dialect == "postgres": sql = _postgres_query_sql(sql) sql_dialect = "duckdb" @@ -229,7 +260,7 @@ def rewrite_semantic_input( or allow_non_additive_unsafe or any(model.security is not None or model.invariant_filters for model in graph.models.values()) ) - args = [graph_to_semantic_json(graph, input_dialect=input_dialect), sql] + args = [_compiler_graph_json(graph, input_dialect=input_dialect), sql] entrypoint = "rewrite_with_semantic_input" if context_required: entrypoint = ( diff --git a/sidemantic/sql/query_rewriter.py b/sidemantic/sql/query_rewriter.py index 7f6e9adff..df3312423 100644 --- a/sidemantic/sql/query_rewriter.py +++ b/sidemantic/sql/query_rewriter.py @@ -182,6 +182,12 @@ def __init__( def _route_rust(self, sql: str, user_attributes: dict | None, strict: bool) -> str | None: """Attempt the complete Rust rewrite before any Python semantic expansion.""" + from sidemantic.core.required_filters import has_required_filters, validate_required_filters_sql + + if has_required_filters(self.graph): + if self.would_use_yardstick_rewrite(sql): + raise ValueError("Required model filters require standard semantic SQL; Yardstick SQL is unsupported") + validate_required_filters_sql(self.graph, sql, dialect=self.dialect) self.rust_fallback_reason = None self.last_engine_selection = {"engine": "python", "reason": None} if not self._use_rust_rewriter: diff --git a/tests/core/test_required_filters.py b/tests/core/test_required_filters.py new file mode 100644 index 000000000..4f80729b1 --- /dev/null +++ b/tests/core/test_required_filters.py @@ -0,0 +1,298 @@ +"""Required filters are caller obligations, checked before compiler dispatch.""" + +import json +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from sidemantic import Dimension, Metric, Model, Relationship, SemanticLayer +from sidemantic.adapters.sidemantic import SidemanticAdapter +from sidemantic.core.consumption import Explore +from sidemantic.core.inheritance import merge_model +from sidemantic.core.segment import Segment +from sidemantic.sql.query_rewriter import QueryRewriter + + +@pytest.fixture +def required_layer(): + layer = SemanticLayer(auto_register=False, engine="python") + layer.add_model( + Model( + name="events", + table="event_rows", + primary_key="id", + required_filters=["events.event_date"], + dimensions=[ + Dimension(name="event_date", type="time", granularity="day"), + Dimension(name="status", type="categorical"), + Dimension(name="test_name", type="categorical"), + ], + metrics=[Metric(name="count", agg="count")], + segments=[Segment(name="recent", sql="{model}.event_date >= '2024-01-01'")], + ) + ) + layer.adapter.execute("create table event_rows (id int, event_date date, status text, test_name text)") + layer.adapter.execute("insert into event_rows values (1, '2024-01-01', 'ok', 'a'), (2, '2023-01-01', 'ok', 'b')") + yield layer + layer.adapter.close() + + +@pytest.fixture +def native_dispatch(required_layer, monkeypatch): + """Exercise engine selection and handoff with only native execution replaced.""" + expected = required_layer.compile(metrics=["events.count"], filters=["events.event_date >= '2024-01-01'"]) + module = SimpleNamespace( + validate_with_semantic_input=Mock(return_value=[]), + compile_with_semantic_input=Mock(return_value=expected), + rewrite_with_semantic_input=Mock(return_value=expected), + ) + monkeypatch.setattr("sidemantic.core.semantic_layer.get_rust_module", lambda: module) + monkeypatch.setattr("sidemantic.rust_bridge.get_rust_module", lambda: module) + return module + + +@pytest.mark.parametrize("engine", ["python", "rust", "auto"]) +def test_engine_dispatch_cannot_bypass_requirements(required_layer, native_dispatch, engine): + layer = SemanticLayer(auto_register=False, engine=engine) + layer.add_model(required_layer.graph.models["events"]) + try: + with pytest.raises(ValueError, match="a query on events must filter events.event_date"): + layer.compile(metrics=["events.count"]) + native_dispatch.compile_with_semantic_input.assert_not_called() + native_dispatch.validate_with_semantic_input.assert_not_called() + sql = layer.compile(metrics=["events.count"], filters=["events.event_date >= '2024-01-01'"]) + assert required_layer.adapter.execute(sql).fetchall() == [(1,)] + if engine != "python": + native_dispatch.compile_with_semantic_input.assert_called_once() + snapshot = json.loads(native_dispatch.compile_with_semantic_input.call_args.args[0]) + assert "required_filters" not in snapshot["models"][0] + assert layer.last_engine_selection["engine"] == "rust" + finally: + layer.adapter.close() + + +@pytest.mark.parametrize( + "filters", + [[], ["events.status = 'event_date'"], ["events.status = 'events.event_date'"], ["'events.event_date' = 'x'"]], +) +def test_unrelated_filters_and_literals_do_not_satisfy(required_layer, filters): + with pytest.raises(ValueError, match="must filter events.event_date"): + required_layer.compile(metrics=["events.count"], filters=filters) + + +@pytest.mark.parametrize("field", ["events.event_date", '"events"."event_date"', "events.event_date__day"]) +def test_date_picker_filter(required_layer, field): + assert required_layer.query(metrics=["events.count"], filters=[f"{field} >= '2024-01-01'"]).fetchall() == [(1,)] + + +def test_grouping_is_per_field_and_only_for_or_groups(required_layer): + model = required_layer.graph.models["events"] + with pytest.raises(ValueError, match="must filter events.event_date"): + required_layer.compile(metrics=["events.count"], dimensions=["events.event_date__month"]) + model.required_filters = [] + model.required_filters_or_groups = ["event_date", "test_name"] + with pytest.raises(ValueError, match="must filter or group by events.test_name"): + required_layer.compile(metrics=["events.count"], dimensions=["events.event_date__month"]) + assert ( + required_layer.query( + metrics=["events.count"], dimensions=["events.event_date__month"], filters=["events.test_name = 'a'"] + ).fetchall()[0][-1] + == 1 + ) + with pytest.raises(ValueError, match="must filter or group by events.event_date"): + required_layer.compile(dimensions=["events.event_date", "events.test_name"], ungrouped=True) + + +def test_segments_explore_defaults_and_mandatory_filters(required_layer): + assert required_layer.query(metrics=["events.count"], segments=["events.recent"]).fetchall() == [(1,)] + required_layer.add_explore( + Explore( + name="recent", model="events", default_metrics=["count"], default_filters=["event_date >= '2024-01-01'"] + ) + ) + assert required_layer.query(explore="recent").fetchall() == [(1,)] + with pytest.raises(ValueError, match="must filter events.event_date"): + required_layer.compile(explore="recent", filters=["events.status = 'ok'"]) + required_layer.add_explore( + Explore(name="always_recent", model="events", default_metrics=["count"], filters=["event_date >= '2024-01-01'"]) + ) + assert required_layer.query(explore="always_recent", filters=["events.status = 'ok'"]).fetchall() == [(1,)] + + +def test_invariants_do_not_satisfy_caller_obligations(required_layer): + required_layer.graph.models["events"].invariant_filters = ["event_date >= '2024-01-01'"] + with pytest.raises(ValueError, match="must filter events.event_date"): + required_layer.compile(metrics=["events.count"]) + + +def test_graph_metric_and_intermediate_join_model(required_layer): + layer = required_layer + layer.add_metric(Metric(name="total", type="derived", sql="{events.count} * 2")) + with pytest.raises(ValueError, match="must filter events.event_date"): + layer.compile(metrics=["total"]) + layer.add_metric(Metric(name="stats.total", type="derived", sql="{events.count} * 2")) + with pytest.raises(ValueError, match="must filter events.event_date"): + QueryRewriter(layer.graph).rewrite("select stats.total from metrics") + layer.graph.models["events"].relationships = [Relationship(name="mapping", type="many_to_one", foreign_key="id")] + layer.add_model( + Model( + name="mapping", + table="mapping", + primary_key="id", + required_filters_or_groups=["test_name"], + dimensions=[Dimension(name="test_name", type="categorical")], + relationships=[Relationship(name="users", type="many_to_one", foreign_key="id")], + ) + ) + layer.add_model( + Model(name="users", table="users", primary_key="id", dimensions=[Dimension(name="name", type="categorical")]) + ) + with pytest.raises(ValueError, match="must filter or group by mapping.test_name"): + layer.compile(metrics=["events.count"], dimensions=["users.name"], filters=["events.event_date > '2024-01-01'"]) + sql = layer.compile( + metrics=["events.count"], + dimensions=["users.name", "mapping.test_name"], + filters=["events.event_date > '2024-01-01'"], + ) + assert "mapping" in sql + + +@pytest.mark.parametrize("rust", [False, True]) +def test_sql_scopes_aliases_and_engine_dispatch(required_layer, native_dispatch, rust): + rewriter = QueryRewriter(required_layer.graph, use_rust_rewriter=rust, rust_no_fallback=True) + for sql in ( + "select e.count from events e", + "select * from (select count from events) x where event_date >= '2024-01-01'", + "select e.count from events e where exists (select event_date from events where event_date > '2024-01-01')", + ): + with pytest.raises(ValueError, match="must filter events.event_date"): + rewriter.rewrite(sql, strict=False) + native_dispatch.rewrite_with_semantic_input.assert_not_called() + sql = rewriter.rewrite("select e.count from events e where e.event_date >= '2024-01-01'") + assert required_layer.adapter.execute(sql).fetchall() == [(1,)] + if rust: + native_dispatch.rewrite_with_semantic_input.assert_called_once() + + +@pytest.mark.parametrize("group", ["test_name", "test"]) +def test_sql_group_fields_and_aliases(required_layer, group): + model = required_layer.graph.models["events"] + model.required_filters = [] + model.required_filters_or_groups = ["test_name"] + sql = QueryRewriter(required_layer.graph).rewrite(f"select test_name as test, count from events group by {group}") + assert len(required_layer.adapter.execute(sql).fetchall()) == 2 + + +def test_yaml_roundtrip_and_inheritance(required_layer, tmp_path): + from sidemantic.semantic_handoff import graph_to_semantic_input + + adapter = SidemanticAdapter() + model = required_layer.graph.models["events"] + model.required_filters_or_groups = ["test_name"] + child = merge_model(Model(name="child", extends="events", required_filters=["status"]), model) + assert child.required_filters == ["event_date", "status"] + path = tmp_path / "models.yml" + adapter.export(required_layer.graph, path) + assert adapter.parse(path).models["events"].required_filters == ["event_date"] + assert adapter.parse(path).models["events"].required_filters_or_groups == ["test_name"] + assert graph_to_semantic_input(required_layer.graph)["models"][0]["required_filters"] == ["event_date"] + assert child.required_filters_or_groups == ["test_name"] + assert merge_model(Model(name="child", extends="events", required_filters=[]), model).required_filters == [ + "event_date" + ] + + +def test_default_off(required_layer): + required_layer.graph.models["events"].required_filters = [] + assert required_layer.query(metrics=["events.count"]).fetchall() == [(2,)] + + +def test_cli_filter_error_and_success(required_layer, tmp_path): + from typer.testing import CliRunner + + from sidemantic.cli import app + + SidemanticAdapter().export(required_layer.graph, tmp_path / "models.yml") + args = ["query", "--models", str(tmp_path), "--engine", "python", "--metric", "events.count", "--dry-run"] + runner = CliRunner() + missing = runner.invoke(app, args) + assert missing.exit_code != 0 + assert "must filter events.event_date" in missing.output + present = runner.invoke(app, [*args, "--filter", "events.event_date >= '2024-01-01'"]) + assert present.exit_code == 0, present.output + + +def test_api_returns_400_and_accepts_date_picker(required_layer): + pytest.importorskip("fastapi") + pytest.importorskip("pyarrow") + pytest.importorskip("httpx") + from fastapi.testclient import TestClient + + from sidemantic.api_server import create_app + + with TestClient(create_app(required_layer)) as client: + missing = client.post("/query", json={"metrics": ["events.count"]}) + assert missing.status_code == 400 + assert "must filter events.event_date" in missing.json()["error"] + present = client.post( + "/query", json={"metrics": ["events.count"], "filters": ["events.event_date >= '2024-01-01'"]} + ) + assert present.status_code == 200 + invalid = client.post("/query", json={"metrics": ["events.count"], "filters": ["events.event_date > ("]}) + assert invalid.status_code == 400 + raw = client.post("/raw", json={"query": "select * from event_rows"}) + assert raw.status_code == 403 + + +def test_pgwire_transport_and_raw_bypass(required_layer): + from sidemantic.core.semantic_layer import SecurityError + from sidemantic.core.transport_security import deny_raw_sql, rewrite_transport_sql + + with pytest.raises(ValueError, match="must filter events.event_date"): + rewrite_transport_sql( + required_layer, "select count from events", transport="PostgreSQL", user_attributes=None, strict=False + ) + sql = rewrite_transport_sql( + required_layer, + "select e.count from events e where e.event_date >= '2024-01-01'", + transport="PostgreSQL", + user_attributes=None, + strict=False, + ) + assert required_layer.adapter.execute(sql).fetchall() == [(1,)] + with pytest.raises(SecurityError, match="model required filters"): + deny_raw_sql(required_layer, transport="MCP raw query") + + +def test_filter_only_model_and_multiple_requirements(required_layer): + required_layer.graph.models["events"].required_filters.append("status") + with pytest.raises(ValueError, match="must filter events.status"): + required_layer.compile(metrics=["events.count"], filters=["events.event_date > '2024-01-01'"]) + required_layer.add_model( + Model(name="users", table="users", primary_key="id", metrics=[Metric(name="count", agg="count")]) + ) + with pytest.raises(ValueError, match="must filter events.event_date"): + required_layer.compile(metrics=["users.count"], filters=["events.status = 'ok'"]) + + +def test_predicate_subquery_does_not_satisfy_outer_read(required_layer): + from sidemantic.core.required_filters import validate_required_filters + + with pytest.raises(ValueError, match="must filter events.event_date"): + validate_required_filters( + required_layer.graph, metrics=["events.count"], filters=["exists (select events.event_date from events)"] + ) + + +@pytest.mark.parametrize("rust", [False, True]) +def test_yardstick_wrapper_cannot_bypass_requirements(required_layer, native_dispatch, rust): + rewriter = QueryRewriter(required_layer.graph, use_rust_rewriter=rust) + with pytest.raises(ValueError, match=r"yardstick\(\) is unsupported"): + rewriter.rewrite("select * from yardstick('select count from events')") + native_dispatch.rewrite_with_semantic_input.assert_not_called() + + +def test_output_alias_cannot_satisfy_required_filter(required_layer): + with pytest.raises(ValueError, match="must filter events.event_date"): + required_layer.compile(metrics=["events.count"], aliases={"events.count": "event_date"}) From 844a16fba77978ca46a6f23c6b528d4c676f5428 Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Mon, 19 Oct 2026 13:41:33 -0700 Subject: [PATCH 2/3] Invalidate SQL rewrites when required filters change --- sidemantic/core/semantic_layer.py | 6 ++++++ tests/core/test_required_filters.py | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/sidemantic/core/semantic_layer.py b/sidemantic/core/semantic_layer.py index ff77c06aa..c41e650a3 100644 --- a/sidemantic/core/semantic_layer.py +++ b/sidemantic/core/semantic_layer.py @@ -2301,6 +2301,12 @@ def sql(self, query: str, *, user_attributes: dict | None = None): use_cache = user_attributes is None and not yardstick_warnings_may_apply(self.graph, query) cache_key = ( getattr(self.graph, "_version", 0), + # Policy lists can be assigned or mutated without a graph version + # change. A cached rewrite must still satisfy the current obligations. + tuple( + (name, tuple(model.required_filters), tuple(model.required_filters_or_groups)) + for name, model in self.graph.models.items() + ), self.dialect, self.use_preaggregations, self.enforce_visibility, diff --git a/tests/core/test_required_filters.py b/tests/core/test_required_filters.py index 4f80729b1..4bcd676c6 100644 --- a/tests/core/test_required_filters.py +++ b/tests/core/test_required_filters.py @@ -296,3 +296,27 @@ def test_yardstick_wrapper_cannot_bypass_requirements(required_layer, native_dis def test_output_alias_cannot_satisfy_required_filter(required_layer): with pytest.raises(ValueError, match="must filter events.event_date"): required_layer.compile(metrics=["events.count"], aliases={"events.count": "event_date"}) + + +@pytest.mark.parametrize("policy", ["required_filters", "required_filters_or_groups"]) +@pytest.mark.parametrize("mutation", ["assign", "append", "replace"]) +def test_cached_sql_observes_policy_mutations(required_layer, policy, mutation): + model = required_layer.graph.models["events"] + model.required_filters = [] + if mutation == "replace": + setattr(model, policy, ["status"]) + query = "select count from events where status = 'ok'" + assert required_layer.sql(query).fetchall() == [(2,)] + + if mutation == "assign": + setattr(model, policy, ["event_date"]) + elif mutation == "append": + getattr(model, policy).append("event_date") + else: + getattr(model, policy)[0] = "event_date" + + with pytest.raises(ValueError, match="events.event_date"): + required_layer.sql(query) + + getattr(model, policy).clear() + assert required_layer.sql(query).fetchall() == [(2,)] From 554cd56da4bd9dfac81cc245e174b90cc269d589 Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Mon, 19 Oct 2026 14:22:55 -0700 Subject: [PATCH 3/3] Expose required filters in the model schema --- sidemantic-schema.json | 16 ++++++++++++++++ tests/test_schema_generation.py | 9 +++++++++ 2 files changed, 25 insertions(+) diff --git a/sidemantic-schema.json b/sidemantic-schema.json index 4eed4dbe9..1c2c01309 100644 --- a/sidemantic-schema.json +++ b/sidemantic-schema.json @@ -5453,6 +5453,22 @@ "title": "Relationships", "type": "array" }, + "required_filters": { + "description": "Fields callers must filter", + "items": { + "type": "string" + }, + "title": "Required Filters", + "type": "array" + }, + "required_filters_or_groups": { + "description": "Fields callers must filter or group by, independently for each field", + "items": { + "type": "string" + }, + "title": "Required Filters Or Groups", + "type": "array" + }, "schema_exposure": { "anyOf": [ { diff --git a/tests/test_schema_generation.py b/tests/test_schema_generation.py index ad0484c44..5e3d376e2 100644 --- a/tests/test_schema_generation.py +++ b/tests/test_schema_generation.py @@ -120,3 +120,12 @@ def test_catalog_only_schema_validation(schema_validator, catalog, definition): ) def test_invalid_catalog_schema_validation(schema_validator, document): assert list(schema_validator.iter_errors(document)) + + +@pytest.mark.parametrize("policy", ["required_filters", "required_filters_or_groups"]) +def test_required_filter_schema_validation(schema_validator, policy): + model = {"name": "events", "table": "events", policy: ["event_date"]} + schema_validator.validate({"models": [model]}) + for invalid in ("event_date", [1]): + model[policy] = invalid + assert list(schema_validator.iter_errors({"models": [model]}))