From 826cf1dc419485299264cc197cfcc5a292ac0e33 Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sun, 4 Oct 2026 19:51:31 -0700 Subject: [PATCH] Add semantic SQL discovery and definition lifecycle --- .github/workflows/ci.yml | 4 +- sidemantic-duckdb/CMakeLists.txt | 2 +- sidemantic-duckdb/README.md | 80 +- .../src/include/sidemantic_api.hpp | 7 + .../src/include/sidemantic_parser.hpp | 1 + sidemantic-duckdb/src/sidemantic_api.cpp | 109 ++ .../src/sidemantic_extension.cpp | 1 + sidemantic-duckdb/src/sidemantic_grammar.cpp | 12 +- sidemantic-duckdb/src/sidemantic_parser.cpp | 392 +++++- sidemantic-duckdb/test/README.md | 9 + .../test/sql/sidemantic_lifecycle.test | 159 +++ ...mantic_memory_and_invalid_persistence.test | 54 +- .../test/sql/sidemantic_native_peg.test | 24 + .../test/sql/sidemantic_prefixless.test | 197 +++ sidemantic-duckdb/test/test_catalog_host.py | 220 ++++ .../test/test_duckflight_host.py | 192 +++ sidemantic-rs/include/sidemantic.h | 41 + sidemantic-rs/src/config/sql_parser.rs | 79 +- sidemantic-rs/src/ffi/catalog.rs | 1145 ++++++++++++++++- sidemantic-rs/src/ffi/catalog/api.rs | 433 +++++++ 20 files changed, 3099 insertions(+), 62 deletions(-) create mode 100644 sidemantic-duckdb/src/include/sidemantic_api.hpp create mode 100644 sidemantic-duckdb/src/sidemantic_api.cpp create mode 100644 sidemantic-duckdb/test/sql/sidemantic_lifecycle.test create mode 100644 sidemantic-duckdb/test/sql/sidemantic_prefixless.test create mode 100644 sidemantic-duckdb/test/test_catalog_host.py create mode 100644 sidemantic-duckdb/test/test_duckflight_host.py create mode 100644 sidemantic-rs/src/ffi/catalog/api.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f893954f..fd2a71ce4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -783,7 +783,9 @@ jobs: env: SIDEMANTIC_DUCKDB_BINARY: sidemantic-duckdb/build/release/duckdb SIDEMANTIC_DUCKDB_EXTENSION: sidemantic-duckdb/build/release/extension/sidemantic/sidemantic.duckdb_extension - run: uv run sidemantic-duckdb/test/test_semantic_input_host.py + run: | + uv run sidemantic-duckdb/test/test_semantic_input_host.py + uv run sidemantic-duckdb/test/test_catalog_host.py - name: Save completed build caches if: ${{ !cancelled() }} diff --git a/sidemantic-duckdb/CMakeLists.txt b/sidemantic-duckdb/CMakeLists.txt index 84702efe9..e521acae8 100644 --- a/sidemantic-duckdb/CMakeLists.txt +++ b/sidemantic-duckdb/CMakeLists.txt @@ -145,7 +145,7 @@ add_custom_target(sidemantic_rust_staticlib DEPENDS "${SIDEMANTIC_LIB}") # Include Rust library headers include_directories(${SIDEMANTIC_INCLUDE}) -set(EXTENSION_SOURCES src/sidemantic_extension.cpp src/sidemantic_parser.cpp src/sidemantic_grammar.cpp src/sidemantic_catalog.cpp) +set(EXTENSION_SOURCES src/sidemantic_extension.cpp src/sidemantic_parser.cpp src/sidemantic_grammar.cpp src/sidemantic_catalog.cpp src/sidemantic_api.cpp) build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) diff --git a/sidemantic-duckdb/README.md b/sidemantic-duckdb/README.md index 174605a6d..b1ba7afb1 100644 --- a/sidemantic-duckdb/README.md +++ b/sidemantic-duckdb/README.md @@ -10,6 +10,7 @@ A DuckDB extension that adds a SQL-first semantic layer. Define metrics and dime - **Fan-out Protection**: Uses model keys to protect supported aggregations against duplicated rows from joins - **Definition Files**: Load native YAML, Cube.js YAML, and native SQL definition files - **Transactional Definitions**: Commit and roll back model changes with DuckDB transactions +- **Catalog Discovery**: Inspect definitions, metadata, relationships and compatible dimensions with SQL - **SQL Containers**: Use semantic queries in views, `CREATE TABLE AS`, and `INSERT ... SELECT` - **Native PEG Grammar**: Composable grammar on the pinned Cyanoptera development build, with a DuckDB 1.5.6 compatibility frontend @@ -42,11 +43,8 @@ LOAD 'build/release/extension/sidemantic/sidemantic.duckdb_extension'; For embedded clients, set DuckDB's `allow_unsigned_extensions` database configuration before opening the connection. Community extension installation is planned, but this repository does not yet publish the signed multi-platform artifacts required for `INSTALL sidemantic FROM community`. -In DuckDB 1.5.6 shells that preload `autocomplete` (including Homebrew), use -`SEMANTIC SELECT` for semantic queries. That extension's PEG override consumes -ordinary SQL before Sidemantic receives it, and 1.5.6 has no parser-priority setting. -Clients without that competing override support automatic routing. Explicit -`SEMANTIC SELECT` works in both configurations. +Semantic queries use ordinary `SELECT`, including in shells that preload +`autocomplete`. The older `SEMANTIC SELECT` spelling remains supported. ## Quick Start (Pure SQL) @@ -76,7 +74,7 @@ METRIC avg_order_value AS AVG(amount); DIMENSION (name status, type categorical); -- 5. Query using semantic layer -SEMANTIC SELECT orders_model.status, orders_model.revenue FROM orders_model; +SELECT orders_model.status, orders_model.revenue FROM orders_model; -- Automatically rewrites to: -- SELECT status, SUM(amount) FROM orders GROUP BY 1 @@ -89,6 +87,60 @@ SEMANTIC SELECT orders_model.status, orders_model.revenue FROM orders_model; -- └───────────┴────────────────────┘ ``` +## Discover and manage definitions + +```sql +SHOW MODELS; +SHOW METRICS FROM orders_model; +SHOW DIMENSIONS FROM orders_model; +SHOW RELATIONSHIPS FROM orders_model; +SHOW SEGMENTS FROM orders_model; +DESCRIBE MODEL orders_model; + +-- Dimensions that can be combined with this metric +SHOW DIMENSIONS FOR orders_model.revenue; + +-- Validate the complete selection, predicates and physical database bindings +EXPLAIN SELECT orders_model.status, orders_model.revenue +FROM orders_model WHERE orders_model.status = 'completed'; +``` + +Discovery returns the definition kind, model and qualified name, label, +description, semantic and declared data types, SQL, aggregation, relationship +target and cardinality, granularity, visibility, and full JSON definition. +Absent metadata is SQL `NULL`. `DESCRIBE MODEL` includes the model and all its +fields. `SHOW DIMENSIONS FOR` checks each dimension with the query compiler; +use `EXPLAIN SELECT` to validate a complete combination of metrics and dimensions. +`SHOW SEMANTIC METRICS`, and the corresponding forms for other kinds, are also +accepted. + +Qualified definitions work independently of the connection's active model: + +```sql +CREATE OR REPLACE METRIC orders_model.revenue AS SUM(amount); +CREATE OR REPLACE DIMENSION orders_model.status AS status; +CREATE SEGMENT orders_model.completed AS status = 'completed'; +DROP SEGMENT orders_model.completed; +DROP METRIC IF EXISTS orders_model.old_metric; +DROP MODEL IF EXISTS retired_model; +``` + +Removal uses dependency restrictions: a referenced model or field cannot be +dropped until the dependent definitions are removed or changed. There is no +implicit cascade. Definitions, removal and import participate in transactions; +`PREPARE` and `EXPLAIN` do not apply a mutation. + +```sql +EXPORT SEMANTIC CATALOG; +-- Returns one versioned JSON snapshot in the definition column. +IMPORT SEMANTIC CATALOG ''; +``` + +Export includes models, global metrics, parameters, table calculations and +metadata. Import validates the complete candidate and merges it atomically into +the selected database. A definition with the same name is replaced. Export and +discovery work on read-only databases and observe the current transaction. + ## Versioned SemanticInput The stateless SQL functions `sidemantic_compile_semantic_input(input_json, query_json)` @@ -236,13 +288,13 @@ files raise errors when the semantic catalog is accessed. ## Native PEG frontend DuckDB 1.5.6 uses the compatibility frontend. The pinned `v2.0-cyanoptera` build -registers a `sidemantic` grammar extension for model/item declarations and -`SEMANTIC SELECT`, including nested `PREPARE` and `EXPLAIN`. Definition properties +registers a `sidemantic` grammar extension for definitions, discovery, removal, +import/export and explicit semantic queries, including nested `PREPARE` and `EXPLAIN`. Definition properties still use the shared Rust configuration parser; semantic query compilation still uses the Rust SQL AST. -Loading Sidemantic enables its parser override for automatic routing, subject to -the 1.5.6 autocomplete limitation above. To explicitly compose the native +Loading Sidemantic enables automatic routing, including a planning hook for +hosts with another parser override. To explicitly compose the native grammar with other installed grammars, include it in `active_grammar_extensions`: ```sql @@ -479,10 +531,10 @@ rows, including equal-valued metrics on distinct customers. ## Current support boundaries - The query frontend handles `SELECT`, `CREATE VIEW`, `CREATE TABLE AS`, and - `INSERT ... SELECT`, including `PREPARE` and `EXPLAIN` wrappers. Use the - `SEMANTIC` prefix when another extension takes priority for ordinary SQL. -- Semantic definitions support creation, replacement, loading, and active-model - selection. There is no `DROP MODEL`, `ALTER MODEL`, or item-removal SQL interface. + `INSERT ... SELECT`, including `PREPARE` and `EXPLAIN` wrappers. +- Semantic definitions support creation, replacement, loading, discovery, + export/import, active-model selection and dependency-restricted removal. + Use `CREATE OR REPLACE` to update definitions; there is no `ALTER MODEL` syntax. - Semantic queries support a subset of DuckDB SQL. Correlated semantic subqueries and grouping modifiers such as `ROLLUP` are rejected; explicit `GROUP BY` must repeat the selected semantic dimensions. diff --git a/sidemantic-duckdb/src/include/sidemantic_api.hpp b/sidemantic-duckdb/src/include/sidemantic_api.hpp new file mode 100644 index 000000000..fc762d719 --- /dev/null +++ b/sidemantic-duckdb/src/include/sidemantic_api.hpp @@ -0,0 +1,7 @@ +#pragma once + +#include "duckdb/parser/parser_extension.hpp" + +namespace duckdb { +ParserExtensionPlanResult PlanSidemanticCatalog(const string &kind, const string &model, const string &metric); +} diff --git a/sidemantic-duckdb/src/include/sidemantic_parser.hpp b/sidemantic-duckdb/src/include/sidemantic_parser.hpp index e85df2b1b..8e4de053b 100644 --- a/sidemantic-duckdb/src/include/sidemantic_parser.hpp +++ b/sidemantic-duckdb/src/include/sidemantic_parser.hpp @@ -33,6 +33,7 @@ ParserExtensionParseResult sidemantic_parse(ParserExtensionInfo *, const string ParserExtensionPlanResult sidemantic_plan(ParserExtensionInfo *, ClientContext &, unique_ptr); BoundStatement sidemantic_bind(ClientContext &, Binder &, OperatorExtensionInfo *, SQLStatement &); +void RegisterSidemanticRouting(DatabaseInstance &db); struct SidemanticParserExtension : ParserExtension { SidemanticParserExtension() { diff --git a/sidemantic-duckdb/src/sidemantic_api.cpp b/sidemantic-duckdb/src/sidemantic_api.cpp new file mode 100644 index 000000000..021c92c56 --- /dev/null +++ b/sidemantic-duckdb/src/sidemantic_api.cpp @@ -0,0 +1,109 @@ +#include "sidemantic_api.hpp" +#include "sidemantic_catalog.hpp" +#include "sidemantic_compat.hpp" +#include "sidemantic.h" + +#include "duckdb/function/table_function.hpp" +#include "duckdb/planner/binder.hpp" + +namespace duckdb { +namespace { + +struct CatalogBindData : TableFunctionData { + string kind; + string model; + string metric; +}; + +struct CatalogState : GlobalTableFunctionState { + vector> rows; + idx_t offset = 0; +}; + +static unique_ptr CatalogBind(ClientContext &context, TableFunctionBindInput &input, + vector &types, vector &names) { + if (input.binder) RegisterSidemanticCatalogRead(context, input.binder->GetStatementProperties()); + auto data = make_uniq(); + data->kind = input.inputs[0].GetValue(); + data->model = input.inputs[1].GetValue(); + data->metric = input.inputs[2].GetValue(); + if (data->kind == "export") { + names.emplace_back("definition"); + types.emplace_back(LogicalType::VARCHAR); + return std::move(data); + } + for (auto name : {"kind", "model_name", "name", "qualified_name", "label", "description", "type", "data_type", + "sql", "aggregation", "target_model", "relationship_type", "granularity"}) { + names.emplace_back(name); + types.emplace_back(LogicalType::VARCHAR); + } + names.emplace_back("is_public"); + types.emplace_back(LogicalType::BOOLEAN); + names.emplace_back("definition"); + types.emplace_back(LogicalType::VARCHAR); + return std::move(data); +} + +static unique_ptr CatalogInit(ClientContext &context, TableFunctionInitInput &input) { + auto &data = input.bind_data->Cast(); + auto snapshot = ReadSidemanticCatalog(context); + auto state = make_uniq(); + if (data.kind == "export") { + auto result = sidemantic_snapshot_export(snapshot.payload.c_str()); + if (result.error) { + string error(result.error); + sidemantic_free_result(result); + throw InvalidInputException("Sidemantic: %s", error); + } + string definition(result.sql); + sidemantic_free_result(result); + state->rows.push_back({Value(definition)}); + return std::move(state); + } + struct CatalogResult { + SidemanticCatalogEntries result; + ~CatalogResult() { sidemantic_free_catalog_entries(result); } + } result {sidemantic_snapshot_catalog(snapshot.payload.c_str(), data.kind.c_str(), data.model.c_str(), + data.metric.empty() ? nullptr : data.metric.c_str())}; + if (result.result.error) throw InvalidInputException("Sidemantic: %s", result.result.error); + for (idx_t i = 0; i < result.result.count; ++i) { + auto &entry = result.result.entries[i]; + vector row; + for (auto value : {entry.kind, entry.model_name, entry.name, entry.qualified_name, entry.label, + entry.description, entry.semantic_type, entry.data_type, entry.sql, entry.aggregation, + entry.target_model, entry.relationship_type, entry.granularity}) { + row.push_back(value ? Value(value) : Value(LogicalType::VARCHAR)); + } + row.push_back(Value::BOOLEAN(entry.is_public)); + row.push_back(Value(entry.definition)); + state->rows.push_back(std::move(row)); + } + return std::move(state); +} + +static void CatalogFunction(ClientContext &, TableFunctionInput &input, DataChunk &output) { + auto &state = input.global_state->Cast(); + auto count = MinValue(STANDARD_VECTOR_SIZE, state.rows.size() - state.offset); + for (idx_t row = 0; row < count; ++row) { + auto &values = state.rows[state.offset + row]; + for (idx_t column = 0; column < values.size(); ++column) output.SetValue(column, row, values[column]); + } + state.offset += count; + output.SetCardinality(count); +} + +} // namespace + +ParserExtensionPlanResult PlanSidemanticCatalog(const string &kind, const string &model, const string &metric) { + // Parser-owned lowering only: these functions are never registered in the + // SQL function catalog. Users inspect definitions with SHOW and DESCRIBE. + ParserExtensionPlanResult result; + result.function = TableFunction("sidemantic_catalog_scan", + {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, + CatalogFunction, CatalogBind, CatalogInit); + result.parameters = {Value(kind), Value(model), Value(metric)}; + result.return_type = StatementReturnType::QUERY_RESULT; + return result; +} + +} // namespace duckdb diff --git a/sidemantic-duckdb/src/sidemantic_extension.cpp b/sidemantic-duckdb/src/sidemantic_extension.cpp index f7524622a..e1c0cbaca 100644 --- a/sidemantic-duckdb/src/sidemantic_extension.cpp +++ b/sidemantic-duckdb/src/sidemantic_extension.cpp @@ -160,6 +160,7 @@ static void LoadInternal(ExtensionLoader &loader) { parser.parser_info = RegisterSidemanticGrammar(db); ParserExtension::Register(config, parser); OperatorExtension::Register(config, make_shared_ptr()); + RegisterSidemanticRouting(db); loader.RegisterFunction(TableFunction("sidemantic_load", {LogicalType::VARCHAR}, LoadFunction, LoadBind, LoadInit)); loader.RegisterFunction(TableFunction("sidemantic_load_file", {LogicalType::VARCHAR}, LoadFunction, LoadBind, LoadInit)); diff --git a/sidemantic-duckdb/src/sidemantic_grammar.cpp b/sidemantic-duckdb/src/sidemantic_grammar.cpp index 38afd13d2..74fe50d8e 100644 --- a/sidemantic-duckdb/src/sidemantic_grammar.cpp +++ b/sidemantic-duckdb/src/sidemantic_grammar.cpp @@ -123,11 +123,21 @@ class SidemanticGrammar final : public GrammarExtension { GrammarChange::AddRule("SidemanticModel <- 'MODEL' (SidemanticBlock / ColIdOrString ('FROM' (SidemanticSource / SidemanticBlock) SidemanticBlock / SidemanticBlock)?)"), GrammarChange::AddRule("SidemanticItem <- ('METRIC' / 'DIMENSION' / 'SEGMENT') (SidemanticBlock / SidemanticSource ('AS' Expression / SidemanticBlock))"), GrammarChange::AddRule("SidemanticDefinition <- 'SEMANTIC'? ('CREATE' ('OR' 'REPLACE')?)? (SidemanticModel / SidemanticItem)", StartDefinition), + GrammarChange::AddRule("SidemanticDrop <- 'SEMANTIC'? 'DROP' ('MODEL' / 'METRIC' / 'DIMENSION' / 'SEGMENT') ('IF' 'EXISTS')? SidemanticSource", StartDefinition), + GrammarChange::AddRule("SidemanticShow <- 'SHOW' 'SEMANTIC'? ('MODELS' / 'METRICS' / 'DIMENSIONS' / 'SEGMENTS' / 'RELATIONSHIPS') (('FROM' ColIdOrString) / ('FOR' SidemanticSource))?", StartDefinition), + GrammarChange::AddRule("SidemanticDescribe <- ('DESCRIBE' / 'DESC') 'MODEL' ColIdOrString", StartDefinition), + GrammarChange::AddRule("SidemanticExport <- 'EXPORT' 'SEMANTIC' 'CATALOG'", StartDefinition), + GrammarChange::AddRule("SidemanticImport <- 'IMPORT' 'SEMANTIC' 'CATALOG' ColIdOrString", StartDefinition), GrammarChange::AddRule("SidemanticQuery <- 'SEMANTIC' (SelectStatement / CreateStatement / InsertStatement)", StartSemanticQuery), - GrammarChange::AddRule("SidemanticPreparable <- SidemanticDefinition / SidemanticQuery"), + GrammarChange::AddRule("SidemanticPreparable <- SidemanticShow / SidemanticDescribe / SidemanticExport / SidemanticImport / SidemanticDrop / SidemanticDefinition / SidemanticQuery"), GrammarChange::AddRule("SidemanticPrepare <- 'PREPARE' ColIdOrString 'AS' SidemanticPreparable", StartPrepare), GrammarChange::AddRule("SidemanticExplain <- 'EXPLAIN' AnalyzeKeyword? ExplainOptionList? SidemanticPreparable", StartExplain), GrammarChange::PrependChoice("Statement", "SidemanticDefinition"), + GrammarChange::PrependChoice("Statement", "SidemanticDrop"), + GrammarChange::PrependChoice("Statement", "SidemanticShow"), + GrammarChange::PrependChoice("Statement", "SidemanticDescribe"), + GrammarChange::PrependChoice("Statement", "SidemanticExport"), + GrammarChange::PrependChoice("Statement", "SidemanticImport"), GrammarChange::PrependChoice("Statement", "SidemanticQuery"), GrammarChange::PrependChoice("Statement", "SidemanticPrepare"), GrammarChange::PrependChoice("Statement", "SidemanticExplain"), diff --git a/sidemantic-duckdb/src/sidemantic_parser.cpp b/sidemantic-duckdb/src/sidemantic_parser.cpp index ae63bf205..8a396cab7 100644 --- a/sidemantic-duckdb/src/sidemantic_parser.cpp +++ b/sidemantic-duckdb/src/sidemantic_parser.cpp @@ -1,7 +1,13 @@ #include "sidemantic_parser.hpp" #include "sidemantic_catalog.hpp" +#include "sidemantic_api.hpp" #include "sidemantic.h" +#include "duckdb/common/error_data.hpp" +#include "duckdb/main/config.hpp" +#include "duckdb/main/connection_manager.hpp" +#include "duckdb/main/prepared_statement_data.hpp" +#include "duckdb/main/valid_checker.hpp" #include "duckdb/parser/expression/columnref_expression.hpp" #include "duckdb/parser/expression/subquery_expression.hpp" #include "duckdb/parser/expression/star_expression.hpp" @@ -20,6 +26,10 @@ #include "duckdb/parser/tableref/basetableref.hpp" #include "duckdb/parser/tableref/joinref.hpp" #include "duckdb/parser/tableref/subqueryref.hpp" +#include "duckdb/parser/tableref/showref.hpp" +#include "duckdb/planner/extension_callback.hpp" +#include "duckdb/planner/bound_parameter_map.hpp" +#include "duckdb/planner/planner_extension.hpp" #include "duckdb/planner/operator/logical_extension_operator.hpp" namespace duckdb { @@ -78,6 +88,36 @@ string Literal(const string &text) { return "'" + StringUtil::Replace(text, "'", "''") + "'"; } +string QualifiedNameJson(const vector &tokens, idx_t pos, idx_t max_names) { + string result = "["; + idx_t count = 0; + while (pos < tokens.size()) { + auto token = tokens[pos++].text; + if (token.empty() || token == "." || token == "(" || token == ")" || token == ",") { + throw ParserException("Expected a semantic identifier"); + } + if (count++) result += ","; + result += '"'; + for (auto character : IdentifierText(token)) { + auto byte = static_cast(character); + if (character == '"' || character == '\\') result += '\\'; + if (byte < 0x20) { + static const char hex[] = "0123456789abcdef"; + result += "\\u00"; + result += hex[byte >> 4]; + result += hex[byte & 15]; + } else result += character; + } + result += '"'; + if (pos == tokens.size()) break; + if (tokens[pos++].text != "." || pos == tokens.size()) { + throw ParserException("Expected one qualified semantic name"); + } + } + if (count == 0 || count > max_names) throw ParserException("Expected a model name or model.field"); + return result + "]"; +} + class SidemanticSQLStatement : public ExtensionStatement { public: explicit SidemanticSQLStatement(unique_ptr data) @@ -421,8 +461,251 @@ SelectStatement *QueryInStatement(SQLStatement &statement, bool include_insert_c return nullptr; } +unique_ptr CompileSemanticStatement(SQLStatement &statement, + const SidemanticCatalogSnapshot &snapshot, + bool explicit_semantic = false) { + auto rewritten = statement.Copy(); + auto query = QueryInStatement(*rewritten, true); + if (!query) return nullptr; + ModelListOwner models(snapshot.payload); + if (!SemanticReferences(models.list).Query(*query->node) && !explicit_semantic) return nullptr; + auto sql = query->ToString(); + auto result = sidemantic_snapshot_rewrite(snapshot.payload.c_str(), sql.c_str()); + if (result.error) { + string error(result.error); + sidemantic_free_result(result); + throw BinderException("Sidemantic: %s", error); + } + if (!result.sql) { + sidemantic_free_result(result); + throw InternalException("Sidemantic returned no rewritten SQL"); + } + string compiled(result.sql); + sidemantic_free_result(result); + Parser parser(SidemanticBuiltinParserOptions()); + parser.ParseQuery(compiled); + if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { + throw BinderException("Sidemantic rewrite must produce one SELECT statement"); + } + query->node = std::move(parser.statements[0]->Cast().node); + return rewritten; +} + +// Autocomplete can parse SHOW MODELS as a native SHOW/DESCRIBE reference before +// our parser sees it. Recognize that AST, without reparsing the current query. +unique_ptr SemanticShowStatement(SQLStatement &statement) { + if (statement.type != StatementType::SELECT_STATEMENT) return nullptr; + auto &node = *statement.Cast().node; + if (node.type != QueryNodeType::SELECT_NODE) return nullptr; + auto &select = node.Cast(); + if (!select.from_table || select.from_table->type != TableReferenceType::SHOW_REF) return nullptr; + auto &show = select.from_table->Cast(); +#if SIDEMANTIC_NEW_IDENTIFIER_API + if (show.show_type != ShowType::SHOW || !show.GetCatalogName().empty() || !show.GetSchemaName().empty()) { + return nullptr; + } + auto name = SidemanticName(show.GetTableName()); +#else + // The stable AST represents SHOW name and DESCRIBE name identically. + // Consult only this statement's preserved source to distinguish them. + auto source = statement.query; + if (statement.stmt_location < source.size()) { + source = source.substr(statement.stmt_location, statement.stmt_length ? statement.stmt_length : string::npos); + } + auto tokens = Tokens(source); + if (!Keyword(tokens, 0, "SHOW")) return nullptr; + string name = show.table_name; + if (show.query && show.query->type == QueryNodeType::SELECT_NODE) { + auto &inner = show.query->Cast(); + if (!inner.from_table || inner.from_table->type != TableReferenceType::BASE_TABLE) return nullptr; + auto &table = inner.from_table->Cast(); + if (!table.catalog_name.empty() || !table.schema_name.empty()) return nullptr; + name = table.table_name; + } else if (!show.catalog_name.empty() || !show.schema_name.empty()) return nullptr; +#endif + for (auto plural : {"models", "metrics", "dimensions", "segments", "relationships"}) { + if (!StringUtil::CIEquals(name, plural)) continue; + auto data = make_uniq(); + data->operation = "show_" + string(plural); + data->operation.pop_back(); + return SidemanticStatement(std::move(data)); + } + return nullptr; +} + +constexpr const char *ROUTING_STATE = "sidemantic_native_routing"; +constexpr const char *ROUTING_PROBE = "sidemantic_pristine_statement"; + +// DuckDB's native binder consumes expressions as it visits them. The client +// rebind API protects a pristine statement copy, unlike OperatorExtension's +// failure callback. A private post-bind signal also reaches that API when a +// same-name physical table made the first native bind succeed. +class SidemanticRoutingState : public ClientContextState { +public: + explicit SidemanticRoutingState(ClientContext &context) : context(context) {} + + bool CanRequestRebind() override { + candidate.reset(); + probe_error = false; + probing = false; + has_models = false; + catalog_error = ErrorData(); + snapshot = {}; + // ROLLBACK must remain bindable after a transaction error. Do not read + // the semantic catalog while DuckDB's transaction is invalidated. + if (!context.transaction.HasActiveTransaction() || ValidChecker::IsInvalidated(context.ActiveTransaction())) { + return false; + } + probing = true; + try { + snapshot = ReadSidemanticCatalog(context); + if (!snapshot.payload.empty()) { + ModelListOwner models(snapshot.payload); + has_models = models.list.count != 0; + } + } catch (const std::exception &exception) { + ErrorData error(exception); + switch (error.Type()) { + case ExceptionType::INVALID_INPUT: + case ExceptionType::PARSER: + case ExceptionType::IO: + case ExceptionType::PERMISSION: + // Catalog discovery and explicit semantic statements perform + // their own read. Ordinary SQL and extension introspection + // must remain usable even if legacy definitions are invalid. + catalog_error = std::move(error); + snapshot = {}; + break; + default: + throw; + } + } + // With no models, normal SQL binds once. Keeping a pristine copy still + // lets a native SHOW failure reach our catalog discovery statements. + return true; + } + + RebindQueryInfo OnPlanningError(ClientContext &, SQLStatement &statement, ErrorData &error) override { + if (!probing) return RebindQueryInfo::DO_NOT_REBIND; + bool own_probe = probe_error && error.ExtraInfo().count(ROUTING_PROBE); + probing = false; + probe_error = false; + if (!own_probe && error.Type() != ExceptionType::BINDER && error.Type() != ExceptionType::CATALOG) { + return RebindQueryInfo::DO_NOT_REBIND; + } + candidate = Candidate(statement, !own_probe); + return candidate || own_probe ? RebindQueryInfo::ATTEMPT_TO_REBIND : RebindQueryInfo::DO_NOT_REBIND; + } + + RebindQueryInfo OnRebindPreparedStatement(ClientContext &, BindPreparedStatementCallbackInfo &info, + RebindQueryInfo) override { + // SQL EXECUTE uses a nested Planner instead of the client preparation + // API. Its callback supplies the original prepared AST before binding. + if (!info.prepared_statement.unbound_statement) return RebindQueryInfo::DO_NOT_REBIND; + auto prepared = Candidate(*info.prepared_statement.unbound_statement); + if (!prepared) return RebindQueryInfo::DO_NOT_REBIND; + probing = false; + probe_error = false; + candidate = std::move(prepared); + return RebindQueryInfo::ATTEMPT_TO_REBIND; + } + + RebindQueryInfo OnFinalizePrepare(ClientContext &, PreparedStatementData &, PreparedStatementMode) override { + // C/API Prepare does not necessarily end a query. Disarm successful + // probes here too, so a later direct ExtractPlan never sees our signal. + probing = false; + probe_error = false; + return RebindQueryInfo::DO_NOT_REBIND; + } + + void QueryEnd() override { + candidate.reset(); + probing = false; + probe_error = false; + } + + BoundStatement BindCandidate(Binder &binder) { + // Consume before binding: PREPARE and EXECUTE have nested planners, + // while our rewritten query may itself contain subqueries and views. + auto statement = std::move(candidate); + probing = false; + // The discarded native plan may have inferred parameter types from + // same-name physical columns. Infer them again from the semantic plan, + // retaining only the values/types explicitly supplied by the caller. + if (auto parameters = binder.GetParameters()) { + parameters->GetParametersPtr()->clear(); + parameters->rebind = false; + } + binder.GetStatementProperties() = StatementProperties(); + RegisterSidemanticCatalogRead(context, binder.GetStatementProperties()); + auto child = Binder::CreateBinder(context, &binder); + return child->Bind(*statement); + } + + bool probing = false; + bool probe_error = false; + bool has_models = false; + unique_ptr candidate; + +private: + unique_ptr Candidate(SQLStatement &statement, bool native_failed = false) { + if (statement.type == StatementType::PREPARE_STATEMENT) { + return Candidate(*statement.Cast().statement, native_failed); + } + if (auto show = SemanticShowStatement(statement)) return show; + // A native query that already succeeded needs no recovery. If native + // binding failed, expose the real catalog error instead of pretending + // that semantic definitions were absent or returning a partial rewrite. + if (native_failed && catalog_error.HasError() && QueryInStatement(statement)) catalog_error.Throw(); + if (!has_models) return nullptr; + return CompileSemanticStatement(statement, snapshot); + } + + ClientContext &context; + SidemanticCatalogSnapshot snapshot; + ErrorData catalog_error; +}; + +void SidemanticPostBind(PlannerExtensionInput &input, BoundStatement &bound) { + auto state = input.context.registered_state->Get(ROUTING_STATE); + if (!state) return; + if (state->candidate) { + bound = state->BindCandidate(input.binder); + return; + } + if (!state->probing || state->probe_error) return; + // SHOW of an existing physical table also binds successfully. Probe its + // small fixed description shape even in a database with no semantic models. + bool description = bound.names.size() >= 2 && bound.names[0] == "column_name" && bound.names[1] == "column_type"; + if (!state->has_models && !description) return; + state->probe_error = true; + // INVALID bypasses operator-extension error recovery in both supported + // planners. Only the client rebind callback should consume this signal. + throw Exception(unordered_map {{ROUTING_PROBE, "true"}}, ExceptionType::INVALID, + "Sidemantic requires the pristine statement for semantic binding"); +} + +class SidemanticConnectionCallback : public ExtensionCallback { +public: + void OnConnectionOpened(ClientContext &context) override { + context.registered_state->GetOrCreate(ROUTING_STATE, context); + } +}; + } // namespace +void RegisterSidemanticRouting(DatabaseInstance &db) { + auto &config = DBConfig::GetConfig(db); + auto callback = make_shared_ptr(); + ExtensionCallback::Register(config, callback); + // Register future connections before visiting existing ones. GetOrCreate + // makes connections opened concurrently with LOAD harmless duplicates. + for (auto &context : ConnectionManager::Get(db).GetConnectionList()) callback->OnConnectionOpened(*context); + PlannerExtension planner; + planner.post_bind_function = SidemanticPostBind; + PlannerExtension::Register(config, planner); +} + unique_ptr SidemanticParseData::Copy() const { auto copy = make_uniq(); copy->operation = operation; @@ -458,6 +741,64 @@ unique_ptr ParseSidemanticDefinition(const string &sql) { auto tokens = Tokens(sql, &clean); while (!tokens.empty() && tokens.back().text == ";") tokens.pop_back(); idx_t pos = Keyword(tokens, 0, "SEMANTIC") ? 1 : 0; + if (Keyword(tokens, pos, "SHOW")) { + ++pos; + if (Keyword(tokens, pos, "SEMANTIC")) ++pos; + string kind; + for (auto name : {"MODELS", "METRICS", "DIMENSIONS", "SEGMENTS", "RELATIONSHIPS"}) { + if (Keyword(tokens, pos, name)) kind = StringUtil::Lower(name); + } + if (kind.empty()) return nullptr; + kind.pop_back(); + ++pos; + auto result = make_uniq(); + result->operation = "show_" + kind; + if (Keyword(tokens, pos, "FOR") && kind == "dimension") { + result->operation = "show_compatible_dimensions"; + result->content = QualifiedNameJson(tokens, pos + 1, 2); + } else if (pos < tokens.size()) { + if (!Keyword(tokens, pos, "FROM") || pos + 2 != tokens.size()) { + throw ParserException("SHOW semantic definitions expects FROM model or DIMENSIONS FOR model.metric"); + } + result->content = IdentifierText(tokens[pos + 1].text); + } + return result; + } + if ((Keyword(tokens, pos, "DESCRIBE") || Keyword(tokens, pos, "DESC")) && Keyword(tokens, pos + 1, "MODEL")) { + if (pos + 3 != tokens.size()) throw ParserException("DESCRIBE MODEL expects one model name"); + auto result = make_uniq(); + result->operation = "show_"; + result->content = IdentifierText(tokens[pos + 2].text); + return result; + } + if ((Keyword(tokens, pos, "EXPORT") || Keyword(tokens, pos, "IMPORT")) && + Keyword(tokens, pos + 1, "SEMANTIC") && Keyword(tokens, pos + 2, "CATALOG")) { + bool importing = Keyword(tokens, pos, "IMPORT"); + auto result = make_uniq(); + result->operation = importing ? "import" : "export"; + if (importing) { + if (pos + 4 != tokens.size() || tokens[pos + 3].text.front() != '\'') { + throw ParserException("IMPORT SEMANTIC CATALOG expects a quoted catalog snapshot"); + } + result->content = IdentifierText(tokens[pos + 3].text); + } else if (pos + 3 != tokens.size()) { + throw ParserException("EXPORT SEMANTIC CATALOG takes no arguments"); + } + return result; + } + if (Keyword(tokens, pos, "DROP")) { + ++pos; + if (!(Keyword(tokens, pos, "MODEL") || Keyword(tokens, pos, "METRIC") || + Keyword(tokens, pos, "DIMENSION") || Keyword(tokens, pos, "SEGMENT"))) return nullptr; + auto result = make_uniq(); + result->operation = "drop_" + StringUtil::Lower(tokens[pos++].text); + if (Keyword(tokens, pos, "IF") && Keyword(tokens, pos + 1, "EXISTS")) { + result->replace = true; + pos += 2; + } + result->content = QualifiedNameJson(tokens, pos, result->operation == "drop_model" ? 1 : 2); + return result; + } bool create = Keyword(tokens, pos, "CREATE"); bool replace = false; if (create) { @@ -539,6 +880,9 @@ unique_ptr WrapSidemanticQuery(unique_ptr statement, prepare.statement = WrapSidemanticQuery(std::move(prepare.statement), explicit_semantic); return statement; } + // Ordinary queries retain their native statement type. Routing happens + // after parsing, including when another parser extension accepted them. + if (!explicit_semantic) return statement; if (QueryInStatement(*statement)) { auto data = make_uniq(); data->statement = std::move(statement); @@ -577,6 +921,11 @@ ParserExtensionParseResult sidemantic_parse(ParserExtensionInfo *, const string ParserExtensionPlanResult sidemantic_plan(ParserExtensionInfo *, ClientContext &context, unique_ptr parsed) { auto &data = static_cast(*parsed); + if (data.operation == "export") return PlanSidemanticCatalog("export", "", ""); + if (data.operation == "show_compatible_dimensions") return PlanSidemanticCatalog("dimension", "", data.content); + if (StringUtil::StartsWith(data.operation, "show_")) { + return PlanSidemanticCatalog(data.operation.substr(5), data.content, ""); + } if (!data.operation.empty()) { return PlanSidemanticMutation(context, data.operation, data.content, data.replace, "Semantic definitions updated"); } @@ -588,49 +937,26 @@ ParserExtensionPlanResult sidemantic_plan(ParserExtensionInfo *, ClientContext & } BoundStatement sidemantic_bind(ClientContext &context, Binder &binder, OperatorExtensionInfo *, SQLStatement &statement) { + auto routing = context.registered_state->Get(ROUTING_STATE); + if (routing && routing->probe_error) return {}; + if (routing && routing->candidate) return routing->BindCandidate(binder); if (statement.type != StatementType::EXTENSION_STATEMENT) return {}; auto &extension = statement.Cast(); if (extension.extension.plan_function != sidemantic_plan) return {}; auto state = context.registered_state->Get("sidemantic_bind"); - if (!state || !state->data) throw InternalException("Sidemantic query bind state is missing"); + // Definition and discovery statements bind directly through their plan + // function. Only query wrappers leave deferred bind data; preserve any + // genuine planning error from the other statement forms. + if (!state || !state->data) return {}; // Take ownership before a nested bind can replace the registered state. auto parsed = std::move(state->data); auto &data = static_cast(*parsed); if (!data.statement) throw InternalException("Sidemantic query statement is missing"); + if (routing) routing->probing = false; RegisterSidemanticCatalogRead(context, binder.GetStatementProperties()); auto snapshot = ReadSidemanticCatalog(context); - ModelListOwner models(snapshot.payload); - auto original = data.statement->Copy(); - auto query = QueryInStatement(*original, true); - if (!query) throw InternalException("Sidemantic expected a query-bearing statement"); - // DuckDB identifiers are case-insensitive even when quoted. Normalize only - // semantic bindings to the declared spelling before passing them to Rust. - bool detected = SemanticReferences(models.list).Query(*query->node); - bool semantic = data.explicit_semantic || detected; - if (semantic) { - auto sql = query->ToString(); - auto result = sidemantic_snapshot_rewrite(snapshot.payload.c_str(), sql.c_str()); - if (result.error) { - string error(result.error); - sidemantic_free_result(result); - throw BinderException("Sidemantic: %s", error); - } - if (!result.sql) { - sidemantic_free_result(result); - throw InternalException("Sidemantic returned no rewritten SQL"); - } - string rewritten(result.sql); - sidemantic_free_result(result); - Parser parser(SidemanticBuiltinParserOptions()); - parser.ParseQuery(rewritten); - if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { - throw BinderException("Sidemantic rewrite must produce one SELECT statement"); - } - query->node = std::move(parser.statements[0]->Cast().node); - } else { - // An ordinary SQL statement keeps its exact original scopes and AST. - original = data.statement->Copy(); - } + auto original = CompileSemanticStatement(*data.statement, snapshot, data.explicit_semantic); + if (!original) original = data.statement->Copy(); auto child_binder = Binder::CreateBinder(context, &binder); return child_binder->Bind(*original); } diff --git a/sidemantic-duckdb/test/README.md b/sidemantic-duckdb/test/README.md index ec4f06693..d6713c598 100644 --- a/sidemantic-duckdb/test/README.md +++ b/sidemantic-duckdb/test/README.md @@ -21,5 +21,14 @@ and `SIDEMANTIC_DUCKDB_EXTENSION` to the freshly built artifacts. These tests ex returned SQL and check policy isolation and invalid arguments. See the main README for the complete invocation. +`test_catalog_host.py` uses the same environment variables and verifies +SHOW/DESCRIBE metadata, compatible dimensions, native SELECT, prepared reads, +transaction rollback, and export/import across restart and read-only connections. + +`test_duckflight_host.py` runs the SQL through an authenticated DuckFlight +PostgreSQL listener using psycopg. Set `DUCKFLIGHT_EXTENSION` as well as the two +Sidemantic host variables. It checks ordinary and prepared semantic SELECT, +bound predicates, result metadata, discovery, and definition transactions. + `test_release_workflow.py` exercises the release input allowlist without building or publishing artifacts. diff --git a/sidemantic-duckdb/test/sql/sidemantic_lifecycle.test b/sidemantic-duckdb/test/sql/sidemantic_lifecycle.test new file mode 100644 index 000000000..1213a37eb --- /dev/null +++ b/sidemantic-duckdb/test/sql/sidemantic_lifecycle.test @@ -0,0 +1,159 @@ +# name: test/sql/sidemantic_lifecycle.test +# description: Qualified definition lifecycle is transactional and dependency restricted +# group: [sidemantic] + +require sidemantic + +load __TEST_DIR__/sidemantic_lifecycle.duckdb + +require sidemantic + +statement ok +CREATE TABLE lifecycle_source (id INTEGER, amount INTEGER, region VARCHAR); + +statement ok +INSERT INTO lifecycle_source VALUES (1, 10, 'west'), (2, 30, 'east'); + +statement ok +CREATE MODEL orders (table lifecycle_source, primary_key id); + +statement ok +CREATE MODEL other (table lifecycle_source, primary_key id); + +# Explicit targets must not depend on the active model, including quoted names. +statement ok +CREATE METRIC ORDERS."Gross amount" AS SUM(amount); + +statement ok +SEMANTIC CREATE OR REPLACE METRIC "orders"."Gross amount" AS SUM(amount) * 2; + +query I +SELECT orders."Gross amount" FROM orders; +---- +80 + +statement ok +CREATE DIMENSION orders."sales.region" AS region; + +statement ok +CREATE OR REPLACE DIMENSION ORDERS."sales.region" AS upper(region); + +statement ok +CREATE SEGMENT orders."Open orders" AS amount > 10; + +statement ok +CREATE OR REPLACE SEGMENT ORDERS."Open orders" AS amount >= 10; + +statement ok +EXPLAIN DROP METRIC orders."Gross amount"; + +statement ok +PREPARE remove_metric AS SEMANTIC DROP METRIC orders."Gross amount"; + +query I +SELECT orders."Gross amount" FROM orders; +---- +80 + +statement ok +BEGIN; + +statement ok +EXECUTE remove_metric; + +statement error +SEMANTIC SELECT orders."Gross amount" FROM orders; +---- +not found + +statement ok +ROLLBACK; + +query I +SELECT orders."Gross amount" FROM orders; +---- +80 + +statement ok +SEMANTIC DROP DIMENSION orders."sales.region"; + +statement ok +DROP SEGMENT ORDERS."Open orders"; + +statement error +DROP SEGMENT orders."Open orders"; +---- +not found + +statement ok +DROP SEGMENT IF EXISTS orders."Open orders"; + +statement ok +DROP MODEL IF EXISTS absent; + +statement ok +CREATE METRIC orders.revenue AS SUM(amount); + +statement ok +CREATE METRIC orders.doubled AS revenue * 2; + +statement error +DROP METRIC orders.revenue; +---- +dependent definition + +query I +SELECT orders.revenue FROM orders; +---- +40 + +statement ok +DROP METRIC orders.doubled; + +# A physical input with the same name is not a semantic dependency. +statement ok +CREATE DIMENSION orders.amount AS amount; + +statement ok +DROP DIMENSION orders.amount; + +query I +SELECT orders.revenue FROM orders; +---- +40 + +statement ok +MODEL orders; + +statement ok +BEGIN; + +statement ok +DROP MODEL orders; + +statement error +CREATE METRIC orphan AS COUNT(*); +---- +no active model + +statement ok +ROLLBACK; + +query I +SELECT orders.revenue FROM orders; +---- +40 + +load __TEST_DIR__/sidemantic_lifecycle.duckdb readonly + +require sidemantic + +statement error +DROP METRIC orders.revenue; +---- +read-only + +query I +SELECT orders.revenue FROM orders; +---- +40 diff --git a/sidemantic-duckdb/test/sql/sidemantic_memory_and_invalid_persistence.test b/sidemantic-duckdb/test/sql/sidemantic_memory_and_invalid_persistence.test index 848528438..1222f6672 100644 --- a/sidemantic-duckdb/test/sql/sidemantic_memory_and_invalid_persistence.test +++ b/sidemantic-duckdb/test/sql/sidemantic_memory_and_invalid_persistence.test @@ -23,16 +23,64 @@ SELECT count(*) FROM sidemantic_models() WHERE model_name = 'temp_orders'; load __TEST_DIR__/sidemantic_invalid_persistence.duckdb +statement ok +CREATE TABLE ordinary_data (value INTEGER); + +statement ok +INSERT INTO ordinary_data VALUES (7); + statement ok COPY (SELECT 'MODEL (' AS line) TO '__TEST_DIR__/sidemantic_invalid_persistence.sidemantic.sql' (HEADER false); restart -require sidemantic +# Availability was checked above. An explicit LOAD makes a regression fail +# instead of allowing the test runner to skip the remaining assertions. +statement ok +LOAD sidemantic; + +query I +SELECT 42; +---- +42 -# LOAD only registers the extension. Legacy validation happens when definitions -# are first accessed, without creating or replacing native catalog entries. +query I +SELECT value FROM ordinary_data; +---- +7 + +statement ok +DESCRIBE ordinary_data; + +# LOAD only registers the extension. Legacy validation errors are reported when +# definitions are accessed, without creating or replacing native catalog entries. statement error SELECT * FROM sidemantic_models(); ---- Sidemantic + +statement error +SHOW MODELS; +---- +Sidemantic + +statement error +SELECT broken_model.revenue FROM broken_model; +---- +Sidemantic + +statement ok +BEGIN; + +statement error +SEMANTIC SELECT broken_model.revenue FROM broken_model; +---- +Sidemantic + +statement ok +ROLLBACK; + +query I +SELECT value FROM ordinary_data; +---- +7 diff --git a/sidemantic-duckdb/test/sql/sidemantic_native_peg.test b/sidemantic-duckdb/test/sql/sidemantic_native_peg.test index 4255fb5a7..609e64c45 100644 --- a/sidemantic-duckdb/test/sql/sidemantic_native_peg.test +++ b/sidemantic-duckdb/test/sql/sidemantic_native_peg.test @@ -136,3 +136,27 @@ query I SELECT 42; ---- 42 + +statement ok +SHOW MODELS; + +statement ok +SHOW METRICS FROM peg_orders; + +statement ok +SHOW DIMENSIONS FOR peg_orders.revenue; + +statement ok +DESCRIBE MODEL peg_orders; + +statement ok +EXPORT SEMANTIC CATALOG; + +statement ok +PREPARE inspect_peg AS SHOW METRICS FROM peg_orders; + +statement ok +EXECUTE inspect_peg; + +statement ok +DROP MODEL prepared_peg; diff --git a/sidemantic-duckdb/test/sql/sidemantic_prefixless.test b/sidemantic-duckdb/test/sql/sidemantic_prefixless.test new file mode 100644 index 000000000..6b770003b --- /dev/null +++ b/sidemantic-duckdb/test/sql/sidemantic_prefixless.test @@ -0,0 +1,197 @@ +# name: test/sql/sidemantic_prefixless.test +# description: Prefixless semantic SQL retains native statement behavior with autocomplete loaded first +# group: [sidemantic] + +require autocomplete + +require sidemantic + +# Native SQL in an empty semantic catalog needs no semantic retry. +query I +SELECT 42; +---- +42 + +statement ok +CREATE TABLE models (dummy INTEGER); + +# SHOW has reserved semantic catalog meaning even if a physical table matches. +query TTTTTTTTTTTTTIT +SHOW MODELS; +---- + +statement ok +DESCRIBE models; + +statement ok +CREATE TABLE prefixless_orders (id INTEGER, status VARCHAR, amount INTEGER, revenue INTEGER); + +statement ok +INSERT INTO prefixless_orders VALUES (1, 'paid', 10, 999), (2, 'paid', 30, 999), (3, 'pending', 5, 999); + +statement ok +MODEL prefixless_orders FROM prefixless_orders ( + primary key (id) + status + sum(amount) AS revenue + sum(amount) AS semantic_total +); + +statement ok +SELECT 1; SHOW MODELS; + +# The native binder can resolve both columns. Semantic dimensions still group, +# and a same-name physical metric column must not change the query's meaning. +query TI +SELECT status, revenue FROM prefixless_orders ORDER BY status; +---- +paid 40 +pending 5 + +query T +SELECT status FROM prefixless_orders ORDER BY status; +---- +paid +pending + +# Binding the missing physical metric fails after binding WHERE. Retry from the +# pristine statement must preserve the filter instead of aggregating all rows. +query TI +SELECT status, semantic_total FROM prefixless_orders WHERE status = 'paid'; +---- +paid 40 + +query I +SELECT o.semantic_total FROM prefixless_orders o WHERE o.status = 'pending'; +---- +5 + +statement ok +PREPARE prefixless_parameter AS SELECT semantic_total FROM prefixless_orders WHERE status = $1; + +query I +EXECUTE prefixless_parameter('paid'); +---- +40 + +query I +EXECUTE prefixless_parameter('pending'); +---- +5 + +statement ok +PREPARE prefixless_native_success AS SELECT revenue FROM prefixless_orders WHERE status = $1; + +query I +EXECUTE prefixless_native_success('paid'); +---- +40 + +statement ok +BEGIN; + +statement ok +CREATE OR REPLACE METRIC prefixless_orders.revenue AS COUNT(*); + +query I +EXECUTE prefixless_native_success('paid'); +---- +2 + +statement ok +ROLLBACK; + +query I +EXECUTE prefixless_native_success('paid'); +---- +40 + +statement ok +CREATE TABLE prefixless_totals AS SELECT status, revenue FROM prefixless_orders; + +query TI +SELECT * FROM prefixless_totals ORDER BY status; +---- +paid 40 +pending 5 + +statement ok +CREATE VIEW prefixless_view AS SELECT status, semantic_total FROM prefixless_orders; + +query TI +SELECT * FROM prefixless_view ORDER BY status; +---- +paid 40 +pending 5 + +statement ok +CREATE TABLE prefixless_inserted (status VARCHAR, revenue BIGINT); + +statement ok +INSERT INTO prefixless_inserted SELECT status, revenue FROM prefixless_orders WHERE status = 'paid'; + +query TI +SELECT * FROM prefixless_inserted; +---- +paid 40 + +statement ok +EXPLAIN SELECT status, revenue FROM prefixless_orders; + +# Explicit physical qualification and local scopes keep native meanings. +query I +SELECT revenue FROM main.prefixless_orders ORDER BY id; +---- +999 +999 +999 + +query I +WITH prefixless_orders AS (SELECT 7 AS revenue) SELECT revenue FROM prefixless_orders; +---- +7 + +query I +SELECT prefixless_orders.revenue FROM (SELECT 8 AS revenue) prefixless_orders; +---- +8 + +statement error +SELECT missing FROM prefixless_orders; +---- +missing + +statement error +SELECT revenue, missing FROM prefixless_orders; +---- +Sidemantic: + +# A failed query must not leak a private routing signal into the next query. +query I +SELECT 43; +---- +43 + +query I +SEMANTIC SELECT revenue FROM prefixless_orders; +---- +45 + +statement ok +BEGIN; + +statement ok +CREATE TABLE prefixless_abort (i INTEGER CHECK (i > 0)); + +statement error +INSERT INTO prefixless_abort VALUES (-1); +---- +CHECK + +statement ok +ROLLBACK; + +query I +SELECT revenue FROM prefixless_orders; +---- +45 diff --git a/sidemantic-duckdb/test/test_catalog_host.py b/sidemantic-duckdb/test/test_catalog_host.py new file mode 100644 index 000000000..b6c5d2220 --- /dev/null +++ b/sidemantic-duckdb/test/test_catalog_host.py @@ -0,0 +1,220 @@ +# /// script +# dependencies = ["pytest"] +# /// +"""Exercise semantic catalog SQL through each built DuckDB shell.""" + +import json +import os +import subprocess +from decimal import Decimal +from pathlib import Path + +import pytest + + +def literal(value): + return "'" + value.replace("'", "''") + "'" + + +DEFINITIONS = """ +models: + - name: orders + table: raw_orders + primary_key: id + label: Orders + description: Order facts + dimensions: + - name: status + type: categorical + description: Fulfilment status + metrics: + - name: revenue + agg: sum + sql: amount + label: Revenue + description: Gross revenue + segments: + - name: completed + sql: status = 'complete' + relationships: + - name: customers + type: many_to_one + foreign_key: customer_id + - name: customers + table: raw_customers + primary_key: id + dimensions: + - name: country + type: categorical + - name: unrelated + table: raw_unrelated + primary_key: id + dimensions: + - name: name + type: categorical +parameters: + - name: region + type: string + default_value: US +metadata: + owner: finance +""" + +SETUP = f""" +create table raw_orders(id integer, customer_id integer, status varchar, amount decimal(10,2)); +insert into raw_orders values (1, 1, 'complete', 10), (2, 1, 'complete', 30), (3, 2, 'pending', 5); +create table raw_customers(id integer, country varchar); +insert into raw_customers values (1, 'US'), (2, 'CA'); +select * from sidemantic_load({literal(DEFINITIONS)}); +""" + + +def execute(sql, *, setup=SETUP, database=":memory:", readonly=False, error=None): + binary = Path(os.environ["SIDEMANTIC_DUCKDB_BINARY"]).resolve() + extension = Path(os.environ["SIDEMANTIC_DUCKDB_EXTENSION"]).resolve() + args = [str(binary), "-unsigned", "-no-init", "-json", "-bail"] + if readonly: + args.append("-readonly") + args.append(str(database)) + result = subprocess.run( + args, + input=f".output /dev/null\nload {literal(str(extension))};\n{setup}\n.output stdout\n{sql}\n", + text=True, + capture_output=True, + timeout=30, + ) + if error: + assert result.returncode != 0, result.stdout + assert "INTERNAL Error" not in result.stderr, result.stderr + assert error in result.stderr, result.stderr + return [] + assert result.returncode == 0, result.stderr + decoder = json.JSONDecoder() + results = [] + remaining = result.stdout.strip() + while remaining: + rows, end = decoder.raw_decode(remaining) + results.append(rows) + remaining = remaining[end:].strip() + return results + + +def test_catalog_discovery_and_select_combinations(): + models, metrics, dimensions, relationships, segments, description, compatible, rows = execute(""" + show models; + show metrics from ORDERS; + show dimensions from orders; + show relationships from orders; + show segments from orders; + describe model orders; + show dimensions for ORDERS.REVENUE; + select customers.country, orders.revenue from orders order by customers.country; + """) + assert {row["name"] for row in models} == {"orders", "customers", "unrelated"} + metric = metrics[0] + assert (metric["qualified_name"], metric["label"], metric["description"]) == ( + "orders.revenue", + "Revenue", + "Gross revenue", + ) + assert metric["aggregation"] == "sum" + assert metric["is_public"] is True + assert json.loads(metric["definition"])["sql"] == "amount" + assert dimensions[0]["description"] == "Fulfilment status" + assert dimensions[0]["type"] == "categorical" + assert relationships[0]["target_model"] == "customers" + assert relationships[0]["relationship_type"] == "many_to_one" + assert segments[0]["sql"] == "status = 'complete'" + assert {row["kind"] for row in description} == {"model", "metric", "dimension", "relationship", "segment"} + assert [row["qualified_name"] for row in compatible] == ["customers.country", "orders.status"] + assert [(row["country"], Decimal(str(row["revenue"]))) for row in rows] == [ + ("CA", Decimal("5.00")), + ("US", Decimal("40.00")), + ] + + +@pytest.mark.parametrize( + ("sql", "error"), + [ + ("show metrics from missing;", "model 'missing' not found"), + ("show dimensions for orders.missing;", "metric 'orders.missing' not found"), + ("select orders.revenue, unrelated.name from orders;", "Sidemantic"), + ("import semantic catalog '{\"version\":99}';", "unsupported catalog snapshot version"), + ], +) +def test_invalid_catalog_requests_surface_errors(sql, error): + execute(sql, error=error) + + +def test_prepared_catalog_reads_and_rollback(): + results = execute( + """ + prepare inspect_metrics as show metrics from orders; + execute inspect_metrics; + begin; + create metric orders.order_count as count(*); + execute inspect_metrics; + rollback; + execute inspect_metrics; + """, + setup=SETUP + "explain drop metric orders.revenue;", + ) + metric_results = [rows for rows in results if rows and rows[0].get("kind") == "metric"] + assert [[row["name"] for row in rows] for rows in metric_results] == [ + ["revenue"], + ["order_count", "revenue"], + ["revenue"], + ] + + +def test_export_import_roundtrip_and_readonly(tmp_path): + source = tmp_path / "source.duckdb" + snapshot = execute("export semantic catalog;", database=source)[0][0]["definition"] + decoded = json.loads(snapshot) + assert decoded["version"] == 1 + assert decoded["metadata"]["owner"] == "finance" + assert decoded["parameters"][0]["name"] == "region" + destination = tmp_path / "destination.duckdb" + imported = execute( + "export semantic catalog;", + setup=f"import semantic catalog {literal(snapshot)};", + database=destination, + )[0][0]["definition"] + assert json.loads(imported) == decoded + restarted = execute("export semantic catalog;", setup="", database=destination, readonly=True) + assert json.loads(restarted[0][0]["definition"]) == decoded + execute("drop metric orders.revenue;", setup="", database=destination, readonly=True, error="read-only") + + +def test_prefixless_physical_collision_and_prepared_catalog_changes(): + results = execute( + """ + select status, revenue from orders order by status; + select semantic_total from orders where status = 'paid'; + prepare by_status as select revenue from orders where status = $1; + execute by_status('paid'); + begin; + create or replace metric orders.revenue as count(*); + execute by_status('paid'); + rollback; + execute by_status('paid'); + """, + setup=""" + create table orders(id integer, status varchar, amount integer, revenue integer); + insert into orders values (1, 'paid', 10, 999), (2, 'paid', 30, 999), (3, 'pending', 5, 999); + MODEL orders FROM orders ( + primary key(id) + status + sum(amount) AS revenue + sum(amount) AS semantic_total + ); + """, + ) + assert [(row["status"], int(row["revenue"])) for row in results[0]] == [("paid", 40), ("pending", 5)] + assert int(results[1][0]["semantic_total"]) == 40 + totals = [int(rows[0]["revenue"]) for rows in results[2:] if rows and "revenue" in rows[0]] + assert totals == [40, 2, 40] + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v", "-o", "addopts="])) diff --git a/sidemantic-duckdb/test/test_duckflight_host.py b/sidemantic-duckdb/test/test_duckflight_host.py new file mode 100644 index 000000000..eb1823153 --- /dev/null +++ b/sidemantic-duckdb/test/test_duckflight_host.py @@ -0,0 +1,192 @@ +# /// script +# dependencies = ["psycopg[binary]>=3.2,<4", "pytest"] +# /// +"""Exercise Sidemantic through the real DuckFlight extension's PostgreSQL listener.""" + +import hashlib +import json +import os +import secrets +import shutil +import socket +import subprocess +import time +from decimal import Decimal +from pathlib import Path + +import pytest + +psycopg = pytest.importorskip("psycopg") + + +def literal(value): + return "'" + str(value).replace("'", "''") + "'" + + +@pytest.fixture +def postgres(tmp_path): + required = ("DUCKFLIGHT_EXTENSION", "SIDEMANTIC_DUCKDB_EXTENSION") + if any(not os.environ.get(name) for name in (*required, "SIDEMANTIC_DUCKDB_BINARY")): + pytest.skip("Set SIDEMANTIC_DUCKDB_BINARY, SIDEMANTIC_DUCKDB_EXTENSION, and DUCKFLIGHT_EXTENSION") + password = secrets.token_urlsafe(24) + salt = secrets.token_bytes(16) + digest = hashlib.pbkdf2_hmac("sha256", password.encode(), salt, 10000).hex() + config = tmp_path / "users.toml" + config.write_text(f'[users.smoke]\npassword_hash = "{digest}"\nsalt = {list(salt)}\niterations = 10000\n') + # Bind-check within the project's permitted local port range before starting. + for port in range(5400, 5500): + with socket.socket() as probe: + try: + probe.bind(("127.0.0.1", port)) + except OSError: + continue + break + else: + pytest.fail("No available localhost port in 5400-5499") + address = f"127.0.0.1:{port}" + binary = Path(os.environ["SIDEMANTIC_DUCKDB_BINARY"]).resolve() + assert binary.is_file(), binary + setup = "" + for name in required: + path = Path(os.environ[name]).resolve() + assert path.is_file(), path + if name == "SIDEMANTIC_DUCKDB_EXTENSION": + # Keep a concurrent native rebuild from replacing a loaded library. + snapshot = tmp_path / "sidemantic.duckdb_extension" + shutil.copyfile(path, snapshot) + path = snapshot + setup += f"load {literal(path)};\n" + setup += f"select * from duckflight_pg_serve({literal(address)}, {literal(config)});\n" + with (tmp_path / "host.log").open("w+") as log: + host = subprocess.Popen( + [str(binary), "-unsigned", "-no-init", "-bail", ":memory:"], + stdin=subprocess.PIPE, + stdout=log, + stderr=log, + text=True, + ) + try: + host.stdin.write(setup) + host.stdin.flush() + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if host.poll() is not None: + log.seek(0) + pytest.fail(log.read()) + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + break + except OSError: + time.sleep(0.1) + else: + pytest.fail("DuckFlight listener startup timed out") + with psycopg.connect( + host="127.0.0.1", + port=port, + user="smoke", + password=password, + dbname="memory", + sslmode="disable", + connect_timeout=10, + autocommit=True, + ) as connection: + yield connection + finally: + if host.poll() is None: + host.stdin.write(f"select * from duckflight_stop('pgwire', {literal(address)});\n.quit\n") + host.stdin.flush() + try: + host.wait(timeout=10) + except subprocess.TimeoutExpired: + host.kill() + host.wait() + log.seek(0) + assert host.returncode == 0, log.read() + + +@pytest.fixture +def orders(postgres): + postgres.execute("create table orders(order_id integer, status varchar, amount decimal(10,2))") + postgres.execute("insert into orders values (1, 'complete', 100), (2, 'complete', 50), (3, 'pending', 75)") + postgres.execute(""" + MODEL orders FROM orders ( + primary key(order_id) + status + sum(amount) AS revenue + ) + """) + return postgres + + +def test_semantic_queries_and_postgres_metadata(orders): + postgres = orders + query = "select orders.status, orders.revenue from orders order by orders.status" + for prepared in (False, True): + result = postgres.execute(query, prepare=prepared) + assert [column.name for column in result.description] == ["status", "revenue"] + assert result.fetchall() == [("complete", Decimal("150.00")), ("pending", Decimal("75.00"))] + assert postgres.execute( + "select column_name from information_schema.columns where table_name = 'orders' order by ordinal_position" + ).fetchall() == [("order_id",), ("status",), ("amount",)] + + +@pytest.mark.parametrize("prepared", [False, True]) +def test_discovery_and_catalog_transactions(orders, prepared): + metrics = orders.execute("SHOW SEMANTIC METRICS FROM orders", prepare=prepared) + names = [column.name for column in metrics.description] + rows = [dict(zip(names, row)) for row in metrics.fetchall()] + assert [row["name"] for row in rows] == ["revenue"] + dimensions = orders.execute("SHOW SEMANTIC DIMENSIONS FROM orders", prepare=prepared) + names = [column.name for column in dimensions.description] + assert [dict(zip(names, row))["name"] for row in dimensions.fetchall()] == ["status"] + assert orders.execute("SHOW SEMANTIC RELATIONSHIPS FROM orders", prepare=prepared).fetchall() == [] + assert orders.execute("SHOW SEMANTIC SEGMENTS FROM orders", prepare=prepared).fetchall() == [] + assert orders.execute("DESCRIBE MODEL orders", prepare=prepared).fetchall() + snapshot = orders.execute("EXPORT SEMANTIC CATALOG", prepare=prepared).fetchone()[0] + assert isinstance(json.loads(snapshot), dict) + orders.execute("begin") + try: + orders.execute("drop model orders") + assert orders.execute("SHOW SEMANTIC MODELS").fetchall() == [] + finally: + orders.execute("rollback") + assert orders.execute("SHOW SEMANTIC MODELS").fetchall() + orders.execute("drop model orders") + orders.execute(f"IMPORT SEMANTIC CATALOG {literal(snapshot)}", prepare=prepared) + assert orders.execute("select revenue from orders").fetchall() == [(Decimal("225.00"),)] + + +@pytest.mark.parametrize("prepared", [False, True]) +def test_prefixless_bound_predicate_and_result_metadata(orders, prepared): + result = orders.execute( + "select orders.status, orders.revenue from orders where orders.status = %s", + ("pending",), + prepare=prepared, + ) + assert [column.name for column in result.description] == ["status", "revenue"] + assert [column.type_code for column in result.description] == [25, 1700] + assert result.fetchall() == [("pending", Decimal("75.00"))] + + +def test_native_model_prepare_and_describe_do_not_mutate(orders): + statement = b""" + MODEL prepared_orders FROM orders ( + primary key(order_id) + status + sum(amount) AS revenue + ) + """ + result = orders.pgconn.prepare(b"native_model", statement) + assert result.status == psycopg.pq.ExecStatus.COMMAND_OK, result.error_message.decode() + models = orders.execute("SHOW SEMANTIC MODELS") + names = [column.name for column in models.description] + assert "prepared_orders" not in [dict(zip(names, row))["name"] for row in models.fetchall()] + result = orders.pgconn.exec_prepared(b"native_model", []) + assert result.status in (psycopg.pq.ExecStatus.COMMAND_OK, psycopg.pq.ExecStatus.TUPLES_OK), ( + result.error_message.decode() + ) + assert orders.execute("select revenue from prepared_orders").fetchall() == [(Decimal("225.00"),)] + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v", "-o", "addopts="])) diff --git a/sidemantic-rs/include/sidemantic.h b/sidemantic-rs/include/sidemantic.h index 576cd3d5d..917a7ace6 100644 --- a/sidemantic-rs/include/sidemantic.h +++ b/sidemantic-rs/include/sidemantic.h @@ -70,6 +70,47 @@ typedef struct { SidemanticModelList sidemantic_snapshot_list_models(const char *snapshot); void sidemantic_free_model_list(SidemanticModelList result); +/* Rich catalog metadata. Optional strings are NULL when not declared. The + * semantic_type describes a metric/dimension; data_type is an authored logical + * type, not a guess about the physical result. definition is complete JSON. + */ +typedef struct { + char *kind; + char *model_name; + char *name; + char *qualified_name; + char *label; + char *description; + char *semantic_type; + char *data_type; + char *sql; + char *aggregation; + char *target_model; + char *relationship_type; + char *granularity; + bool is_public; + char *definition; +} SidemanticCatalogEntry; + +typedef struct { + SidemanticCatalogEntry *entries; + size_t count; + char *error; +} SidemanticCatalogEntries; + +/* Empty kind/model selects all. Non-NULL metric is a JSON array of model and + * metric identifiers and selects compatible dimensions. Results own all memory. + */ +SidemanticCatalogEntries sidemantic_snapshot_catalog(const char *snapshot, const char *kind, + const char *model, const char *metric); +void sidemantic_free_catalog_entries(SidemanticCatalogEntries result); + +/* Export returns a lossless versioned snapshot for the 'import' operation. Free + * the text result + * with sidemantic_free_result(); the text is in the result's sql member. + */ +SidemanticRewriteResult sidemantic_snapshot_export(const char *snapshot); + /* * Load semantic models from YAML string. * diff --git a/sidemantic-rs/src/config/sql_parser.rs b/sidemantic-rs/src/config/sql_parser.rs index 65c8ba970..2ae6ce6f1 100644 --- a/sidemantic-rs/src/config/sql_parser.rs +++ b/sidemantic-rs/src/config/sql_parser.rs @@ -1978,6 +1978,41 @@ fn build_metric(props: &HashMap) -> Option { metric.agg = None; } + // An expression containing authored aggregates is a calculation over + // grouped results, not a physical row input for the simple metric path. + // Keep non-aggregate expressions (e.g. `METRIC value AS amount`) as row + // expressions; the parser cannot infer semantic names from bare columns. + if metric.r#type == MetricType::Simple && metric.agg == Some(Aggregation::Expression) { + fn has_aggregate(value: &serde_json::Value) -> bool { + if value.get("window").is_some() || value.get("window_function").is_some() { + return false; + } + crate::core::is_aggregate_ast_node(value) + || match value { + serde_json::Value::Object(fields) => fields.values().any(has_aggregate), + serde_json::Value::Array(values) => values.iter().any(has_aggregate), + _ => false, + } + } + let aggregate = metric + .sql + .as_ref() + .and_then(|sql| { + crate::semantic_input::with_semantic_stack(|| { + let ast = crate::core::parse_semantic_expression(sql)?; + let ast = serde_json::to_value(ast) + .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + Ok(has_aggregate(&ast)) + }) + .ok() + }) + .unwrap_or(false); + if aggregate { + metric.r#type = MetricType::Derived; + metric.agg = None; + } + } + Some(metric) } @@ -2405,13 +2440,55 @@ mod tests { assert_eq!(amount_variance_pop.sql, Some("amount".to_string())); let approximate_customers = model.get_metric("approximate_customers").unwrap(); - assert_eq!(approximate_customers.agg, Some(Aggregation::Expression)); + assert_eq!(approximate_customers.r#type, MetricType::Derived); + assert_eq!(approximate_customers.agg, None); assert_eq!( approximate_customers.sql, Some("APPROX_COUNT_DISTINCT(customer_id)".to_string()) ); } + #[test] + fn inline_aggregate_calculations_preserve_raw_row_expressions() { + let model = parse_sql_model( + "MODEL (name orders, table raw_orders); + METRIC doubled AS SUM(amount) * 2; + METRIC rows_plus_one AS COUNT(*) + 1; + METRIC raw_amount AS amount; + METRIC raw_doubled AS amount * 2; + METRIC raw_window AS SUM(amount) OVER ();", + ) + .unwrap(); + for name in ["doubled", "rows_plus_one"] { + let metric = model.get_metric(name).unwrap(); + assert_eq!(metric.r#type, MetricType::Derived); + assert_eq!(metric.agg, None); + } + for name in ["raw_amount", "raw_doubled", "raw_window"] { + let metric = model.get_metric(name).unwrap(); + assert_eq!(metric.r#type, MetricType::Simple); + assert_eq!(metric.agg, Some(Aggregation::Expression)); + } + } + + #[test] + fn inline_aggregate_classification_runs_off_small_host_stacks() { + std::thread::Builder::new() + .stack_size(512 * 1024) + .spawn(|| { + let sql = "MODEL (name orders, table raw_orders); + METRIC doubled AS SUM(CASE WHEN amount > 0 THEN amount ELSE 0 END) * 2;"; + let model = parse_sql_model(sql).unwrap(); + assert_eq!( + model.get_metric("doubled").unwrap().r#type, + MetricType::Derived + ); + }) + .unwrap() + .join() + .unwrap(); + } + #[test] fn test_parse_compact_sql_model() { let sql = r#" diff --git a/sidemantic-rs/src/ffi/catalog.rs b/sidemantic-rs/src/ffi/catalog.rs index d6da5b0ca..56a14a210 100644 --- a/sidemantic-rs/src/ffi/catalog.rs +++ b/sidemantic-rs/src/ffi/catalog.rs @@ -21,10 +21,12 @@ use crate::core::{Metric, Model, Parameter, SemanticGraph, TableCalculation}; use crate::sql::QueryRewriter; use super::{ - active_model_for_loaded_models, add_item_definition, semantic_error, semantic_result, - sidemantic_free, FfiState, SidemanticRewriteResult, + active_model_for_loaded_models, semantic_error, semantic_result, sidemantic_free, FfiState, + SidemanticRewriteResult, }; +mod api; + type CatalogResult = Result; #[derive(Default, Serialize, Deserialize)] @@ -272,6 +274,645 @@ pub extern "C" fn sidemantic_snapshot_load_sources( }) } +// DROP operates on declarations, then reconstructs all indexes before publishing. +// The boolean slot in snapshot_apply means IF EXISTS for drop operations. +fn drop_definition( + original: &str, + active: &str, + kind: &str, + content: &str, + if_exists: bool, +) -> CatalogResult<(String, String)> { + let mut snapshot = Snapshot::parse(original)?; + let names: Vec = serde_json::from_str(content).map_err(|e| e.to_string())?; + let (model_name, field) = + match (kind, names.as_slice()) { + ("model", [model]) => (model.as_str(), None), + ("metric" | "dimension" | "segment", [model, field]) => { + (model.as_str(), Some(field.as_str())) + } + ("metric" | "dimension" | "segment", [field]) if !active.is_empty() => { + (active, Some(field.as_str())) + } + _ => return Err( + "DROP requires a model name or a qualified model.field name (or an active model)" + .into(), + ), + }; + let missing = || { + if if_exists { + Ok((original.to_owned(), active.to_owned())) + } else { + Err(format!("{kind} '{}' not found", names.join("."))) + } + }; + let Some(index) = snapshot + .models + .iter() + .position(|m| m.name.eq_ignore_ascii_case(model_name)) + else { + return missing(); + }; + let model = &snapshot.models[index]; + let canonical_field = match kind { + "metric" => model + .metrics + .iter() + .map(|f| &f.name) + .find(|n| n.eq_ignore_ascii_case(field.unwrap())), + "dimension" => model + .dimensions + .iter() + .map(|f| &f.name) + .find(|n| n.eq_ignore_ascii_case(field.unwrap())), + "segment" => model + .segments + .iter() + .map(|f| &f.name) + .find(|n| n.eq_ignore_ascii_case(field.unwrap())), + _ => None, + }; + if field.is_some() && canonical_field.is_none() { + return missing(); + } + let target_model = model.clone(); + let target_field = canonical_field.cloned(); + if let Some(field) = &target_field { + let model = &mut snapshot.models[index]; + match kind { + "metric" => model.metrics.retain(|f| &f.name != field), + "dimension" => model.dimensions.retain(|f| &f.name != field), + "segment" => model.segments.retain(|f| &f.name != field), + _ => unreachable!(), + } + } else { + snapshot.models.remove(index); + } + restrict_drop(&snapshot, &target_model, target_field.as_deref())?; + let graph = snapshot.into_graph()?; + let json = serde_json::to_string(&Snapshot::from_graph(&graph)).map_err(|e| e.to_string())?; + let active = if target_field.is_none() && active.eq_ignore_ascii_case(&target_model.name) { + "" + } else { + active + }; + Ok((json, active.to_owned())) +} + +fn restrict_drop( + snapshot: &Snapshot, + removed_model: &Model, + target_field: Option<&str>, +) -> CatalogResult<()> { + let target_model = removed_model.name.as_str(); + let target = target_field.map_or_else( + || target_model.to_owned(), + |f| format!("{target_model}.{f}"), + ); + let matches = |model: Option<&str>, field: &str, context: Option<&str>, bare: bool| { + let field_matches = target_field.is_none_or(|target| target.eq_ignore_ascii_case(field)); + field_matches + && match model { + Some(model) => model.eq_ignore_ascii_case(target_model), + None => { + bare && match context { + Some(model) => model.eq_ignore_ascii_case(target_model), + None => removed_model + .metrics + .iter() + .map(|f| &f.name) + .chain(removed_model.dimensions.iter().map(|f| &f.name)) + .chain(removed_model.segments.iter().map(|f| &f.name)) + .any(|name| name.eq_ignore_ascii_case(field)), + } + } + } + }; + let reject = |owner: &str| { + Err(format!( + "cannot drop '{target}': dependent definition '{owner}' (RESTRICT)" + )) + }; + let reference = |value: &str, context: Option<&str>, owner: &str| -> CatalogResult<()> { + let (model, field) = value + .rsplit_once('.') + .map_or((None, value), |(m, f)| (Some(m), f)); + if matches(model, field, context, true) { + reject(owner) + } else { + Ok(()) + } + }; + let expression = + |sql: &str, context: Option<&str>, bare: bool, owner: &str| -> CatalogResult<()> { + // These documented row-alias placeholders are physical references. + // Other templates remain unprovable and are rejected explicitly. + let sql = sql + .replace("{model}", "__physical_row") + .replace("${CUBE}", "__physical_row"); + let refs = match crate::core::semantic_column_references(&sql) { + Ok(refs) => refs, + Err(_) => { + if drop_sql_dependency(&sql, removed_model, target_field, context, bare, false) + .map_err(|e| { + format!( + "cannot drop '{target}': cannot prove dependencies of '{owner}': {e}" + ) + })? + { + return reject(owner); + } + return Ok(()); + } + }; + for column in refs { + // Aggregate arguments and unqualified row expressions are physical + // columns, even when a semantic field has the same spelling. + if (target_field.is_none() || !column.aggregate_input) + && matches(column.model.as_deref(), &column.field, context, bare) + { + return reject(owner); + } + } + Ok(()) + }; + let metric = |metric: &Metric, context: Option<&str>, owner: &str| -> CatalogResult<()> { + let semantic_expression = + metric.agg.is_none() || metric.agg == Some(crate::core::Aggregation::Expression); + for value in [ + &metric.base_metric, + &metric.numerator, + &metric.denominator, + &metric.extends, + &metric.non_additive_dimension, + ] + .into_iter() + .flatten() + { + reference(value, context, owner)?; + } + for value in [ + &metric.entity_dimensions, + &metric.non_additive_window_groupings, + &metric.drill_fields, + ] + .into_iter() + .flatten() + .flatten() + { + reference(value, context, owner)?; + } + if let Some(sql) = metric + .sql + .as_ref() + .filter(|_| semantic_expression || target_field.is_none()) + { + expression(sql, context, semantic_expression, owner)?; + } + for sql in metric + .filters + .iter() + .chain(metric.having.iter()) + .chain(metric.window_expression.iter()) + .chain(metric.window_order.iter()) + .chain(metric.base_event.iter()) + .chain(metric.conversion_event.iter()) + .chain(metric.cohort_event.iter()) + .chain(metric.activity_event.iter()) + .chain(metric.steps.iter().flatten()) + { + expression(sql, context, false, owner)?; + } + Ok(()) + }; + for model in &snapshot.models { + let context = Some(model.name.as_str()); + if target_field.is_none() + && (model + .extends + .as_deref() + .is_some_and(|v| v.eq_ignore_ascii_case(target_model)) + || model.relationships.iter().any(|r| { + r.related_model().eq_ignore_ascii_case(target_model) + || r.through + .as_deref() + .is_some_and(|v| v.eq_ignore_ascii_case(target_model)) + })) + { + return reject(&model.name); + } + if let Some(sql) = &model.sql { + if drop_sql_dependency(sql, removed_model, target_field, None, false, true).map_err( + |e| { + format!( + "cannot drop '{target}': cannot prove dependencies of SQL model '{}': {e}", + model.name + ) + }, + )? { + return reject(&model.name); + } + } + if let Some(time) = &model.default_time_dimension { + reference(time, context, &model.name)?; + } + for dimension in &model.dimensions { + let owner = format!("{}.{}", model.name, dimension.name); + if let Some(parent) = &dimension.parent { + reference(parent, context, &owner)?; + } + for sql in dimension.sql.iter().chain(dimension.window.iter()) { + expression(sql, context, false, &owner)?; + } + } + for item in &model.metrics { + metric(item, context, &format!("{}.{}", model.name, item.name))?; + } + for segment in &model.segments { + expression( + &segment.sql, + context, + false, + &format!("{}.{}", model.name, segment.name), + )?; + } + for relationship in &model.relationships { + if let Some(sql) = &relationship.sql { + expression( + sql, + context, + false, + &format!("{}.{}", model.name, relationship.name), + )?; + } + } + for preagg in &model.pre_aggregations { + let owner = format!("{}.{}", model.name, preagg.name); + for value in preagg + .measures + .iter() + .flatten() + .chain(preagg.dimensions.iter().flatten()) + .chain(preagg.time_dimension.iter()) + { + reference(value, context, &owner)?; + } + if let Some(sql) = &preagg.sql { + if drop_sql_dependency(sql, removed_model, target_field, None, false, true) + .map_err(|e| format!("cannot drop '{target}': cannot prove dependencies of SQL pre-aggregation '{owner}': {e}"))? { + return reject(&owner); + } + } + } + } + for item in &snapshot.metrics { + metric(item, None, &item.name)?; + } + for calc in &snapshot.table_calculations { + for value in calc + .field + .iter() + .chain(calc.partition_by.iter().flatten()) + .chain(calc.order_by.iter().flatten()) + { + reference(value, None, &calc.name)?; + } + if let Some(sql) = &calc.expression { + expression(sql, None, true, &calc.name)?; + } + } + Ok(()) +} + +// Query dependencies need SQL scope: a physical table aliased to a model name +// and a CTE with that name must not create semantic dependencies. The serialized +// AST includes typed expression children omitted by polyglot's public walker. +fn drop_sql_dependency( + sql: &str, + model: &Model, + field: Option<&str>, + context: Option<&str>, + bare: bool, + query: bool, +) -> CatalogResult { + use serde_json::Value; + use std::collections::{HashMap, HashSet}; + + type Scope = HashMap; + struct References<'a> { + model: &'a Model, + field: Option<&'a str>, + context: Option<&'a str>, + bare: bool, + } + impl References<'_> { + fn source(&self, node: &Value, scope: &mut Scope, ctes: &HashSet) -> bool { + if let Some(table) = node.get("table") { + let name = table["name"]["name"].as_str().unwrap_or(""); + let target = table["schema"].is_null() + && table["catalog"].is_null() + && name.eq_ignore_ascii_case(&self.model.name) + && !ctes.contains(&name.to_ascii_lowercase()); + let alias = table["alias"]["name"].as_str().unwrap_or(name); + scope.insert(alias.to_ascii_lowercase(), target); + return target && self.field.is_none(); + } + if let Some(subquery) = node.get("subquery") { + let found = self.visit(&subquery["this"], scope, ctes, false); + if let Some(alias) = subquery["alias"]["name"].as_str() { + scope.insert(alias.to_ascii_lowercase(), false); + } + return found; + } + if let Some(alias) = node.get("alias") { + let mut child_scope = Scope::new(); + let found = self.source(&alias["this"], &mut child_scope, ctes); + if let Some(name) = alias["alias"]["name"].as_str() { + scope.insert(name.to_ascii_lowercase(), child_scope.values().any(|v| *v)); + } + return found; + } + match node { + Value::Array(values) => { + let mut found = false; + for child in values { + found |= self.source(child, scope, ctes); + } + found + } + Value::Object(values) => { + let mut found = false; + for child in values.values() { + found |= self.source(child, scope, ctes); + } + found + } + _ => false, + } + } + + fn visit( + &self, + node: &Value, + scope: &Scope, + ctes: &HashSet, + aggregate: bool, + ) -> bool { + let Value::Object(fields) = node else { + return node.as_array().is_some_and(|children| { + children + .iter() + .any(|child| self.visit(child, scope, ctes, aggregate)) + }); + }; + let mut ctes = ctes.clone(); + if let Some(with) = fields.get("with").filter(|value| !value.is_null()) { + for cte in with["ctes"].as_array().into_iter().flatten() { + let name = cte["alias"]["name"] + .as_str() + .unwrap_or("") + .to_ascii_lowercase(); + if with["recursive"].as_bool() == Some(true) { + ctes.insert(name.clone()); + } + if self.visit(&cte["this"], scope, &ctes, false) { + return true; + } + ctes.insert(name); + } + } + if let Some(select) = fields.get("select") { + // Process WITH before collecting sources so CTEs shadow model names. + let mut select = select.clone(); + if let Some(with) = select.get_mut("with") { + if let Some(definitions) = with["ctes"].as_array() { + for cte in definitions { + let name = cte["alias"]["name"] + .as_str() + .unwrap_or("") + .to_ascii_lowercase(); + if with["recursive"].as_bool() == Some(true) { + ctes.insert(name.clone()); + } + if self.visit(&cte["this"], scope, &ctes, false) { + return true; + } + ctes.insert(name); + } + } + *with = Value::Null; + } + let mut scope = scope.clone(); + if self.source(&select["from"], &mut scope, &ctes) { + return true; + } + for join in select["joins"].as_array().into_iter().flatten() { + if self.source(&join["this"], &mut scope, &ctes) { + return true; + } + } + return self.visit(&select, &scope, &ctes, false); + } + let aggregate = aggregate + || crate::core::is_aggregate_ast_node(node) + || fields.contains_key("window") + || fields.contains_key("window_function"); + if let Some(column) = fields.get("column") { + let name = column["name"]["name"].as_str().unwrap_or(""); + if self + .field + .is_some_and(|field| !field.eq_ignore_ascii_case(name)) + || (aggregate && self.field.is_some()) + { + return false; + } + if let Some(table) = column["table"]["name"].as_str() { + return scope + .get(&table.to_ascii_lowercase()) + .copied() + .unwrap_or_else(|| { + table.eq_ignore_ascii_case(&self.model.name) + && !ctes.contains(&table.to_ascii_lowercase()) + }); + } + if !scope.is_empty() { + return scope.values().any(|target| *target); + } + return self.bare + && self.context.map_or_else( + || { + self.model + .metrics + .iter() + .any(|f| f.name.eq_ignore_ascii_case(name)) + || self + .model + .dimensions + .iter() + .any(|f| f.name.eq_ignore_ascii_case(name)) + || self + .model + .segments + .iter() + .any(|f| f.name.eq_ignore_ascii_case(name)) + }, + |context| context.eq_ignore_ascii_case(&self.model.name), + ); + } + if fields.contains_key("star") && !aggregate { + return scope.values().any(|target| *target); + } + fields + .iter() + .filter(|(key, _)| key.as_str() != "with") + .any(|(_, child)| self.visit(child, scope, &ctes, aggregate)) + } + } + let input = if query { + sql.to_owned() + } else { + format!("SELECT {sql}") + }; + match crate::semantic_input::dialects::parse(&input, polyglot_sql::DialectType::DuckDB) { + Ok(ast) => { + let ast = serde_json::to_value(ast).map_err(|e| e.to_string())?; + Ok(References { + model, + field, + context, + bare, + } + .visit(&ast, &Scope::new(), &HashSet::new(), false)) + } + Err(error) => { + // Unsupported syntax cannot establish a dependency, but must not + // freeze unrelated definitions. Tokenize only to decide whether the + // unresolved scope could mention this target; never accept it as a + // proven dependency or scan literals/comments as identifiers. + use polyglot_sql::tokens::TokenType; + let tokens = polyglot_sql::Dialect::get(polyglot_sql::DialectType::DuckDB) + .tokenize(sql) + .map_err(|e| e.to_string())?; + let identifier = |name: &str| { + tokens.iter().any(|token| { + matches!( + token.token_type, + TokenType::Identifier | TokenType::QuotedIdentifier | TokenType::Var + ) && token.text.eq_ignore_ascii_case(name) + }) + }; + let model_mentioned = identifier(&model.name); + let in_context = + context.is_none_or(|context| context.eq_ignore_ascii_case(&model.name)); + let possible = field.map_or(model_mentioned, |field| { + (identifier(field) && (model_mentioned || bare && in_context)) + || model_mentioned + && tokens + .iter() + .any(|token| token.token_type == TokenType::Star) + }); + if possible { + Err(error.to_string()) + } else { + Ok(false) + } + } + } +} + +// Read a declaration identifier without splitting quoted names on dots/spaces. +fn declaration_identifier(input: &str) -> Option<(String, &str)> { + let input = input.trim_start(); + let first = input.chars().next()?; + if matches!(first, '\'' | '"' | '`') { + let mut value = String::new(); + let mut chars = input.char_indices().skip(1).peekable(); + while let Some((index, character)) = chars.next() { + if character == first { + if chars.peek().is_some_and(|(_, next)| *next == first) { + chars.next(); + value.push(first); + } else { + return Some((value, &input[index + character.len_utf8()..])); + } + } else { + value.push(character); + } + } + None + } else { + let end = input + .find(|c: char| c.is_whitespace() || matches!(c, '.' | '(' | ')' | ';')) + .unwrap_or(input.len()); + (end > 0).then(|| (input[..end].to_owned(), &input[end..])) + } +} + +fn apply_item(state: &mut FfiState, content: &str, replace: bool) -> CatalogResult<()> { + let (keyword, body) = declaration_identifier(content).ok_or("missing definition kind")?; + let mut model_name = state.active_model.clone(); + let mut adjusted = content.to_owned(); + let mut declared_name = None; + if let Some((first, remainder)) = declaration_identifier(body) { + let (field, rest) = if let Some(remainder) = remainder.trim_start().strip_prefix('.') { + let (field, rest) = + declaration_identifier(remainder).ok_or("missing qualified field name")?; + model_name = Some(first); + (field, rest) + } else { + (first, remainder) + }; + declared_name = Some(field); + adjusted = if let Some(properties) = rest.trim_start().strip_prefix('(') { + format!("{keyword} (name __field, {properties}") + } else { + format!("{keyword} __field {rest}") + }; + } + let model_name = model_name.ok_or("no active model; use a qualified model.field name")?; + let mut model = state + .graph + .models() + .find(|model| model.name.eq_ignore_ascii_case(&model_name)) + .cloned() + .ok_or_else(|| format!("model '{model_name}' not found"))?; + let parsed = parse_sql_model(&format!( + "MODEL (name __definition, table dummy);\n{adjusted}" + )) + .map_err(|e| format!("parsing definition: {e}"))?; + // Preserve canonical spelling on replacement and reject case-only duplicates, + // matching DuckDB's identifier contract, including quoted identifiers. + macro_rules! merge_items { + ($items:ident, $kind:literal) => { + for mut item in parsed.$items { + if let Some(name) = &declared_name { + item.name = name.clone(); + } + if let Some(existing) = model + .$items + .iter() + .position(|old| old.name.eq_ignore_ascii_case(&item.name)) + { + if !replace { + return Err(format!("duplicate {} '{}'", $kind, item.name)); + } + item.name = model.$items[existing].name.clone(); + model.$items[existing] = item; + } else { + model.$items.push(item); + } + } + }; + } + merge_items!(metrics, "metric"); + merge_items!(dimensions, "dimension"); + merge_items!(segments, "segment"); + state + .graph + .replace_model(model) + .map_err(|e| format!("updating model: {e}")) +} + fn apply( snapshot: &str, active_model: &str, @@ -279,6 +920,47 @@ fn apply( content: &str, replace: bool, ) -> CatalogResult<(String, String)> { + // DuckDB workers can have only 512 KiB of stack. Keep dependency parsing, + // AST walking/serialization, and graph reconstruction on the established + // semantic worker; nested compiler operations reuse that protected stack. + crate::semantic_input::with_semantic_stack(|| { + Ok(apply_on_semantic_worker( + snapshot, + active_model, + operation, + content, + replace, + )) + }) + .map_err(|error| error.to_string())? +} + +fn apply_on_semantic_worker( + snapshot: &str, + active_model: &str, + operation: &str, + content: &str, + replace: bool, +) -> CatalogResult<(String, String)> { + if let Some(kind) = operation.strip_prefix("drop_") { + return drop_definition(snapshot, active_model, kind, content, replace); + } + if operation == "import" { + let incoming = Snapshot::parse(content)?; + let active = if incoming.models.len() == 1 { + incoming.models[0].name.clone() + } else { + String::new() + }; + let incoming = Snapshot::from_graph(&incoming.into_graph()?); + let mut merged = Snapshot::parse(snapshot)?; + merged.merge(incoming); + let graph = merged.into_graph()?; + return Ok(( + serde_json::to_string(&Snapshot::from_graph(&graph)).map_err(|e| e.to_string())?, + active, + )); + } let mut state = FfiState { graph: Snapshot::parse(snapshot)?.into_graph()?, active_model: if active_model.is_empty() { @@ -289,7 +971,18 @@ fn apply( }; match operation { "model" => { - let model = parse_sql_model(content).map_err(|e| format!("parsing definition: {e}"))?; + let mut model = + parse_sql_model(content).map_err(|e| format!("parsing definition: {e}"))?; + if let Some(existing) = state + .graph + .models() + .find(|old| old.name.eq_ignore_ascii_case(&model.name)) + { + if !replace { + return Err(format!("model '{}' already exists", model.name)); + } + model.name = existing.name.clone(); + } let name = model.name.clone(); if replace { state.graph.replace_model(model) @@ -300,14 +993,16 @@ fn apply( state.active_model = Some(name); } "item" => { - add_item_definition(&mut state, content, replace)?; + apply_item(&mut state, content, replace)?; } "use" => { let name = content.trim(); - if state.graph.get_model(name).is_none() { - return Err(format!("model '{name}' not found")); - } - state.active_model = Some(name.to_owned()); + let model = state + .graph + .models() + .find(|model| model.name.eq_ignore_ascii_case(name)) + .ok_or_else(|| format!("model '{name}' not found"))?; + state.active_model = Some(model.name.clone()); } "yaml" | "file" | "legacy_sql" => { let loaded = match operation { @@ -541,6 +1236,440 @@ mod tests { .unwrap() } + #[test] + fn qualified_lifecycle_preserves_active_model_and_quoted_names() { + let (snapshot, _) = orders(); + let (mut snapshot, active) = apply_ffi( + &snapshot, + "orders", + "model", + "MODEL (name other, table other_rows);", + false, + ) + .unwrap(); + for (kind, definition, field) in [ + ( + "metric", + "METRIC \"ORDERS\".\"Gross amount\" AS SUM(amount)", + "Gross amount", + ), + ( + "dimension", + "DIMENSION ORDERS.\"sales.region\" AS region", + "sales.region", + ), + ( + "segment", + "SEGMENT orders.\"Open orders\" AS status = 'open'", + "Open orders", + ), + ] { + for replace in [false, true] { + let result = apply_ffi(&snapshot, &active, "item", definition, replace).unwrap(); + snapshot = result.0; + assert_eq!(result.1, "other"); + } + let target = serde_json::to_string(&["ORDERS", field]).unwrap(); + let result = + apply_ffi(&snapshot, &active, &format!("drop_{kind}"), &target, false).unwrap(); + snapshot = result.0; + assert_eq!(result.1, "other"); + assert!( + apply_ffi(&snapshot, &active, &format!("drop_{kind}"), &target, false) + .unwrap_err() + .contains("not found") + ); + assert_eq!( + apply_ffi(&snapshot, &active, &format!("drop_{kind}"), &target, true) + .unwrap() + .0, + snapshot + ); + } + let (snapshot, active) = + apply_ffi(&snapshot, &active, "drop_model", "[\"OTHER\"]", false).unwrap(); + assert!(active.is_empty()); + assert_eq!(Snapshot::parse(&snapshot).unwrap().models.len(), 1); + } + + #[test] + fn qualified_replacement_compiles_inline_aggregates_as_grouped_calculations() { + let (snapshot, active) = orders(); + let (snapshot, _) = apply_ffi( + &snapshot, + &active, + "item", + "METRIC orders.\"Gross amount\" AS SUM(amount)", + false, + ) + .unwrap(); + let (snapshot, _) = apply_ffi( + &snapshot, + &active, + "item", + "METRIC orders.\"Gross amount\" AS SUM(amount) * 2", + true, + ) + .unwrap(); + let query = "SELECT orders.\"Gross amount\" FROM orders"; + let actual = rewrite(&snapshot, query).unwrap(); + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("raw_orders") + .with_metric(Metric::derived("Gross amount", "SUM(amount) * 2")), + ) + .unwrap(); + let expected = QueryRewriter::new(&graph) + .rewrite_with_dialect(query, polyglot_sql::DialectType::DuckDB) + .unwrap(); + assert_eq!(actual, expected); + assert!(actual.contains("SUM("), "{actual}"); + assert!(!actual.contains("SELECT id,"), "{actual}"); + } + + #[test] + fn lifecycle_dependency_parsing_runs_off_small_host_stacks() { + std::thread::Builder::new() + .stack_size(512 * 1024) + .spawn(|| { + let (snapshot, active) = orders(); + let calculated = "METRIC orders.calculated AS + SUM(CASE WHEN amount > 0 THEN amount ELSE 0 END) * 2"; + let (snapshot, _) = + apply_ffi(&snapshot, &active, "item", calculated, false).unwrap(); + let (snapshot, _) = apply_ffi( + &snapshot, + &active, + "item", + "METRIC orders.doubled AS revenue * 2", + false, + ) + .unwrap(); + // Scalar dependency analysis must reject safely on a DuckDB-sized + // stack, and the worker must remain usable after that error. + let revenue = r#"["orders","revenue"]"#; + let doubled = r#"["orders","doubled"]"#; + let calculated = r#"["orders","calculated"]"#; + let error = + apply_ffi(&snapshot, &active, "drop_metric", revenue, false).unwrap_err(); + assert!(error.contains("dependent definition")); + let (snapshot, _) = + apply_ffi(&snapshot, &active, "drop_metric", doubled, false).unwrap(); + // The SQL-source scope walker and unsupported-expression fallback + // use the same protected boundary, including their AST traversal. + let source = r#" +models: + - name: source + sql: SELECT o.revenue FROM orders o + primary_key: id +"#; + let (snapshot, _) = apply_ffi(&snapshot, &active, "yaml", source, false).unwrap(); + let (snapshot, _) = + apply_ffi(&snapshot, &active, "drop_metric", calculated, false).unwrap(); + let error = + apply_ffi(&snapshot, &active, "drop_metric", revenue, false).unwrap_err(); + assert!(error.contains("source")); + let malformed = r#" +models: + - name: unsupported + table: raw_other + primary_key: id + metrics: + - name: invalid_sql + type: derived + sql: 'unrelated_value + (' +"#; + let (snapshot, _) = + apply_ffi(&snapshot, &active, "yaml", malformed, false).unwrap(); + assert!( + apply_ffi(&snapshot, &active, "drop_model", r#"["source"]"#, false).is_ok() + ); + }) + .unwrap() + .join() + .unwrap(); + } + + #[test] + fn drop_restrict_checks_dependencies_and_preserves_original_snapshot() { + let (original, active) = orders(); + for definition in [ + "METRIC doubled AS revenue * 2;", + "METRIC (name yoy, type time_comparison, base_metric revenue, comparison_type yoy);", + "SEGMENT valuable AS orders.revenue > 10;", + ] { + let (snapshot, _) = apply_ffi(&original, &active, "item", definition, false).unwrap(); + let error = apply_ffi( + &snapshot, + &active, + "drop_metric", + "[\"orders\",\"revenue\"]", + false, + ) + .unwrap_err(); + assert!(error.contains("dependent definition"), "{error}"); + assert!(rewrite(&snapshot, "SELECT orders.revenue FROM orders") + .unwrap() + .contains("SUM(")); + } + let (snapshot, _) = apply_ffi( + &original, + &active, + "item", + "DIMENSION amount AS amount", + false, + ) + .unwrap(); + let (snapshot, _) = apply_ffi( + &snapshot, + &active, + "drop_dimension", + "[\"orders\",\"amount\"]", + false, + ) + .unwrap(); + assert!(rewrite(&snapshot, "SELECT orders.revenue FROM orders") + .unwrap() + .contains("SUM(")); + let mut declared = Snapshot::parse(&original).unwrap(); + declared + .metrics + .push(Metric::derived("global_total", "orders.revenue")); + let snapshot = serde_json::to_string(&declared).unwrap(); + for (operation, target) in [ + ("drop_metric", "[\"orders\",\"revenue\"]"), + ("drop_model", "[\"orders\"]"), + ] { + assert!(apply_ffi(&snapshot, &active, operation, target, false) + .unwrap_err() + .contains("global_total")); + } + declared.metrics.clear(); + declared.models.push( + Model::new("customers", "id") + .with_table("raw_customers") + .with_relationship(crate::core::Relationship::many_to_one("orders")), + ); + let snapshot = serde_json::to_string(&declared).unwrap(); + assert!( + apply_ffi(&snapshot, &active, "drop_model", "[\"orders\"]", false) + .unwrap_err() + .contains("customers") + ); + } + + #[test] + fn snapshot_import_is_lossless_and_validates_before_merge() { + let (snapshot, _) = orders(); + let (restored, active) = apply_ffi("", "", "import", &snapshot, false).unwrap(); + assert_eq!(snapshot, restored); + assert_eq!(active, "orders"); + assert!(apply_ffi(&snapshot, &active, "import", "{\"version\":2}", false).is_err()); + } + + #[test] + fn drop_checks_dimension_calculation_and_dynamic_sql_dependencies() { + let (snapshot, active) = orders(); + let mut declaration = Snapshot::parse(&snapshot).unwrap(); + declaration.models[0] + .dimensions + .push(Dimension::time("created_at")); + declaration.models[0].default_time_dimension = Some("created_at".into()); + let with_time = serde_json::to_string(&declaration).unwrap(); + assert!(apply_ffi( + &with_time, + &active, + "drop_dimension", + "[\"orders\",\"created_at\"]", + false + ) + .unwrap_err() + .contains("dependent definition")); + declaration.models[0].default_time_dimension = None; + declaration.table_calculations.push( + TableCalculation::new("running_revenue", TableCalcType::RunningTotal) + .with_field("revenue"), + ); + let with_calculation = serde_json::to_string(&declaration).unwrap(); + assert!(apply_ffi( + &with_calculation, + &active, + "drop_metric", + "[\"orders\",\"revenue\"]", + false + ) + .unwrap_err() + .contains("running_revenue")); + declaration.table_calculations.clear(); + declaration + .models + .push(Model::new("dynamic_source", "id").with_sql("SELECT * FROM orders")); + let with_sql = serde_json::to_string(&declaration).unwrap(); + assert!( + apply_ffi(&with_sql, &active, "drop_model", "[\"orders\"]", false) + .unwrap_err() + .contains("dependent definition") + ); + } + + #[test] + fn drop_sql_sources_respect_aliases_ctes_and_physical_inputs() { + let model = Model::new("orders", "id") + .with_metric(Metric::sum("revenue", "amount")) + .with_dimension(Dimension::categorical("amount")); + for (sql, model_dependency, revenue_dependency) in [ + ("SELECT r.revenue FROM raw_orders r", false, false), + ("SELECT orders.revenue FROM raw_orders orders", false, false), + ( + "WITH orders AS (SELECT revenue FROM raw_orders) SELECT orders.revenue FROM orders", + false, + false, + ), + ("SELECT o.revenue FROM orders o", true, true), + ("SELECT revenue FROM orders", true, true), + ("SELECT * FROM orders", true, true), + ( + "WITH totals AS (SELECT o.revenue FROM orders o) SELECT * FROM totals", + true, + true, + ), + ( + "SELECT * FROM (SELECT o.revenue FROM orders o) totals", + true, + true, + ), + ( + "SELECT * FROM (SELECT revenue FROM raw_orders) orders", + false, + false, + ), + ("SELECT sum(o.amount) FROM orders o", true, false), + ("SELECT 'orders.revenue' FROM raw_orders", false, false), + ("SELECT o.revenue FROM main.orders o", false, false), + ] { + assert_eq!( + drop_sql_dependency(sql, &model, None, None, false, true).unwrap(), + model_dependency, + "{sql}" + ); + assert_eq!( + drop_sql_dependency(sql, &model, Some("revenue"), None, false, true).unwrap(), + revenue_dependency, + "{sql}" + ); + } + assert!(!drop_sql_dependency( + "SELECT sum(o.amount) FROM orders o", + &model, + Some("amount"), + None, + false, + true + ) + .unwrap()); + let quoted = + Model::new("Orders Archive", "id").with_metric(Metric::sum("Gross Amount", "amount")); + assert!(drop_sql_dependency( + "SELECT o.\"Gross Amount\" FROM \"Orders Archive\" o", + "ed, + Some("Gross Amount"), + None, + false, + true + ) + .unwrap()); + } + + #[test] + fn unrelated_sql_definitions_do_not_block_drop() { + let (original, active) = orders(); + let mut declaration = Snapshot::parse(&original).unwrap(); + declaration.models[0].metrics.push(Metric::count("unused")); + declaration + .models + .push(Model::new("sql_source", "id").with_sql("SELECT id, amount FROM raw_source")); + declaration + .models + .push(Model::new("unrelated", "id").with_table("raw_unrelated")); + declaration.models[2].pre_aggregations.push(serde_json::from_value(serde_json::json!({ + "name": "physical_rollup", "type": "original_sql", "sql": "SELECT sum(amount) FROM raw_rollup" + })).unwrap()); + let snapshot = serde_json::to_string(&declaration).unwrap(); + assert!(apply_ffi( + &snapshot, + &active, + "drop_metric", + "[\"orders\",\"unused\"]", + false + ) + .is_ok()); + assert!(apply_ffi(&snapshot, &active, "drop_model", "[\"unrelated\"]", false).is_ok()); + declaration.models[2].pre_aggregations[0].sql = + Some("SELECT o.unused FROM orders o".into()); + let with_dependency = serde_json::to_string(&declaration).unwrap(); + assert!(apply_ffi( + &with_dependency, + &active, + "drop_metric", + "[\"orders\",\"unused\"]", + false + ) + .unwrap_err() + .contains("physical_rollup")); + declaration.models[2].pre_aggregations.clear(); + declaration.models[1].sql = Some("SELECT o.revenue FROM orders o".into()); + let snapshot = serde_json::to_string(&declaration).unwrap(); + assert!(apply_ffi( + &snapshot, + &active, + "drop_metric", + "[\"orders\",\"unused\"]", + false + ) + .is_ok()); + assert!(apply_ffi( + &snapshot, + &active, + "drop_metric", + "[\"orders\",\"revenue\"]", + false + ) + .unwrap_err() + .contains("sql_source")); + // An unrelated unsupported expression must not freeze catalog changes; + // an unsupported expression mentioning this metric still fails closed. + declaration.models[1].sql = None; + declaration.models[1].table = Some("raw_source".into()); + declaration.models[1] + .metrics + .push(Metric::derived("unsupported", "unrelated_value + (")); + let snapshot = serde_json::to_string(&declaration).unwrap(); + assert!(apply_ffi( + &snapshot, + &active, + "drop_metric", + "[\"orders\",\"unused\"]", + false + ) + .is_ok()); + declaration.models[1].metrics[0].sql = Some("orders.unused + (".into()); + let snapshot = serde_json::to_string(&declaration).unwrap(); + let error = apply_ffi( + &snapshot, + &active, + "drop_metric", + "[\"orders\",\"unused\"]", + false, + ) + .unwrap_err(); + assert!( + error.contains("cannot prove dependencies") || error.contains("dependent definition"), + "{error}" + ); + } + fn load_sources( snapshot: &str, sources: &[(&str, &str)], diff --git a/sidemantic-rs/src/ffi/catalog/api.rs b/sidemantic-rs/src/ffi/catalog/api.rs new file mode 100644 index 000000000..2323e6a64 --- /dev/null +++ b/sidemantic-rs/src/ffi/catalog/api.rs @@ -0,0 +1,433 @@ +//! Read-only catalog discovery for native semantic SQL hosts. + +use serde_json::{json, Value}; + +use super::*; + +fn canonical_snapshot(snapshot: &str) -> CatalogResult { + Ok(Snapshot::from_graph( + &Snapshot::parse(snapshot)?.into_graph()?, + )) +} + +fn text_result(operation: impl FnOnce() -> CatalogResult) -> SidemanticRewriteResult { + semantic_result(guard(operation).map_err(semantic_error)) +} + +/// Return the complete, versioned catalog, including non-model declarations. +#[no_mangle] +pub extern "C" fn sidemantic_snapshot_export(snapshot: *const c_char) -> SidemanticRewriteResult { + text_result(|| { + serde_json::to_string(&canonical_snapshot(&optional_arg(snapshot, "snapshot")?)?) + .map_err(|error| error.to_string()) + }) +} + +#[derive(Debug)] +struct CatalogEntry { + kind: &'static str, + model_name: Option, + name: String, + qualified_name: String, + definition: Value, +} + +impl CatalogEntry { + fn new( + kind: &'static str, + model: Option<&str>, + name: &str, + definition: &T, + ) -> CatalogResult { + Ok(Self { + kind, + model_name: model.map(str::to_owned), + name: name.to_owned(), + qualified_name: model + .map_or_else(|| name.to_owned(), |model| format!("{model}.{name}")), + definition: serde_json::to_value(definition).map_err(|error| error.to_string())?, + }) + } +} + +fn entries(snapshot: &Snapshot, kind: &str, model: &str) -> CatalogResult> { + if ![ + "", + "model", + "metric", + "dimension", + "segment", + "relationship", + "parameter", + "table_calculation", + ] + .contains(&kind) + { + return Err(format!("unknown catalog kind '{kind}'")); + } + let selected = if model.is_empty() { + None + } else { + Some( + snapshot + .models + .iter() + .find(|candidate| candidate.name.eq_ignore_ascii_case(model)) + .ok_or_else(|| format!("model '{model}' not found"))? + .name + .as_str(), + ) + }; + let mut entries = Vec::new(); + for model in &snapshot.models { + if selected.is_some_and(|selected| selected != model.name) { + continue; + } + entries.push(CatalogEntry::new("model", None, &model.name, model)?); + for dimension in &model.dimensions { + entries.push(CatalogEntry::new( + "dimension", + Some(&model.name), + &dimension.name, + dimension, + )?); + } + for metric in &model.metrics { + entries.push(CatalogEntry::new( + "metric", + Some(&model.name), + &metric.name, + metric, + )?); + } + for segment in &model.segments { + entries.push(CatalogEntry::new( + "segment", + Some(&model.name), + &segment.name, + segment, + )?); + } + for relationship in &model.relationships { + let mut entry = CatalogEntry::new( + "relationship", + Some(&model.name), + &relationship.name, + relationship, + )?; + entry.definition["target_model"] = json!(relationship.related_model()); + entries.push(entry); + } + } + if selected.is_none() { + for metric in &snapshot.metrics { + entries.push(CatalogEntry::new("metric", None, &metric.name, metric)?); + } + for parameter in &snapshot.parameters { + entries.push(CatalogEntry::new( + "parameter", + None, + ¶meter.name, + parameter, + )?); + } + for calculation in &snapshot.table_calculations { + entries.push(CatalogEntry::new( + "table_calculation", + None, + &calculation.name, + calculation, + )?); + } + } + entries.retain(|entry| kind.is_empty() || entry.kind == kind); + entries.sort_by(|a, b| (&a.kind, &a.qualified_name).cmp(&(&b.kind, &b.qualified_name))); + Ok(entries) +} + +#[repr(C)] +pub struct SidemanticCatalogEntry { + pub kind: *mut c_char, + pub model_name: *mut c_char, + pub name: *mut c_char, + pub qualified_name: *mut c_char, + pub label: *mut c_char, + pub description: *mut c_char, + pub semantic_type: *mut c_char, + pub data_type: *mut c_char, + pub sql: *mut c_char, + pub aggregation: *mut c_char, + pub target_model: *mut c_char, + pub relationship_type: *mut c_char, + pub granularity: *mut c_char, + pub is_public: bool, + pub definition: *mut c_char, +} + +impl SidemanticCatalogEntry { + fn from_entry(entry: CatalogEntry) -> CatalogResult { + // Validate every allocation before transferring ownership to C. Null + // means absent metadata; it is distinct from an authored empty string. + let field = |value: Option<&str>| -> CatalogResult> { + value + .map(CString::new) + .transpose() + .map_err(|_| "catalog metadata contains a NUL byte".into()) + }; + let property = |name: &str| field(entry.definition.get(name).and_then(Value::as_str)); + let kind = field(Some(entry.kind))?; + let model_name = field(entry.model_name.as_deref())?; + let name = field(Some(&entry.name))?; + let qualified_name = field(Some(&entry.qualified_name))?; + let label = property("label")?; + let description = property("description")?; + let semantic_type = property("type")?; + let data_type = property("logical_data_type")?; + let sql = property("sql")?; + let aggregation = property("agg")?; + let target_model = property("target_model")?; + let relationship_type = if entry.kind == "relationship" { + property("type")? + } else { + None + }; + let granularity = property("granularity")?; + let definition = field(Some(&entry.definition.to_string()))?; + let raw = |value: Option| value.map_or(ptr::null_mut(), CString::into_raw); + Ok(Self { + kind: raw(kind), + model_name: raw(model_name), + name: raw(name), + qualified_name: raw(qualified_name), + label: raw(label), + description: raw(description), + semantic_type: raw(semantic_type), + data_type: raw(data_type), + sql: raw(sql), + aggregation: raw(aggregation), + target_model: raw(target_model), + relationship_type: raw(relationship_type), + granularity: raw(granularity), + is_public: entry + .definition + .get("public") + .and_then(Value::as_bool) + .unwrap_or(true), + definition: raw(definition), + }) + } +} + +impl Drop for SidemanticCatalogEntry { + fn drop(&mut self) { + for value in [ + self.kind, + self.model_name, + self.name, + self.qualified_name, + self.label, + self.description, + self.semantic_type, + self.data_type, + self.sql, + self.aggregation, + self.target_model, + self.relationship_type, + self.granularity, + self.definition, + ] { + sidemantic_free(value); + } + } +} + +#[repr(C)] +pub struct SidemanticCatalogEntries { + pub entries: *mut SidemanticCatalogEntry, + pub count: usize, + pub error: *mut c_char, +} + +fn compatible_dimensions(snapshot: &str, metric: &str) -> CatalogResult> { + let snapshot = canonical_snapshot(snapshot)?; + let names: Vec = serde_json::from_str(metric).map_err(|e| e.to_string())?; + let [model_name, metric_name] = names.as_slice() else { + return Err("SHOW DIMENSIONS FOR requires model.metric".into()); + }; + let metric = entries(&snapshot, "metric", model_name)? + .into_iter() + .find(|entry| entry.name.eq_ignore_ascii_case(metric_name)) + .ok_or_else(|| format!("metric '{model_name}.{metric_name}' not found"))?; + let candidates = entries(&snapshot, "dimension", "")?; + let graph = snapshot.into_graph()?; + let rewriter = QueryRewriter::new(&graph); + let quote = |name: &str| format!("\"{}\"", name.replace('"', "\"\"")); + let owner = quote(metric.model_name.as_deref().unwrap()); + let projection = format!("{owner}.{}", quote(&metric.name)); + let compile = |selection: &str| { + rewriter.rewrite_with_dialect( + &format!("select {selection} from {owner}"), + polyglot_sql::DialectType::DuckDB, + ) + }; + // Invalid metrics must surface their error rather than look like an empty + // dimension list. Each candidate then uses the same compiler as SELECT. + compile(&projection).map_err(|error| error.to_string())?; + Ok(candidates + .into_iter() + .filter(|entry| { + let dimension = format!( + "{}.{}", + quote(entry.model_name.as_deref().unwrap()), + quote(&entry.name) + ); + compile(&format!("{projection}, {dimension}")).is_ok() + }) + .collect()) +} + +/// A non-null metric holds the parsed model and metric identifiers as JSON. +/// Return dimensions whose SELECT combination compiles, not just graph neighbors. +#[no_mangle] +pub extern "C" fn sidemantic_snapshot_catalog( + snapshot: *const c_char, + kind: *const c_char, + model: *const c_char, + metric: *const c_char, +) -> SidemanticCatalogEntries { + let result = guard(|| { + let source = optional_arg(snapshot, "snapshot")?; + let snapshot = canonical_snapshot(&source)?; + let kind = optional_arg(kind, "kind")?; + let model = optional_arg(model, "model")?; + let rows = if metric.is_null() { + entries(&snapshot, &kind, &model)? + } else { + compatible_dimensions(&source, &required_arg(metric, "metric")?)? + }; + rows.into_iter() + .map(SidemanticCatalogEntry::from_entry) + .collect::>>() + }); + match result { + Ok(entries) => { + let entries = entries.into_boxed_slice(); + SidemanticCatalogEntries { + count: entries.len(), + entries: Box::into_raw(entries).cast(), + error: ptr::null_mut(), + } + } + Err(error) => SidemanticCatalogEntries { + entries: ptr::null_mut(), + count: 0, + error: semantic_error(error), + }, + } +} + +#[no_mangle] +pub extern "C" fn sidemantic_free_catalog_entries(result: SidemanticCatalogEntries) { + if !result.entries.is_null() { + unsafe { + drop(Box::from_raw(ptr::slice_from_raw_parts_mut( + result.entries, + result.count, + ))); + } + } + sidemantic_free(result.error); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn snapshot() -> Snapshot { + let (snapshot, _) = apply( + "", + "", + "yaml", + r#" +models: + - name: orders + table: raw_orders + primary_key: id + label: Orders + description: Order facts + dimensions: + - name: status + type: categorical + description: Fulfilment status + metrics: + - name: revenue + agg: sum + sql: amount + label: Revenue + description: Gross revenue + relationships: + - name: customers + type: many_to_one + foreign_key: customer_id + - name: customers + table: raw_customers + primary_key: id + dimensions: + - name: country + type: categorical + - name: unrelated + table: unrelated + primary_key: id + dimensions: + - name: name + type: categorical +metrics: + - name: total + type: derived + sql: orders.revenue +parameters: + - name: region + type: string + default_value: US +metadata: + owner: finance +"#, + false, + ) + .unwrap(); + canonical_snapshot(&snapshot).unwrap() + } + + #[test] + fn discovery_checks_select_combinations_with_the_query_compiler() { + let snapshot = snapshot(); + let source = serde_json::to_string(&snapshot).unwrap(); + let rows = compatible_dimensions(&source, r#"["ORDERS","REVENUE"]"#).unwrap(); + let names: Vec<_> = rows.iter().map(|row| row.qualified_name.as_str()).collect(); + assert_eq!(names, ["customers.country", "orders.status"]); + assert!(compatible_dimensions(&source, r#"["orders","missing"]"#).is_err()); + assert!(compatible_dimensions(&source, r#"["missing","revenue"]"#).is_err()); + } + + #[test] + fn discovery_preserves_definitions_and_scopes() { + let snapshot = snapshot(); + let rows = entries(&snapshot, "", "ORDERS").unwrap(); + assert!(rows + .iter() + .any(|row| row.kind == "model" && row.definition["description"] == "Order facts")); + assert!(rows + .iter() + .any(|row| row.qualified_name == "orders.revenue" + && row.definition["label"] == "Revenue")); + assert!( + rows.iter() + .any(|row| row.kind == "relationship" + && row.definition["target_model"] == "customers") + ); + // The loader exposes the global derived metric in its owning model too. + assert_eq!(entries(&snapshot, "metric", "").unwrap().len(), 3); + assert!(entries(&snapshot, "metric", "missing").is_err()); + assert!(entries(&snapshot, "unknown", "").is_err()); + } +}