diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db78bc08c..6f893954f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -732,12 +732,18 @@ jobs: uv-key: ${{ steps.uv.outputs.cache-key }} duckdb-extension: - name: DuckDB Extension + name: DuckDB Extension (${{ matrix.duckdb }}) needs: check-rust-changes if: needs.check-rust-changes.outputs.duckdb_changed == 'true' runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + duckdb: [v1.5.6, v2.0-cyanoptera] env: GEN: ninja + DUCKDB_VERSION: ${{ matrix.duckdb }} + SIDEMANTIC_NATIVE_PEG: ${{ matrix.duckdb == 'v2.0-cyanoptera' && '1' || '' }} steps: - uses: actions/checkout@v4 @@ -753,7 +759,7 @@ jobs: - name: Fetch DuckDB dependencies working-directory: sidemantic-duckdb - run: make deps DUCKDB_VERSION=v1.5.5 + run: make deps - name: Install build dependencies run: sudo apt-get update && sudo apt-get install -y ninja-build diff --git a/.github/workflows/duckdb-extension-release.yml b/.github/workflows/duckdb-extension-release.yml index a59741b72..9e0d051ec 100644 --- a/.github/workflows/duckdb-extension-release.yml +++ b/.github/workflows/duckdb-extension-release.yml @@ -4,9 +4,9 @@ on: workflow_dispatch: inputs: duckdb_version: - description: "DuckDB tag used for extension build and tests; currently v1.5.5 only" + description: "Stable DuckDB release to package (v1.5.6); Cyanoptera is tested separately in CI" required: true - default: "v1.5.5" + default: "v1.5.6" type: string create_github_release: description: "Create or update a GitHub release with extension artifacts" @@ -55,11 +55,11 @@ jobs: if [ "$EVENT_NAME" = "workflow_dispatch" ]; then VERSION="$REQUESTED_VERSION" else - VERSION="v1.5.5" + VERSION="v1.5.6" fi - # Keep in sync with SUPPORTED_DUCKDB_VERSION in the extension Makefile. - if [ "$VERSION" != "v1.5.5" ]; then + # Publish stable artifacts only. CI separately tests the pinned PEG development build. + if [ "$VERSION" != "v1.5.6" ]; then echo "Unsupported DuckDB version" >&2 exit 1 fi diff --git a/docs/rust-native-runtime-packaging.md b/docs/rust-native-runtime-packaging.md index d335dd711..5e1b55028 100644 --- a/docs/rust-native-runtime-packaging.md +++ b/docs/rust-native-runtime-packaging.md @@ -16,7 +16,7 @@ Python remains the default engine; installing the Rust bindings does not promote Workflow files live in `.github/workflows/`. Rust crate and wheel versions must match; package metadata tests enforce this. Root Python and WASM versions are separate. The DuckDB extension is tied to the DuckDB build version, currently -`v1.5.5`. Do not use a release intended for a different DuckDB ABI. +`v1.5.6`. Do not use a release intended for a different DuckDB ABI. ## Install from a checkout or downloaded artifact diff --git a/sidemantic-duckdb/CMakeLists.txt b/sidemantic-duckdb/CMakeLists.txt index a13a8f4b3..84702efe9 100644 --- a/sidemantic-duckdb/CMakeLists.txt +++ b/sidemantic-duckdb/CMakeLists.txt @@ -9,6 +9,78 @@ set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) project(${TARGET_NAME}) include_directories(src/include) +# Probe the host headers: extension builds can contain multiple DuckDB checkouts. +include(CheckCXXSourceCompiles) +include(CMakePushCheckState) +function(sidemantic_detect_apis) + cmake_push_check_state(RESET) + get_directory_property(CMAKE_REQUIRED_INCLUDES INCLUDE_DIRECTORIES) + set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + set(host_api "${CMAKE_SOURCE_DIR};${CMAKE_CXX_COMPILER}") + foreach(header parser/parser_extension.hpp function/table_function.hpp + parser/expression/columnref_expression.hpp parser/tableref/basetableref.hpp + parser/parser_options.hpp parser/expression/constant_expression.hpp + parser/statement/insert_statement.hpp + parser/grammar_extension.hpp parser/peg/compiled_grammar.hpp) + if(EXISTS "${CMAKE_SOURCE_DIR}/src/include/duckdb/${header}") + file(SHA256 "${CMAKE_SOURCE_DIR}/src/include/duckdb/${header}" header_hash) + string(APPEND host_api ";${header_hash}") + endif() + endforeach() + if(NOT SIDEMANTIC_HOST_API STREQUAL host_api) + foreach(api TOKEN_PARSE_FN NEW_IDENTIFIER_API NEW_EXPRESSION_API QUALIFIED_TABLE_API BUILTIN_PARSER_OPTIONS LITERAL_API GRAMMAR_EXTENSION INSERT_QUERY_NODE) + unset(SIDEMANTIC_${api} CACHE) + endforeach() + set(SIDEMANTIC_HOST_API "${host_api}" CACHE INTERNAL "DuckDB headers used for compatibility probes" FORCE) + endif() + check_cxx_source_compiles(" + #include + #include + using namespace duckdb; + using Callback = ParserExtensionParseResult (*)(ParserExtensionInfo *, const vector &); + static_assert(std::is_same::value, \"token callback\"); + " SIDEMANTIC_TOKEN_PARSE_FN) + check_cxx_source_compiles(" + #include + #include + using namespace duckdb; + using Callback = unique_ptr (*)(ClientContext &, TableFunctionBindInput &, vector &, vector &); + static_assert(std::is_same::value, \"identifier names\"); + " SIDEMANTIC_NEW_IDENTIFIER_API) + check_cxx_source_compiles(" + #include + void probe(duckdb::ColumnRefExpression &expr) { expr.ColumnNames(); } + " SIDEMANTIC_NEW_EXPRESSION_API) + check_cxx_source_compiles(" + #include + void probe(duckdb::BaseTableRef &ref) { ref.GetQualifiedName(); } + " SIDEMANTIC_QUALIFIED_TABLE_API) + check_cxx_source_compiles(" + #include + void probe() { duckdb::ParserOptions::Builtin(); } + " SIDEMANTIC_BUILTIN_PARSER_OPTIONS) + check_cxx_source_compiles(" + #include + void probe() { duckdb::ConstantExpression::String(duckdb::string()); } + " SIDEMANTIC_LITERAL_API) + check_cxx_source_compiles(" + #include + void probe(duckdb::InsertStatement &stmt) { (void)stmt.node->select_statement; } + " SIDEMANTIC_INSERT_QUERY_NODE) + check_cxx_source_compiles(" + #include + #include + void probe(const duckdb::ClientContext &context) { + duckdb::CompiledGrammar::Create(context, {\"sidemantic\"}); + duckdb::GrammarChange::PrependChoice(\"Statement\", \"SidemanticStatement\"); + } + " SIDEMANTIC_GRAMMAR_EXTENSION) + configure_file(src/include/sidemantic_compat.hpp.in sidemantic_compat.hpp) + cmake_pop_check_state() +endfunction() +sidemantic_detect_apis() +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + # Path to sidemantic-rs set(SIDEMANTIC_RS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../sidemantic-rs") set(SIDEMANTIC_CARGO_TARGET_DIR "${SIDEMANTIC_RS_DIR}/target" CACHE PATH "Cargo target directory for sidemantic-rs") @@ -73,7 +145,7 @@ add_custom_target(sidemantic_rust_staticlib DEPENDS "${SIDEMANTIC_LIB}") # Include Rust library headers include_directories(${SIDEMANTIC_INCLUDE}) -set(EXTENSION_SOURCES src/sidemantic_extension.cpp) +set(EXTENSION_SOURCES src/sidemantic_extension.cpp src/sidemantic_parser.cpp src/sidemantic_grammar.cpp src/sidemantic_catalog.cpp) build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) diff --git a/sidemantic-duckdb/Makefile b/sidemantic-duckdb/Makefile index ae7c19c7f..a8cd3c1a5 100644 --- a/sidemantic-duckdb/Makefile +++ b/sidemantic-duckdb/Makefile @@ -4,19 +4,27 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) # Configuration of extension EXT_NAME=sidemantic EXT_CONFIG=${PROJ_DIR}extension_config.cmake -SUPPORTED_DUCKDB_VERSION := v1.5.5 +SUPPORTED_DUCKDB_VERSION := v1.5.6 +SUPPORTED_DUCKDB_NEXT := v2.0-cyanoptera +# Pin the moving development branch to the native PEG API exercised by CI. +DUCKDB_NEXT_COMMIT := 80e17fc252edd6d9e9b090ae00a1100daef4876a DUCKDB_VERSION ?= $(SUPPORTED_DUCKDB_VERSION) +# Exercise coexistence with the parser override preloaded by packaged shells. +DEFAULT_TEST_EXTENSION_DEPS := autocomplete .PHONY: deps deps: - @if [ "$(DUCKDB_VERSION)" != "$(SUPPORTED_DUCKDB_VERSION)" ]; then \ - echo "DUCKDB_VERSION=$(DUCKDB_VERSION) is unsupported by the vendored extension-ci-tools checkout."; \ - echo "Use DUCKDB_VERSION=$(SUPPORTED_DUCKDB_VERSION), or update extension-ci-tools and this guard together."; \ + @if [ "$(DUCKDB_VERSION)" != "$(SUPPORTED_DUCKDB_VERSION)" ] && [ "$(DUCKDB_VERSION)" != "$(SUPPORTED_DUCKDB_NEXT)" ]; then \ + echo "Use DUCKDB_VERSION=$(SUPPORTED_DUCKDB_VERSION) or $(SUPPORTED_DUCKDB_NEXT)."; \ exit 1; \ fi @if [ ! -d duckdb/.git ]; then \ rm -rf duckdb; \ git clone --depth 1 --branch $(DUCKDB_VERSION) https://github.com/duckdb/duckdb.git duckdb; \ + fi + @if [ "$(DUCKDB_VERSION)" = "$(SUPPORTED_DUCKDB_NEXT)" ]; then \ + git -C duckdb fetch --depth 1 origin $(DUCKDB_NEXT_COMMIT); \ + git -C duckdb checkout --detach $(DUCKDB_NEXT_COMMIT); \ else \ git -C duckdb fetch --depth 1 origin refs/tags/$(DUCKDB_VERSION):refs/tags/$(DUCKDB_VERSION); \ git -C duckdb checkout $(DUCKDB_VERSION); \ diff --git a/sidemantic-duckdb/README.md b/sidemantic-duckdb/README.md index 83a1847d2..174605a6d 100644 --- a/sidemantic-duckdb/README.md +++ b/sidemantic-duckdb/README.md @@ -7,8 +7,11 @@ A DuckDB extension that adds a SQL-first semantic layer. Define metrics and dime - **Pure SQL Definition**: Define models, metrics, and dimensions using SQL statements - **Automatic Query Rewriting**: Query qualified model fields directly and get proper aggregations automatically - **Cross-Model JOINs**: Automatically generates JOINs when querying across related models -- **Fan-out Detection**: Warns when joins may cause metric inflation +- **Fan-out Protection**: Uses model keys to protect supported aggregations against duplicated rows from joins - **Definition Files**: Load native YAML, Cube.js YAML, and native SQL definition files +- **Transactional Definitions**: Commit and roll back model changes with DuckDB transactions +- **SQL Containers**: Use semantic queries in views, `CREATE TABLE AS`, and `INSERT ... SELECT` +- **Native PEG Grammar**: Composable grammar on the pinned Cyanoptera development build, with a DuckDB 1.5.6 compatibility frontend ## Installation @@ -27,7 +30,7 @@ LOAD '/absolute/path/to/sidemantic.duckdb_extension'; For local development: ```bash -make deps DUCKDB_VERSION=v1.5.5 +make deps DUCKDB_VERSION=v1.5.6 make make test ./build/release/duckdb -unsigned @@ -39,6 +42,12 @@ LOAD 'build/release/extension/sidemantic/sidemantic.duckdb_extension'; For embedded clients, set DuckDB's `allow_unsigned_extensions` database configuration before opening the connection. Community extension installation is planned, but this repository does not yet publish the signed multi-platform artifacts required for `INSTALL sidemantic FROM community`. +In DuckDB 1.5.6 shells that preload `autocomplete` (including Homebrew), use +`SEMANTIC SELECT` for semantic queries. That extension's PEG override consumes +ordinary SQL before Sidemantic receives it, and 1.5.6 has no parser-priority setting. +Clients without that competing override support automatic routing. Explicit +`SEMANTIC SELECT` works in both configurations. + ## Quick Start (Pure SQL) Define your semantic layer entirely in SQL, no YAML required: @@ -67,7 +76,7 @@ METRIC avg_order_value AS AVG(amount); DIMENSION (name status, type categorical); -- 5. Query using semantic layer -SELECT orders_model.status, orders_model.revenue FROM orders_model; +SEMANTIC SELECT orders_model.status, orders_model.revenue FROM orders_model; -- Automatically rewrites to: -- SELECT status, SUM(amount) FROM orders GROUP BY 1 @@ -89,7 +98,7 @@ SQL and do not change models loaded through the session APIs. ```sql select sidemantic_compile_semantic_input( - '{"version":1,"models":[{"name":"sales","table":"orders","primary_key":"order_id", + '{"version":1,"input_dialect":"duckdb","models":[{"name":"sales","table":"orders","primary_key":"order_id", "metrics":[{"name":"revenue","agg":"sum","sql":"amount"}]}]}', '{"metrics":["sales.revenue"]}' ); @@ -184,6 +193,65 @@ ORDER BY model.revenue DESC; The older `SEMANTIC SELECT ...` form is still supported as a compatibility fallback. +Aliases and quoted identifiers work with automatic routing: + +```sql +SELECT o.status, o.revenue FROM orders_model AS o; +SELECT "o"."revenue" FROM "orders_model" AS "o"; +``` + +Routing inspects DuckDB's parsed table and field references. Comments and strings +do not trigger rewriting; physical tables, CTEs and subqueries can shadow model +names. Once a semantic field is detected, compiler errors are reported directly. +Use `SEMANTIC SELECT` to explicitly request semantic compilation. Unsupported +semantic SQL still raises an error; the extension does not implement every DuckDB +query construct in the Rust compiler. + +## Transactions and persistence + +Definitions and loaded files are stored in the selected DuckDB database as a +versioned snapshot in the `main.__sidemantic_catalog()` macro. This reserved macro +is extension-owned and should not be edited manually. DuckDB provides persistence, +rollback, transaction isolation and conflicting-writer detection for the snapshot. +In-memory databases keep their definitions in memory. + +```sql +BEGIN; +CREATE OR REPLACE METRIC orders_model.revenue AS SUM(amount); +ROLLBACK; +``` + +Parsing, `EXPLAIN`, and `PREPARE` do not apply definitions or load files. Mutations +happen during execution; prepared semantic queries rebind to current definitions. +`MODEL model_name` selects a connection-local active model, and rollback restores +the prior selection. Definitions are shared through the database catalog, while +active model selections are independent between connections. + +Existing `.sidemantic.sql` sidecars remain readable when no native snapshot exists. +The first successful definition update stores the imported definitions in DuckDB. +The sidecar is left untouched and no new sidecars are written. After migration, +edit definitions through SQL or load an updated file explicitly. Invalid legacy +files raise errors when the semantic catalog is accessed. + +## Native PEG frontend + +DuckDB 1.5.6 uses the compatibility frontend. The pinned `v2.0-cyanoptera` build +registers a `sidemantic` grammar extension for model/item declarations and +`SEMANTIC SELECT`, including nested `PREPARE` and `EXPLAIN`. Definition properties +still use the shared Rust configuration parser; semantic query compilation still +uses the Rust SQL AST. + +Loading Sidemantic enables its parser override for automatic routing, subject to +the 1.5.6 autocomplete limitation above. To explicitly compose the native +grammar with other installed grammars, include it in `active_grammar_extensions`: + +```sql +SET active_grammar_extensions = ['sidemantic']; +``` + +The native grammar test disables parser overrides and executes custom syntax, +so it verifies native grammar integration independently of the compatibility path. + ## Alternative: Definition Files For larger deployments or version-controlled definitions, load models from YAML or SQL files. @@ -278,6 +346,32 @@ SELECT * FROM sidemantic_load_file('/path/to/orders.sql'); SELECT * FROM sidemantic_load_file('/path/to/models/'); ``` +File imports use DuckDB's filesystem and obey `enable_external_access`, +`allowed_paths`, `allowed_directories`, and `disabled_filesystems`. Directory +imports resolve inheritance across files and apply atomically. SQL-host imports +treat YAML values literally; they do not expand process environment variables. + +### Views, tables, and inserts + +Prefix the complete statement with `SEMANTIC` to work with either parser frontend: + +```sql +SEMANTIC CREATE VIEW revenue_by_status AS +SELECT orders_model.status, orders_model.revenue FROM orders_model; + +SEMANTIC CREATE TABLE revenue_snapshot AS +SELECT orders_model.status, orders_model.revenue FROM orders_model; + +SEMANTIC INSERT INTO revenue_snapshot +SELECT orders_model.status, orders_model.revenue FROM orders_model; +``` + +DuckDB retains control of destination columns, `RETURNING`, transactions, and +read-only restrictions. Prepared inserts accept query parameters. `EXPLAIN` +plans the statement without executing it. Views persist the compiled SQL at +creation time, so replacing a metric does not silently change an existing view; +recreate the view to use the new definition. + ### YAML Format Reference ```yaml @@ -376,7 +470,34 @@ SELECT orders.revenue, customers.country FROM orders; | `one_to_one` | One-to-one mapping | No | | `many_to_many` | Many-to-many (requires bridge table) | Yes | -**Fan-out Warning**: When joining from "one" to "many" side, metrics from the "one" side may be inflated. The extension adds a SQL comment warning when this is detected. +The Rust compiler uses model keys and protected aggregation plans to prevent +supported metrics from being multiplied by relationship joins. Define correct +primary keys and relationship cardinalities; the compiler relies on that metadata. +The extension suite executes cross-model sums and counts over repeated customer +rows, including equal-valued metrics on distinct customers. + +## Current support boundaries + +- The query frontend handles `SELECT`, `CREATE VIEW`, `CREATE TABLE AS`, and + `INSERT ... SELECT`, including `PREPARE` and `EXPLAIN` wrappers. Use the + `SEMANTIC` prefix when another extension takes priority for ordinary SQL. +- Semantic definitions support creation, replacement, loading, and active-model + selection. There is no `DROP MODEL`, `ALTER MODEL`, or item-removal SQL interface. +- Semantic queries support a subset of DuckDB SQL. Correlated semantic subqueries + and grouping modifiers such as `ROLLUP` are rejected; explicit `GROUP BY` must + repeat the selected semantic dimensions. +- Native execution tests cover derived, ratio, cumulative, time-comparison, + conversion, retention, and cohort metrics. Retention produces a table with + cohort date, elapsed periods, active users, cohort size, and retention percentage; + use `sidemantic_compile_semantic_input` and execute its returned SQL. A scalar projection such as + `SEMANTIC SELECT events.retention FROM events` is rejected. These contracts + do not establish full parity with every Python adapter or API. +- DuckDB 1.5.6 uses the compatibility frontend. Native PEG support is tested against + the pinned 2.x commit, not an arbitrary future 2.x build. Rust still parses model + properties and compiles semantic queries. +- Distribution currently targets unsigned Linux amd64 release packages. Signed + community installation and a published multi-platform binary matrix are not + provided by this repository. ## Utility Functions @@ -401,10 +522,10 @@ SELECT sidemantic_rewrite_sql('SELECT orders.revenue FROM orders'); ## How It Works -1. **Parser Override**: DuckDB 1.5+ lets the extension intercept qualified semantic `SELECT` queries before native parsing -2. **Parser Extension Fallback**: The legacy `SEMANTIC` prefix still routes through the parser extension -3. **Query Rewriting**: The Rust-based sidemantic library rewrites `model.metric` references to actual SQL aggregations -4. **Execution**: The rewritten SQL is parsed by DuckDB and executed normally +1. DuckDB parses queries; the native PEG or compatibility frontend captures definitions without applying them. +2. At bind time, parsed field references are resolved against the caller's transactional catalog snapshot. +3. The Rust compiler rewrites semantic SQL using a private graph. Stateful FFI calls release the registry lock before compilation; native parser workers reuse their large stacks across calls. +4. DuckDB binds and executes generated SQL in the same client context. Definition statements update the catalog during execution. ## Building from Source @@ -413,8 +534,8 @@ SELECT sidemantic_rewrite_sql('SELECT orders.revenue FROM orders'); cd sidemantic-duckdb # Fetch the DuckDB source version used by CI. -# extension-ci-tools is vendored in this directory and pinned to v1.5.5. -make deps DUCKDB_VERSION=v1.5.5 +# Stable builds and release artifacts target v1.5.6. +make deps DUCKDB_VERSION=v1.5.6 # Build the extension. CMake builds the sibling sidemantic-rs static library automatically. make @@ -426,13 +547,20 @@ make test ./build/release/duckdb -unsigned ``` +CI also builds `make deps DUCKDB_VERSION=v2.0-cyanoptera`, pinned in the Makefile to +commit `80e17fc252edd6d9e9b090ae00a1100daef4876a`. Run its tests with +`SIDEMANTIC_NATIVE_PEG=1 make test`. Use separate build directories/checkouts when +switching DuckDB versions. CMake probes host APIs rather than assuming C++ ABI +compatibility across releases. Release packaging targets unsigned Linux amd64 +builds for 1.5.6; development-version CI does not publish stable artifacts. + ## Architecture ``` ┌─────────────────────────────────────────────────────────────┐ -│ DuckDB Extension (C++) │ -│ - Parser override (intercepts qualified semantic SELECTs) │ -│ - Parser extension (supports legacy SEMANTIC queries) │ +│ DuckDB Extension (C++) │ +│ - Native PEG / compatibility parser and AST routing │ +│ - Transactional catalog definitions and legacy migration │ │ - Table functions (sidemantic_load, sidemantic_models) │ │ - Scalar function (sidemantic_rewrite_sql) │ └─────────────────────────────────────────────────────────────┘ @@ -440,7 +568,7 @@ make test │ C FFI ▼ ┌─────────────────────────────────────────────────────────────┐ -│ sidemantic-rs (Rust) │ +│ sidemantic-rs (Rust) │ │ - YAML parsing (native + Cube.js formats) │ │ - Semantic graph (models, relationships) │ │ - SQL generation and query rewriting │ diff --git a/sidemantic-duckdb/docs/UPDATING.md b/sidemantic-duckdb/docs/UPDATING.md index a3ac73eff..e5d43f401 100644 --- a/sidemantic-duckdb/docs/UPDATING.md +++ b/sidemantic-duckdb/docs/UPDATING.md @@ -1,23 +1,26 @@ -# Extension updating -When cloning this template, the target version of DuckDB should be the latest stable release of DuckDB. However, there -will inevitably come a time when a new DuckDB is released and the extension repository needs updating. This process goes -as follows: +# Updating DuckDB compatibility -- Bump submodules - - `./duckdb` should be set to latest tagged release - - `./extension-ci-tools` should be set to updated branch corresponding to latest DuckDB release. So if you're building for DuckDB `v1.1.0` there will be a branch in `extension-ci-tools` named `v1.1.0` to which you should check out. -- Bump versions in `./github/workflows` - - `duckdb_version` input in `duckdb-stable-build` job in `MainDistributionPipeline.yml` should be set to latest tagged release - - `duckdb_version` input in `duckdb-stable-deploy` job in `MainDistributionPipeline.yml` should be set to latest tagged release - - the reusable workflow `duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml` for the `duckdb-stable-build` job should be set to latest tagged release +The extension builds against DuckDB's internal C++ API. A loadable extension must +match its host version and platform; source compatibility is not binary compatibility. -# API changes -DuckDB extensions built with this extension template are built against the internal C++ API of DuckDB. This API is not guaranteed to be stable. -What this means for extension development is that when updating your extensions DuckDB target version using the above steps, you may run into the fact that your extension no longer builds properly. +The Makefile selects stable DuckDB 1.5.6 or a pinned Cyanoptera development commit. +CI builds both and runs the SQL and SemanticInput host suites. The development job +sets `SIDEMANTIC_NATIVE_PEG=1` to require tests that disable parser overrides and use +Sidemantic's registered native grammar. Release packaging intentionally accepts only +the stable version and currently produces unsigned Linux amd64 artifacts. -Currently, DuckDB does not (yet) provide a specific change log for these API changes, but it is generally not too hard to figure out what has changed. +When updating: -For figuring out how and why the C++ API changed, we recommend using the following resources: -- DuckDB's [Release Notes](https://github.com/duckdb/duckdb/releases) -- DuckDB's history of [Core extension patches](https://github.com/duckdb/duckdb/commits/main/.github/patches/extensions) -- The git history of the relevant C++ Header file of the API that has changed \ No newline at end of file +1. Change the stable version or `DUCKDB_NEXT_COMMIT` in the Makefile and the CI matrix + as appropriate. Keep the release workflow's stable allowlist and input tests aligned. +2. Use fresh build directories for each host. `src/include/sidemantic_compat.hpp.in` + is generated from the API probes in CMake; adapt those probes and frontend code + when upstream interfaces change. +3. Build and run the extension SQL suites, including transactions, restart, legacy + migration, routing and native PEG tests. Run `test/test_semantic_input_host.py` + against the freshly built shell and extension. +4. Update the README with the verified versions and distribution scope. Do not + infer additional platform support or signed/community availability from compilation. + +The vendored extension-ci-tools supplies the build harness. Update it only when +changes to that harness are required by the selected DuckDB versions. diff --git a/sidemantic-duckdb/src/include/sidemantic_catalog.hpp b/sidemantic-duckdb/src/include/sidemantic_catalog.hpp new file mode 100644 index 000000000..278f65fe0 --- /dev/null +++ b/sidemantic-duckdb/src/include/sidemantic_catalog.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include "duckdb/main/client_context.hpp" +#include "duckdb/parser/parser_extension.hpp" + +namespace duckdb { + +struct SidemanticCatalogSnapshot { + string payload; + string active_model; +}; + +// Resolve definitions against the caller's current catalog transaction. Legacy +// sidecars are read only when no native snapshot exists and are never modified. +SidemanticCatalogSnapshot ReadSidemanticCatalog(ClientContext &context); + +void RegisterSidemanticCatalogRead(ClientContext &context, StatementProperties &properties); +void RegisterSidemanticCatalogWrite(ClientContext &context, StatementProperties &properties); + +// Supported operations: model, item, use, yaml, file. +void ExecuteSidemanticMutation(ClientContext &context, const string &operation, + const string &content, bool replace); + +ParserExtensionPlanResult PlanSidemanticMutation(ClientContext &context, const string &operation, + const string &content, bool replace, + const string &result_message); + +} // namespace duckdb diff --git a/sidemantic-duckdb/src/include/sidemantic_compat.hpp.in b/sidemantic-duckdb/src/include/sidemantic_compat.hpp.in new file mode 100644 index 000000000..4a14a41ff --- /dev/null +++ b/sidemantic-duckdb/src/include/sidemantic_compat.hpp.in @@ -0,0 +1,33 @@ +#pragma once + +#cmakedefine01 SIDEMANTIC_TOKEN_PARSE_FN +#cmakedefine01 SIDEMANTIC_NEW_IDENTIFIER_API +#cmakedefine01 SIDEMANTIC_NEW_EXPRESSION_API +#cmakedefine01 SIDEMANTIC_QUALIFIED_TABLE_API +#cmakedefine01 SIDEMANTIC_BUILTIN_PARSER_OPTIONS +#cmakedefine01 SIDEMANTIC_LITERAL_API +#cmakedefine01 SIDEMANTIC_GRAMMAR_EXTENSION +#cmakedefine01 SIDEMANTIC_INSERT_QUERY_NODE + +#include "duckdb/parser/parser_options.hpp" +#if SIDEMANTIC_NEW_IDENTIFIER_API +#include "duckdb/common/identifier.hpp" +#endif + +namespace duckdb { +#if SIDEMANTIC_NEW_IDENTIFIER_API +using SidemanticIdentifier = Identifier; +inline const string &SidemanticName(const Identifier &name) { return name.GetIdentifierName(); } +#else +using SidemanticIdentifier = string; +#endif +inline const string &SidemanticName(const string &name) { return name; } + +inline ParserOptions SidemanticBuiltinParserOptions() { +#if SIDEMANTIC_BUILTIN_PARSER_OPTIONS + return ParserOptions::Builtin(); +#else + return ParserOptions(); +#endif +} +} // namespace duckdb diff --git a/sidemantic-duckdb/src/include/sidemantic_extension.hpp b/sidemantic-duckdb/src/include/sidemantic_extension.hpp index fe62c95ee..c93e0db9b 100644 --- a/sidemantic-duckdb/src/include/sidemantic_extension.hpp +++ b/sidemantic-duckdb/src/include/sidemantic_extension.hpp @@ -1,16 +1,9 @@ #pragma once #include "duckdb.hpp" -#include "duckdb/parser/parser.hpp" -#include "duckdb/parser/parser_extension.hpp" -#include "duckdb/parser/statement/extension_statement.hpp" -#include "duckdb/planner/binder.hpp" -#include "duckdb/planner/operator/logical_extension_operator.hpp" -#include "duckdb/planner/operator_extension.hpp" namespace duckdb { -// Main extension class class SidemanticExtension : public Extension { public: void Load(ExtensionLoader &loader) override; @@ -18,69 +11,4 @@ class SidemanticExtension : public Extension { std::string Version() const override; }; -// Forward declarations -BoundStatement sidemantic_bind(ClientContext &context, Binder &binder, - OperatorExtensionInfo *info, SQLStatement &statement); - -ParserExtensionParseResult sidemantic_parse(ParserExtensionInfo *, - const std::string &query); - -ParserOverrideResult sidemantic_parser_override(ParserExtensionInfo *info, - const std::string &query, - ParserOptions &options); - -ParserExtensionPlanResult sidemantic_plan(ParserExtensionInfo *, ClientContext &, - unique_ptr); - -struct SidemanticParserInfo : ParserExtensionInfo { - SidemanticParserInfo(string db_path, string context_key) - : db_path(std::move(db_path)), context_key(std::move(context_key)) {} - - string db_path; - string context_key; -}; - -// Operator extension: handles binding after parsing -struct SidemanticOperatorExtension : public OperatorExtension { - SidemanticOperatorExtension() : OperatorExtension() { Bind = sidemantic_bind; } - std::string GetName() override { return "sidemantic"; } - unique_ptr - Deserialize(Deserializer &deserializer) override { - throw InternalException("sidemantic operator should not be serialized"); - } -}; - -// Parser extension: intercepts query strings -struct SidemanticParserExtension : public ParserExtension { - SidemanticParserExtension(string db_path, string context_key) : ParserExtension() { - parse_function = sidemantic_parse; - plan_function = sidemantic_plan; - parser_override = sidemantic_parser_override; - parser_info = - make_shared_ptr(std::move(db_path), std::move(context_key)); - } -}; - -// Container for parsed statement (passed between parse and bind phases) -struct SidemanticParseData : ParserExtensionParseData { - unique_ptr statement; - - unique_ptr Copy() const override { - return make_uniq_base( - statement->Copy()); - } - string ToString() const override { return "SidemanticParseData"; } - SidemanticParseData(unique_ptr statement) - : statement(std::move(statement)) {} -}; - -// State stored in ClientContext between parse and bind -class SidemanticState : public ClientContextState { -public: - explicit SidemanticState(unique_ptr parse_data) - : parse_data(std::move(parse_data)) {} - void QueryEnd() override { parse_data.reset(); } - unique_ptr parse_data; -}; - } // namespace duckdb diff --git a/sidemantic-duckdb/src/include/sidemantic_parser.hpp b/sidemantic-duckdb/src/include/sidemantic_parser.hpp new file mode 100644 index 000000000..e85df2b1b --- /dev/null +++ b/sidemantic-duckdb/src/include/sidemantic_parser.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include "sidemantic_compat.hpp" +#include "duckdb/parser/parser.hpp" +#include "duckdb/parser/statement/extension_statement.hpp" +#include "duckdb/planner/binder.hpp" +#include "duckdb/planner/operator_extension.hpp" + +namespace duckdb { + +struct SidemanticParseData : ParserExtensionParseData { + // Definitions carry source only. Mutation is deferred until execution. + string operation; + string content; + bool replace = false; + bool explicit_semantic = false; + unique_ptr statement; + + unique_ptr Copy() const override; + string ToString() const override; +}; + +unique_ptr SidemanticStatement(unique_ptr data); +unique_ptr ParseSidemanticDefinition(const string &sql); +unique_ptr WrapSidemanticQuery(unique_ptr statement, bool explicit_semantic = false); + +ParserOverrideResult sidemantic_parser_override(ParserExtensionInfo *, const string &, ParserOptions &); +#if SIDEMANTIC_TOKEN_PARSE_FN +ParserExtensionParseResult sidemantic_parse(ParserExtensionInfo *, const vector &); +#else +ParserExtensionParseResult sidemantic_parse(ParserExtensionInfo *, const string &); +#endif +ParserExtensionPlanResult sidemantic_plan(ParserExtensionInfo *, ClientContext &, + unique_ptr); +BoundStatement sidemantic_bind(ClientContext &, Binder &, OperatorExtensionInfo *, SQLStatement &); + +struct SidemanticParserExtension : ParserExtension { + SidemanticParserExtension() { + parse_function = sidemantic_parse; + plan_function = sidemantic_plan; + parser_override = sidemantic_parser_override; + } +}; + +struct SidemanticOperatorExtension : OperatorExtension { + SidemanticOperatorExtension() { Bind = sidemantic_bind; } + string GetName() override { return "sidemantic"; } + unique_ptr Deserialize(Deserializer &) override; +}; + +// On hosts with the native grammar API, LOAD registers a composable grammar. +// Parsing retains an explicitly selected caller grammar and never changes it. +shared_ptr RegisterSidemanticGrammar(DatabaseInstance &db); +bool ParseSidemanticGrammar(ParserExtensionInfo *, const string &, const ParserOptions &, + vector> &statements); + +} // namespace duckdb diff --git a/sidemantic-duckdb/src/sidemantic_catalog.cpp b/sidemantic-duckdb/src/sidemantic_catalog.cpp new file mode 100644 index 000000000..b4c1668b7 --- /dev/null +++ b/sidemantic-duckdb/src/sidemantic_catalog.cpp @@ -0,0 +1,333 @@ +#include "sidemantic_catalog.hpp" +#include "sidemantic_compat.hpp" +#include "sidemantic.h" + +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp" +#include "duckdb/common/file_system.hpp" +#include "duckdb/function/scalar_macro_function.hpp" +#include "duckdb/main/attached_database.hpp" +#include "duckdb/main/client_context_state.hpp" +#include "duckdb/main/database_manager.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/parser/parsed_data/create_macro_info.hpp" +#include "duckdb/planner/binder.hpp" +#include "duckdb/storage/storage_manager.hpp" +#include + +namespace duckdb { + +static constexpr const char *CATALOG_MACRO = "__sidemantic_catalog"; +static constexpr const char *SESSION_STATE = "sidemantic_catalog_session"; + +// Only session preferences live outside DuckDB's catalog. Capture their values +// lazily: the first semantic statement may arrive after BEGIN already ran. +class SidemanticCatalogSession : public ClientContextState { +public: + unordered_map active_models; + unordered_map previous_active_models; + unordered_set previously_unset; + + void SetActive(idx_t catalog_oid, string model) { + if (!previous_active_models.count(catalog_oid) && !previously_unset.count(catalog_oid)) { + auto current = active_models.find(catalog_oid); + if (current == active_models.end()) { + previously_unset.insert(catalog_oid); + } else { + previous_active_models.emplace(catalog_oid, current->second); + } + } + active_models[catalog_oid] = std::move(model); + } + + void TransactionCommit(MetaTransaction &, ClientContext &) override { + previous_active_models.clear(); + previously_unset.clear(); + } + + void TransactionRollback(MetaTransaction &, ClientContext &) override { + for (auto &entry : previous_active_models) { + active_models[entry.first] = std::move(entry.second); + } + for (auto oid : previously_unset) { + active_models.erase(oid); + } + previous_active_models.clear(); + previously_unset.clear(); + } +}; + +static Catalog &SemanticCatalog(ClientContext &context) { + return Catalog::GetCatalog(context, DatabaseManager::GetDefaultDatabase(context)); +} + +static shared_ptr Session(ClientContext &context) { + return context.registered_state->GetOrCreate(SESSION_STATE); +} + +struct SnapshotResultOwner { + explicit SnapshotResultOwner(SidemanticSnapshotResult result) : result(result) { + } + ~SnapshotResultOwner() { + sidemantic_free_snapshot_result(result); + } + SidemanticSnapshotResult result; + + SidemanticCatalogSnapshot Get() const { + if (result.error) { + throw InvalidInputException("Sidemantic: %s", result.error); + } + if (!result.snapshot) { + throw InternalException("Sidemantic returned no catalog snapshot"); + } + return {result.snapshot, result.active_model ? result.active_model : ""}; + } +}; + +static string ReadDefinitionFile(FileSystem &fs, const string &path) { + auto file = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ); + auto size = file->GetFileSize(); + if (size < 0 || file->GetType() != FileType::FILE_TYPE_REGULAR) { + throw InvalidInputException("Sidemantic definitions require a regular file: %s", path); + } + string content(size, '\0'); + if (!content.empty()) file->Read(&content[0], content.size(), 0); + if (content.find('\0') != string::npos) { + throw InvalidInputException("Sidemantic definitions contain a NUL byte: %s", path); + } + return content; +} + +static void CollectDefinitionFiles(FileSystem &fs, const string &directory, vector &paths, idx_t depth = 0) { + // Bound recursive traversal, including directory symlink cycles. + static constexpr idx_t MAX_DIRECTORY_DEPTH = 64; + if (depth >= MAX_DIRECTORY_DEPTH) { + throw InvalidInputException("Sidemantic model directory exceeds maximum nesting depth: %s", directory); + } + if (!fs.ListFiles(directory, [&](const string &name, bool is_directory) { + auto path = fs.JoinPath(directory, name); + if (is_directory) { + CollectDefinitionFiles(fs, path, paths, depth + 1); + } else { + auto lower = StringUtil::Lower(name); + if (StringUtil::EndsWith(lower, ".yml") || StringUtil::EndsWith(lower, ".yaml") || + StringUtil::EndsWith(lower, ".sql")) paths.push_back(std::move(path)); + } + })) { + throw IOException("Could not list Sidemantic model directory: %s", directory); + } +} + +static SidemanticSnapshotResult ImportDefinitionFiles(ClientContext &context, const string &snapshot, + const string &path) { + // The context filesystem enforces enable_external_access, allowed paths + // and disabled filesystems for every listing and open. Rust receives only + // captured bytes and cannot bypass the host's file or environment boundary. + auto &fs = FileSystem::GetFileSystem(context); + bool directory = fs.DirectoryExists(path); + vector paths; + if (directory) { + CollectDefinitionFiles(fs, path, paths); + std::sort(paths.begin(), paths.end()); + } else { + paths.push_back(path); + } + vector contents; + for (auto &source : paths) contents.push_back(ReadDefinitionFile(fs, source)); + vector sources; + for (idx_t i = 0; i < paths.size(); ++i) { + sources.push_back({paths[i].c_str(), contents[i].c_str()}); + } + return sidemantic_snapshot_load_sources(snapshot.c_str(), sources.data(), sources.size(), directory); +} + +static SidemanticCatalogSnapshot ReadLegacySnapshot(ClientContext &context, Catalog &catalog) { + auto &database = catalog.GetAttached(); + if (catalog.InMemory() || !database.HasStorageManager()) { + return {}; + } + auto path = database.GetStorageManager().GetDBPath(); + if (path.empty() || path == ":memory:") { + return {}; + } + + // Match the historical Rust sidecar path: replace the database filename's + // final extension, leaving directory names (including dots) untouched. + auto separator = path.find_last_of("/\\"); + auto filename_start = separator == string::npos ? 0 : separator + 1; + auto extension = path.find_last_of('.'); + if (extension != string::npos && extension > filename_start) { + path.resize(extension); + } + path += ".sidemantic.sql"; + + // The context filesystem supplies its opener and access checks itself. + auto &fs = FileSystem::GetFileSystem(context); + if (!fs.FileExists(path)) { + return {}; + } + auto content = ReadDefinitionFile(fs, path); + SnapshotResultOwner imported(sidemantic_snapshot_apply(nullptr, nullptr, "legacy_sql", content.c_str(), false)); + return imported.Get(); +} + +SidemanticCatalogSnapshot ReadSidemanticCatalog(ClientContext &context) { + auto &catalog = SemanticCatalog(context); + auto state = Session(context); + const string macro_name = CATALOG_MACRO; +#if SIDEMANTIC_NEW_IDENTIFIER_API + const QualifiedName qualified_name(catalog.GetName(), Identifier::DefaultSchema(), Identifier(macro_name)); + EntryLookupInfo lookup(CatalogType::MACRO_ENTRY, qualified_name); + auto entry = Catalog::GetEntry(context, lookup, OnEntryNotFound::RETURN_NULL); +#else + EntryLookupInfo lookup(CatalogType::MACRO_ENTRY, macro_name); + auto entry = Catalog::GetEntry(context, catalog.GetName(), DEFAULT_SCHEMA, lookup, OnEntryNotFound::RETURN_NULL); +#endif + SidemanticCatalogSnapshot snapshot; + if (entry) { + if (entry->type != CatalogType::MACRO_ENTRY) { + throw InvalidInputException("Invalid Sidemantic catalog entry type"); + } + auto ¯o_entry = entry->Cast(); + if (macro_entry.macros.size() != 1 || macro_entry.macros[0]->type != MacroType::SCALAR_MACRO) { + throw InvalidInputException("Invalid Sidemantic catalog snapshot macro"); + } + auto ¯o = macro_entry.macros[0]->Cast(); + if (!macro.expression || macro.expression->GetExpressionClass() != ExpressionClass::CONSTANT) { + throw InvalidInputException("Invalid Sidemantic catalog snapshot expression"); + } + auto &constant = macro.expression->Cast(); +#if SIDEMANTIC_LITERAL_API + auto &literal = constant.GetLiteral(); + if (literal.kind != LiteralKind::STRING) { + throw InvalidInputException("Invalid Sidemantic catalog snapshot value"); + } + snapshot.payload = literal.text; +#else + auto &value = constant.value; + if (value.IsNull() || value.type() != LogicalType::VARCHAR) { + throw InvalidInputException("Invalid Sidemantic catalog snapshot value"); + } + snapshot.payload = value.GetValue(); +#endif + } else { + snapshot = ReadLegacySnapshot(context, catalog); + } + if (snapshot.payload.find('\0') != string::npos) { + throw InvalidInputException("Sidemantic catalog snapshot contains a NUL byte"); + } + auto active = state->active_models.find(catalog.GetOid()); + if (active != state->active_models.end()) { + snapshot.active_model = active->second; + } + return snapshot; +} + +void RegisterSidemanticCatalogRead(ClientContext &context, StatementProperties &properties) { + properties.RegisterDBRead(SemanticCatalog(context), context); + // A prepared semantic query must also observe connection-local MODEL changes. + properties.always_require_rebind = true; +} + +void RegisterSidemanticCatalogWrite(ClientContext &context, StatementProperties &properties) { + properties.RegisterDBModify(SemanticCatalog(context), context, DatabaseModificationType::CREATE_CATALOG_ENTRY); + properties.always_require_rebind = true; +} + +void ExecuteSidemanticMutation(ClientContext &context, const string &operation, + const string &content, bool replace) { + if (content.find('\0') != string::npos) { + throw InvalidInputException("Sidemantic definition contains a NUL byte"); + } + auto &catalog = SemanticCatalog(context); + auto snapshot = ReadSidemanticCatalog(context); + SnapshotResultOwner applied(operation == "file" + ? ImportDefinitionFiles(context, snapshot.payload, content) + : sidemantic_snapshot_apply(snapshot.payload.c_str(), snapshot.active_model.c_str(), + operation.c_str(), content.c_str(), replace)); + auto candidate = applied.Get(); + if (operation != "use") { + CreateMacroInfo info(CatalogType::MACRO_ENTRY); +#if SIDEMANTIC_NEW_IDENTIFIER_API + info.SetQualifiedName(QualifiedName(catalog.GetName(), Identifier::DefaultSchema(), Identifier(CATALOG_MACRO))); +#else + info.catalog = catalog.GetName(); + info.schema = DEFAULT_SCHEMA; + info.name = CATALOG_MACRO; +#endif + info.on_conflict = OnCreateConflict::REPLACE_ON_CONFLICT; + // Not internal: internal catalog entries are omitted from checkpoints. +#if SIDEMANTIC_LITERAL_API + auto expression = ConstantExpression::String(candidate.payload); +#else + auto expression = make_uniq(Value(candidate.payload)); +#endif + info.macros.push_back(make_uniq(std::move(expression))); + catalog.CreateFunction(context, info); + } + // Publish the session preference only after the catalog write succeeds. + Session(context)->SetActive(catalog.GetOid(), std::move(candidate.active_model)); +} + +struct MutationBindData : public TableFunctionData { + string operation; + string content; + bool replace; + string result_message; +}; + +struct MutationGlobalState : public GlobalTableFunctionState { + bool done = false; +}; + +static unique_ptr BindMutation(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, + vector &names) { + auto data = make_uniq(); + data->operation = input.inputs[0].GetValue(); + data->content = input.inputs[1].GetValue(); + data->replace = input.inputs[2].GetValue(); + data->result_message = input.inputs[3].GetValue(); + return_types.emplace_back(LogicalType::VARCHAR); + names.emplace_back("result"); + if (input.binder) { + auto &properties = input.binder->GetStatementProperties(); + RegisterSidemanticCatalogRead(context, properties); + } + return std::move(data); +} + +static unique_ptr InitMutation(ClientContext &, TableFunctionInitInput &) { + return make_uniq(); +} + +static void RunMutation(ClientContext &context, TableFunctionInput &input, DataChunk &output) { + auto &state = input.global_state->Cast(); + if (state.done) { + return; + } + auto &data = input.bind_data->Cast(); + ExecuteSidemanticMutation(context, data.operation, data.content, data.replace); + state.done = true; + output.SetCardinality(1); + output.SetValue(0, 0, Value(data.result_message)); +} + +ParserExtensionPlanResult PlanSidemanticMutation(ClientContext &context, const string &operation, + const string &content, bool replace, + const string &result_message) { + ParserExtensionPlanResult result; + result.function = TableFunction("sidemantic_definition", + {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::VARCHAR}, + RunMutation, BindMutation, InitMutation); + result.parameters = {Value(operation), Value(content), Value::BOOLEAN(replace), Value(result_message)}; + result.return_type = StatementReturnType::QUERY_RESULT; + if (operation != "use") { + StatementProperties properties; + RegisterSidemanticCatalogWrite(context, properties); + result.modified_databases = std::move(properties.modified_databases); + } + return result; +} + +} // namespace duckdb diff --git a/sidemantic-duckdb/src/sidemantic_extension.cpp b/sidemantic-duckdb/src/sidemantic_extension.cpp index bfa1460aa..f7524622a 100644 --- a/sidemantic-duckdb/src/sidemantic_extension.cpp +++ b/sidemantic-duckdb/src/sidemantic_extension.cpp @@ -1,212 +1,107 @@ #define DUCKDB_EXTENSION_MAIN #include "sidemantic_extension.hpp" -#include "duckdb/parser/parser.hpp" -#include "duckdb/parser/statement/extension_statement.hpp" +#include "sidemantic_parser.hpp" +#include "sidemantic_catalog.hpp" +#include "sidemantic.h" #include "duckdb/function/table_function.hpp" #include "duckdb/common/vector_operations/binary_executor.hpp" #include "duckdb/common/vector_operations/ternary_executor.hpp" -#include "sidemantic.h" - -#include -#include namespace duckdb { -static std::string DatabasePath(DatabaseInstance &db) { - auto &db_config = db.config; - if (!db_config.options.database_path.empty()) { - return db_config.options.database_path; - } - return ""; -} - -static std::string ContextKey(DatabaseInstance &db) { - auto path = DatabasePath(db); - if (!path.empty()) { - return "duckdb:" + path; - } - return "duckdb:memory:" + std::to_string(reinterpret_cast(&db)); -} - -static const char *ContextKeyPtr(const std::string &context_key) { - return context_key.empty() ? nullptr : context_key.c_str(); -} - -static const SidemanticParserInfo *ParserInfo(ParserExtensionInfo *info) { - return dynamic_cast(info); -} - -//============================================================================= -// TABLE FUNCTION: sidemantic_load(yaml) -//============================================================================= - -struct SidemanticLoadData : public TableFunctionData { - string yaml_content; - bool done = false; +struct LoadData : public TableFunctionData { + string operation; + string content; }; -static unique_ptr SidemanticLoadBind(ClientContext &context, - TableFunctionBindInput &input, - vector &return_types, - vector &names) { - auto result = make_uniq(); - result->yaml_content = input.inputs[0].GetValue(); - - return_types.push_back(LogicalType::VARCHAR); - names.push_back("result"); - - return std::move(result); -} - -static void SidemanticLoadFunction(ClientContext &context, TableFunctionInput &data_p, - DataChunk &output) { - auto &data = data_p.bind_data->CastNoConst(); - if (data.done) { - return; - } - data.done = true; - - auto context_key = ContextKey(*context.db); - char *error = sidemantic_load_yaml_for_context(ContextKeyPtr(context_key), data.yaml_content.c_str()); - if (error) { - string error_msg(error); - sidemantic_free(error); - throw InvalidInputException("Failed to load semantic models: %s", error_msg); - } - - output.SetCardinality(1); - output.SetValue(0, 0, Value("Models loaded successfully")); -} - -//============================================================================= -// TABLE FUNCTION: sidemantic_load_file(path) -//============================================================================= - -struct SidemanticLoadFileData : public TableFunctionData { - string file_path; +struct LoadState : public GlobalTableFunctionState { bool done = false; }; -static unique_ptr SidemanticLoadFileBind(ClientContext &context, - TableFunctionBindInput &input, - vector &return_types, - vector &names) { - auto result = make_uniq(); - result->file_path = input.inputs[0].GetValue(); - - return_types.push_back(LogicalType::VARCHAR); - names.push_back("result"); - - return std::move(result); -} - -static void SidemanticLoadFileFunction(ClientContext &context, TableFunctionInput &data_p, - DataChunk &output) { - auto &data = data_p.bind_data->CastNoConst(); - if (data.done) { - return; - } - data.done = true; - - auto context_key = ContextKey(*context.db); - char *error = sidemantic_load_file_for_context(ContextKeyPtr(context_key), data.file_path.c_str()); - if (error) { - string error_msg(error); - sidemantic_free(error); - throw InvalidInputException("Failed to load semantic models: %s", error_msg); - } - +template +static unique_ptr LoadBind(ClientContext &context, TableFunctionBindInput &input, + vector &types, vector &names) { + auto data = make_uniq(); + data->operation = FROM_FILE ? "file" : "yaml"; + if (input.inputs[0].IsNull()) throw InvalidInputException("Sidemantic load argument cannot be NULL"); + data->content = input.inputs[0].GetValue(); + types.push_back(LogicalType::VARCHAR); + names.emplace_back("result"); + if (input.binder) RegisterSidemanticCatalogWrite(context, input.binder->GetStatementProperties()); + return std::move(data); +} + +static unique_ptr LoadInit(ClientContext &, TableFunctionInitInput &) { + return make_uniq(); +} + +static void LoadFunction(ClientContext &context, TableFunctionInput &input, DataChunk &output) { + auto &state = input.global_state->Cast(); + if (state.done) return; + auto &data = input.bind_data->Cast(); + ExecuteSidemanticMutation(context, data.operation, data.content, false); + state.done = true; output.SetCardinality(1); - output.SetValue(0, 0, Value("Models loaded from: " + data.file_path)); + output.SetValue(0, 0, Value("Models loaded successfully")); } -//============================================================================= -// TABLE FUNCTION: sidemantic_models() -//============================================================================= - -struct SidemanticModelsData : public TableFunctionData { - bool done = false; +struct ModelsState : public GlobalTableFunctionState { + vector names; + idx_t offset = 0; }; -static unique_ptr SidemanticModelsBind(ClientContext &context, - TableFunctionBindInput &input, - vector &return_types, - vector &names) { - return_types.push_back(LogicalType::VARCHAR); - names.push_back("model_name"); - return make_uniq(); -} - -static void SidemanticModelsFunction(ClientContext &context, TableFunctionInput &data_p, - DataChunk &output) { - auto &data = data_p.bind_data->CastNoConst(); - if (data.done) { - return; - } - data.done = true; - - auto context_key = ContextKey(*context.db); - char *models_str = sidemantic_list_models_for_context(ContextKeyPtr(context_key)); - if (!models_str) { - output.SetCardinality(0); - return; - } - - string models(models_str); - sidemantic_free(models_str); - - if (models.empty()) { - output.SetCardinality(0); - return; - } - - // Split by comma - vector model_names; - size_t pos = 0; - while ((pos = models.find(',')) != string::npos) { - model_names.push_back(models.substr(0, pos)); - models.erase(0, pos + 1); - } - if (!models.empty()) { - model_names.push_back(models); - } - - output.SetCardinality(model_names.size()); - for (idx_t i = 0; i < model_names.size(); i++) { - output.SetValue(0, i, Value(model_names[i])); - } -} - -//============================================================================= -// SCALAR FUNCTION: sidemantic_rewrite_sql(sql) -//============================================================================= - -static void SidemanticRewriteSqlFunction(DataChunk &args, ExpressionState &state, - Vector &result) { - auto &sql_vector = args.data[0]; - UnaryExecutor::Execute( - sql_vector, result, args.size(), [&](string_t sql) { - std::string context_key; - if (state.HasContext()) { - context_key = ContextKey(*state.GetContext().db); - } - SidemanticRewriteResult res = - sidemantic_rewrite_for_context(ContextKeyPtr(context_key), sql.GetString().c_str()); - - if (res.error) { - string error_msg(res.error); - sidemantic_free_result(res); - throw InvalidInputException("Rewrite failed: %s", error_msg); - } - - string rewritten(res.sql); - sidemantic_free_result(res); - return StringVector::AddString(result, rewritten); - }); +static unique_ptr ModelsBind(ClientContext &context, TableFunctionBindInput &input, + vector &types, vector &names) { + types.push_back(LogicalType::VARCHAR); + names.emplace_back("model_name"); + if (input.binder) RegisterSidemanticCatalogRead(context, input.binder->GetStatementProperties()); + return nullptr; +} + +static unique_ptr ModelsInit(ClientContext &context, TableFunctionInitInput &) { + auto state = make_uniq(); + auto snapshot = ReadSidemanticCatalog(context); + auto models = sidemantic_snapshot_list_models(snapshot.payload.c_str()); + if (models.error) { + string error(models.error); + sidemantic_free_model_list(models); + throw InvalidInputException("Sidemantic: %s", error); + } + for (idx_t i = 0; i < models.count; ++i) state->names.emplace_back(models.models[i].name); + sidemantic_free_model_list(models); + return std::move(state); +} + +static void ModelsFunction(ClientContext &, TableFunctionInput &input, DataChunk &output) { + auto &state = input.global_state->Cast(); + auto count = MinValue(STANDARD_VECTOR_SIZE, state.names.size() - state.offset); + for (idx_t i = 0; i < count; ++i) output.SetValue(0, i, Value(state.names[state.offset + i])); + state.offset += count; + output.SetCardinality(count); +} + +static void RewriteFunction(DataChunk &args, ExpressionState &state, Vector &result) { + auto snapshot = ReadSidemanticCatalog(state.GetContext()); + UnaryExecutor::Execute(args.data[0], result, args.size(), [&](string_t input) { + auto sql = input.GetString(); + if (sql.find('\0') != string::npos) throw InvalidInputException("Sidemantic SQL contains a NUL byte"); + auto rewritten = sidemantic_snapshot_rewrite(snapshot.payload.c_str(), sql.c_str()); + if (rewritten.error) { + string error(rewritten.error); + sidemantic_free_result(rewritten); + throw InvalidInputException("Sidemantic: %s", error); + } + if (!rewritten.sql) { + sidemantic_free_result(rewritten); + throw InternalException("Sidemantic returned no SQL"); + } + string sql_result(rewritten.sql); + sidemantic_free_result(rewritten); + return StringVector::AddString(result, sql_result); + }); } -//============================================================================= // Stateless SemanticInput entrypoints retain policies in the canonical input. // Reject NULs before crossing the C string boundary instead of truncating input. static string SemanticInputArgument(string_t value) { @@ -256,896 +151,36 @@ static void SidemanticRewriteSemanticInputFunction(DataChunk &args, ExpressionSt }); } -// PARSER EXTENSION -//============================================================================= - -// Check if query starts with SEMANTIC keyword (case insensitive) -static bool StartsWithSemantic(const std::string &query, std::string &stripped_query) { - // Skip leading whitespace - size_t start = 0; - while (start < query.size() && std::isspace(query[start])) { - start++; - } - - // Check for "SEMANTIC" prefix (case insensitive) - const char *prefix = "SEMANTIC"; - size_t prefix_len = 8; - - if (query.size() - start < prefix_len) { - return false; - } - - for (size_t i = 0; i < prefix_len; i++) { - if (std::toupper(query[start + i]) != prefix[i]) { - return false; - } - } - - // Must be followed by whitespace - if (start + prefix_len < query.size() && !std::isspace(query[start + prefix_len])) { - return false; - } - - // Strip the SEMANTIC prefix - stripped_query = query.substr(start + prefix_len); - return true; -} - -// Check if a string starts with a keyword (case insensitive), skipping whitespace -static bool StartsWithKeyword(const std::string &str, const std::string &keyword, size_t &end_pos) { - size_t start = 0; - while (start < str.size() && std::isspace(str[start])) { - start++; - } - - if (str.size() - start < keyword.size()) { - return false; - } - - for (size_t i = 0; i < keyword.size(); i++) { - if (std::toupper(str[start + i]) != std::toupper(keyword[i])) { - return false; - } - } - - // Must be followed by whitespace or end of string - if (start + keyword.size() < str.size() && !std::isspace(str[start + keyword.size()])) { - return false; - } - - end_pos = start + keyword.size(); - return true; -} - -static std::string TrimCopy(const std::string &value) { - size_t start = 0; - while (start < value.size() && std::isspace(value[start])) { - start++; - } - size_t end = value.size(); - while (end > start && std::isspace(value[end - 1])) { - end--; - } - return value.substr(start, end - start); -} - -static std::string LowerCopy(const std::string &value) { - std::string result; - result.reserve(value.size()); - for (auto ch : value) { - result.push_back(static_cast(std::tolower(static_cast(ch)))); - } - return result; -} - -static bool StartsWithSemanticQueryKeyword(const std::string &query) { - size_t pos = 0; - return StartsWithKeyword(query, "SELECT", pos) || StartsWithKeyword(query, "WITH", pos); -} - -static bool QueryContainsLoadedModelQualifier(const char *context_key_ptr, - const std::string &query) { - char *models_ptr = sidemantic_list_models_for_context(context_key_ptr); - if (!models_ptr) { - return false; - } - - std::string models(models_ptr); - sidemantic_free(models_ptr); - auto lower_query = LowerCopy(query); - - size_t start = 0; - while (start < models.size()) { - auto comma = models.find(',', start); - auto model = TrimCopy(models.substr( - start, comma == std::string::npos ? std::string::npos : comma - start)); - if (!model.empty()) { - auto qualifier = LowerCopy(model) + "."; - if (lower_query.find(qualifier) != std::string::npos) { - return true; - } - } - if (comma == std::string::npos) { - break; - } - start = comma + 1; - } - - return false; -} - -static bool IsCompactModelStatement(const std::string &query) { - size_t model_pos = 0; - if (!StartsWithKeyword(query, "MODEL", model_pos)) { - return false; - } - - auto rest = query.substr(model_pos); - size_t start = 0; - while (start < rest.size() && std::isspace(static_cast(rest[start]))) { - start++; - } - - size_t end = start; - while (end < rest.size() && - (std::isalnum(static_cast(rest[end])) || rest[end] == '_')) { - end++; - } - if (end == start) { - return false; - } - - size_t from_pos = 0; - return StartsWithKeyword(rest.substr(end), "FROM", from_pos); -} - -static bool IsModelBlockStatement(const std::string &query) { - size_t model_pos = 0; - if (!StartsWithKeyword(query, "MODEL", model_pos)) { - return false; - } - - auto rest = query.substr(model_pos); - size_t start = 0; - while (start < rest.size() && std::isspace(static_cast(rest[start]))) { - start++; - } - return start < rest.size() && rest[start] == '('; -} - -static std::string NativeItemDefinitionType(const std::string &query) { - size_t pos = 0; - if (StartsWithKeyword(query, "METRIC", pos)) { - return "METRIC"; - } - if (StartsWithKeyword(query, "DIMENSION", pos)) { - return "DIMENSION"; - } - if (StartsWithKeyword(query, "SEGMENT", pos)) { - return "SEGMENT"; - } - return ""; -} - -static bool IsBareIdentifier(const std::string &value) { - if (value.empty() || !(std::isalpha(value[0]) || value[0] == '_')) { - return false; - } - for (auto ch : value) { - if (!(std::isalnum(ch) || ch == '_')) { - return false; - } - } - return true; -} - -static bool StartsWithNameProperty(const std::string &value, size_t pos) { - const std::string keyword = "name"; - if (pos + keyword.size() >= value.size()) { - return false; - } - for (size_t i = 0; i < keyword.size(); i++) { - if (std::tolower(value[pos + i]) != keyword[i]) { - return false; - } - } - auto separator = value[pos + keyword.size()]; - return std::isspace(separator) || separator == ':' || separator == '='; -} - -static bool ExtractModelNameProperty(const std::string &body, std::string &name, std::string &error) { - auto open = body.find('('); - if (open == std::string::npos) { - return false; - } - - size_t pos = open + 1; - bool in_single_quote = false; - bool in_double_quote = false; - int paren_depth = 0; - int bracket_depth = 0; - int brace_depth = 0; - - while (pos < body.size()) { - while (pos < body.size() && paren_depth == 0 && bracket_depth == 0 && - brace_depth == 0 && (std::isspace(body[pos]) || body[pos] == ',')) { - pos++; - } - if (pos >= body.size() || - (paren_depth == 0 && bracket_depth == 0 && brace_depth == 0 && body[pos] == ')')) { - return false; - } - - if (paren_depth == 0 && bracket_depth == 0 && brace_depth == 0 && - StartsWithNameProperty(body, pos)) { - pos += 4; - while (pos < body.size() && std::isspace(body[pos])) { - pos++; - } - if (pos < body.size() && (body[pos] == ':' || body[pos] == '=')) { - pos++; - while (pos < body.size() && std::isspace(body[pos])) { - pos++; - } - } - if (pos < body.size() && (body[pos] == '\'' || body[pos] == '"')) { - auto quote = body[pos]; - pos++; - std::string quoted_name; - while (pos < body.size()) { - auto ch = body[pos]; - if (ch == quote) { - if (pos + 1 < body.size() && body[pos + 1] == quote) { - quoted_name.push_back(quote); - pos += 2; - continue; - } - name = quoted_name; - return true; - } - quoted_name.push_back(ch); - pos++; - } - error = "CREATE MODEL body name is unterminated"; - return false; - } - size_t name_start = pos; - while (pos < body.size() && (std::isalnum(body[pos]) || body[pos] == '_')) { - pos++; - } - if (pos > name_start) { - name = body.substr(name_start, pos - name_start); - return true; - } - error = "CREATE MODEL body name must be a bare or quoted identifier"; - return false; - } - - while (pos < body.size()) { - auto ch = body[pos]; - if (in_single_quote) { - if (ch == '\'' && pos + 1 < body.size() && body[pos + 1] == '\'') { - pos += 2; - continue; - } - if (ch == '\'') { - in_single_quote = false; - } - pos++; - continue; - } - if (in_double_quote) { - if (ch == '"' && pos + 1 < body.size() && body[pos + 1] == '"') { - pos += 2; - continue; - } - if (ch == '"') { - in_double_quote = false; - } - pos++; - continue; - } - - if (ch == '\'') { - in_single_quote = true; - } else if (ch == '"') { - in_double_quote = true; - } else if (ch == '(') { - paren_depth++; - } else if (ch == ')') { - if (paren_depth > 0) { - paren_depth--; - } else if (bracket_depth == 0 && brace_depth == 0) { - return false; - } - } else if (ch == '[') { - bracket_depth++; - } else if (ch == ']') { - if (bracket_depth > 0) { - bracket_depth--; - } - } else if (ch == '{') { - brace_depth++; - } else if (ch == '}') { - if (brace_depth > 0) { - brace_depth--; - } - } else if (ch == ',' && paren_depth == 0 && bracket_depth == 0 && - brace_depth == 0) { - break; - } - pos++; - } - } - - return false; -} - -// Check if query is a CREATE [OR REPLACE] METRIC/DIMENSION/SEGMENT statement -// Returns the statement type or empty string if not matched -// Handles syntaxes like: -// - "CREATE METRIC name AS expr" -// - "CREATE OR REPLACE METRIC name AS expr" -// - "CREATE METRIC (...)" -// - "CREATE METRIC model.name AS expr" -static std::string IsDefinitionStatement(const std::string &query, std::string &definition, bool &is_replace) { - size_t pos = 0; - is_replace = false; - - // Must start with CREATE - if (!StartsWithKeyword(query, "CREATE", pos)) { - return ""; - } - - std::string rest = query.substr(pos); - - // Check for optional OR REPLACE - size_t or_pos = 0; - if (StartsWithKeyword(rest, "OR", or_pos)) { - rest = rest.substr(or_pos); - size_t replace_pos = 0; - if (StartsWithKeyword(rest, "REPLACE", replace_pos)) { - rest = rest.substr(replace_pos); - is_replace = true; - } else { - return ""; // "CREATE OR" without "REPLACE" is invalid - } - } - - // Helper to check for AS keyword (case-insensitive) - auto is_as_keyword = [](const std::string &s, size_t p) -> bool { - if (p + 2 > s.size()) return false; - return (s[p] == 'A' || s[p] == 'a') && - (s[p + 1] == 'S' || s[p + 1] == 's') && - (p + 2 >= s.size() || std::isspace(s[p + 2])); - }; - - // Helper lambda to check for definition patterns after keyword - auto check_definition = [&is_as_keyword](const std::string &after_kw, const std::string &keyword, std::string &def) -> bool { - size_t start = 0; - while (start < after_kw.size() && std::isspace(after_kw[start])) start++; - - // Case 1: Direct opening paren - "KEYWORD (..." - if (start < after_kw.size() && after_kw[start] == '(') { - def = keyword + " " + after_kw.substr(start); - return true; - } - - // Read first identifier - size_t name_start = start; - while (start < after_kw.size() && (std::isalnum(after_kw[start]) || after_kw[start] == '_')) start++; - if (start == name_start) { - return false; - } - - // Skip whitespace - while (start < after_kw.size() && std::isspace(after_kw[start])) start++; - - // Case 2: Check for "AS" keyword - simple SQL syntax "KEYWORD name AS expr" - if (is_as_keyword(after_kw, start)) { - def = keyword + " " + after_kw.substr(name_start); - return true; - } - - // Case 3: Check for dot (model.name syntax) - if (start < after_kw.size() && after_kw[start] == '.') { - start++; // skip dot - // Read second identifier (field name) - size_t field_start = start; - while (start < after_kw.size() && (std::isalnum(after_kw[start]) || after_kw[start] == '_')) start++; - if (start == field_start) { - return false; - } - // Skip whitespace - while (start < after_kw.size() && std::isspace(after_kw[start])) start++; - - // Check for AS (model.name AS expr syntax) - if (is_as_keyword(after_kw, start)) { - def = keyword + " " + after_kw.substr(name_start); - return true; - } - - // Check for paren (model.name (props) syntax) - if (start < after_kw.size() && after_kw[start] == '(') { - def = keyword + " " + after_kw.substr(name_start); - return true; - } - } - - return false; - }; - - // Check for METRIC - size_t metric_pos = 0; - if (StartsWithKeyword(rest, "METRIC", metric_pos)) { - std::string after_metric = rest.substr(metric_pos); - if (check_definition(after_metric, "METRIC", definition)) { - return "METRIC"; - } - } - - // Check for DIMENSION - size_t dim_pos = 0; - if (StartsWithKeyword(rest, "DIMENSION", dim_pos)) { - std::string after_dim = rest.substr(dim_pos); - if (check_definition(after_dim, "DIMENSION", definition)) { - return "DIMENSION"; - } - } - - // Check for SEGMENT - size_t seg_pos = 0; - if (StartsWithKeyword(rest, "SEGMENT", seg_pos)) { - std::string after_seg = rest.substr(seg_pos); - if (check_definition(after_seg, "SEGMENT", definition)) { - return "SEGMENT"; - } - } - - return ""; -} - -// Check if stripped query is a CREATE [OR REPLACE] MODEL statement -// Returns: 0 = not a create model, 1 = create model, 2 = create or replace model -// Sets definition to be in nom-parser format: "MODEL (name ..., ...)" -static int IsCreateModelStatement(const std::string &query, std::string &definition, std::string &error) { - size_t pos = 0; - - // Check for CREATE - if (!StartsWithKeyword(query, "CREATE", pos)) { - return 0; - } - - std::string rest = query.substr(pos); - bool is_replace = false; - - // Check for optional OR REPLACE - size_t or_pos = 0; - if (StartsWithKeyword(rest, "OR", or_pos)) { - rest = rest.substr(or_pos); - size_t replace_pos = 0; - if (StartsWithKeyword(rest, "REPLACE", replace_pos)) { - rest = rest.substr(replace_pos); - is_replace = true; - } else { - return 0; // "CREATE OR" without "REPLACE" is invalid - } - } - - // Check for MODEL - size_t model_pos = 0; - if (!StartsWithKeyword(rest, "MODEL", model_pos)) { - return 0; - } - - // Skip whitespace after MODEL - rest = rest.substr(model_pos); - size_t start = 0; - while (start < rest.size() && std::isspace(rest[start])) { - start++; - } - rest = rest.substr(start); - - // Find the opening parenthesis - everything from there is the definition body - size_t paren_pos = rest.find('('); - if (paren_pos == std::string::npos) { - return 0; // No parenthesis found - } - - auto outer_name = TrimCopy(rest.substr(0, paren_pos)); - auto body = rest.substr(paren_pos); - if (!outer_name.empty() && !IsBareIdentifier(outer_name)) { - error = "Invalid CREATE MODEL name: " + outer_name; - return -1; - } - - std::string inner_name; - std::string name_error; - if (!outer_name.empty() && ExtractModelNameProperty(body, inner_name, name_error)) { - if (inner_name != outer_name) { - error = "CREATE MODEL name '" + outer_name + "' does not match body name '" + inner_name + "'"; - return -1; - } - definition = "MODEL " + body; - return is_replace ? 2 : 1; - } - if (!name_error.empty()) { - error = name_error; - return -1; - } - - if (!outer_name.empty()) { - auto after_open = body.substr(1); - auto trimmed_after_open = TrimCopy(after_open); - if (!trimmed_after_open.empty() && trimmed_after_open[0] == ')') { - definition = "MODEL (name " + outer_name + after_open; - } else { - definition = "MODEL (name " + outer_name + ", " + after_open; - } - return is_replace ? 2 : 1; - } - - definition = "MODEL " + body; - return is_replace ? 2 : 1; -} - -ParserOverrideResult sidemantic_parser_override(ParserExtensionInfo *info, - const std::string &query, - ParserOptions &options) { - (void)options; - - // Keep the legacy SEMANTIC prefix on the parse_function path so definitions - // and explicit SEMANTIC SELECT keep their existing behavior. - std::string stripped_query; - if (StartsWithSemantic(query, stripped_query)) { - return ParserOverrideResult(); - } - - // Sidemantic SQL is valid DuckDB SQL, so avoid intercepting broad table scans - // such as SELECT * FROM model. No-prefix mode only takes over qualified model - // references like SELECT orders.revenue FROM orders. - if (!StartsWithSemanticQueryKeyword(query)) { - return ParserOverrideResult(); - } - - auto parser_info = ParserInfo(info); - const char *context_key_ptr = parser_info ? ContextKeyPtr(parser_info->context_key) : nullptr; - if (!QueryContainsLoadedModelQualifier(context_key_ptr, query)) { - return ParserOverrideResult(); - } - - SidemanticRewriteResult result = sidemantic_rewrite_for_context(context_key_ptr, query.c_str()); - if (result.error) { - sidemantic_free_result(result); - return ParserOverrideResult(); - } - if (!result.was_rewritten || !result.sql) { - sidemantic_free_result(result); - return ParserOverrideResult(); - } - - string rewritten_sql(result.sql); - sidemantic_free_result(result); - if (TrimCopy(rewritten_sql) == TrimCopy(query)) { - return ParserOverrideResult(); - } - - try { - Parser parser; - parser.ParseQuery(rewritten_sql); - if (parser.statements.empty()) { - return ParserOverrideResult(); - } - return ParserOverrideResult(std::move(parser.statements)); - } catch (...) { - return ParserOverrideResult(); - } -} - -ParserExtensionParseResult sidemantic_parse(ParserExtensionInfo *info, - const std::string &query) { - auto parser_info = ParserInfo(info); - const char *context_key_ptr = parser_info ? ContextKeyPtr(parser_info->context_key) : nullptr; - const char *db_path_ptr = - parser_info && !parser_info->db_path.empty() ? parser_info->db_path.c_str() : nullptr; - - std::string stripped_query; - bool had_semantic_prefix = StartsWithSemantic(query, stripped_query); - if (!had_semantic_prefix) { - stripped_query = query; - } - - // Check if this is a CREATE [OR REPLACE] MODEL statement - std::string definition; - std::string create_model_error; - int create_type = IsCreateModelStatement(stripped_query, definition, create_model_error); - - if (create_type < 0) { - return ParserExtensionParseResult(create_model_error); - } - if (create_type > 0) { - // This is a CREATE MODEL statement - handle specially - bool replace = (create_type == 2); - - char *error = sidemantic_define_for_context(context_key_ptr, definition.c_str(), db_path_ptr, replace); - if (error) { - string error_msg(error); - sidemantic_free(error); - return ParserExtensionParseResult(error_msg); - } - - // Return a simple SELECT statement as acknowledgment - Parser parser; - parser.ParseQuery("SELECT 'Model created successfully' AS result"); - auto statements = std::move(parser.statements); - - return ParserExtensionParseResult( - make_uniq_base( - std::move(statements[0]))); - } - - // Check if this is a native SQL model block: - // MODEL (name orders, table orders, primary_key order_id) - if (IsModelBlockStatement(stripped_query)) { - char *error = - sidemantic_define_for_context(context_key_ptr, stripped_query.c_str(), db_path_ptr, false); - if (error) { - string error_msg(error); - sidemantic_free(error); - return ParserExtensionParseResult(error_msg); - } - - Parser parser; - parser.ParseQuery("SELECT 'Model created successfully' AS result"); - auto statements = std::move(parser.statements); - - return ParserExtensionParseResult( - make_uniq_base( - std::move(statements[0]))); - } - - // Check if this is a compact native SQL model block: - // model orders from orders ( - // primary key (order_id) - // sum(amount) as revenue - // ) - if (IsCompactModelStatement(stripped_query)) { - char *error = - sidemantic_define_for_context(context_key_ptr, stripped_query.c_str(), db_path_ptr, false); - if (error) { - string error_msg(error); - sidemantic_free(error); - return ParserExtensionParseResult(error_msg); - } - - Parser parser; - parser.ParseQuery("SELECT 'Model created successfully' AS result"); - auto statements = std::move(parser.statements); - - return ParserExtensionParseResult( - make_uniq_base( - std::move(statements[0]))); - } - - // Check if this is a MODEL statement (to switch active model) - // Using "SEMANTIC MODEL orders" instead of "SEMANTIC USE orders" because DuckDB - // handles USE statements specially before parser extensions are called - size_t model_pos = 0; - if (StartsWithKeyword(stripped_query, "MODEL", model_pos)) { - std::string rest = stripped_query.substr(model_pos); - // Trim whitespace and get model name (until semicolon, paren, or end) - size_t start = 0; - while (start < rest.size() && std::isspace(rest[start])) start++; - size_t end = start; - while (end < rest.size() && !std::isspace(rest[end]) && rest[end] != ';' && rest[end] != '(') end++; - std::string model_name = rest.substr(start, end - start); - - // Skip if there's a paren after - that's CREATE MODEL syntax - size_t paren_check = end; - while (paren_check < rest.size() && std::isspace(rest[paren_check])) paren_check++; - if (paren_check < rest.size() && rest[paren_check] == '(') { - // This looks like a CREATE MODEL with inline parens, skip - goto not_model_switch; - } - - if (!model_name.empty()) { - char *error = sidemantic_use_for_context(context_key_ptr, model_name.c_str()); - if (error) { - string error_msg(error); - sidemantic_free(error); - return ParserExtensionParseResult(error_msg); - } - - // Return acknowledgment - Parser parser; - parser.ParseQuery("SELECT 'Using model: " + model_name + "' AS result"); - auto statements = std::move(parser.statements); - - return ParserExtensionParseResult( - make_uniq_base( - std::move(statements[0]))); - } - } - -not_model_switch: - // Check if this is a native SQL metric/dimension/segment definition: - // METRIC (name revenue, agg sum, sql amount) - // METRIC revenue AS SUM(amount) - std::string native_def_type = NativeItemDefinitionType(stripped_query); - if (!native_def_type.empty()) { - char *error = - sidemantic_add_definition_for_context(context_key_ptr, stripped_query.c_str(), db_path_ptr, false); - if (error) { - string error_msg(error); - sidemantic_free(error); - return ParserExtensionParseResult(error_msg); - } - - Parser parser; - parser.ParseQuery("SELECT '" + native_def_type + " created successfully' AS result"); - auto statements = std::move(parser.statements); - - return ParserExtensionParseResult( - make_uniq_base( - std::move(statements[0]))); - } - - // Check if this is a CREATE [OR REPLACE] METRIC/DIMENSION/SEGMENT statement - bool is_replace = false; - std::string def_type = IsDefinitionStatement(stripped_query, definition, is_replace); - if (!def_type.empty()) { - char *error = - sidemantic_add_definition_for_context(context_key_ptr, definition.c_str(), db_path_ptr, is_replace); - if (error) { - string error_msg(error); - sidemantic_free(error); - return ParserExtensionParseResult(error_msg); - } - - // Return acknowledgment - Parser parser; - std::string action = is_replace ? "replaced" : "created"; - parser.ParseQuery("SELECT '" + def_type + " " + action + " successfully' AS result"); - auto statements = std::move(parser.statements); - - return ParserExtensionParseResult( - make_uniq_base( - std::move(statements[0]))); - } - - if (!had_semantic_prefix) { - // No-prefix parser fallback only handles Sidemantic statements that DuckDB - // cannot parse natively. Valid SELECTs are handled by parser_override. - return ParserExtensionParseResult(); - } - - // Regular SEMANTIC SELECT query - try to rewrite using sidemantic - SidemanticRewriteResult result = sidemantic_rewrite_for_context(context_key_ptr, stripped_query.c_str()); - - // If there was an error, return it - if (result.error) { - string error_msg(result.error); - sidemantic_free_result(result); - return ParserExtensionParseResult(error_msg); - } - - // Parse the rewritten SQL using DuckDB's parser - string rewritten_sql(result.sql); - sidemantic_free_result(result); - - Parser parser; - parser.ParseQuery(rewritten_sql); - auto statements = std::move(parser.statements); - - if (statements.empty()) { - return ParserExtensionParseResult("Rewritten query produced no statements"); - } - - // Return the parsed statement - return ParserExtensionParseResult( - make_uniq_base( - std::move(statements[0]))); -} - -ParserExtensionPlanResult sidemantic_plan(ParserExtensionInfo *, - ClientContext &context, - unique_ptr parse_data) { - // Store parse data in client context state - auto state = make_shared_ptr(std::move(parse_data)); - context.registered_state->Remove("sidemantic"); - context.registered_state->Insert("sidemantic", state); - - // Throw to trigger the operator extension's Bind function - throw BinderException("Use sidemantic_bind instead"); -} - -BoundStatement sidemantic_bind(ClientContext &context, Binder &binder, - OperatorExtensionInfo *info, SQLStatement &statement) { - switch (statement.type) { - case StatementType::EXTENSION_STATEMENT: { - auto &ext_statement = dynamic_cast(statement); - - // Check this is our extension's statement - if (ext_statement.extension.parse_function == sidemantic_parse) { - // Retrieve stashed parse data - auto lookup = context.registered_state->Get("sidemantic"); - if (lookup) { - auto state = (SidemanticState *)lookup.get(); - auto sidemantic_binder = Binder::CreateBinder(context, &binder); - auto parse_data = dynamic_cast(state->parse_data.get()); - - // Bind the SQL statement we generated - return sidemantic_binder->Bind(*(parse_data->statement)); - } - throw BinderException("Registered state not found"); - } - } - default: - return {}; // Not ours - } -} - -//============================================================================= -// EXTENSION LOADING -//============================================================================= static void LoadInternal(ExtensionLoader &loader) { auto &db = loader.GetDatabaseInstance(); auto &config = DBConfig::GetConfig(db); - - auto db_path = DatabasePath(db); - auto context_key = ContextKey(db); - - // Auto-load definitions from file if it exists - const char *db_path_ptr = db_path.empty() ? nullptr : db_path.c_str(); - char *error = sidemantic_autoload_for_context(ContextKeyPtr(context_key), db_path_ptr); - if (error) { - std::string message = "Failed to autoload sidemantic definitions: " + std::string(error); - sidemantic_free(error); - throw InvalidInputException("%s", message.c_str()); - } - - // Enable parser_override so qualified semantic SELECTs can omit the SEMANTIC prefix. - // FALLBACK mode lets ordinary DuckDB SQL continue through the native parser. config.SetOptionByName("allow_parser_override_extension", Value("fallback")); - - // Register parser extension - SidemanticParserExtension parser(db_path, context_key); + SidemanticParserExtension parser; + parser.parser_info = RegisterSidemanticGrammar(db); ParserExtension::Register(config, parser); - - // Register operator extension OperatorExtension::Register(config, make_shared_ptr()); - // Register table functions - TableFunction load_func("sidemantic_load", {LogicalType::VARCHAR}, - SidemanticLoadFunction, SidemanticLoadBind); - loader.RegisterFunction(load_func); - - TableFunction load_file_func("sidemantic_load_file", {LogicalType::VARCHAR}, - SidemanticLoadFileFunction, SidemanticLoadFileBind); - loader.RegisterFunction(load_file_func); - - TableFunction models_func("sidemantic_models", {}, - SidemanticModelsFunction, SidemanticModelsBind); - loader.RegisterFunction(models_func); - - // Register scalar function for manual rewriting - auto rewrite_func = ScalarFunction("sidemantic_rewrite_sql", - {LogicalType::VARCHAR}, - LogicalType::VARCHAR, - SidemanticRewriteSqlFunction); - loader.RegisterFunction(rewrite_func); - loader.RegisterFunction(ScalarFunction("sidemantic_compile_semantic_input", - {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, - SidemanticCompileSemanticInputFunction)); - loader.RegisterFunction(ScalarFunction("sidemantic_rewrite_semantic_input", + loader.RegisterFunction(TableFunction("sidemantic_load", {LogicalType::VARCHAR}, LoadFunction, LoadBind, LoadInit)); + loader.RegisterFunction(TableFunction("sidemantic_load_file", {LogicalType::VARCHAR}, LoadFunction, LoadBind, LoadInit)); + loader.RegisterFunction(TableFunction("sidemantic_models", {}, ModelsFunction, ModelsBind, ModelsInit)); + + auto rewrite = ScalarFunction("sidemantic_rewrite_sql", {LogicalType::VARCHAR}, LogicalType::VARCHAR, RewriteFunction); + rewrite.SetStability(FunctionStability::VOLATILE); + rewrite.SetFallible(); + loader.RegisterFunction(rewrite); + auto compile_input = ScalarFunction("sidemantic_compile_semantic_input", + {LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, SidemanticCompileSemanticInputFunction); + compile_input.SetFallible(); + loader.RegisterFunction(compile_input); + auto rewrite_input = ScalarFunction("sidemantic_rewrite_semantic_input", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, - SidemanticRewriteSemanticInputFunction)); + SidemanticRewriteSemanticInputFunction); + rewrite_input.SetFallible(); + loader.RegisterFunction(rewrite_input); } -void SidemanticExtension::Load(ExtensionLoader &loader) { - LoadInternal(loader); -} +void SidemanticExtension::Load(ExtensionLoader &loader) { LoadInternal(loader); } std::string SidemanticExtension::Version() const { #ifdef EXT_VERSION_SIDEMANTIC @@ -1158,9 +193,5 @@ std::string SidemanticExtension::Version() const { } // namespace duckdb extern "C" { - -DUCKDB_CPP_EXTENSION_ENTRY(sidemantic, loader) { - duckdb::LoadInternal(loader); -} - +DUCKDB_CPP_EXTENSION_ENTRY(sidemantic, loader) { duckdb::LoadInternal(loader); } } diff --git a/sidemantic-duckdb/src/sidemantic_grammar.cpp b/sidemantic-duckdb/src/sidemantic_grammar.cpp new file mode 100644 index 000000000..38afd13d2 --- /dev/null +++ b/sidemantic-duckdb/src/sidemantic_grammar.cpp @@ -0,0 +1,180 @@ +#include "sidemantic_parser.hpp" + +#if SIDEMANTIC_GRAMMAR_EXTENSION +#include "duckdb/main/database.hpp" +#include "duckdb/parser/grammar_extension.hpp" +#include "duckdb/parser/peg/compiled_grammar.hpp" +#include "duckdb/parser/peg/transformer/peg_transformer.hpp" +#include "duckdb/parser/statement/prepare_statement.hpp" +#include "duckdb/parser/statement/select_statement.hpp" + +namespace duckdb { +namespace { + +// The native grammar owns statement structure, quoting, comments and balanced +// definition bodies. Rust remains the shared authority for model properties. +class DefinitionAtomMatcher final : public AtomicMatcher { +public: + DefinitionAtomMatcher() : AtomicMatcher(MatcherType::CUSTOM) {} + MatcherResult MatchAtomic(MatchState &state) const override { + auto token = state.token_iterator.Current(); + if (!token || token->type == TokenType::END_OF_INPUT || token->type == TokenType::TOKEN_ERROR || + token->text == ";" || token->text == "(" || token->text == ")" || token->text == "[" || + token->text == "]" || token->text == "{" || token->text == "}") { + return MatcherResult::Failure(); + } + auto result = state.AllocateParseResult(token->text, token->offset, token->length); + state.token_iterator.Advance(); + state.UpdateMaxTokenIndex(); + return result; + } + SuggestionType AddSuggestionInternal(MatchState &) const override { return SuggestionType::OPTIONAL; } + string ToString() const override { return "semantic definition property"; } +}; + +static unique_ptr TransformDefinition(PEGTransformer &transformer, ParseResult &result) { + auto location = result.GetLocation(); + string source(location.length, ' '); + // Preserve token positions and quoted source; comments are intentionally + // omitted before passing the definition to the shared configuration parser. + for (idx_t i = 0; i < transformer.token_iterator.Size(); ++i) { + auto &token = transformer.token_iterator.GetToken(i); + if (token.offset < location.offset || token.offset >= location.End() || token.type == TokenType::COMMENT) continue; + if (token.preceded_by_newline && token.offset > location.offset) { + // Compact model declarations use line breaks to separate fields. + source[token.offset - location.offset - 1] = '\n'; + } + source.replace(token.offset - location.offset, token.length, token.text); + } + auto definition = ParseSidemanticDefinition(source); + if (!definition) throw InternalException("Sidemantic grammar produced an invalid declaration"); + return make_uniq>>(SidemanticStatement(std::move(definition))); +} + +static unique_ptr StartDefinition(PEGTransformer &transformer, ParseResult &result) { + return make_uniq(transformer, result, TransformDefinition); +} + +static unique_ptr TransformSemanticQuery(PEGTransformer &transformer, ParseResult &result) { + auto &list = result.Cast(); + auto &choice = list.Child(1).Child(0); + auto statement = transformer.Transform>(choice.GetResult()); + return make_uniq>>(WrapSidemanticQuery(std::move(statement), true)); +} + +static unique_ptr StartSemanticQuery(PEGTransformer &transformer, ParseResult &result) { + return make_uniq(transformer, result, TransformSemanticQuery); +} + +static unique_ptr TransformPreparable(PEGTransformer &transformer, ParseResult &result) { + auto &choice = result.Cast().Child(0); + return transformer.Transform>(choice.GetResult()); +} + +static unique_ptr TransformPrepare(PEGTransformer &transformer, ParseResult &result) { + auto &list = result.Cast(); + auto statement = make_uniq(); + statement->name = transformer.Transform(list.GetChild(1)); + statement->statement = TransformPreparable(transformer, list.GetChild(3)); + statement->statement->named_param_map = transformer.named_parameter_map; + statement->statement->has_anonymous_parameters = transformer.has_anonymous_parameters; + transformer.ClearParameters(); + unique_ptr prepared = std::move(statement); + return make_uniq>>(std::move(prepared)); +} + +static unique_ptr StartPrepare(PEGTransformer &transformer, ParseResult &result) { + return make_uniq(transformer, result, TransformPrepare); +} + +static unique_ptr TransformExplain(PEGTransformer &transformer, ParseResult &result) { + auto &list = result.Cast(); + optional analyze; + auto &analyze_result = list.Child(1); + if (analyze_result.HasResult()) analyze = transformer.Transform(analyze_result.GetResult()); + optional> options; + auto &options_result = list.Child(2); + if (options_result.HasResult()) { + options = transformer.Transform>(options_result.GetResult()); + } + auto inner = TransformPreparable(transformer, list.GetChild(3)); + auto statement = PEGTransformerFactory::TransformExplainStatement(transformer, analyze, options, std::move(inner)); + return make_uniq>>(WrapSidemanticQuery(std::move(statement))); +} + +static unique_ptr StartExplain(PEGTransformer &transformer, ParseResult &result) { + return make_uniq(transformer, result, TransformExplain); +} + +class SidemanticGrammar final : public GrammarExtension { +public: + SidemanticGrammar() : GrammarExtension("sidemantic", "Semantic models, metrics, dimensions, segments and queries") {} + vector GetChanges() const override { + return { + GrammarChange::AddRule("SidemanticAtom <- Identifier"), + GrammarChange::AddTerminalRuleOverride("SidemanticAtom", [](const PEGKeywordHelper &) { + return make_uniq(); + }), + GrammarChange::AddRule("SidemanticBody <- SidemanticBlock / SidemanticList / SidemanticObject / SidemanticAtom"), + GrammarChange::AddRule("SidemanticBlock <- '(' SidemanticBody* ')'"), + GrammarChange::AddRule("SidemanticList <- '[' SidemanticBody* ']'"), + GrammarChange::AddRule("SidemanticObject <- '{' SidemanticBody* '}'"), + GrammarChange::AddRule("SidemanticSource <- ColIdOrString ('.' ColIdOrString)*"), + GrammarChange::AddRule("SidemanticModel <- 'MODEL' (SidemanticBlock / ColIdOrString ('FROM' (SidemanticSource / SidemanticBlock) SidemanticBlock / SidemanticBlock)?)"), + GrammarChange::AddRule("SidemanticItem <- ('METRIC' / 'DIMENSION' / 'SEGMENT') (SidemanticBlock / SidemanticSource ('AS' Expression / SidemanticBlock))"), + GrammarChange::AddRule("SidemanticDefinition <- 'SEMANTIC'? ('CREATE' ('OR' 'REPLACE')?)? (SidemanticModel / SidemanticItem)", StartDefinition), + GrammarChange::AddRule("SidemanticQuery <- 'SEMANTIC' (SelectStatement / CreateStatement / InsertStatement)", StartSemanticQuery), + GrammarChange::AddRule("SidemanticPreparable <- SidemanticDefinition / SidemanticQuery"), + GrammarChange::AddRule("SidemanticPrepare <- 'PREPARE' ColIdOrString 'AS' SidemanticPreparable", StartPrepare), + GrammarChange::AddRule("SidemanticExplain <- 'EXPLAIN' AnalyzeKeyword? ExplainOptionList? SidemanticPreparable", StartExplain), + GrammarChange::PrependChoice("Statement", "SidemanticDefinition"), + GrammarChange::PrependChoice("Statement", "SidemanticQuery"), + GrammarChange::PrependChoice("Statement", "SidemanticPrepare"), + GrammarChange::PrependChoice("Statement", "SidemanticExplain"), + }; + } +}; + +struct SidemanticGrammarInfo : public ParserExtensionInfo { + shared_ptr default_grammar; + shared_ptr grammar; +}; + +} // namespace + +shared_ptr RegisterSidemanticGrammar(DatabaseInstance &db) { + auto extension = make_shared_ptr(); + GrammarExtension::Register(db, extension); + auto info = make_shared_ptr(); + info->default_grammar = db.GetParserCache().GetMatcher(); + info->grammar = CompiledGrammar::Create(vector> {*extension}); + return info; +} + +bool ParseSidemanticGrammar(ParserExtensionInfo *info, const string &sql, const ParserOptions &options, + vector> &statements) { + auto grammar_info = dynamic_cast(info); + if (!grammar_info) return false; + auto native = options; + native.extensions = nullptr; + // Respect explicitly composed grammars. Callers may opt in with + // SET active_grammar_extensions = ['sidemantic', ...]. + // Use the database's cached default: statically linked loadable extensions + // can have a different DefaultGrammar() singleton from the host DuckDB. + if (!native.compiled_grammar || native.compiled_grammar == grammar_info->default_grammar) { + native.compiled_grammar = grammar_info->grammar; + } + Parser parser(native); + parser.ParseQuery(sql); + statements = std::move(parser.statements); + return true; +} + +} // namespace duckdb +#else +namespace duckdb { +shared_ptr RegisterSidemanticGrammar(DatabaseInstance &) { return nullptr; } +bool ParseSidemanticGrammar(ParserExtensionInfo *, const string &, const ParserOptions &, + vector> &) { return false; } +} // namespace duckdb +#endif diff --git a/sidemantic-duckdb/src/sidemantic_parser.cpp b/sidemantic-duckdb/src/sidemantic_parser.cpp new file mode 100644 index 000000000..ae63bf205 --- /dev/null +++ b/sidemantic-duckdb/src/sidemantic_parser.cpp @@ -0,0 +1,642 @@ +#include "sidemantic_parser.hpp" +#include "sidemantic_catalog.hpp" +#include "sidemantic.h" + +#include "duckdb/parser/expression/columnref_expression.hpp" +#include "duckdb/parser/expression/subquery_expression.hpp" +#include "duckdb/parser/expression/star_expression.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" +#include "duckdb/parser/parsed_expression_iterator.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/query_node/set_operation_node.hpp" +#include "duckdb/parser/query_node/recursive_cte_node.hpp" +#include "duckdb/parser/query_node/cte_node.hpp" +#include "duckdb/parser/statement/explain_statement.hpp" +#include "duckdb/parser/statement/create_statement.hpp" +#include "duckdb/parser/statement/insert_statement.hpp" +#include "duckdb/parser/statement/prepare_statement.hpp" +#include "duckdb/parser/statement/select_statement.hpp" +#include "duckdb/parser/tableref/basetableref.hpp" +#include "duckdb/parser/tableref/joinref.hpp" +#include "duckdb/parser/tableref/subqueryref.hpp" +#include "duckdb/planner/operator/logical_extension_operator.hpp" + +namespace duckdb { +namespace { + +struct SourceToken { + string text; + idx_t start; + idx_t end; +}; + +// Use DuckDB's lexer for strings, quoted names and comments on both hosts. +// Offsets remain byte offsets into the original SQL, including UTF-8 text. +vector Tokens(const string &sql, string *without_comments = nullptr) { + auto tokens = Parser::Tokenize(sql); + vector result; + if (without_comments) { + *without_comments = sql; + } + for (idx_t i = 0; i < tokens.size(); ++i) { + auto start = tokens[i].start; + auto end = i + 1 < tokens.size() ? tokens[i + 1].start : sql.size(); + if (tokens[i].type == SimplifiedTokenType::SIMPLIFIED_TOKEN_COMMENT) { + if (without_comments) { + for (auto j = start; j < end; ++j) { + if ((*without_comments)[j] != '\n') { + (*without_comments)[j] = ' '; + } + } + } + continue; + } + while (end > start && StringUtil::CharacterIsSpace(sql[end - 1])) { + --end; + } + if (end > start) { + result.push_back({sql.substr(start, end - start), start, end}); + } + } + return result; +} + +bool Keyword(const vector &tokens, idx_t index, const char *keyword) { + return index < tokens.size() && StringUtil::CIEquals(tokens[index].text, keyword); +} + +string IdentifierText(const string &text) { + if (text.size() >= 2 && (text.front() == '"' || text.front() == '\'') && text.back() == text.front()) { + string quote(1, text.front()); + return StringUtil::Replace(text.substr(1, text.size() - 2), quote + quote, quote); + } + return text; +} + +string Literal(const string &text) { + return "'" + StringUtil::Replace(text, "'", "''") + "'"; +} + +class SidemanticSQLStatement : public ExtensionStatement { +public: + explicit SidemanticSQLStatement(unique_ptr data) + : ExtensionStatement(SidemanticParserExtension(), std::move(data)) {} + + unique_ptr Copy() const override { + auto result = make_uniq(parse_data->Copy()); + result->named_param_map = named_param_map; + result->query = query; + result->stmt_location = stmt_location; +#if SIDEMANTIC_NEW_IDENTIFIER_API + result->has_anonymous_parameters = has_anonymous_parameters; +#else + result->stmt_length = stmt_length; +#endif + return std::move(result); + } +}; + +ParserOptions WithoutOverrides(const ParserOptions &options) { + auto result = options; + result.extensions = nullptr; + return result; +} + +// The compatibility frontend only recognizes Sidemantic declarations. Ordinary +// SQL, including PREPARE/EXPLAIN syntax, is still parsed by DuckDB. +unique_ptr ParseCompatibilityStatement(const string &sql, const ParserOptions &options) { + auto tokens = Tokens(sql); + if (tokens.empty()) { + return nullptr; + } + auto definition = ParseSidemanticDefinition(sql); + if (definition) { + return SidemanticStatement(std::move(definition)); + } + bool explicit_semantic = Keyword(tokens, 0, "SEMANTIC"); + auto input = explicit_semantic ? sql.substr(tokens[0].end) : sql; + if (Keyword(tokens, 0, "EXPLAIN") || Keyword(tokens, 0, "PREPARE")) { + // Replace only the nested statement when it is custom syntax. Let the + // native parser validate all surrounding options and parameter syntax. + idx_t nested = 1; + if (Keyword(tokens, 0, "PREPARE")) { + int depth = 0; + for (; nested < tokens.size(); ++nested) { + if (tokens[nested].text == "(") ++depth; + if (tokens[nested].text == ")") --depth; + if (depth == 0 && Keyword(tokens, nested, "AS")) { + ++nested; + break; + } + } + } else { + if (Keyword(tokens, nested, "ANALYZE") || Keyword(tokens, nested, "ANALYSE")) ++nested; + if (nested < tokens.size() && tokens[nested].text == "(") { + int depth = 0; + do { + if (tokens[nested].text == "(") ++depth; + if (tokens[nested].text == ")") --depth; + ++nested; + } while (nested < tokens.size() && depth > 0); + } + } + if (nested < tokens.size()) { + auto nested_sql = sql.substr(tokens[nested].start); + if (ParseSidemanticDefinition(nested_sql) || Keyword(tokens, nested, "SEMANTIC")) { + auto child = ParseCompatibilityStatement(nested_sql, options); + Parser parser(WithoutOverrides(options)); + parser.ParseQuery(sql.substr(0, tokens[nested].start) + "SELECT 1"); + auto statement = std::move(parser.statements.at(0)); + if (statement->type == StatementType::EXPLAIN_STATEMENT) { + statement->Cast().stmt = std::move(child); + } else { + statement->Cast().statement = std::move(child); + } + return WrapSidemanticQuery(std::move(statement)); + } + } + } + Parser parser(WithoutOverrides(options)); + parser.ParseQuery(input); + if (parser.statements.size() != 1) { + throw ParserException("Expected one Sidemantic statement"); + } + return WrapSidemanticQuery(std::move(parser.statements[0]), explicit_semantic); +} + +void ParseCompatibility(const string &sql, const ParserOptions &options, + vector> &statements) { + auto tokens = Tokens(sql); + idx_t start = 0; + for (auto &token : tokens) { + if (token.text != ";") continue; + auto statement = ParseCompatibilityStatement(sql.substr(start, token.start - start), options); + if (statement) statements.push_back(std::move(statement)); + start = token.end; + } + auto statement = ParseCompatibilityStatement(sql.substr(start), options); + if (statement) statements.push_back(std::move(statement)); +} + +struct ModelListOwner { + explicit ModelListOwner(const string &snapshot) : list(sidemantic_snapshot_list_models(snapshot.c_str())) { + if (list.error) { + string error(list.error); + sidemantic_free_model_list(list); + throw InvalidInputException("Sidemantic catalog: %s", error); + } + } + ~ModelListOwner() { sidemantic_free_model_list(list); } + SidemanticModelList list; +}; + +// A null binding shadows a semantic model name (e.g. a CTE or a physical table +// aliased to that name). Each query has its own scope, including nested SELECTs. +using ModelScope = case_insensitive_map_t; +using CteNames = case_insensitive_set_t; + +class SemanticReferences { +public: + explicit SemanticReferences(const SidemanticModelList &list) { + for (idx_t i = 0; i < list.count; ++i) { + models[list.models[i].name] = &list.models[i]; + } + } + + bool Query(QueryNode &node, ModelScope inherited = {}, CteNames ctes = {}) { + bool found = false; + for (auto &cte : node.cte_map.map) { +#if SIDEMANTIC_NEW_IDENTIFIER_API + found |= Query(*cte.second->query_node, inherited, ctes); +#else + found |= Query(*cte.second->query->node, inherited, ctes); +#endif + ctes.insert(SidemanticName(cte.first)); + } + if (node.type == QueryNodeType::SET_OPERATION_NODE) { + for (auto &child : node.Cast().children) found |= Query(*child, inherited, ctes); + return found; + } + if (node.type == QueryNodeType::RECURSIVE_CTE_NODE) { + auto &recursive = node.Cast(); + ctes.insert(SidemanticName(recursive.ctename)); + return Query(*recursive.left, inherited, ctes) | Query(*recursive.right, inherited, ctes) | found; + } + if (node.type == QueryNodeType::CTE_NODE) { + auto &cte = node.Cast(); + found |= Query(*cte.query, inherited, ctes); + ctes.insert(SidemanticName(cte.ctename)); + return Query(*cte.child, inherited, ctes) | found; + } + if (node.type != QueryNodeType::SELECT_NODE) return found; + auto &select = node.Cast(); + auto scope = std::move(inherited); + for (auto &name : ctes) scope[name] = nullptr; + if (select.from_table) CollectTables(*select.from_table, scope, ctes, found); + auto inspect = [&](unique_ptr &expression) { + found |= Expression(*expression, scope, ctes); + }; + for (auto &expression : select.select_list) inspect(expression); + for (auto &expression : select.groups.group_expressions) inspect(expression); + if (select.where_clause) inspect(select.where_clause); + if (select.having) inspect(select.having); + if (select.qualify) inspect(select.qualify); + if (select.from_table) found |= TableExpressions(*select.from_table, scope, ctes); + ParsedExpressionIterator::EnumerateQueryNodeModifiers(node, inspect); + return found; + } + +private: + ModelScope models; + + bool CanonicalField(const SidemanticModelInfo *model, string &name) { + if (!model) return false; + for (idx_t i = 0; i < model->field_count; ++i) { + if (StringUtil::CIEquals(name, model->fields[i])) { + name = model->fields[i]; + return true; + } + } + auto field = name; + auto suffix = field.rfind("__"); + string grain_suffix; + if (suffix != string::npos) { + auto grain = StringUtil::Lower(field.substr(suffix + 2)); + static const case_insensitive_set_t grains {"year", "quarter", "month", "week", "day", "hour", "minute", "second"}; + if (grains.count(grain)) { + field.resize(suffix); + grain_suffix = "__" + grain; + } + } + for (idx_t i = 0; i < model->field_count; ++i) { + if (StringUtil::CIEquals(field, model->fields[i])) { + name = model->fields[i] + grain_suffix; + return true; + } + } + return false; + } + + void CollectTables(TableRef &ref, ModelScope &scope, const CteNames &ctes, bool &found) { + if (ref.type == TableReferenceType::JOIN) { + auto &join = ref.Cast(); + CollectTables(*join.left, scope, ctes, found); + CollectTables(*join.right, scope, ctes, found); + return; + } + if (ref.type == TableReferenceType::SUBQUERY) { + found |= Query(*ref.Cast().subquery->node, scope, ctes); + } + const SidemanticModelInfo *model = nullptr; + string name; + if (ref.type == TableReferenceType::BASE_TABLE) { + auto &table = ref.Cast(); +#if SIDEMANTIC_QUALIFIED_TABLE_API + name = SidemanticName(table.Table()); + bool qualified = !table.GetQualifiedName().Schema().empty() || !table.GetQualifiedName().Catalog().empty(); +#else + name = table.table_name; + bool qualified = !table.schema_name.empty() || !table.catalog_name.empty(); +#endif + auto entry = models.find(name); + if (!qualified && !ctes.count(name) && entry != models.end()) model = entry->second; + if (model) { + name = model->name; +#if SIDEMANTIC_QUALIFIED_TABLE_API + table.SetTable(Identifier(name)); +#else + table.table_name = name; +#endif + } + } + auto alias = ref.alias.empty() ? name : SidemanticName(ref.alias); + if (!alias.empty()) scope[alias] = model; + } + + bool TableExpressions(TableRef &ref, const ModelScope &scope, const CteNames &ctes) { + if (ref.type != TableReferenceType::JOIN) return false; + auto &join = ref.Cast(); + bool found = join.condition && Expression(*join.condition, scope, ctes); + return TableExpressions(*join.left, scope, ctes) | TableExpressions(*join.right, scope, ctes) | found; + } + + bool Expression(ParsedExpression &expression, const ModelScope &scope, const CteNames &ctes) { + if (expression.GetExpressionClass() == ExpressionClass::COLUMN_REF) { + auto &column = expression.Cast(); +#if SIDEMANTIC_NEW_EXPRESSION_API + auto &names = column.ColumnNamesMutable(); +#else + auto &names = column.column_names; +#endif + if (names.size() == 2) { + auto qualifier = SidemanticName(names[0]); + auto local = scope.find(qualifier); + const SidemanticModelInfo *model = nullptr; + if (local != scope.end()) { + model = local->second; + qualifier = local->first; + } else { + auto entry = models.find(qualifier); + if (entry != models.end()) { + model = entry->second; + qualifier = entry->first; + } + } + if (!model) return false; + names[0] = SidemanticIdentifier(qualifier); + auto field = SidemanticName(names[1]); + bool found = CanonicalField(model, field); + if (found) names[1] = SidemanticIdentifier(field); + return found; + } + if (names.size() == 1) { + for (auto &binding : scope) { + auto field = SidemanticName(names[0]); + if (CanonicalField(binding.second, field)) { + names[0] = SidemanticIdentifier(field); + return true; + } + } + } + return false; + } + bool found = false; + if (expression.GetExpressionClass() == ExpressionClass::SUBQUERY) { +#if SIDEMANTIC_NEW_EXPRESSION_API + auto &query = expression.Cast().Subquery(); +#else + auto &query = expression.Cast().subquery; +#endif + found |= Query(*query->node, scope, ctes); + } + ParsedExpressionIterator::EnumerateChildren(expression, [&](ParsedExpression &child) { + found |= Expression(child, scope, ctes); + }); + return found; + } +}; + +class SidemanticBindState : public ClientContextState { +public: + explicit SidemanticBindState(unique_ptr data) : data(std::move(data)) {} + void QueryEnd() override { data.reset(); } + unique_ptr data; +}; + +// Keep the native container intact: DuckDB owns destination names, column +// mapping, RETURNING, write properties, transactions and view persistence. +SelectStatement *QueryInStatement(SQLStatement &statement, bool include_insert_ctes = false) { + if (statement.type == StatementType::EXPLAIN_STATEMENT) { + return QueryInStatement(*statement.Cast().stmt, include_insert_ctes); + } + if (statement.type == StatementType::SELECT_STATEMENT) { + return &statement.Cast(); + } + if (statement.type == StatementType::CREATE_STATEMENT) { + auto &info = *statement.Cast().info; + if (info.type == CatalogType::VIEW_ENTRY) return info.Cast().query.get(); + if (info.type == CatalogType::TABLE_ENTRY) return info.Cast().query.get(); + } + if (statement.type == StatementType::INSERT_STATEMENT) { +#if SIDEMANTIC_INSERT_QUERY_NODE + auto &insert = *statement.Cast().node; +#else + auto &insert = statement.Cast(); +#endif + auto &query = insert.select_statement; + if (query && include_insert_ctes && !insert.cte_map.map.empty()) { + // WITH before INSERT is an outer scope. Preserve it around the + // source SELECT instead of merging it with that SELECT's own WITH. + auto wrapper = make_uniq(); + wrapper->select_list.push_back(make_uniq()); + wrapper->cte_map = std::move(insert.cte_map); + auto inner = make_uniq(); + inner->node = std::move(query->node); + wrapper->from_table = make_uniq(std::move(inner), "__sidemantic_insert_source"); + query->node = std::move(wrapper); + } + return query.get(); + } + return nullptr; +} + +} // namespace + +unique_ptr SidemanticParseData::Copy() const { + auto copy = make_uniq(); + copy->operation = operation; + copy->content = content; + copy->replace = replace; + copy->explicit_semantic = explicit_semantic; + if (statement) copy->statement = statement->Copy(); + return std::move(copy); +} + +string SidemanticParseData::ToString() const { + return statement ? statement->ToString() : content; +} + +unique_ptr SidemanticStatement(unique_ptr data) { + auto result = make_uniq(std::move(data)); + auto &parsed = static_cast(*result->parse_data); + if (parsed.statement) { + result->named_param_map = parsed.statement->named_param_map; + result->query = parsed.statement->query; + result->stmt_location = parsed.statement->stmt_location; +#if SIDEMANTIC_NEW_IDENTIFIER_API + result->has_anonymous_parameters = parsed.statement->has_anonymous_parameters; +#else + result->stmt_length = parsed.statement->stmt_length; +#endif + } + return std::move(result); +} + +unique_ptr ParseSidemanticDefinition(const string &sql) { + string clean; + auto tokens = Tokens(sql, &clean); + while (!tokens.empty() && tokens.back().text == ";") tokens.pop_back(); + idx_t pos = Keyword(tokens, 0, "SEMANTIC") ? 1 : 0; + bool create = Keyword(tokens, pos, "CREATE"); + bool replace = false; + if (create) { + ++pos; + if (Keyword(tokens, pos, "OR") && Keyword(tokens, pos + 1, "REPLACE")) { + pos += 2; + replace = true; + } + } + if (pos >= tokens.size()) return nullptr; + bool model = Keyword(tokens, pos, "MODEL"); + bool item = Keyword(tokens, pos, "METRIC") || Keyword(tokens, pos, "DIMENSION") || Keyword(tokens, pos, "SEGMENT"); + if (!model && !item) return nullptr; + auto result = make_uniq(); + result->replace = replace; + result->operation = model ? "model" : "item"; + result->content = clean.substr(tokens[pos].start, tokens.back().end - tokens[pos].start); + if (model && !create && pos + 2 == tokens.size() && tokens[pos + 1].text != "(") { + result->operation = "use"; + result->content = IdentifierText(tokens[pos + 1].text); + } else if (model && create && pos + 2 < tokens.size() && tokens[pos + 2].text == "(") { + auto name = IdentifierText(tokens[pos + 1].text); + int depth = 0; + bool has_name = false; + bool property_start = false; + for (idx_t i = pos + 2; i < tokens.size(); ++i) { + auto &token = tokens[i].text; + if (token == "(" || token == "[" || token == "{") { + ++depth; + property_start = i == pos + 2; + continue; + } + if (token == ")" || token == "]" || token == "}") --depth; + if (depth == 1 && property_start && Keyword(tokens, i, "NAME") && i + 1 < tokens.size()) { + auto value = i + 1; + if (tokens[value].text == ":" || tokens[value].text == "=") ++value; + if (value >= tokens.size()) { + throw ParserException("CREATE MODEL name '%s' is missing a body name", name); + } + auto body_name = IdentifierText(tokens[value].text); + if (body_name != name) { + throw ParserException("CREATE MODEL name '%s' does not match body name '%s'", name, body_name); + } + has_name = true; + break; + } + property_start = depth == 1 && token == ","; + } + auto open = tokens[pos + 2].start; + if (has_name) { + result->content = "MODEL " + clean.substr(open, tokens.back().end - open); + } else { + auto rest = clean.substr(tokens[pos + 2].end, tokens.back().end - tokens[pos + 2].end); + auto separator = pos + 3 < tokens.size() && tokens[pos + 3].text == ")" ? "" : ", "; + result->content = "MODEL (name " + Literal(name) + separator + rest; + } + } else if (model && create) { + result->content = "MODEL " + clean.substr(tokens[pos].end, tokens.back().end - tokens[pos].end); + } + return result; +} + +unique_ptr WrapSidemanticQuery(unique_ptr statement, bool explicit_semantic) { + if (statement->type == StatementType::EXPLAIN_STATEMENT) { + auto &explain = statement->Cast(); + if (explain.stmt->type == StatementType::EXTENSION_STATEMENT) { + auto &extension = explain.stmt->Cast(); + if (extension.extension.plan_function == sidemantic_plan) { + auto &data = static_cast(*extension.parse_data); + if (data.statement) { + explicit_semantic |= data.explicit_semantic; + auto inner = std::move(data.statement); + explain.stmt = std::move(inner); + } + } + } + } else if (statement->type == StatementType::PREPARE_STATEMENT) { + auto &prepare = statement->Cast(); + prepare.statement = WrapSidemanticQuery(std::move(prepare.statement), explicit_semantic); + return statement; + } + if (QueryInStatement(*statement)) { + auto data = make_uniq(); + data->statement = std::move(statement); + data->explicit_semantic = explicit_semantic; + return SidemanticStatement(std::move(data)); + } else if (explicit_semantic && statement->type != StatementType::EXTENSION_STATEMENT) { + throw ParserException("SEMANTIC expects SELECT, CREATE VIEW, CREATE TABLE AS, INSERT SELECT or a semantic definition"); + } + return statement; +} + +ParserOverrideResult sidemantic_parser_override(ParserExtensionInfo *info, const string &query, ParserOptions &options) { + vector> statements; + if (ParseSidemanticGrammar(info, query, options, statements)) { + for (auto &statement : statements) statement = WrapSidemanticQuery(std::move(statement)); + } else { + ParseCompatibility(query, options, statements); + } + return ParserOverrideResult(std::move(statements)); +} + +#if SIDEMANTIC_TOKEN_PARSE_FN +ParserExtensionParseResult sidemantic_parse(ParserExtensionInfo *, const vector &) { + // All custom forms, including SEMANTIC, are handled by the full-source + // override. Never reconstruct source from a post-failure token tail. + return ParserExtensionParseResult(); +} +#else +ParserExtensionParseResult sidemantic_parse(ParserExtensionInfo *, const string &query) { + auto definition = ParseSidemanticDefinition(query); + if (definition) return ParserExtensionParseResult(std::move(definition)); + return ParserExtensionParseResult(); +} +#endif + +ParserExtensionPlanResult sidemantic_plan(ParserExtensionInfo *, ClientContext &context, + unique_ptr parsed) { + auto &data = static_cast(*parsed); + if (!data.operation.empty()) { + return PlanSidemanticMutation(context, data.operation, data.content, data.replace, "Semantic definitions updated"); + } + context.registered_state->Remove("sidemantic_bind"); + context.registered_state->Insert("sidemantic_bind", make_shared_ptr(std::move(parsed))); + // DuckDB's operator extension binds the original or rewritten AST using the + // same ClientContext, preserving temporary tables, parameters and snapshots. + throw BinderException("Use sidemantic_bind instead"); +} + +BoundStatement sidemantic_bind(ClientContext &context, Binder &binder, OperatorExtensionInfo *, SQLStatement &statement) { + if (statement.type != StatementType::EXTENSION_STATEMENT) return {}; + auto &extension = statement.Cast(); + if (extension.extension.plan_function != sidemantic_plan) return {}; + auto state = context.registered_state->Get("sidemantic_bind"); + if (!state || !state->data) throw InternalException("Sidemantic query bind state is missing"); + // Take ownership before a nested bind can replace the registered state. + auto parsed = std::move(state->data); + auto &data = static_cast(*parsed); + if (!data.statement) throw InternalException("Sidemantic query statement is missing"); + RegisterSidemanticCatalogRead(context, binder.GetStatementProperties()); + auto snapshot = ReadSidemanticCatalog(context); + ModelListOwner models(snapshot.payload); + auto original = data.statement->Copy(); + auto query = QueryInStatement(*original, true); + if (!query) throw InternalException("Sidemantic expected a query-bearing statement"); + // DuckDB identifiers are case-insensitive even when quoted. Normalize only + // semantic bindings to the declared spelling before passing them to Rust. + bool detected = SemanticReferences(models.list).Query(*query->node); + bool semantic = data.explicit_semantic || detected; + if (semantic) { + auto sql = query->ToString(); + auto result = sidemantic_snapshot_rewrite(snapshot.payload.c_str(), sql.c_str()); + if (result.error) { + string error(result.error); + sidemantic_free_result(result); + throw BinderException("Sidemantic: %s", error); + } + if (!result.sql) { + sidemantic_free_result(result); + throw InternalException("Sidemantic returned no rewritten SQL"); + } + string rewritten(result.sql); + sidemantic_free_result(result); + Parser parser(SidemanticBuiltinParserOptions()); + parser.ParseQuery(rewritten); + if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { + throw BinderException("Sidemantic rewrite must produce one SELECT statement"); + } + query->node = std::move(parser.statements[0]->Cast().node); + } else { + // An ordinary SQL statement keeps its exact original scopes and AST. + original = data.statement->Copy(); + } + auto child_binder = Binder::CreateBinder(context, &binder); + return child_binder->Bind(*original); +} + +unique_ptr SidemanticOperatorExtension::Deserialize(Deserializer &) { + throw InternalException("Sidemantic operator should not be serialized"); +} + +} // namespace duckdb diff --git a/sidemantic-duckdb/test/README.md b/sidemantic-duckdb/test/README.md index fb5e514f9..ec4f06693 100644 --- a/sidemantic-duckdb/test/README.md +++ b/sidemantic-duckdb/test/README.md @@ -1,11 +1,25 @@ # Testing this extension -This directory contains all the tests for this extension. The `sql` directory holds tests that are written as [SQLLogicTests](https://duckdb.org/dev/sqllogictest/intro.html). DuckDB aims to have most its tests in this format as SQL statements, so for the quack extension, this should probably be the goal too. + +The `sql` directory contains executable DuckDB SQLLogicTests for semantic queries, +definitions, native contracts, routing, transactions, restart and legacy migration. The root makefile contains targets to build and run all of these tests. To run the SQLLogicTests: ```bash make test ``` -or +or ```bash make test_debug -``` \ No newline at end of file +``` + +For the pinned Cyanoptera build, use `SIDEMANTIC_NATIVE_PEG=1 make test`. Its native +grammar suite explicitly disables parser overrides before executing custom syntax. +The same suite is skipped on DuckDB 1.5.6. + +Run `test_semantic_input_host.py` with `uv run` and set `SIDEMANTIC_DUCKDB_BINARY` +and `SIDEMANTIC_DUCKDB_EXTENSION` to the freshly built artifacts. These tests execute +returned SQL and check policy isolation and invalid arguments. See the main README +for the complete invocation. + +`test_release_workflow.py` exercises the release input allowlist without building +or publishing artifacts. diff --git a/sidemantic-duckdb/test/fixtures/host_import/child.yml b/sidemantic-duckdb/test/fixtures/host_import/child.yml new file mode 100644 index 000000000..9e87e917b --- /dev/null +++ b/sidemantic-duckdb/test/fixtures/host_import/child.yml @@ -0,0 +1,4 @@ +models: + - name: import_child + extends: import_parent + table: import_rows diff --git a/sidemantic-duckdb/test/fixtures/host_import/parent.yml b/sidemantic-duckdb/test/fixtures/host_import/parent.yml new file mode 100644 index 000000000..d1df85df5 --- /dev/null +++ b/sidemantic-duckdb/test/fixtures/host_import/parent.yml @@ -0,0 +1,8 @@ +models: + - name: import_parent + table: import_rows + primary_key: id + metrics: + - name: revenue + agg: sum + sql: amount diff --git a/sidemantic-duckdb/test/sql/sidemantic_advanced_metrics.test b/sidemantic-duckdb/test/sql/sidemantic_advanced_metrics.test new file mode 100644 index 000000000..9be2b26ca --- /dev/null +++ b/sidemantic-duckdb/test/sql/sidemantic_advanced_metrics.test @@ -0,0 +1,130 @@ +# name: test/sql/sidemantic_advanced_metrics.test +# description: Advanced metrics execute over real rows through the native extension +# group: [sidemantic] + +require sidemantic + +statement ok +CREATE TABLE metric_sales (id INTEGER, day DATE, category VARCHAR, amount INTEGER); + +statement ok +INSERT INTO metric_sales VALUES + (1, '2024-01-01', 'A', 10), (2, '2024-01-02', 'A', 30), + (3, '2024-02-01', 'A', 60), (4, '2024-04-01', 'A', 100), + (5, '2024-01-01', 'B', 20), (6, '2024-02-01', 'B', 10); + +statement ok +SELECT * FROM sidemantic_load(' +models: + - name: sales_metrics + table: metric_sales + primary_key: id + dimensions: + - {name: day, type: time} + - {name: category, type: categorical} + metrics: + - {name: revenue, agg: sum, sql: amount} + - {name: orders, agg: count} + - {name: doubled, type: derived, sql: "revenue * 2"} + - {name: average_order, type: ratio, numerator: revenue, denominator: orders} + - {name: running, type: cumulative, base_metric: revenue} + - {name: month_change, type: time_comparison, base_metric: revenue, comparison_type: mom, calculation: difference} +'); + +query TIR +SEMANTIC SELECT s.category, s.doubled, s.average_order FROM sales_metrics s ORDER BY s.category; +---- +A 400 50.000 +B 60 15.000 + +# Running totals partition by category. Calendar comparisons must not substitute +# February for the missing March value when evaluating April. +query TTII +SEMANTIC SELECT s.category, s.day__month, s.running, s.month_change FROM sales_metrics s ORDER BY s.category, s.day__month; +---- +A 2024-01-01 00:00:00 40 NULL +A 2024-02-01 00:00:00 100 20 +A 2024-04-01 00:00:00 200 NULL +B 2024-01-01 00:00:00 20 NULL +B 2024-02-01 00:00:00 30 -10 + +statement ok +CREATE TABLE metric_events (id INTEGER, user_id VARCHAR, event_type VARCHAR, event_date DATE, region VARCHAR, platform VARCHAR); + +statement ok +INSERT INTO metric_events VALUES + (1, 'u1', 'signup', '2024-01-01', 'US', 'web'), + (2, 'u1', 'purchase', '2024-01-02', 'US', 'app'), + (3, 'u1', 'active', '2024-01-02', 'US', 'app'), + (4, 'u2', 'signup', '2024-01-01', 'US', 'web'), + (5, 'u2', 'purchase', '2024-01-10', 'US', 'web'), + (6, 'u2', 'active', '2024-01-03', 'US', 'web'), + (7, 'u3', 'signup', '2024-01-01', 'CA', 'web'); + +statement ok +SELECT * FROM sidemantic_load(' +models: + - name: event_metrics + table: metric_events + primary_key: id + dimensions: + - {name: user_id, type: categorical} + - {name: event_type, type: categorical} + - {name: event_date, type: time} + - {name: region, type: categorical} + - {name: platform, type: categorical} + metrics: + - name: purchase_conversion + type: conversion + entity: user_id + base_event: signup + conversion_event: purchase + conversion_window: 7 days + - name: active_retention + type: retention + entity: user_id + cohort_event: "event_type = ''signup''" + activity_event: "event_type = ''active''" + periods: 7 + retention_granularity: day + - name: multiple_platforms + type: cohort + agg: count + entity: user_id + inner_metrics: + - {name: platform_count, agg: count_distinct, sql: platform} + having: platform_count >= 2 +'); + +query TR +SEMANTIC SELECT e.region, e.purchase_conversion FROM event_metrics e ORDER BY e.region; +---- +CA 0.000 +US 0.500 + +# Retention returns a table and is covered by the structured host contract suite. +statement error +SEMANTIC SELECT event_metrics.active_retention FROM event_metrics; +---- +returns a table + +query I +SEMANTIC SELECT event_metrics.multiple_platforms FROM event_metrics; +---- +1 + +# Invalid user inputs are fallible SQL errors, never internal DuckDB errors. +statement error +SELECT sidemantic_compile_semantic_input('{}', '{}'); +---- +SemanticInput failed: + +query I +SELECT try(sidemantic_compile_semantic_input('{}', '{}')) IS NULL; +---- +true + +query I +SELECT try(sidemantic_rewrite_semantic_input('{}', 'SELECT 1', '{}')) IS NULL; +---- +true diff --git a/sidemantic-duckdb/test/sql/sidemantic_catalog_names.test b/sidemantic-duckdb/test/sql/sidemantic_catalog_names.test new file mode 100644 index 000000000..b75f9ab85 --- /dev/null +++ b/sidemantic-duckdb/test/sql/sidemantic_catalog_names.test @@ -0,0 +1,44 @@ +# name: test/sql/sidemantic_catalog_names.test +# description: Native catalog model names are structured rather than comma-delimited +# group: [sidemantic] + +require sidemantic + +statement ok +CREATE TABLE name_source (id INTEGER, amount INTEGER); + +statement ok +INSERT INTO name_source VALUES (1, 10), (2, 30); + +statement ok +SELECT * FROM sidemantic_load(' +models: + - name: "orders, archive" + table: name_source + primary_key: id + metrics: + - name: revenue + agg: sum + sql: amount +'); + +query T +SELECT * FROM sidemantic_models(); +---- +orders, archive + +query I +SELECT "o"."revenue" FROM "orders, archive" AS "o"; +---- +40 + +statement ok +MODEL "orders, archive"; + +statement ok +METRIC rows AS COUNT(*); + +query I +SELECT "o"."rows" FROM "orders, archive" AS "o"; +---- +2 diff --git a/sidemantic-duckdb/test/sql/sidemantic_containers.test b/sidemantic-duckdb/test/sql/sidemantic_containers.test new file mode 100644 index 000000000..620e4d19b --- /dev/null +++ b/sidemantic-duckdb/test/sql/sidemantic_containers.test @@ -0,0 +1,152 @@ +# name: test/sql/sidemantic_containers.test +# description: Semantic source queries preserve native SQL container behavior +# group: [sidemantic] + +load __TEST_DIR__/sidemantic_containers.duckdb + +require sidemantic + +statement ok +CREATE TABLE container_rows (id INTEGER, amount INTEGER); + +statement ok +INSERT INTO container_rows VALUES (1, 10), (2, 30); + +statement ok +MODEL (name container_orders, table container_rows, primary_key id); + +statement ok +DIMENSION (name id, type numeric); + +statement ok +METRIC revenue AS SUM(amount); + +statement ok +SEMANTIC CREATE VIEW container_view (total) AS SELECT container_orders.revenue FROM container_orders; + +query I +SELECT * FROM container_view; +---- +40 + +statement ok +SEMANTIC CREATE TABLE container_saved AS SELECT container_orders.revenue AS total FROM container_orders; + +query I +SELECT * FROM container_saved; +---- +40 + +statement ok +EXPLAIN SEMANTIC CREATE TABLE container_explained AS SELECT container_orders.revenue FROM container_orders; + +query I +SELECT count(*) FROM duckdb_tables() WHERE table_name = 'container_explained'; +---- +0 + +statement ok +EXPLAIN SEMANTIC INSERT INTO container_saved SELECT container_orders.revenue FROM container_orders; + +query I +SELECT count(*) FROM container_saved; +---- +1 + +statement ok +BEGIN; + +query I +SEMANTIC INSERT INTO container_saved (total) SELECT container_orders.revenue FROM container_orders RETURNING total; +---- +40 + +statement ok +SEMANTIC CREATE TABLE container_rollback AS SELECT container_orders.revenue FROM container_orders; + +statement ok +ROLLBACK; + +query I +SELECT count(*) FROM container_saved; +---- +1 + +query I +SELECT count(*) FROM duckdb_tables() WHERE table_name = 'container_rollback'; +---- +0 + +statement ok +PREPARE container_insert AS SEMANTIC INSERT INTO container_saved SELECT o.revenue FROM container_orders o WHERE o.id = $1 RETURNING total; + +query I +EXECUTE container_insert(1); +---- +10 + +query I +EXECUTE container_insert(2); +---- +30 + +# Leading WITH belongs to the INSERT and remains outside the source's own scope. +query I +SEMANTIC WITH totals AS (SELECT o.revenue AS total FROM container_orders o) INSERT INTO container_saved SELECT total FROM totals RETURNING total; +---- +40 + +query I +SEMANTIC WITH container_orders AS (SELECT 17 AS revenue) INSERT INTO container_saved SELECT container_orders.revenue FROM container_orders RETURNING total; +---- +17 + +query I +SEMANTIC WITH totals AS (SELECT 17 AS total) INSERT INTO container_saved WITH totals AS (SELECT o.revenue AS total FROM container_orders o) SELECT total FROM totals RETURNING total; +---- +40 + +statement ok +CREATE TABLE container_named (label VARCHAR, total BIGINT); + +query TI +SEMANTIC INSERT INTO container_named BY NAME SELECT o.revenue AS total, 'sales' AS label FROM container_orders o RETURNING label, total; +---- +sales 40 + +statement error +SEMANTIC INSERT INTO container_saved SELECT o.missing_metric FROM container_orders o; +---- +missing_metric + +# Views store compiled SQL; later model changes do not change their meaning. +statement ok +CREATE OR REPLACE METRIC container_orders.revenue AS COUNT(*); + +query I +SELECT * FROM container_view; +---- +40 + +restart + +require sidemantic + +query I +SELECT * FROM container_view; +---- +40 + +query I +SELECT sum(total) FROM container_saved; +---- +177 + +load __TEST_DIR__/sidemantic_containers.duckdb readonly + +require sidemantic + +statement error +SEMANTIC INSERT INTO container_saved SELECT o.revenue FROM container_orders o; +---- +read-only diff --git a/sidemantic-duckdb/test/sql/sidemantic_file_access.test b/sidemantic-duckdb/test/sql/sidemantic_file_access.test new file mode 100644 index 000000000..b1c762209 --- /dev/null +++ b/sidemantic-duckdb/test/sql/sidemantic_file_access.test @@ -0,0 +1,67 @@ +# name: test/sql/sidemantic_file_access.test +# description: Definition imports obey DuckDB file permissions and remain atomic +# group: [sidemantic] + +require sidemantic + +statement ok +CREATE TABLE import_rows (id INTEGER, amount INTEGER); + +statement ok +INSERT INTO import_rows VALUES (1, 10), (2, 30); + +statement ok +COPY (SELECT 'MODEL (name imported, table import_rows, primary_key id); METRIC revenue AS SUM(amount);' AS line) TO '__TEST_DIR__/allowed_import.sql' (HEADER false, QUOTE ''); + +statement ok +COPY (SELECT 'MODEL (name forbidden, table import_rows, primary_key id);' AS line) TO '__TEST_DIR__/denied_import.sql' (HEADER false, QUOTE ''); + +statement ok +SET allowed_paths = ['__TEST_DIR__/allowed_import.sql']; + +statement ok +SET allowed_directories = ['test/fixtures/host_import']; + +statement ok +SET enable_external_access = false; + +statement error +SELECT * FROM sidemantic_load_file('__TEST_DIR__/denied_import.sql'); +---- +disabled by configuration + +query I +SELECT count(*) FROM sidemantic_models(); +---- +0 + +statement ok +SELECT * FROM sidemantic_load_file('__TEST_DIR__/allowed_import.sql'); + +query I +SEMANTIC SELECT imported.revenue FROM imported; +---- +40 + +# A directory is captured through DuckDB's filesystem before Rust resolves +# inheritance across files. The child sorts before its parent deliberately. +statement ok +SELECT * FROM sidemantic_load_file('test/fixtures/host_import'); + +query I +SEMANTIC SELECT import_child.revenue FROM import_child; +---- +40 + +query I +SELECT count(*) FROM sidemantic_models(); +---- +3 + +statement ok +SET disabled_filesystems = 'LocalFileSystem'; + +statement error +SELECT * FROM sidemantic_load_file('__TEST_DIR__/allowed_import.sql'); +---- +disabled diff --git a/sidemantic-duckdb/test/sql/sidemantic_legacy_migration.test b/sidemantic-duckdb/test/sql/sidemantic_legacy_migration.test new file mode 100644 index 000000000..017de868e --- /dev/null +++ b/sidemantic-duckdb/test/sql/sidemantic_legacy_migration.test @@ -0,0 +1,82 @@ +# name: test/sql/sidemantic_legacy_migration.test +# description: Legacy sidecars seed native transactionally persisted catalog snapshots without being rewritten +# group: [sidemantic] + +load __TEST_DIR__/sidemantic_legacy_migration.duckdb + +require sidemantic + +statement ok +CREATE TABLE legacy_events (id INTEGER, amount INTEGER); + +statement ok +INSERT INTO legacy_events VALUES (1, 10), (2, 20); + +statement ok +COPY (SELECT 'MODEL (name legacy_events, table legacy_events, primary_key id); METRIC total AS SUM(amount);' AS line) TO '__TEST_DIR__/sidemantic_legacy_migration.sidemantic.sql' (HEADER false, QUOTE ''); + +query I +SEMANTIC SELECT legacy_events.total FROM legacy_events; +---- +30 + +# Reading the legacy seed does not write the database. +query I +SELECT count(*) FROM duckdb_functions() WHERE function_name = '__sidemantic_catalog'; +---- +0 + +statement ok +BEGIN; + +statement ok +CREATE OR REPLACE METRIC legacy_events.total AS COUNT(*); + +query I +SEMANTIC SELECT legacy_events.total FROM legacy_events; +---- +2 + +statement ok +ROLLBACK; + +query I +SEMANTIC SELECT legacy_events.total FROM legacy_events; +---- +30 + +query I +SELECT count(*) FROM duckdb_functions() WHERE function_name = '__sidemantic_catalog'; +---- +0 + +statement ok +CREATE OR REPLACE METRIC legacy_events.total AS COUNT(*); + +# The old SQL file remains intact for recovery or manual export comparison. +query I +SELECT contains(content, 'SUM(amount)') AND NOT contains(content, 'COUNT(*)') FROM read_text('__TEST_DIR__/sidemantic_legacy_migration.sidemantic.sql'); +---- +true + +restart + +require sidemantic + +query I +SEMANTIC SELECT legacy_events.total FROM legacy_events; +---- +2 + +# Once migrated, even a malformed legacy file cannot supersede native state. +statement ok +COPY (SELECT 'MODEL (' AS line) TO '__TEST_DIR__/sidemantic_legacy_migration.sidemantic.sql' (HEADER false); + +restart + +require sidemantic + +query I +SEMANTIC SELECT legacy_events.total FROM legacy_events; +---- +2 diff --git a/sidemantic-duckdb/test/sql/sidemantic_memory_and_invalid_persistence.test b/sidemantic-duckdb/test/sql/sidemantic_memory_and_invalid_persistence.test index 9f07a5468..848528438 100644 --- a/sidemantic-duckdb/test/sql/sidemantic_memory_and_invalid_persistence.test +++ b/sidemantic-duckdb/test/sql/sidemantic_memory_and_invalid_persistence.test @@ -1,5 +1,5 @@ # name: test/sql/sidemantic_memory_and_invalid_persistence.test -# description: Test in-memory session-local definitions and invalid persisted definitions +# description: In-memory lifecycle and errors reading malformed unmigrated legacy definitions # group: [sidemantic] require sidemantic @@ -23,29 +23,16 @@ SELECT count(*) FROM sidemantic_models() WHERE model_name = 'temp_orders'; load __TEST_DIR__/sidemantic_invalid_persistence.duckdb -statement ok -CREATE TABLE valid_events (event_id INT); - -statement ok -INSERT INTO valid_events VALUES (1), (2); - -statement ok -SEMANTIC CREATE MODEL valid_events (name valid_events, table valid_events, primary_key event_id); - -statement ok -SEMANTIC CREATE METRIC event_count AS COUNT(*); - -query I -SEMANTIC SELECT valid_events.event_count FROM valid_events; ----- -2 - statement ok COPY (SELECT 'MODEL (' AS line) TO '__TEST_DIR__/sidemantic_invalid_persistence.sidemantic.sql' (HEADER false); -restart no_extension_load +restart + +require sidemantic +# LOAD only registers the extension. Legacy validation happens when definitions +# are first accessed, without creating or replacing native catalog entries. statement error -LOAD '__BUILD_DIRECTORY__/extension/sidemantic/sidemantic.duckdb_extension'; +SELECT * FROM sidemantic_models(); ---- -Failed to autoload sidemantic definitions +Sidemantic diff --git a/sidemantic-duckdb/test/sql/sidemantic_native_contract.test b/sidemantic-duckdb/test/sql/sidemantic_native_contract.test index ff9ec72c7..22edc9cc8 100644 --- a/sidemantic-duckdb/test/sql/sidemantic_native_contract.test +++ b/sidemantic-duckdb/test/sql/sidemantic_native_contract.test @@ -7,13 +7,13 @@ load __TEST_DIR__/sidemantic_native_contract.duckdb require sidemantic statement ok -CREATE TABLE native_customers (customer_id INT, country VARCHAR); +CREATE TABLE native_customers (customer_id INT, country VARCHAR, credit_limit INT); statement ok CREATE TABLE native_orders (order_id INT, customer_id INT, status VARCHAR, amount INT); statement ok -INSERT INTO native_customers VALUES (1, 'CA'), (2, 'US'); +INSERT INTO native_customers VALUES (1, 'CA', 100), (2, 'US', 100); statement ok INSERT INTO native_orders VALUES (101, 1, 'paid', 120), (102, 2, 'paid', 200), (103, 2, 'pending', 50); @@ -45,6 +45,12 @@ models: dimensions: - name: country type: categorical + metrics: + - name: total_credit + agg: sum + sql: credit_limit + - name: customer_count + agg: count '); query IR rowsort @@ -64,6 +70,20 @@ SELECT sidemantic_rewrite_sql('SELECT native_orders.revenue, native_customers.co ---- true +# Customer 2 repeats across two orders. Each customer's credit must contribute +# once, even though both customers have the same credit value (SUM DISTINCT +# would incorrectly collapse them). Order revenue retains all three rows. +query III +SEMANTIC SELECT native_orders.revenue, native_customers.total_credit, native_customers.customer_count FROM native_orders; +---- +370 200 2 + +query TII rowsort +SEMANTIC SELECT native_orders.status, native_customers.total_credit, native_customers.customer_count FROM native_orders; +---- +paid 200 2 +pending 100 1 + statement ok COPY (SELECT 'MODEL (name native_events, table native_events, primary_key event_id); METRIC (name event_count, agg count);' AS line) TO '__TEST_DIR__/native_definitions.sidemantic.sql' (HEADER false, QUOTE ''); diff --git a/sidemantic-duckdb/test/sql/sidemantic_native_peg.test b/sidemantic-duckdb/test/sql/sidemantic_native_peg.test new file mode 100644 index 000000000..4255fb5a7 --- /dev/null +++ b/sidemantic-duckdb/test/sql/sidemantic_native_peg.test @@ -0,0 +1,138 @@ +# name: test/sql/sidemantic_native_peg.test +# description: Native grammar parses custom syntax without the parser override +# group: [sidemantic] + +require-env SIDEMANTIC_NATIVE_PEG 1 + +require sidemantic + +statement ok +SET active_grammar_extensions = ['sidemantic']; + +statement ok +SET allow_parser_override_extension = 'default'; + +statement ok +CREATE TABLE peg_raw (id INTEGER, amount INTEGER); + +statement ok +INSERT INTO peg_raw VALUES (1, 10), (2, 30); + +statement ok +/* Native grammar declaration */ MODEL peg_orders FROM peg_raw ( + primary key (id) + id + sum(amount) AS revenue +); + +query I +SEMANTIC SELECT "o"."revenue" FROM "peg_orders" o; +---- +40 + +statement ok +SEMANTIC CREATE VIEW peg_view AS SELECT peg_orders.revenue AS total FROM peg_orders; + +statement ok +SEMANTIC CREATE TABLE peg_saved AS SELECT peg_orders.revenue AS total FROM peg_orders; + +query I +SEMANTIC INSERT INTO peg_saved SELECT peg_orders.revenue FROM peg_orders RETURNING total; +---- +40 + +query I +SELECT total FROM peg_view; +---- +40 + +statement ok +PREPARE peg_query AS SEMANTIC SELECT o.revenue FROM peg_orders o; + +query I +EXECUTE peg_query; +---- +40 + +statement ok +PREPARE peg_parameter AS SEMANTIC SELECT o.revenue FROM peg_orders o WHERE o.id = $1; + +query I +EXECUTE peg_parameter(1); +---- +10 + +query I +EXECUTE peg_parameter(2); +---- +30 + +statement ok +EXPLAIN (FORMAT JSON) SEMANTIC SELECT o.revenue FROM peg_orders o; + +statement ok +MODEL peg_derived FROM (SELECT * FROM peg_raw WHERE amount > 10) ( + primary key (id) + sum(amount) AS revenue +); + +query I +SEMANTIC SELECT peg_derived.revenue FROM peg_derived; +---- +30 + +statement ok +CREATE OR REPLACE METRIC peg_orders.revenue AS COUNT(*); + +query I +SEMANTIC SELECT peg_orders.revenue FROM peg_orders; +---- +2 + +statement ok +PREPARE peg_definition AS CREATE MODEL prepared_peg (table peg_raw, primary_key id); + +query I +SELECT count(*) FROM sidemantic_models() WHERE model_name = 'prepared_peg'; +---- +0 + +statement ok +EXECUTE peg_definition; + +statement ok +MODEL prepared_peg; + +statement ok +METRIC revenue AS SUM(amount); + +query I +SEMANTIC SELECT prepared_peg.revenue FROM prepared_peg; +---- +40 + +statement ok +EXPLAIN MODEL explained_peg FROM peg_raw ( + primary key (id) + count(*) AS rows +); + +query I +SELECT count(*) FROM sidemantic_models() WHERE model_name = 'explained_peg'; +---- +0 + +statement error +MODEL broken FROM peg_raw (primary key (id) count(*) AS rows; +---- +syntax error + +query I +SELECT count(*) FROM sidemantic_models() WHERE model_name = 'broken'; +---- +0 + +query I +SELECT 42; +---- +42 diff --git a/sidemantic-duckdb/test/sql/sidemantic_parser_coexistence.test b/sidemantic-duckdb/test/sql/sidemantic_parser_coexistence.test new file mode 100644 index 000000000..c5c1d2629 --- /dev/null +++ b/sidemantic-duckdb/test/sql/sidemantic_parser_coexistence.test @@ -0,0 +1,45 @@ +# name: test/sql/sidemantic_parser_coexistence.test +# description: Explicit semantic queries coexist with a previously loaded parser override +# group: [sidemantic] + +# DuckDB 1.5.6 autocomplete claims ordinary SELECT before later overrides run. +# This is the same registration order used by packaged interactive shells. +require autocomplete + +require sidemantic + +statement ok +CREATE TABLE coexist_raw (id INTEGER, amount INTEGER); + +statement ok +INSERT INTO coexist_raw VALUES (1, 10), (2, 30); + +statement ok +MODEL coexist_orders FROM coexist_raw ( + primary key (id) + sum(amount) AS revenue +); + +query I +SEMANTIC SELECT "o"."revenue" FROM "coexist_orders" AS "o"; +---- +40 + +statement error +SEMANTIC SELECT o.revenue, o.missing FROM coexist_orders o; +---- +Sidemantic: + +statement ok +PREPARE coexist_query AS SEMANTIC SELECT o.revenue FROM coexist_orders o; + +query I +EXECUTE coexist_query; +---- +40 + +query I +SELECT amount FROM coexist_raw ORDER BY id; +---- +10 +30 diff --git a/sidemantic-duckdb/test/sql/sidemantic_persistence.test b/sidemantic-duckdb/test/sql/sidemantic_persistence.test index 05810dcd6..10ebab02a 100644 --- a/sidemantic-duckdb/test/sql/sidemantic_persistence.test +++ b/sidemantic-duckdb/test/sql/sidemantic_persistence.test @@ -51,6 +51,24 @@ SEMANTIC SELECT events.total_amount FROM events; ---- 60 +# Definitions and table changes share the same DuckDB transaction. +statement ok +BEGIN; + +statement ok +INSERT INTO events VALUES (4, 40); + +statement ok +SEMANTIC CREATE OR REPLACE METRIC events.total_amount AS COUNT(*); + +query I +SEMANTIC SELECT events.total_amount FROM events; +---- +4 + +statement ok +ROLLBACK; + restart require sidemantic @@ -59,3 +77,32 @@ query I SEMANTIC SELECT events.total_amount FROM events; ---- 60 + +query I +SELECT count(*) FROM events; +---- +3 + +# Definitions remain readable when the database is opened read-only; attempts +# to mutate definitions fail through DuckDB's ordinary catalog write checks. +load __TEST_DIR__/sidemantic_persistence.duckdb readonly + +require sidemantic + +query I +SEMANTIC SELECT events.total_amount FROM events; +---- +60 + +statement ok +SEMANTIC MODEL events; + +statement error +SEMANTIC CREATE METRIC events.forbidden AS COUNT(*); +---- +read-only + +query I +SEMANTIC SELECT events.total_amount FROM events; +---- +60 diff --git a/sidemantic-duckdb/test/sql/sidemantic_routing.test b/sidemantic-duckdb/test/sql/sidemantic_routing.test new file mode 100644 index 000000000..ad059e49a --- /dev/null +++ b/sidemantic-duckdb/test/sql/sidemantic_routing.test @@ -0,0 +1,175 @@ +# name: test/sql/sidemantic_routing.test +# description: Semantic routing uses parsed names, scopes and fields +# group: [sidemantic] + +require sidemantic + +statement ok +CREATE TABLE routing_raw (id INTEGER, status VARCHAR, amount INTEGER); + +statement ok +INSERT INTO routing_raw VALUES (1, 'paid', 10), (2, 'paid', 30), (3, 'pending', 5); + +statement ok +/* Leading comments are accepted. */ +MODEL routing_orders FROM routing_raw ( + primary key (id) + status + sum(amount) AS revenue +); + +query TI +SELECT o.status, o.revenue FROM routing_orders AS o ORDER BY o.status; +---- +paid 40 +pending 5 + +query TI +SELECT "o"."status", "o" /* qualifier comment */ . "revenue" +FROM "routing_orders" AS "o" ORDER BY "o"."status"; +---- +paid 40 +pending 5 + +query I +SELECT revenue FROM routing_orders; +---- +45 + +query I +SELECT O.REVENUE FROM ROUTING_ORDERS O; +---- +45 + +query I +SELECT "O"."REVENUE" FROM "ROUTING_ORDERS" "O"; +---- +45 + +# Text inside a comment or string cannot cause semantic dispatch. +query T +SELECT 'routing_orders.revenue; MODEL bogus' /* routing_orders.revenue */; +---- +routing_orders.revenue; MODEL bogus + +query I +SELECT routing_orders.amount FROM routing_raw AS routing_orders ORDER BY id; +---- +10 +30 +5 + +# A CTE or derived table shadows a model name. +query I +WITH routing_orders AS (SELECT 7 AS revenue) SELECT routing_orders.revenue FROM routing_orders; +---- +7 + +query I +SELECT routing_orders.revenue FROM (SELECT 8 AS revenue) routing_orders; +---- +8 + +statement ok +CREATE SCHEMA physical; + +statement ok +CREATE TABLE physical.routing_orders AS SELECT 9 AS revenue; + +query I +SELECT routing_orders.revenue FROM physical.routing_orders; +---- +9 + +query IIII +SELECT model, metric, dimension, semantic FROM (VALUES (1, 2, 3, 4)) t(model, metric, dimension, semantic); +---- +1 2 3 4 + +# Prepared queries bind semantic definitions at each execution. +statement ok +PREPARE routing_query AS SELECT o.revenue FROM routing_orders o; + +query I +EXECUTE routing_query; +---- +45 + +statement ok +PREPARE routing_parameter AS SELECT o.revenue FROM routing_orders o WHERE o.status = $1; + +query I +EXECUTE routing_parameter('paid'); +---- +40 + +query I +EXECUTE routing_parameter('pending'); +---- +5 + +statement ok +EXPLAIN SELECT o.revenue FROM routing_orders o; + +# EXPLAIN/PREPARE of custom definitions do not mutate the catalog. +statement ok +EXPLAIN MODEL explain_only FROM routing_raw ( + primary key (id) + count(*) AS rows +); + +statement ok +PREPARE define_model AS MODEL prepared_definition FROM routing_raw ( + primary key (id) + count(*) AS rows +); + +query I +SELECT count(*) FROM sidemantic_models() WHERE model_name IN ('explain_only', 'prepared_definition'); +---- +0 + +statement ok +EXECUTE define_model; + +query I +SELECT prepared_definition.rows FROM prepared_definition; +---- +3 + +statement ok +PREPARE replace_metric AS CREATE OR REPLACE METRIC prepared_definition.rows AS SUM(amount); + +statement ok +EXECUTE replace_metric; + +query I +SELECT prepared_definition.rows FROM prepared_definition; +---- +45 + +statement ok +EXECUTE replace_metric; + +# One parse of several statements still executes mutations in order. +statement ok +MODEL batch_model FROM routing_raw ( + primary key (id) + count(*) AS rows +); METRIC revenue AS SUM(amount); + +query I +SELECT batch_model.revenue FROM batch_model; +---- +45 + +statement error +SEMANTIC SELECT missing_model.revenue FROM missing_model; +---- +Table with name missing_model does not exist + +# A detected semantic query must preserve the semantic compiler's diagnostic. +statement error +SELECT o.revenue, o.no_such_field FROM routing_orders o; +---- +Sidemantic: diff --git a/sidemantic-duckdb/test/sql/sidemantic_transactions.test b/sidemantic-duckdb/test/sql/sidemantic_transactions.test new file mode 100644 index 000000000..5b8f79c15 --- /dev/null +++ b/sidemantic-duckdb/test/sql/sidemantic_transactions.test @@ -0,0 +1,209 @@ +# name: test/sql/sidemantic_transactions.test +# description: Semantic definitions execute transactionally and active models belong to connections +# group: [sidemantic] + +require sidemantic + +statement ok +CREATE TABLE lifecycle_orders (id INTEGER, amount INTEGER); + +statement ok +INSERT INTO lifecycle_orders VALUES (1, 10), (2, 30); + +statement ok +BEGIN; + +statement ok +MODEL lifecycle_orders FROM lifecycle_orders ( + primary key (id) + sum(amount) AS revenue +); + +query I +SEMANTIC SELECT lifecycle_orders.revenue FROM lifecycle_orders; +---- +40 + +statement ok +ROLLBACK; + +query I +SELECT count(*) FROM sidemantic_models(); +---- +0 + +statement ok +BEGIN; + +statement ok +MODEL lifecycle_orders FROM lifecycle_orders ( + primary key (id) + sum(amount) AS revenue +); + +statement ok +COMMIT; + +# Planning and preparing a load must not execute it. +statement ok +EXPLAIN SELECT * FROM sidemantic_load('models: [{name: planned_only, table: lifecycle_orders, primary_key: id}]'); + +statement ok +PREPARE load_models AS SELECT * FROM sidemantic_load('models: [{name: prepared_only, table: lifecycle_orders, primary_key: id}]'); + +query I +SELECT count(*) FROM sidemantic_models() WHERE model_name IN ('planned_only', 'prepared_only'); +---- +0 + +statement ok +BEGIN; + +statement ok +EXECUTE load_models; + +query I +SELECT count(*) FROM sidemantic_models() WHERE model_name = 'prepared_only'; +---- +1 + +statement ok +ROLLBACK; + +query I +SELECT count(*) FROM sidemantic_models() WHERE model_name = 'prepared_only'; +---- +0 + +# Execution state must reset when a prepared load is executed again. +statement ok +EXECUTE load_models; + +query I +SELECT count(*) FROM sidemantic_models() WHERE model_name = 'prepared_only'; +---- +1 + +statement ok +PREPARE semantic_total AS SELECT lifecycle_orders.revenue FROM lifecycle_orders; + +statement ok +BEGIN; + +statement ok +CREATE OR REPLACE METRIC lifecycle_orders.revenue AS COUNT(*); + +query I +EXECUTE semantic_total; +---- +2 + +statement ok +ROLLBACK; + +query I +EXECUTE semantic_total; +---- +40 + +# Each connection chooses its own active model, and rollback restores it. +statement ok con1 +MODEL lifecycle_orders; + +statement ok con2 +MODEL prepared_only; + +statement ok con1 +BEGIN; + +statement ok con1 +MODEL prepared_only; + +statement ok con1 +ROLLBACK; + +statement ok con1 +METRIC connection_one AS COUNT(*); + +statement ok con2 +METRIC connection_two AS COUNT(*); + +query I +SEMANTIC SELECT lifecycle_orders.connection_one FROM lifecycle_orders; +---- +2 + +query I +SEMANTIC SELECT prepared_only.connection_two FROM prepared_only; +---- +2 + +# Another transaction cannot observe an uncommitted definition, and its snapshot +# stays unchanged after the writer commits until the reader ends its transaction. +statement ok con2 +BEGIN; + +query I con2 +SELECT count(*) FROM sidemantic_models() WHERE model_name = 'isolated_model'; +---- +0 + +statement ok con1 +BEGIN; + +statement ok con1 +MODEL isolated_model FROM lifecycle_orders ( + primary key (id) + count(*) AS rows +); + +query I con1 +SELECT count(*) FROM sidemantic_models() WHERE model_name = 'isolated_model'; +---- +1 + +query I con2 +SELECT count(*) FROM sidemantic_models() WHERE model_name = 'isolated_model'; +---- +0 + +statement ok con1 +COMMIT; + +query I con2 +SELECT count(*) FROM sidemantic_models() WHERE model_name = 'isolated_model'; +---- +0 + +statement ok con2 +COMMIT; + +query I con2 +SELECT count(*) FROM sidemantic_models() WHERE model_name = 'isolated_model'; +---- +1 + +# A concurrent catalog writer conflicts instead of silently losing an update. +statement ok con1 +BEGIN; + +statement ok con1 +METRIC isolated_model.first_writer AS COUNT(*); + +statement error con2 +METRIC isolated_model.second_writer AS COUNT(*); +---- +Catalog write-write conflict + +statement ok con1 +COMMIT; + +query I con2 +SEMANTIC SELECT isolated_model.first_writer FROM isolated_model; +---- +2 + +statement error con2 +SEMANTIC SELECT isolated_model.second_writer FROM isolated_model; +---- +second_writer diff --git a/sidemantic-duckdb/test/test_release_workflow.py b/sidemantic-duckdb/test/test_release_workflow.py index 094e24a73..cd3357faa 100644 --- a/sidemantic-duckdb/test/test_release_workflow.py +++ b/sidemantic-duckdb/test/test_release_workflow.py @@ -12,7 +12,7 @@ @pytest.mark.parametrize( "version", - ["v1.5.4", 'v1.5.5"; touch injected; #', "$(touch injected)", "v1.5.5\nextra=value"], + ["v1.5.4", 'v1.5.6"; touch injected; #', "$(touch injected)", "v1.5.6\nextra=value"], ) def test_release_rejects_unsupported_input_without_shell_execution(tmp_path, version): workflow = yaml.safe_load(WORKFLOW.read_text()) @@ -44,9 +44,9 @@ def test_release_accepts_supported_version(tmp_path, event): result = subprocess.run( ["bash", "-e", "-c", step["run"]], cwd=tmp_path, - env={**os.environ, "EVENT_NAME": event, "REQUESTED_VERSION": "v1.5.5", "GITHUB_OUTPUT": str(output)}, + env={**os.environ, "EVENT_NAME": event, "REQUESTED_VERSION": "v1.5.6", "GITHUB_OUTPUT": str(output)}, capture_output=True, text=True, ) assert result.returncode == 0, result.stderr - assert output.read_text() == "version=v1.5.5\n" + assert output.read_text() == "version=v1.5.6\n" diff --git a/sidemantic-duckdb/test/test_semantic_input_host.py b/sidemantic-duckdb/test/test_semantic_input_host.py index 25499cd9c..4402c8814 100644 --- a/sidemantic-duckdb/test/test_semantic_input_host.py +++ b/sidemantic-duckdb/test/test_semantic_input_host.py @@ -39,6 +39,8 @@ def execute(sql, error=False): ) if error: assert result.returncode != 0, result.stdout + assert "INTERNAL Error" not in result.stderr, result.stderr + assert '"exception_type":"Internal"' not in result.stderr, result.stderr assert "SemanticInput" in result.stderr, result.stderr return result.stderr assert result.returncode == 0, result.stderr @@ -57,6 +59,53 @@ def call(query=None, source=None, sql=None, context=None, error=False): return result if error else result[0]["sql"] +def test_retention_table_keeps_inactive_cohort_members_in_denominator(): + source = { + "version": 1, + "input_dialect": "duckdb", + "models": [ + { + "name": "events", + "table": "retention_events", + "primary_key": "id", + "dimensions": [{"name": "event_date", "type": "time"}], + "metrics": [ + { + "name": "retention", + "type": "retention", + "entity": "user_id", + "cohort_event": "event_type = 'signup'", + "activity_event": "event_type = 'active'", + "periods": 7, + "retention_granularity": "day", + } + ], + } + ], + } + compiled = call(source=source, query={"metrics": ["events.retention"]}) + rows = execute( + """ + create table retention_events(id integer, user_id varchar, event_type varchar, event_date date); + insert into retention_events values + (1, 'u1', 'signup', '2024-01-01'), (2, 'u2', 'signup', '2024-01-01'), + (3, 'u3', 'signup', '2024-01-01'), (4, 'u1', 'active', '2024-01-02'), + (5, 'u2', 'active', '2024-01-03'); + """ + + compiled + ) + assert rows == [ + { + "cohort_date": "2024-01-01", + "days_since": day, + "active_users": 1, + "cohort_size": 3, + "retention_pct": 33.3, + } + for day in (1, 2) + ] + + @pytest.mark.parametrize("rewrite", [False, True]) def test_population_and_injection_value(rewrite): for tenant, expected in [("a", 10), ("b", 100), ("a' OR 1=1 --", None)]: diff --git a/sidemantic-rs/include/sidemantic.h b/sidemantic-rs/include/sidemantic.h index c329b5907..576cd3d5d 100644 --- a/sidemantic-rs/include/sidemantic.h +++ b/sidemantic-rs/include/sidemantic.h @@ -8,6 +8,7 @@ #define SIDEMANTIC_H #include +#include #ifdef __cplusplus extern "C" { @@ -20,6 +21,55 @@ typedef struct { bool was_rewritten; /* Whether the query was rewritten (false = passthrough) */ } SidemanticRewriteResult; +/* Stateless catalog operations. Snapshot and active_model may be NULL or empty. + * All other inputs must be NUL-terminated UTF-8 strings. Operations are model, + * item, use, yaml, file, and legacy_sql. No context registry or sidecar is changed. + * Publish a successful snapshot in the host transaction; active_model belongs to + * the calling session. Free every result with sidemantic_free_snapshot_result. + */ +typedef struct { + char *snapshot; + char *active_model; + char *error; +} SidemanticSnapshotResult; + +SidemanticSnapshotResult sidemantic_snapshot_apply(const char *snapshot, const char *active_model, + const char *operation, const char *content, bool replace); +void sidemantic_free_snapshot_result(SidemanticSnapshotResult result); + +/* Import host-authorized UTF-8 contents without accessing files or environment. + * Directory imports retain cross-file inheritance and relationship inference. + * Inputs are borrowed for the call; free the returned snapshot normally. + */ +typedef struct { + const char *path; + const char *content; +} SidemanticSource; + +SidemanticSnapshotResult sidemantic_snapshot_load_sources(const char *snapshot, const SidemanticSource *sources, + size_t count, bool directory); + +/* Direct rewrite after the host has identified semantic field references. + * Returns compiler errors instead of falling back to ordinary SQL. + */ +SidemanticRewriteResult sidemantic_snapshot_rewrite(const char *snapshot, const char *sql); + +typedef struct { + char *name; + char **fields; + size_t field_count; +} SidemanticModelInfo; + +typedef struct { + SidemanticModelInfo *models; + size_t count; + char *error; +} SidemanticModelList; + +/* Model names and semantic fields, sorted and owned by the returned result. */ +SidemanticModelList sidemantic_snapshot_list_models(const char *snapshot); +void sidemantic_free_model_list(SidemanticModelList result); + /* * Load semantic models from YAML string. * diff --git a/sidemantic-rs/src/config/loader.rs b/sidemantic-rs/src/config/loader.rs index 8c693eaec..bf98c5fc7 100644 --- a/sidemantic-rs/src/config/loader.rs +++ b/sidemantic-rs/src/config/loader.rs @@ -447,14 +447,50 @@ pub fn load_from_directory(dir: impl AsRef) -> Result { /// Load all YAML files from a directory into a semantic graph with metadata. pub fn load_from_directory_with_metadata(dir: impl AsRef) -> Result { let dir = dir.as_ref(); - if !dir.is_dir() { return Err(SidemanticError::Validation(format!( "Path is not a directory: {}", dir.display() ))); } + let mut sources = Vec::new(); + for path in walkdir(dir)? { + let extension = path + .extension() + .and_then(|value| value.to_str()) + .unwrap_or_default(); + if !["yml", "yaml", "sql"] + .iter() + .any(|supported| extension.eq_ignore_ascii_case(supported)) + { + continue; + } + let content = fs::read_to_string(&path).map_err(|error| { + SidemanticError::Validation(format!("Failed to read {}: {error}", path.display())) + })?; + let name = path + .strip_prefix(dir) + .unwrap_or(&path) + .to_string_lossy() + .into_owned(); + sources.push((name, content)); + } + load_sources_with_metadata(sources, true) +} + +/// Load host-supplied file contents without reading files or process environment. +/// The host owns access checks and ordering; directory-level inheritance and +/// relationship inference retain the same semantics as the filesystem loader. +pub fn load_literal_sources_with_metadata( + sources: impl IntoIterator, +) -> Result { + load_sources_with_metadata(sources, false) +} +fn load_sources_with_metadata( + sources: impl IntoIterator, + expand_environment: bool, +) -> Result { let mut all_models: HashMap = HashMap::new(); let mut all_extends_map: HashMap = HashMap::new(); let mut all_top_level_metrics: Vec = Vec::new(); @@ -467,27 +503,22 @@ pub fn load_from_directory_with_metadata(dir: impl AsRef) -> Result = HashSet::new(); let mut merged_graph_metadata: Option = None; - // Recursively find and parse model files. - for entry in walkdir(dir)? { - let path = entry; - let ext = path + for (path, content) in sources { + let ext = Path::new(&path) .extension() .and_then(|e| e.to_str()) .map(str::to_ascii_lowercase); match ext.as_deref() { Some("yml") | Some("yaml") => { - let content = fs::read_to_string(&path).map_err(|e| { - SidemanticError::Validation(format!("Failed to read {}: {}", path.display(), e)) - })?; - let format = detect_format(&content); - let parsed = parse_content(&content, format)?; + let parsed = if expand_environment { + parse_content(&content, format)? + } else { + parse_literal_content_with_extends(&content, format)? + }; let source_format = format.source_label(); - let source_file = path - .strip_prefix(dir) - .ok() - .map(|value| value.to_string_lossy().to_string()); + let source_file = Some(path.clone()); let ParsedConfig { models, extends_map, @@ -525,14 +556,8 @@ pub fn load_from_directory_with_metadata(dir: impl AsRef) -> Result { - let content = fs::read_to_string(&path).map_err(|e| { - SidemanticError::Validation(format!("Failed to read {}: {}", path.display(), e)) - })?; let parsed = parse_sql_content(&content)?; - let source_file = path - .strip_prefix(dir) - .ok() - .map(|value| value.to_string_lossy().to_string()); + let source_file = Some(path.clone()); let ParsedConfig { models, extends_map, diff --git a/sidemantic-rs/src/config/mod.rs b/sidemantic-rs/src/config/mod.rs index 5074a5a68..c62040439 100644 --- a/sidemantic-rs/src/config/mod.rs +++ b/sidemantic-rs/src/config/mod.rs @@ -11,8 +11,8 @@ pub use crate::adapters::cube::CubeConfig; pub use loader::{ load_from_directory, load_from_directory_with_metadata, load_from_file, load_from_file_with_metadata, load_from_sql_string_with_metadata, load_from_string, - load_from_string_with_metadata, load_literal_yaml_with_metadata, ConfigFormat, - LoadedGraphMetadata, LoadedModelSource, + load_from_string_with_metadata, load_literal_sources_with_metadata, + load_literal_yaml_with_metadata, ConfigFormat, LoadedGraphMetadata, LoadedModelSource, }; pub use schema::{ModelConfig, SidemanticConfig}; pub use sql_parser::{ diff --git a/sidemantic-rs/src/core/graph.rs b/sidemantic-rs/src/core/graph.rs index a51df6676..cb9ad3e4e 100644 --- a/sidemantic-rs/src/core/graph.rs +++ b/sidemantic-rs/src/core/graph.rs @@ -563,6 +563,11 @@ impl SemanticGraph { self.metrics.values().chain(self.model_metrics.values()) } + /// Graph declarations only, excluding indexes rebuilt from model metrics. + pub(crate) fn graph_metrics(&self) -> impl Iterator { + self.metrics.values() + } + /// Add a graph-level table calculation. pub fn add_table_calculation(&mut self, calc: TableCalculation) -> Result<()> { if self.table_calculations.contains_key(&calc.name) { diff --git a/sidemantic-rs/src/core/key_expression.rs b/sidemantic-rs/src/core/key_expression.rs index 38bc2636b..92f393f0f 100644 --- a/sidemantic-rs/src/core/key_expression.rs +++ b/sidemantic-rs/src/core/key_expression.rs @@ -59,12 +59,19 @@ fn deterministic_scalar(expression: &Expression) -> bool { } } -fn source_column(alias: Option<&str>, key: &str) -> Result { - fn identifier(name: &str) -> String { +fn source_column(alias: Option<&str>, key: &str, dialect: DialectType) -> Result { + fn identifier(name: &str, dialect: DialectType) -> String { + // CTE declarations preserve case for these targets, including DuckDB + // intermediate SQL that is later emitted as PostgreSQL or Redshift. + let preserve_case = matches!( + dialect, + DialectType::DuckDB | DialectType::PostgreSQL | DialectType::Redshift + ) && name.chars().any(|character| character.is_ascii_uppercase()); let mut chars = name.chars(); - if chars - .next() - .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') + if !preserve_case + && chars + .next() + .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') && chars.all(|c| c.is_ascii_alphanumeric() || c == '_') { name.to_string() @@ -73,8 +80,14 @@ fn source_column(alias: Option<&str>, key: &str) -> Result { } } parse_semantic_expression(&alias.map_or_else( - || identifier(key), - |alias| format!("{}.{}", identifier(alias), identifier(key)), + || identifier(key, dialect), + |alias| { + format!( + "{}.{}", + identifier(alias, dialect), + identifier(key, dialect) + ) + }, )) } @@ -87,7 +100,7 @@ pub fn key_expression( dialect: DialectType, ) -> Result { let Some(dimension) = model.get_dimension(key) else { - return source_column(alias, key); + return source_column(alias, key, dialect); }; if dimension.window.is_some() { return Err(unsupported()); @@ -126,7 +139,7 @@ pub fn key_expression( { return Err(unsupported()); } - let replacement = source_column(alias, &column.field)?; + let replacement = source_column(alias, &column.field, dialect)?; let replacement = polyglot_sql::generate(&replacement, dialect) .map_err(|error| SidemanticError::SqlGeneration(error.to_string()))?; replacements.insert((column.model, column.field), replacement); @@ -167,6 +180,29 @@ mod tests { use super::*; use crate::core::Dimension; + #[test] + fn key_inputs_preserve_quoted_cte_case() { + for dialect in [ + DialectType::DuckDB, + DialectType::PostgreSQL, + DialectType::Redshift, + ] { + for sql in [None, Some("id"), Some("tenant * 100 + id")] { + let mut model = Model::new("ACCOUNTS", "id"); + if let Some(sql) = sql { + model.dimensions.push(Dimension::new("id").with_sql(sql)); + } + let expression = + key_expression(&model, "id", Some("ACCOUNTS_cte"), dialect).unwrap(); + let sql = polyglot_sql::generate(&expression, dialect).unwrap(); + assert!(sql.contains("\"ACCOUNTS_cte\".id"), "{sql}"); + if sql.contains("tenant") { + assert!(sql.contains("\"ACCOUNTS_cte\".tenant"), "{sql}"); + } + } + } + } + #[test] fn date_bucket_keys_keep_physical_input_scope() { let model = Model::new("monthly_sales", "month") diff --git a/sidemantic-rs/src/ffi.rs b/sidemantic-rs/src/ffi.rs index 0ea2791b5..ff904e972 100644 --- a/sidemantic-rs/src/ffi.rs +++ b/sidemantic-rs/src/ffi.rs @@ -28,6 +28,8 @@ use crate::config::{ use crate::core::SemanticGraph; use crate::sql::QueryRewriter; +mod catalog; + const DEFAULT_CONTEXT_KEY: &str = "__sidemantic_default_context__"; const DEFINITIONS_LOCK_TIMEOUT: Duration = Duration::from_secs(10); const DEFINITIONS_STALE_LOCK_AFTER: Duration = Duration::from_secs(300); @@ -1075,8 +1077,6 @@ pub extern "C" fn sidemantic_add_definition_for_context( db_path: *const c_char, is_replace: bool, ) -> *mut c_char { - use crate::config::parse_sql_model; - let key = match context_key(context) { Ok(key) => key, Err(error) => return error, @@ -1086,12 +1086,70 @@ pub extern "C" fn sidemantic_add_definition_for_context( Err(error) => return error, }; - // Parse to determine what type it is and extract properties - let sql_trimmed = sql_str.trim(); - let sql_upper = sql_trimmed.to_uppercase(); - let mut states = FFI_STATES.lock().unwrap(); let state = states.entry(key).or_default(); + let mut candidate_state = state.clone(); + let change = match add_item_definition(&mut candidate_state, &sql_str, is_replace) { + Ok(change) => change, + Err(error) => return to_c_string(&format!("Error: {error}")), + }; + + // Persist the definition with the owning model so autoload sees the same graph. + if let Some(definitions_path) = get_definitions_path(db_path) { + let _definitions_lock = match lock_definitions_file(&definitions_path) { + Ok(lock) => lock, + Err(e) => return to_c_string(&format!("Error locking definitions file: {e}")), + }; + let content = match read_definitions_file(&definitions_path) { + Ok(content) => content, + Err(e) => return to_c_string(&format!("Error reading definitions file: {e}")), + }; + if !content_has_model_block(&content, &change.model_name) { + return to_c_string(&format!( + "Error: model '{}' is not present in the persisted definitions file", + change.model_name + )); + } + let candidate_content = if let Some(kind) = change.kind { + persist_model_item_definition_to_content( + &content, + &change.model_name, + kind, + &change.item_names, + &sql_str, + is_replace, + ) + } else { + append_definition_to_content(&content, &sql_str) + }; + + if let Err(e) = validate_definitions_content(&candidate_content) { + return to_c_string(&format!("Error validating definitions file: {e}")); + } + if let Err(e) = write_definitions_file_atomic(&definitions_path, &candidate_content) { + return to_c_string(&format!("Error writing to definitions file: {e}")); + } + } + + *state = candidate_state; + ptr::null_mut() +} + +struct ItemDefinitionChange { + model_name: String, + kind: Option, + item_names: Vec, +} + +// Shared by the legacy sidecar API and the stateless DuckDB catalog API. Neither +// caller publishes its candidate state until all validation/persistence succeeds. +fn add_item_definition( + state: &mut FfiState, + sql: &str, + is_replace: bool, +) -> Result { + let sql_trimmed = sql.trim(); + let sql_upper = sql_trimmed.to_uppercase(); // Check for model.name syntax: "METRIC model.name (...)" or "DIMENSION model.name (...)" // Extract model name if present, otherwise use ACTIVE_MODEL @@ -1100,7 +1158,7 @@ pub extern "C" fn sidemantic_add_definition_for_context( let model_name = if let Some(explicit_model) = target_model_name { // Verify the model exists if state.graph.get_model(&explicit_model).is_none() { - return to_c_string(&format!("Error: model '{explicit_model}' not found")); + return Err(format!("model '{explicit_model}' not found")); } explicit_model } else { @@ -1108,25 +1166,27 @@ pub extern "C" fn sidemantic_add_definition_for_context( if let Some(ref name) = state.active_model { name.clone() } else { - return to_c_string("Error: no active model. Create a model first with CREATE MODEL, select one with MODEL , or use METRIC/DIMENSION/SEGMENT model.name syntax."); + return Err("no active model. Create a model first with CREATE MODEL, select one with MODEL , or use METRIC/DIMENSION/SEGMENT model.name syntax.".into()); } }; // Get the model to modify let model = match state.graph.get_model(&model_name) { Some(m) => m.clone(), - None => return to_c_string(&format!("Error: could not find model '{model_name}'")), + None => return Err(format!("could not find model '{model_name}'")), }; // Parse the definition using a dummy model wrapper - let dummy_sql = format!("MODEL (name {model_name}, table dummy);\n{adjusted_sql}"); + // The wrapper is only a parser container. Do not interpolate a model name + // loaded from YAML (it may contain punctuation or spaces) into SQL syntax. + let dummy_sql = format!("MODEL (name __definition, table dummy);\n{adjusted_sql}"); let parsed = match parse_sql_model(&dummy_sql) { Ok(m) => m, - Err(e) => return to_c_string(&format!("Error parsing definition: {e}")), + Err(e) => return Err(format!("parsing definition: {e}")), }; // Extract what was added and update the model - let mut updated_model = model.clone(); + let mut updated_model = model; let mut persisted_kind = None; let mut persisted_item_names = Vec::new(); @@ -1163,50 +1223,15 @@ pub extern "C" fn sidemantic_add_definition_for_context( } } - let mut candidate_state = state.clone(); - if let Err(e) = candidate_state.graph.replace_model(updated_model) { - return to_c_string(&format!("Error updating model: {e}")); - } - - // Persist the definition with the owning model so autoload sees the same graph. - if let Some(definitions_path) = get_definitions_path(db_path) { - let _definitions_lock = match lock_definitions_file(&definitions_path) { - Ok(lock) => lock, - Err(e) => return to_c_string(&format!("Error locking definitions file: {e}")), - }; - let content = match read_definitions_file(&definitions_path) { - Ok(content) => content, - Err(e) => return to_c_string(&format!("Error reading definitions file: {e}")), - }; - if !content_has_model_block(&content, &model_name) { - return to_c_string(&format!( - "Error: model '{model_name}' is not present in the persisted definitions file" - )); - } - let candidate_content = if let Some(kind) = persisted_kind { - persist_model_item_definition_to_content( - &content, - &model_name, - kind, - &persisted_item_names, - &sql_str, - is_replace, - ) - } else { - append_definition_to_content(&content, &sql_str) - }; - - if let Err(e) = validate_definitions_content(&candidate_content) { - return to_c_string(&format!("Error validating definitions file: {e}")); - } - if let Err(e) = write_definitions_file_atomic(&definitions_path, &candidate_content) { - return to_c_string(&format!("Error writing to definitions file: {e}")); - } - } - - *state = candidate_state; - - ptr::null_mut() // Success + state + .graph + .replace_model(updated_model) + .map_err(|e| format!("updating model: {e}"))?; + Ok(ItemDefinitionChange { + model_name, + kind: persisted_kind, + item_names: persisted_item_names, + }) } /// Extract model prefix from "METRIC model.name (...)" or "METRIC model.name AS expr" syntax @@ -1397,8 +1422,9 @@ pub extern "C" fn sidemantic_rewrite_for_context( } }; - let states = FFI_STATES.lock().unwrap(); - let Some(state) = states.get(&key) else { + // A rewrite owns its graph snapshot: concurrent loads/clears cannot invalidate + // it, and parsing does not hold the process-wide context registry lock. + let Some(graph) = graph_snapshot(&key) else { return SidemanticRewriteResult { sql: to_c_string(&sql_str), error: ptr::null_mut(), @@ -1407,7 +1433,7 @@ pub extern "C" fn sidemantic_rewrite_for_context( }; // Check if query references any semantic models - if !query_references_models(&sql_str, &state.graph) { + if !query_references_models(&sql_str, &graph) { // Passthrough - not a semantic query return SidemanticRewriteResult { sql: to_c_string(&sql_str), @@ -1417,7 +1443,7 @@ pub extern "C" fn sidemantic_rewrite_for_context( } // Rewrite the query - let rewriter = QueryRewriter::new(&state.graph); + let rewriter = QueryRewriter::new(&graph); match rewriter.rewrite(&sql_str) { Ok(rewritten) => SidemanticRewriteResult { sql: to_c_string(&rewritten), @@ -1432,6 +1458,14 @@ pub extern "C" fn sidemantic_rewrite_for_context( } } +fn graph_snapshot(key: &str) -> Option { + FFI_STATES + .lock() + .unwrap() + .get(key) + .map(|state| state.graph.clone()) +} + fn semantic_result(result: std::result::Result) -> SidemanticRewriteResult { let result = result.and_then(|sql| { CString::new(sql).map_err(|_| to_c_string("Error: generated SQL contains a NUL byte")) @@ -2299,6 +2333,51 @@ models: remove_definitions_file(&db_path); } + #[test] + fn rewrite_snapshot_survives_context_clear_and_replacement() { + let _guard = test_lock(); + let context = CString::new("duckdb:snapshot").unwrap(); + let yaml = CString::new( + "models:\n - name: orders\n table: original_orders\n primary_key: id\n metrics:\n - name: revenue\n agg: sum\n sql: amount\n", + ) + .unwrap(); + assert_success(sidemantic_load_yaml_for_context( + context.as_ptr(), + yaml.as_ptr(), + )); + let graph = graph_snapshot("duckdb:snapshot").unwrap(); + // This context mutation would deadlock if a snapshot retained the lock. + std::thread::spawn(|| { + let context = CString::new("duckdb:snapshot").unwrap(); + sidemantic_clear_for_context(context.as_ptr()); + let yaml = CString::new( + "models:\n - name: orders\n table: replacement_orders\n primary_key: id\n", + ) + .unwrap(); + assert_success(sidemantic_load_yaml_for_context( + context.as_ptr(), + yaml.as_ptr(), + )); + }) + .join() + .unwrap(); + let rewritten = QueryRewriter::new(&graph) + .rewrite("SELECT orders.revenue FROM orders") + .unwrap(); + assert!(rewritten.contains("original_orders"), "{rewritten}"); + assert!(!rewritten.contains("replacement_orders"), "{rewritten}"); + assert_eq!( + graph_snapshot("duckdb:snapshot") + .unwrap() + .get_model("orders") + .unwrap() + .table + .as_deref(), + Some("replacement_orders") + ); + sidemantic_clear_for_context(context.as_ptr()); + } + #[test] fn test_context_keyed_state_isolates_models_and_active_model() { let _guard = test_lock(); diff --git a/sidemantic-rs/src/ffi/catalog.rs b/sidemantic-rs/src/ffi/catalog.rs new file mode 100644 index 000000000..d6da5b0ca --- /dev/null +++ b/sidemantic-rs/src/ffi/catalog.rs @@ -0,0 +1,950 @@ +//! Stateless semantic catalog payloads for transactional host catalogs. +//! +//! A payload stores declarations, not graph indexes or session state. Every +//! operation validates a private graph and returns a candidate for the host to +//! publish atomically. Failed operations have no observable mutation. + +use std::ffi::{CStr, CString}; +use std::os::raw::c_char; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::path::Path; +use std::ptr; + +use serde::{Deserialize, Serialize}; + +use crate::config::{ + load_from_directory_with_metadata, load_from_file_with_metadata, + load_from_sql_string_with_metadata, load_literal_sources_with_metadata, + load_literal_yaml_with_metadata, parse_sql_model, LoadedGraphMetadata, +}; +use crate::core::{Metric, Model, Parameter, SemanticGraph, TableCalculation}; +use crate::sql::QueryRewriter; + +use super::{ + active_model_for_loaded_models, add_item_definition, semantic_error, semantic_result, + sidemantic_free, FfiState, SidemanticRewriteResult, +}; + +type CatalogResult = Result; + +#[derive(Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Snapshot { + version: u32, + #[serde(default)] + models: Vec, + #[serde(default)] + metrics: Vec, + #[serde(default)] + parameters: Vec, + #[serde(default)] + table_calculations: Vec, + #[serde(default)] + metadata: Option, +} + +impl Snapshot { + fn parse(json: &str) -> CatalogResult { + if json.trim().is_empty() { + return Ok(Self { + version: 1, + ..Self::default() + }); + } + let snapshot: Self = + serde_json::from_str(json).map_err(|e| format!("invalid catalog snapshot: {e}"))?; + if snapshot.version != 1 { + return Err(format!( + "unsupported catalog snapshot version {}", + snapshot.version + )); + } + Ok(snapshot) + } + + fn from_graph(graph: &SemanticGraph) -> Self { + let mut snapshot = Self { + version: 1, + models: graph.models().cloned().collect(), + metrics: graph.graph_metrics().cloned().collect(), + parameters: graph.parameters().cloned().collect(), + table_calculations: graph.table_calculations().cloned().collect(), + metadata: graph.metadata().cloned(), + }; + snapshot.models.sort_by(|a, b| a.name.cmp(&b.name)); + snapshot.metrics.sort_by(|a, b| a.name.cmp(&b.name)); + snapshot.parameters.sort_by(|a, b| a.name.cmp(&b.name)); + snapshot + .table_calculations + .sort_by(|a, b| a.name.cmp(&b.name)); + snapshot + } + + fn into_graph(self) -> CatalogResult { + let mut graph = SemanticGraph::new(); + // Register globals before model indexes, then validate dependencies once + // the full graph exists. Declaration order must not affect reconstruction. + for metric in self.metrics { + graph + .add_metric_unvalidated(metric) + .map_err(|e| e.to_string())?; + } + for parameter in self.parameters { + graph.add_parameter(parameter).map_err(|e| e.to_string())?; + } + for calculation in self.table_calculations { + graph + .add_table_calculation(calculation) + .map_err(|e| e.to_string())?; + } + for model in self.models { + graph.add_model(model).map_err(|e| e.to_string())?; + } + for metric in graph.graph_metrics() { + graph + .validate_metric_dependencies(metric) + .map_err(|e| e.to_string())?; + } + if let Some(metadata) = self.metadata { + graph.set_metadata(metadata); + } + Ok(graph) + } + + fn merge(&mut self, incoming: Self) { + for model in incoming.models { + self.models.retain(|current| current.name != model.name); + self.models.push(model); + } + for metric in incoming.metrics { + self.metrics.retain(|current| current.name != metric.name); + self.metrics.push(metric); + } + for parameter in incoming.parameters { + self.parameters + .retain(|current| current.name != parameter.name); + self.parameters.push(parameter); + } + for calculation in incoming.table_calculations { + self.table_calculations + .retain(|current| current.name != calculation.name); + self.table_calculations.push(calculation); + } + if let Some(incoming) = incoming.metadata { + merge_metadata( + self.metadata.get_or_insert(serde_json::Value::Null), + incoming, + ); + } + } +} + +fn merge_metadata(current: &mut serde_json::Value, incoming: serde_json::Value) { + if let (Some(current), Some(incoming)) = (current.as_object_mut(), incoming.as_object()) { + for (name, value) in incoming { + merge_metadata( + current + .entry(name.clone()) + .or_insert(serde_json::Value::Null), + value.clone(), + ); + } + } else { + *current = incoming; + } +} + +fn required_arg(value: *const c_char, name: &str) -> CatalogResult { + if value.is_null() { + return Err(format!("null {name} pointer")); + } + unsafe { CStr::from_ptr(value) } + .to_str() + .map(str::to_owned) + .map_err(|e| format!("invalid UTF-8 in {name}: {e}")) +} + +fn optional_arg(value: *const c_char, name: &str) -> CatalogResult { + if value.is_null() { + Ok(String::new()) + } else { + required_arg(value, name) + } +} + +fn guard(operation: impl FnOnce() -> CatalogResult) -> CatalogResult { + catch_unwind(AssertUnwindSafe(operation)) + .map_err(|_| "semantic catalog operation panicked".to_string())? +} + +#[repr(C)] +pub struct SidemanticSnapshotResult { + pub snapshot: *mut c_char, + pub active_model: *mut c_char, + pub error: *mut c_char, +} + +fn merge_loaded(snapshot: &str, loaded: LoadedGraphMetadata) -> CatalogResult<(String, String)> { + let mut merged = Snapshot::parse(snapshot)?; + merged.merge(Snapshot::from_graph(&loaded.graph)); + let graph = merged.into_graph()?; + let snapshot = + serde_json::to_string(&Snapshot::from_graph(&graph)).map_err(|e| e.to_string())?; + let active = active_model_for_loaded_models(&loaded.model_order).unwrap_or_default(); + Ok((snapshot, active)) +} + +fn snapshot_result( + operation: impl FnOnce() -> CatalogResult<(String, String)>, +) -> SidemanticSnapshotResult { + let result = guard(|| { + let (snapshot, active_model) = operation()?; + let snapshot = + CString::new(snapshot).map_err(|_| "snapshot contains a NUL byte".to_string())?; + let active_model = + CString::new(active_model).map_err(|_| "model name contains a NUL byte".to_string())?; + Ok((snapshot, active_model)) + }); + match result { + Ok((snapshot, active_model)) => SidemanticSnapshotResult { + snapshot: snapshot.into_raw(), + active_model: active_model.into_raw(), + error: ptr::null_mut(), + }, + Err(error) => SidemanticSnapshotResult { + snapshot: ptr::null_mut(), + active_model: ptr::null_mut(), + error: semantic_error(error), + }, + } +} + +#[repr(C)] +pub struct SidemanticSource { + pub path: *const c_char, + pub content: *const c_char, +} + +/// Import bytes already read and authorized by the embedding host. +#[no_mangle] +pub extern "C" fn sidemantic_snapshot_load_sources( + snapshot: *const c_char, + sources: *const SidemanticSource, + count: usize, + directory: bool, +) -> SidemanticSnapshotResult { + snapshot_result(|| { + if sources.is_null() && count != 0 { + return Err("null sources pointer".into()); + } + let sources = if count == 0 { + &[] + } else { + unsafe { std::slice::from_raw_parts(sources, count) } + }; + let sources = sources + .iter() + .map(|source| { + Ok(( + required_arg(source.path, "source path")?, + required_arg(source.content, "source content")?, + )) + }) + .collect::>>()?; + let loaded = if directory { + load_literal_sources_with_metadata(sources) + } else { + let [(path, content)] = sources.as_slice() else { + return Err("a single-file import requires exactly one source".into()); + }; + if Path::new(path) + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("sql")) + { + load_from_sql_string_with_metadata(content) + } else { + load_literal_yaml_with_metadata(content) + } + } + .map_err(|error| error.to_string())?; + merge_loaded(&optional_arg(snapshot, "snapshot")?, loaded) + }) +} + +fn apply( + snapshot: &str, + active_model: &str, + operation: &str, + content: &str, + replace: bool, +) -> CatalogResult<(String, String)> { + let mut state = FfiState { + graph: Snapshot::parse(snapshot)?.into_graph()?, + active_model: if active_model.is_empty() { + None + } else { + Some(active_model.to_owned()) + }, + }; + match operation { + "model" => { + let model = parse_sql_model(content).map_err(|e| format!("parsing definition: {e}"))?; + let name = model.name.clone(); + if replace { + state.graph.replace_model(model) + } else { + state.graph.add_model(model) + } + .map_err(|e| e.to_string())?; + state.active_model = Some(name); + } + "item" => { + add_item_definition(&mut state, content, replace)?; + } + "use" => { + let name = content.trim(); + if state.graph.get_model(name).is_none() { + return Err(format!("model '{name}' not found")); + } + state.active_model = Some(name.to_owned()); + } + "yaml" | "file" | "legacy_sql" => { + let loaded = match operation { + "yaml" => load_literal_yaml_with_metadata(content), + "legacy_sql" => load_from_sql_string_with_metadata(content), + _ if Path::new(content).is_dir() => load_from_directory_with_metadata(content), + _ => load_from_file_with_metadata(content), + } + .map_err(|e| e.to_string())?; + return merge_loaded(snapshot, loaded); + } + _ => return Err(format!("unknown catalog operation '{operation}'")), + } + for metric in state.graph.graph_metrics() { + state + .graph + .validate_metric_dependencies(metric) + .map_err(|e| e.to_string())?; + } + let snapshot = + serde_json::to_string(&Snapshot::from_graph(&state.graph)).map_err(|e| e.to_string())?; + Ok((snapshot, state.active_model.unwrap_or_default())) +} + +#[no_mangle] +pub extern "C" fn sidemantic_snapshot_apply( + snapshot: *const c_char, + active_model: *const c_char, + operation: *const c_char, + content: *const c_char, + replace: bool, +) -> SidemanticSnapshotResult { + snapshot_result(|| { + apply( + &optional_arg(snapshot, "snapshot")?, + &optional_arg(active_model, "active_model")?, + &required_arg(operation, "operation")?, + &required_arg(content, "content")?, + replace, + ) + }) +} + +#[no_mangle] +pub extern "C" fn sidemantic_free_snapshot_result(result: SidemanticSnapshotResult) { + sidemantic_free(result.snapshot); + sidemantic_free(result.active_model); + sidemantic_free(result.error); +} + +#[no_mangle] +pub extern "C" fn sidemantic_snapshot_rewrite( + snapshot: *const c_char, + sql: *const c_char, +) -> SidemanticRewriteResult { + semantic_result( + guard(|| { + let graph = Snapshot::parse(&optional_arg(snapshot, "snapshot")?)?.into_graph()?; + let sql = required_arg(sql, "sql")?; + QueryRewriter::new(&graph) + .rewrite_with_dialect(&sql, polyglot_sql::DialectType::DuckDB) + .map_err(|e| e.to_string()) + }) + .map_err(semantic_error), + ) +} + +#[repr(C)] +pub struct SidemanticModelInfo { + pub name: *mut c_char, + pub fields: *mut *mut c_char, + pub field_count: usize, +} + +#[repr(C)] +pub struct SidemanticModelList { + pub models: *mut SidemanticModelInfo, + pub count: usize, + pub error: *mut c_char, +} + +#[no_mangle] +pub extern "C" fn sidemantic_snapshot_list_models(snapshot: *const c_char) -> SidemanticModelList { + let result = guard(|| { + let graph = Snapshot::parse(&optional_arg(snapshot, "snapshot")?)?.into_graph()?; + let snapshot = Snapshot::from_graph(&graph); + snapshot + .models + .into_iter() + .map(|model| { + let mut fields: Vec<_> = model + .dimensions + .into_iter() + .map(|field| field.name) + .chain(model.metrics.into_iter().map(|field| field.name)) + .chain(model.segments.into_iter().map(|field| field.name)) + .collect(); + fields.sort(); + fields.dedup(); + let fields: Vec = fields + .into_iter() + .map(|field| { + CString::new(field) + .map_err(|_| "field name contains a NUL byte".to_string()) + }) + .collect::>()?; + let name = CString::new(model.name) + .map_err(|_| "model name contains a NUL byte".to_string())?; + Ok((name, fields)) + }) + .collect::>>() + }); + match result { + Ok(models) => { + // Convert ownership only after every name has passed validation. + let models: Box<[_]> = models + .into_iter() + .map(|(name, fields)| { + let fields: Box<[_]> = fields.into_iter().map(CString::into_raw).collect(); + SidemanticModelInfo { + name: name.into_raw(), + field_count: fields.len(), + fields: Box::into_raw(fields).cast(), + } + }) + .collect(); + SidemanticModelList { + count: models.len(), + models: Box::into_raw(models).cast(), + error: ptr::null_mut(), + } + } + Err(error) => SidemanticModelList { + models: ptr::null_mut(), + count: 0, + error: semantic_error(error), + }, + } +} + +#[no_mangle] +pub extern "C" fn sidemantic_free_model_list(result: SidemanticModelList) { + if !result.models.is_null() { + let models = + unsafe { Box::from_raw(ptr::slice_from_raw_parts_mut(result.models, result.count)) }; + for model in models { + sidemantic_free(model.name); + let fields = unsafe { + Box::from_raw(ptr::slice_from_raw_parts_mut( + model.fields, + model.field_count, + )) + }; + for field in fields { + sidemantic_free(field); + } + } + } + sidemantic_free(result.error); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{ComparisonType, Dimension, ParameterType, TableCalcType}; + + fn take_apply(result: SidemanticSnapshotResult) -> CatalogResult<(String, String)> { + let value = if result.error.is_null() { + assert!(!result.snapshot.is_null()); + assert!(!result.active_model.is_null()); + Ok(unsafe { + ( + CStr::from_ptr(result.snapshot).to_str().unwrap().to_owned(), + CStr::from_ptr(result.active_model) + .to_str() + .unwrap() + .to_owned(), + ) + }) + } else { + assert!(result.snapshot.is_null()); + assert!(result.active_model.is_null()); + Err(unsafe { CStr::from_ptr(result.error).to_str().unwrap().to_owned() }) + }; + sidemantic_free_snapshot_result(result); + value + } + + fn apply_ffi( + snapshot: &str, + active: &str, + operation: &str, + content: &str, + replace: bool, + ) -> CatalogResult<(String, String)> { + let strings: Vec<_> = [snapshot, active, operation, content] + .map(|value| CString::new(value).unwrap()) + .into(); + take_apply(sidemantic_snapshot_apply( + strings[0].as_ptr(), + strings[1].as_ptr(), + strings[2].as_ptr(), + strings[3].as_ptr(), + replace, + )) + } + + fn rewrite(snapshot: &str, sql: &str) -> CatalogResult { + let snapshot = CString::new(snapshot).unwrap(); + let sql = CString::new(sql).unwrap(); + let result = sidemantic_snapshot_rewrite(snapshot.as_ptr(), sql.as_ptr()); + let value = if result.error.is_null() { + assert!(result.was_rewritten); + Ok(unsafe { CStr::from_ptr(result.sql).to_str().unwrap().to_owned() }) + } else { + assert!(!result.was_rewritten); + Err(unsafe { CStr::from_ptr(result.error).to_str().unwrap().to_owned() }) + }; + super::super::sidemantic_free_result(result); + value + } + + fn orders() -> (String, String) { + apply_ffi( + "", + "", + "legacy_sql", + "MODEL (name orders, table raw_orders, primary_key id); METRIC revenue AS SUM(amount);", + false, + ) + .unwrap() + } + + fn load_sources( + snapshot: &str, + sources: &[(&str, &str)], + directory: bool, + ) -> CatalogResult<(String, String)> { + let snapshot = CString::new(snapshot).unwrap(); + let strings: Vec<_> = sources + .iter() + .map(|(path, content)| { + ( + CString::new(*path).unwrap(), + CString::new(*content).unwrap(), + ) + }) + .collect(); + let inputs: Vec<_> = strings + .iter() + .map(|(path, content)| SidemanticSource { + path: path.as_ptr(), + content: content.as_ptr(), + }) + .collect(); + take_apply(sidemantic_snapshot_load_sources( + snapshot.as_ptr(), + inputs.as_ptr(), + inputs.len(), + directory, + )) + } + + #[test] + fn host_sources_preserve_directory_semantics_without_files_or_environment() { + let sources = [ + ("child.yml", "models:\n - name: child\n extends: parent\n table: child_rows\n"), + ("parent.yaml", "models:\n - name: parent\n table: '${SIDEMANTIC_HOST_IMPORT_TEST:-not_expanded}'\n primary_key: id\n metrics:\n - name: revenue\n agg: sum\n sql: amount\n"), + ("events.SQL", "MODEL (name events, table raw_events, primary_key id); METRIC rows AS COUNT(*);"), + ]; + let (snapshot, active) = load_sources("", &sources, true).unwrap(); + assert!(active.is_empty()); + let graph = Snapshot::parse(&snapshot).unwrap().into_graph().unwrap(); + assert_eq!(graph.models().count(), 3); + assert!(graph + .get_model("child") + .unwrap() + .get_metric("revenue") + .is_some()); + assert_eq!( + graph.get_model("parent").unwrap().table_name(), + "${SIDEMANTIC_HOST_IMPORT_TEST:-not_expanded}" + ); + assert!(rewrite(&snapshot, "SELECT events.rows FROM events") + .unwrap() + .contains("COUNT(")); + let (single, active) = load_sources("", &sources[2..], false).unwrap(); + assert_eq!(active, "events"); + assert_eq!(Snapshot::parse(&single).unwrap().models.len(), 1); + assert!(load_sources(&snapshot, &[("broken.yml", "models: [")], true).is_err()); + assert_eq!(Snapshot::parse(&snapshot).unwrap().models.len(), 3); + } + + #[test] + fn host_source_boundary_rejects_missing_inputs() { + assert!(take_apply(sidemantic_snapshot_load_sources( + ptr::null(), + ptr::null(), + 1, + true + )) + .is_err()); + assert!(take_apply(sidemantic_snapshot_load_sources( + ptr::null(), + ptr::null(), + 0, + false + )) + .is_err()); + let (snapshot, active) = take_apply(sidemantic_snapshot_load_sources( + ptr::null(), + ptr::null(), + 0, + true, + )) + .unwrap(); + assert!(active.is_empty()); + assert!(Snapshot::parse(&snapshot).unwrap().models.is_empty()); + let invalid = SidemanticSource { + path: ptr::null(), + content: ptr::null(), + }; + assert!(take_apply(sidemantic_snapshot_load_sources( + ptr::null(), + &invalid, + 1, + true + )) + .is_err()); + } + + #[test] + fn snapshot_mutations_are_private_and_errors_preserve_original() { + let (snapshot, active) = orders(); + assert_eq!(active, "orders"); + let duplicate = apply_ffi( + &snapshot, + &active, + "item", + "METRIC revenue AS SUM(other_amount);", + false, + ) + .unwrap_err(); + assert!(duplicate.contains("duplicate metric"), "{duplicate}"); + let (replacement, _) = apply_ffi( + &snapshot, + &active, + "item", + "METRIC revenue AS SUM(other_amount);", + true, + ) + .unwrap(); + let sql = "SELECT \"o\".\"revenue\" FROM \"orders\" AS \"o\""; + let original_sql = rewrite(&snapshot, sql).unwrap(); + let replaced_sql = rewrite(&replacement, sql).unwrap(); + assert!( + original_sql.contains("amount AS revenue_raw") + && !original_sql.contains("other_amount"), + "{original_sql}" + ); + assert!( + replaced_sql.contains("other_amount AS revenue_raw"), + "{replaced_sql}" + ); + assert!(rewrite( + &snapshot, + "SELECT orders.revenue, orders.missing FROM orders" + ) + .is_err()); + assert!(apply_ffi( + &snapshot, + &active, + "model", + "MODEL (name orders, table other_orders, primary_key id);", + false + ) + .unwrap_err() + .contains("already exists")); + } + + #[test] + fn imports_preserve_globals_and_require_selection_for_multiple_models() { + let yaml = r#" +models: + - name: orders + table: raw_orders + primary_key: id + metrics: + - name: revenue + agg: sum + sql: amount + - name: customers + table: raw_customers + primary_key: id +parameters: + - name: currency + type: string + default_value: USD +metadata: + owner: finance +sql_metrics: | + METRIC total AS orders.revenue; +"#; + let (snapshot, active) = apply_ffi("", "orders", "yaml", yaml, false).unwrap(); + assert!(active.is_empty()); + let error = apply_ffi( + &snapshot, + &active, + "item", + "DIMENSION (name region, type categorical);", + false, + ) + .unwrap_err(); + assert!(error.contains("no active model"), "{error}"); + let (snapshot, active) = apply_ffi( + &snapshot, + &active, + "item", + "DIMENSION customers.region (type categorical);", + false, + ) + .unwrap(); + assert!(active.is_empty()); + let (snapshot, active) = apply_ffi(&snapshot, &active, "use", "orders", false).unwrap(); + assert_eq!(active, "orders"); + let (snapshot, _) = apply_ffi( + &snapshot, + &active, + "model", + "MODEL (name events, table raw_events, primary_key id);", + false, + ) + .unwrap(); + let graph = Snapshot::parse(&snapshot).unwrap().into_graph().unwrap(); + assert_eq!(graph.models().count(), 3); + assert!(graph.get_metric("total").is_some()); + assert_eq!( + graph.get_parameter("currency").unwrap().default_value, + Some(serde_json::json!("USD")) + ); + assert_eq!(graph.metadata().unwrap()["owner"], "finance"); + assert!(graph + .get_model("customers") + .unwrap() + .get_dimension("region") + .is_some()); + let (merged, _) = apply_ffi( + &snapshot, + "", + "yaml", + "metadata:\n department: analytics\nmodels: []\n", + false, + ) + .unwrap(); + let graph = Snapshot::parse(&merged).unwrap().into_graph().unwrap(); + assert_eq!(graph.metadata().unwrap()["owner"], "finance"); + assert_eq!(graph.metadata().unwrap()["department"], "analytics"); + } + + #[test] + fn snapshot_roundtrip_rebuilds_indexes_without_promoting_model_metrics() { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("orders", "id") + .with_table("raw_orders") + .with_dimension(Dimension::time("created_at")) + .with_metric(Metric::sum("revenue", "amount")) + .with_metric(Metric::time_comparison( + "revenue_yoy", + "revenue", + ComparisonType::Yoy, + )), + ) + .unwrap(); + graph + .add_metric(Metric::derived("total", "orders.revenue")) + .unwrap(); + graph + .add_parameter(Parameter { + name: "currency".into(), + parameter_type: ParameterType::String, + description: None, + label: None, + default_value: Some(serde_json::json!("USD")), + allowed_values: None, + default_to_today: false, + }) + .unwrap(); + graph + .add_table_calculation( + TableCalculation::new("running_revenue", TableCalcType::RunningTotal) + .with_field("revenue"), + ) + .unwrap(); + graph.set_metadata(serde_json::json!({"owner":"finance"})); + assert_eq!(graph.metrics().count(), 2); + let snapshot = serde_json::to_string(&Snapshot::from_graph(&graph)).unwrap(); + let rebuilt = Snapshot::parse(&snapshot).unwrap().into_graph().unwrap(); + assert_eq!(rebuilt.graph_metrics().count(), 1); + assert_eq!(rebuilt.metrics().count(), 2); + assert!(rebuilt.get_parameter("currency").is_some()); + assert!(rebuilt.get_table_calculation("running_revenue").is_some()); + assert_eq!( + serde_json::to_string(&Snapshot::from_graph(&rebuilt)).unwrap(), + snapshot + ); + let (changed, _) = apply_ffi( + &snapshot, + "orders", + "item", + "DIMENSION (name status, type categorical);", + false, + ) + .unwrap(); + let changed = Snapshot::parse(&changed).unwrap().into_graph().unwrap(); + assert!(changed.get_table_calculation("running_revenue").is_some()); + assert!(changed.get_parameter("currency").is_some()); + } + + #[test] + fn file_import_captures_contents_and_supports_independent_candidates() { + let path = std::env::temp_dir().join(format!( + "sidemantic-snapshot-{}-{}.sql", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::write(&path, "MODEL (name orders, table captured_orders, primary_key id); METRIC revenue AS SUM(amount);").unwrap(); + let (snapshot, _) = apply_ffi("", "", "file", path.to_str().unwrap(), false).unwrap(); + std::fs::remove_file(&path).unwrap(); + let candidates: Vec<_> = ["first", "second"].into_iter().map(|table| { + let snapshot = snapshot.clone(); + std::thread::spawn(move || { + let (snapshot, _) = apply_ffi(&snapshot, "orders", "model", &format!("MODEL (name orders, table {table}, primary_key id); METRIC revenue AS SUM(amount);"), true).unwrap(); + rewrite(&snapshot, "SELECT orders.revenue FROM orders").unwrap() + }) + }).collect(); + let sql = rewrite(&snapshot, "SELECT orders.revenue FROM orders").unwrap(); + assert!(sql.contains("captured_orders"), "{sql}"); + for (candidate, table) in candidates.into_iter().zip(["first", "second"]) { + let sql = candidate.join().unwrap(); + assert!(sql.contains(table), "{sql}"); + assert!(!sql.contains("captured_orders"), "{sql}"); + } + } + + #[test] + fn model_list_preserves_punctuation_and_reports_semantic_fields() { + let yaml = "models:\n - name: 'orders, archive'\n table: raw_orders\n primary_key: id\n dimensions:\n - name: status\n type: categorical\n metrics:\n - name: revenue\n agg: sum\n sql: amount\n segments:\n - name: active\n sql: status = 'active'\n"; + let (snapshot, active) = apply_ffi("", "", "yaml", yaml, false).unwrap(); + let (snapshot, _) = apply_ffi( + &snapshot, + &active, + "item", + "DIMENSION (name channel, type categorical);", + false, + ) + .unwrap(); + let snapshot = CString::new(snapshot).unwrap(); + let result = sidemantic_snapshot_list_models(snapshot.as_ptr()); + assert!(result.error.is_null()); + assert_eq!(result.count, 1); + unsafe { + let model = &*result.models; + assert_eq!( + CStr::from_ptr(model.name).to_str().unwrap(), + "orders, archive" + ); + let fields: Vec<_> = std::slice::from_raw_parts(model.fields, model.field_count) + .iter() + .map(|field| CStr::from_ptr(*field).to_str().unwrap()) + .collect(); + assert_eq!(fields, ["active", "channel", "revenue", "status"]); + } + sidemantic_free_model_list(result); + let empty = sidemantic_snapshot_list_models(ptr::null()); + assert!(empty.error.is_null()); + assert_eq!(empty.count, 0); + sidemantic_free_model_list(empty); + } + + #[test] + fn snapshot_boundary_rejects_invalid_inputs_without_partial_results() { + for snapshot in ["{", "{\"version\":2}", "{\"version\":1,\"surprise\":true}"] { + assert!(apply_ffi( + snapshot, + "", + "model", + "MODEL (name orders, table orders);", + false + ) + .is_err()); + let snapshot = CString::new(snapshot).unwrap(); + let list = sidemantic_snapshot_list_models(snapshot.as_ptr()); + assert!(!list.error.is_null()); + assert!(list.models.is_null()); + sidemantic_free_model_list(list); + } + let operation = CString::new("model").unwrap(); + let content = CString::new("MODEL (name orders, table orders);").unwrap(); + let invalid_utf8 = [0xff_u8, 0]; + assert!(take_apply(sidemantic_snapshot_apply( + ptr::null(), + ptr::null(), + ptr::null(), + content.as_ptr(), + false + )) + .is_err()); + assert!(take_apply(sidemantic_snapshot_apply( + ptr::null(), + ptr::null(), + operation.as_ptr(), + ptr::null(), + false + )) + .is_err()); + assert!(take_apply(sidemantic_snapshot_apply( + invalid_utf8.as_ptr().cast(), + ptr::null(), + operation.as_ptr(), + content.as_ptr(), + false + )) + .is_err()); + assert!(apply_ffi("", "", "unknown", "", false).is_err()); + let created = take_apply(sidemantic_snapshot_apply( + ptr::null(), + ptr::null(), + operation.as_ptr(), + content.as_ptr(), + false, + )) + .unwrap(); + assert_eq!(created.1, "orders"); + } +} diff --git a/sidemantic-rs/src/sql/generator.rs b/sidemantic-rs/src/sql/generator.rs index eee76e391..39363a772 100644 --- a/sidemantic-rs/src/sql/generator.rs +++ b/sidemantic-rs/src/sql/generator.rs @@ -556,8 +556,9 @@ impl<'a> SqlGenerator<'a> { format!("\n WHERE {}", filter_sql.join(" AND ")) } }; + let cte_name = self.quote_identifier(&self.model_alias(model_name)); cte_defs.push(format!( - "{model_name}_cte AS (\n {cte_select}\n FROM {cte_source}{cte_where}\n)" + "{cte_name} AS (\n {cte_select}\n FROM {cte_source}{cte_where}\n)" )); } sql.push_str("WITH "); @@ -614,7 +615,7 @@ impl<'a> SqlGenerator<'a> { } else if dimension.window.is_some() { let column = format!( "{}.{}", - alias, + self.quote_identifier(&alias), self.quote_identifier(&Self::window_dimension_alias(dimension)) ); if let Some(granularity) = dim_ref @@ -643,7 +644,11 @@ impl<'a> SqlGenerator<'a> { self.dimension_select_expression(dimension, &alias)? } } else if model.is_foreign_key_dimension(&dim_ref.name) { - format!("{}.{}", alias, self.quote_identifier(&dim_ref.name)) + format!( + "{}.{}", + self.quote_identifier(&alias), + self.quote_identifier(&dim_ref.name) + ) } else { let available: Vec<&str> = model.dimensions.iter().map(|d| d.name.as_str()).collect(); @@ -676,7 +681,11 @@ impl<'a> SqlGenerator<'a> { let output_alias = self.output_alias(&metric_ref.model, &metric_ref.alias, &alias_collisions); let raw_alias = self.metric_raw_alias(model, &metric_ref.name, metric); - let raw_col = format!("{alias}.{}", self.quote_identifier(&raw_alias)); + let raw_col = format!( + "{}.{}", + self.quote_identifier(&alias), + self.quote_identifier(&raw_alias) + ); let sql_expr = match metric.r#type { MetricType::Simple if query.ungrouped => raw_col.clone(), @@ -736,18 +745,18 @@ impl<'a> SqlGenerator<'a> { } } Some(Aggregation::Count) => build_symmetric_aggregate_sql_with_key_expr( - &raw_alias, + &raw_col, &primary_key_expr, SymmetricAggType::Count, - Some(&alias), + None, self.symmetric_agg_dialect(), ), Some(Aggregation::CountDistinct) => { build_symmetric_aggregate_sql_with_key_expr( - &raw_alias, + &raw_col, &primary_key_expr, SymmetricAggType::CountDistinct, - Some(&alias), + None, self.symmetric_agg_dialect(), ) } @@ -756,7 +765,7 @@ impl<'a> SqlGenerator<'a> { if let Some(agg) = &metric.agg { self.aggregate_sql(agg, &raw_col)? } else { - metric.to_sql(Some(&alias)) + metric.to_sql(Some(&self.quote_identifier(&alias))) } } } @@ -767,7 +776,7 @@ impl<'a> SqlGenerator<'a> { Some(agg) if agg != &Aggregation::Expression => { self.aggregate_sql(agg, &raw_col)? } - _ => metric.to_sql(Some(&alias)), + _ => metric.to_sql(Some(&self.quote_identifier(&alias))), }, MetricType::Derived => { // For derived metrics, we need to expand referenced metrics @@ -817,9 +826,8 @@ impl<'a> SqlGenerator<'a> { // FROM clause let mut source_start = sql.len(); sql.push_str(&format!( - "FROM {}_cte AS {}\n", - base_model, - self.model_alias(&base_model) + "FROM {0} AS {0}\n", + self.quote_identifier(&self.model_alias(&base_model)) )); // JOIN clauses @@ -839,8 +847,8 @@ impl<'a> SqlGenerator<'a> { if step.relationship_type == RelationshipType::Cross { sql.push_str(&format!( - "CROSS JOIN {}_cte AS {}\n", - step.to_model, to_alias + "CROSS JOIN {0} AS {0}\n", + self.quote_identifier(&to_alias) )); continue; } @@ -850,8 +858,8 @@ impl<'a> SqlGenerator<'a> { // Custom predicates bind physical source columns, independently // of semantic key expressions used for entity deduplication. custom - .replace("{from}", &from_alias) - .replace("{to}", &to_alias) + .replace("{from}", &self.quote_identifier(&from_alias)) + .replace("{to}", &self.quote_identifier(&to_alias)) } else { self.build_default_join_condition_sql( &step.from_model, @@ -884,8 +892,8 @@ impl<'a> SqlGenerator<'a> { "LEFT JOIN" }; sql.push_str(&format!( - "{join_type} {}_cte AS {} ON {}\n", - step.to_model, to_alias, join_condition + "{join_type} {0} AS {0} ON {join_condition}\n", + self.quote_identifier(&to_alias) )); } } @@ -2405,7 +2413,7 @@ impl<'a> SqlGenerator<'a> { Ok(format!("CONCAT({})", parts.join(", '|', "))) } - /// Generate alias for a model (first letter lowercase) + /// Raw CTE identifier; SQL emission must quote it for the target dialect. fn model_alias(&self, model_name: &str) -> String { format!("{model_name}_cte") } @@ -5505,7 +5513,11 @@ impl<'a> SqlGenerator<'a> { } inputs.insert( (input.model, input.field.clone()), - format!("{alias}.{}", self.quote_identifier(&input.field)), + format!( + "{}.{}", + self.quote_identifier(&alias), + self.quote_identifier(&input.field) + ), ); } let raw = self.emit_expression(&crate::core::replace_semantic_columns( @@ -5712,7 +5724,7 @@ impl<'a> SqlGenerator<'a> { } fn normalize_select_expression(&self, expr: &str, alias: &str) -> String { - expr.replace("{model}", alias) + expr.replace("{model}", &self.quote_identifier(alias)) } fn dimension_select_expression( @@ -5725,7 +5737,8 @@ impl<'a> SqlGenerator<'a> { // Parsing names such as "Order Date" as expressions loses that // distinction (and can interpret the second word as an alias). return Ok(format!( - "{alias}.{}", + "{}.{}", + self.quote_identifier(alias), self.quote_identifier(&dimension.name) )); } @@ -5744,7 +5757,11 @@ impl<'a> SqlGenerator<'a> { { replacements.insert( (column.model, column.field.clone()), - format!("{alias}.{}", self.quote_identifier(&column.field)), + format!( + "{}.{}", + self.quote_identifier(alias), + self.quote_identifier(&column.field) + ), ); } } @@ -5871,7 +5888,11 @@ impl<'a> SqlGenerator<'a> { alias: &str, ) -> Result { let raw_alias = format!("{metric_name}_raw"); - let raw_col = format!("{alias}.{}", self.quote_identifier(&raw_alias)); + let raw_col = format!( + "{}.{}", + self.quote_identifier(alias), + self.quote_identifier(&raw_alias) + ); Ok(match metric.agg.as_ref() { Some(Aggregation::CountDistinct) => format!("COUNT(DISTINCT {raw_col})"), Some(Aggregation::Count) => format!("COUNT({raw_col})"), @@ -6044,7 +6065,7 @@ impl<'a> SqlGenerator<'a> { let model = column.model.as_deref().unwrap_or(default_model); format!( "{}.{}", - self.model_alias(model), + self.quote_identifier(&self.model_alias(model)), self.quote_identifier(&column.field) ) } else { @@ -6095,14 +6116,15 @@ impl<'a> SqlGenerator<'a> { ) -> Result { let mut result = expr.to_string(); if self.graph.get_model(default_model).is_some() { - let default_alias = self.model_alias(default_model); + let default_alias = self.quote_identifier(&self.model_alias(default_model)); result = result.replace("{model}", &default_alias); } for model in self.graph.models() { let alias = self.model_alias(&model.name); + let sql_alias = self.quote_identifier(&alias); for dim in &model.dimensions { - let replacement = format!("{}.{}", alias, self.quote_identifier(&dim.name)); + let replacement = format!("{}.{}", sql_alias, self.quote_identifier(&dim.name)); for token in [ format!("{}.{}", model.name, dim.name), format!("{}_cte.{}", model.name, dim.name), @@ -6115,8 +6137,8 @@ impl<'a> SqlGenerator<'a> { .into_owned(); } } - result = result.replace(&format!("{}.", model.name), &format!("{alias}.")); - result = result.replace(&format!("{}_cte.", model.name), &format!("{alias}.")); + result = result.replace(&format!("{}.", model.name), &format!("{sql_alias}.")); + result = result.replace(&format!("{}_cte.", model.name), &format!("{sql_alias}.")); } Ok(result) @@ -6196,7 +6218,7 @@ impl<'a> SqlGenerator<'a> { (column.model, column.field), format!( "{}.{}", - self.model_alias(&model.name), + self.quote_identifier(&self.model_alias(&model.name)), self.quote_identifier(&Self::window_dimension_alias(dimension)) ), ); @@ -6239,7 +6261,7 @@ impl<'a> SqlGenerator<'a> { return Ok(Expression::Raw(Raw { sql: format!( "{}.{}", - self.model_alias(&model.name), + self.quote_identifier(&self.model_alias(&model.name)), dimension .sql .clone() @@ -6284,7 +6306,8 @@ impl<'a> SqlGenerator<'a> { // Replace model references with aliases for dim in &model.dimensions { - let replacement = format!("{}.{}", alias, dim.sql_expr()); + let replacement = + format!("{}.{}", self.quote_identifier(&alias), dim.sql_expr()); let model_pattern = format!("{}.{}", model.name, dim.name); let cte_pattern = format!("{}.{}", cte_name, dim.name); expanded_filter = expanded_filter.replace(&model_pattern, &replacement); @@ -6433,7 +6456,7 @@ impl<'a> SqlGenerator<'a> { // Get SQL with model alias replaced let alias = self.model_alias(&model_name); - let filter_sql = segment.get_sql(&alias); + let filter_sql = segment.get_sql(&self.quote_identifier(&alias)); filters.push(filter_sql); } @@ -6565,6 +6588,174 @@ mod tests { } } + #[test] + fn quoted_model_names_keep_cte_definitions_and_references_aligned() { + for name in ["orders, archive", "Order Archive", "orders\"archive"] { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new(name, "id") + .with_table("raw_orders") + .with_metric(Metric::sum("revenue", "amount")) + .with_metric(Metric::sum("net revenue", "amount")) + .with_dimension(Dimension::categorical("Order Status")) + .with_dimension( + Dimension::categorical("category").with_sql("{model}.status"), + ), + ) + .unwrap(); + let table = format!("\"{}\"", name.replace('"', "\"\"")); + let cte = format!("\"{}_cte\"", name.replace('"', "\"\"")); + for projection in [ + "\"o\".\"revenue\"", + "\"o\".\"net revenue\", \"o\".\"Order Status\", \"o\".category", + ] { + let sql = crate::sql::QueryRewriter::new(&graph) + .rewrite_with_dialect( + &format!("SELECT {projection} FROM {table} AS \"o\""), + DialectType::DuckDB, + ) + .unwrap(); + crate::semantic_input::dialects::parse(&sql, DialectType::DuckDB).unwrap(); + assert!(sql.contains(&format!("WITH {cte} AS")), "{sql}"); + assert!(sql.contains(&format!("FROM {cte} AS {cte}")), "{sql}"); + assert!(sql.contains(&format!("SUM({cte}.")), "{sql}"); + } + } + } + + #[test] + fn derived_metrics_preserve_quoted_cte_references() { + for strict in [false, true] { + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("ORDERS", "id") + .with_table("orders_table") + .with_dimension(Dimension::new("amount")) + .with_metric(Metric::sum("revenue", "amount")) + .with_metric(Metric::derived("inline_total", "SUM(ORDERS.amount)")) + .with_metric(Metric::derived("derived_total", "ORDERS.revenue * 2")), + ) + .unwrap(); + if strict { + graph.set_metric_scopes(HashMap::new()).unwrap(); + } + for dialect in [ + DialectType::DuckDB, + DialectType::PostgreSQL, + DialectType::Redshift, + ] { + let generator = SqlGenerator::new(&graph).with_dialect(dialect); + for (metric, column) in + [("inline_total", "amount"), ("derived_total", "revenue_raw")] + { + let query = SemanticQuery::new().with_metrics(vec![format!("ORDERS.{metric}")]); + let sql = generator.generate(&query).unwrap(); + crate::semantic_input::dialects::parse(&sql, dialect).unwrap(); + assert!(sql.contains("WITH \"ORDERS_cte\" AS"), "{sql}"); + assert!( + sql.contains(&format!("SUM(\"ORDERS_cte\".{column})")), + "{sql}" + ); + } + } + } + } + + #[test] + fn joins_filters_and_fanout_preserve_quoted_cte_references() { + let mut graph = SemanticGraph::new(); + let mut next_status = Dimension::categorical("next_status").with_sql("status"); + next_status.window = Some("LEAD(status) OVER (ORDER BY id)".into()); + let mut implicit_sum = Metric::new("implicit_sum"); + implicit_sum.sql = Some("amount".into()); + graph + .add_model( + Model::new("ORDERS", "id") + .with_table("orders_table") + .with_dimension(Dimension::new("id")) + .with_dimension(Dimension::new("amount")) + .with_dimension(Dimension::categorical("status")) + .with_dimension(next_status) + .with_metric(Metric::sum("revenue", "amount")) + .with_metric(implicit_sum) + .with_relationship( + Relationship::many_to_one("CUSTOMERS").with_keys("customer_id", "id"), + ) + .with_segment(crate::core::Segment::new( + "selected", + "{model}.status = 'complete' OR CUSTOMERS.region = 'west'", + )), + ) + .unwrap(); + graph + .add_model( + Model::new("CUSTOMERS", "id") + .with_table("customers_table") + .with_dimension(Dimension::new("id")) + .with_dimension(Dimension::categorical("region")) + .with_metric(Metric::count("customer_count")) + .with_metric(Metric::count_distinct("distinct_count", "id")), + ) + .unwrap(); + let queries = [ + SemanticQuery::new().with_metrics(vec!["ORDERS.implicit_sum".into()]), + SemanticQuery::new() + .with_metrics(vec!["ORDERS.revenue".into()]) + .with_filters(vec![ + "ORDERS.status = 'complete' OR CUSTOMERS.region = 'west'".into(), + ]), + SemanticQuery::new() + .with_metrics(vec![ + "CUSTOMERS.customer_count".into(), + "CUSTOMERS.distinct_count".into(), + ]) + .with_dimensions(vec!["ORDERS.status".into()]), + SemanticQuery::new() + .with_metrics(vec![ + "ORDERS.revenue".into(), + "CUSTOMERS.customer_count".into(), + ]) + .with_dimensions(vec!["ORDERS.status".into()]) + .with_filters(vec![ + "ORDERS.next_status = 'complete' OR ORDERS.revenue > 100".into(), + ]), + SemanticQuery::new() + .with_metrics(vec!["ORDERS.revenue".into()]) + .with_segments(vec!["ORDERS.selected".into()]), + ]; + for dialect in [ + DialectType::DuckDB, + DialectType::PostgreSQL, + DialectType::Redshift, + ] { + let generator = SqlGenerator::new(&graph).with_dialect(dialect); + for query in &queries { + let sql = generator.generate(query).unwrap(); + let parsed = crate::semantic_input::dialects::parse(&sql, dialect).unwrap(); + // Parsing alone accepts a wrong unquoted reference: PostgreSQL + // folds it to lowercase and then cannot bind the quoted CTE. + polyglot_sql::transform_map(parsed, &|node| { + let identifiers: Vec<_> = match &node { + Expression::Column(column) => column.table.iter().collect(), + Expression::Table(table) => std::iter::once(&table.name) + .chain(table.alias.iter()) + .collect(), + _ => vec![], + }; + for identifier in identifiers { + if matches!(identifier.name.as_str(), "ORDERS_cte" | "CUSTOMERS_cte") { + assert!(identifier.quoted, "{sql}"); + } + } + Ok(node) + }) + .unwrap(); + } + } + } + #[test] fn computed_dimensions_qualify_each_physical_input() { for expression in [ diff --git a/sidemantic-rs/src/sql/rewriter.rs b/sidemantic-rs/src/sql/rewriter.rs index 51bc9296d..031889861 100644 --- a/sidemantic-rs/src/sql/rewriter.rs +++ b/sidemantic-rs/src/sql/rewriter.rs @@ -15,6 +15,8 @@ use crate::sql::SemanticQuery; type QueryPreparer<'a> = &'a dyn Fn(&SemanticGraph, &mut SemanticQuery) -> Result<()>; mod binding; +#[cfg(not(target_arch = "wasm32"))] +mod parser_worker; mod policy; mod yardstick; @@ -151,14 +153,7 @@ fn run_parser( #[cfg(not(target_arch = "wasm32"))] { - let handle = std::thread::Builder::new() - .stack_size(16 * 1024 * 1024) - .spawn(parse) - .map_err(|e| SidemanticError::SqlParse(e.to_string()))?; - - handle - .join() - .map_err(|_| SidemanticError::SqlParse("Polyglot parser thread panicked".into()))? + parser_worker::run(parse) } } @@ -237,6 +232,38 @@ mod tests { graph } + #[test] + fn retention_table_cannot_be_bound_as_a_scalar_metric() { + let retention: Metric = serde_json::from_value(serde_json::json!({ + "name": "retention", "type": "retention", "entity": "user_id", + "cohort_event": "event_type = 'signup'", "activity_event": "event_type = 'active'", + "periods": 7, "retention_granularity": "day" + })) + .unwrap(); + let mut graph = SemanticGraph::new(); + graph + .add_model( + Model::new("events", "id") + .with_table("raw_events") + .with_dimension(Dimension::time("event_date")) + .with_metric(retention), + ) + .unwrap(); + for sql in [ + "SELECT events.retention FROM events", + "SELECT e.retention AS retained FROM events e", + "SELECT * FROM events", + ] { + let error = QueryRewriter::new(&graph).rewrite(sql).unwrap_err(); + assert!(error.to_string().contains("returns a table"), "{error}"); + } + let query = SemanticQuery::new().with_metrics(vec!["events.retention".into()]); + let sql = crate::sql::SqlGenerator::new(&graph) + .generate(&query) + .unwrap(); + assert!(sql.contains("retention_pct")); + } + #[test] fn test_simple_rewrite() { let graph = create_test_graph(); diff --git a/sidemantic-rs/src/sql/rewriter/binding.rs b/sidemantic-rs/src/sql/rewriter/binding.rs index 98c99fd31..637368512 100644 --- a/sidemantic-rs/src/sql/rewriter/binding.rs +++ b/sidemantic-rs/src/sql/rewriter/binding.rs @@ -183,6 +183,16 @@ impl Bindings { }; let alias = format!("__sd_field_{}", self.references.len()); if metric { + let definition = self.graph.get_metric(&reference).or_else(|| { + let (owner, field) = reference.split_once('.')?; + self.graph.get_model(owner)?.get_metric(field) + }); + if definition.is_some_and(|metric| metric.r#type == crate::core::MetricType::Retention) + { + return Err(SidemanticError::Validation(format!( + "Retention metric '{reference}' returns a table; use structured query compilation instead of a scalar SQL projection" + ))); + } self.query.metrics.push(reference.clone()); } else { self.query.dimensions.push(reference.clone()); diff --git a/sidemantic-rs/src/sql/rewriter/parser_worker.rs b/sidemantic-rs/src/sql/rewriter/parser_worker.rs new file mode 100644 index 000000000..81b82c891 --- /dev/null +++ b/sidemantic-rs/src/sql/rewriter/parser_worker.rs @@ -0,0 +1,144 @@ +//! Reuse a large-stack parser worker per calling thread. Workers terminate when +//! their caller exits; independent callers never serialize behind one worker. + +use std::cell::{Cell, RefCell}; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::mpsc::{self, Sender}; + +use polyglot_sql::Expression; + +use crate::error::{Result, SidemanticError}; + +type Parse = Box Result> + Send>; +struct Job { + parse: Parse, + result: Sender>>, +} + +thread_local! { + static WORKER: RefCell>> = const { RefCell::new(None) }; + static ON_WORKER: Cell = const { Cell::new(false) }; +} + +fn guarded_parse(parse: impl FnOnce() -> Result>) -> Result> { + catch_unwind(AssertUnwindSafe(parse)) + .map_err(|_| SidemanticError::SqlParse("Polyglot parser thread panicked".into()))? +} + +pub(super) fn run( + parse: impl FnOnce() -> Result> + Send + 'static, +) -> Result> { + // Nested parser work is already on the large stack. Dispatching it back to + // the same worker would deadlock; creating another worker is unnecessary. + if ON_WORKER.get() { + return guarded_parse(parse); + } + + WORKER.with(|worker| { + let mut worker = worker.borrow_mut(); + if worker.is_none() { + let (sender, receiver) = mpsc::channel::(); + std::thread::Builder::new() + .name("sidemantic-parser".into()) + .stack_size(16 * 1024 * 1024) + .spawn(move || { + ON_WORKER.set(true); + for job in receiver { + // A malformed input must not kill the reusable worker. + let _ = job.result.send(guarded_parse(job.parse)); + } + }) + .map_err(|error| SidemanticError::SqlParse(error.to_string()))?; + *worker = Some(sender); + } + + let (result, receiver) = mpsc::channel(); + if worker + .as_ref() + .unwrap() + .send(Job { + parse: Box::new(parse), + result, + }) + .is_err() + { + *worker = None; + return Err(SidemanticError::SqlParse("Parser worker stopped".into())); + } + match receiver.recv() { + Ok(result) => result, + Err(_) => { + *worker = None; + Err(SidemanticError::SqlParse("Parser worker stopped".into())) + } + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + fn worker_id() -> std::thread::ThreadId { + let (sender, receiver) = mpsc::channel(); + run(move || { + sender.send(std::thread::current().id()).unwrap(); + Ok(vec![]) + }) + .unwrap(); + receiver.recv().unwrap() + } + + #[test] + fn reuses_worker_after_success_error_and_panic() { + let id = worker_id(); + assert_ne!(id, std::thread::current().id()); + assert_eq!(worker_id(), id); + let error = run(|| Err(SidemanticError::SqlParse("bad SQL".into()))).unwrap_err(); + assert!(error.to_string().contains("bad SQL")); + let error = run(|| panic!("parser panic regression")).unwrap_err(); + assert!(error + .to_string() + .contains("Polyglot parser thread panicked")); + assert_eq!(worker_id(), id); + } + + #[test] + fn nested_parser_calls_use_current_worker() { + run(|| { + assert_eq!(worker_id(), std::thread::current().id()); + Ok(vec![]) + }) + .unwrap(); + } + + #[test] + fn independent_callers_parse_concurrently() { + let (started, started_receiver) = mpsc::channel(); + let (release, release_receiver) = mpsc::channel(); + let blocked = std::thread::spawn(move || { + run(move || { + started.send(()).unwrap(); + release_receiver.recv().unwrap(); + Ok(vec![]) + }) + }); + started_receiver + .recv_timeout(Duration::from_secs(5)) + .unwrap(); + let (finished, finished_receiver) = mpsc::channel(); + let independent = std::thread::spawn(move || { + let result = super::super::parse_sql_with_large_stack("SELECT 1"); + finished + .send(result.map(|statements| statements.len())) + .unwrap(); + }); + let result = finished_receiver.recv_timeout(Duration::from_secs(5)); + // Always unblock the first worker, including when this regression fails. + release.send(()).unwrap(); + blocked.join().unwrap().unwrap(); + independent.join().unwrap(); + assert_eq!(result.unwrap().unwrap(), 1); + } +} diff --git a/sidemantic-rs/tests/many_role_contracts.rs b/sidemantic-rs/tests/many_role_contracts.rs index 2c1f55766..e773d1ab2 100644 --- a/sidemantic-rs/tests/many_role_contracts.rs +++ b/sidemantic-rs/tests/many_role_contracts.rs @@ -53,11 +53,11 @@ fn junction_policies_filter_links_without_changing_source_preservation() { let sql = compile_with_semantic_input(&source.to_string(), &query.to_string()).unwrap(); for role in ["primary_tags", "secondary_tags"] { assert!( - sql.contains(&format!("LEFT JOIN {role}$through_cte")), + sql.contains(&format!("LEFT JOIN \"{role}$through_cte\"")), "{sql}" ); assert!( - !sql.contains(&format!("INNER JOIN {role}$through_cte")), + !sql.contains(&format!("INNER JOIN \"{role}$through_cte\"")), "{sql}" ); } diff --git a/tests/adapters/tmdl/test_parsing.py b/tests/adapters/tmdl/test_parsing.py index 56aaf2e0d..3ff0e4017 100644 --- a/tests/adapters/tmdl/test_parsing.py +++ b/tests/adapters/tmdl/test_parsing.py @@ -3037,8 +3037,8 @@ def test_tmdl_keyless_many_to_many_joins_without_keying_off_endpoints(): layer = SemanticLayer() layer.graph = graph sql = layer.compile(dimensions=["Authors.region", "Books.genre"]) - assert "Authors_cte.author_id" in sql - assert "Books_cte.book_author_id" in sql + assert "Authors_cte.author_id" in sql.replace('"', "") + assert "Books_cte.book_author_id" in sql.replace('"', "") layer.conn.execute("CREATE TABLE Authors (author_id VARCHAR, region VARCHAR)") layer.conn.execute("CREATE TABLE Books (book_author_id VARCHAR, genre VARCHAR)") layer.conn.execute("INSERT INTO Authors VALUES ('a', 'US'), ('a', 'EU')") @@ -3152,8 +3152,8 @@ def test_tmdl_many_to_many_on_non_primary_key_column(): sql = layer.compile(metrics=["A.a_count"], dimensions=["B.b_label"]) # The join pairs each side's alternate column (not the declared id key), and each CTE projects # its alternate column, so neither side joins on a column missing from its CTE. - assert re.search(r"ON\s+B_cte\.b_alt\s*=\s*A_cte\.a_alt", sql) or re.search( - r"ON\s+A_cte\.a_alt\s*=\s*B_cte\.b_alt", sql + assert re.search(r'ON\s+"?B_cte"?\.b_alt\s*=\s*"?A_cte"?\.a_alt', sql) or re.search( + r'ON\s+"?A_cte"?\.a_alt\s*=\s*"?B_cte"?\.b_alt', sql ), sql layer.adapter.execute("create table A(id varchar, a_alt varchar)") layer.adapter.execute("create table B(id varchar, b_alt varchar, b_label varchar)") @@ -3210,7 +3210,7 @@ def test_tmdl_one_to_one_recovers_keyless_source_key(): layer = SemanticLayer() layer.graph = graph sql = layer.compile(metrics=["A.a_count"], dimensions=["B.b_label"]) - assert "A_cte.a_key" in sql + assert "A_cte.a_key" in sql.replace('"', "") layer.adapter.execute("create table A(a_key varchar)") layer.adapter.execute("create table B(b_key varchar, b_label varchar)") layer.adapter.execute("insert into A values ('joined')") @@ -3279,7 +3279,7 @@ def test_tmdl_one_to_one_alternate_key_does_not_shadow_real_key(): layer = SemanticLayer() layer.graph = graph sql = layer.compile(metrics=["Orders.cnt"], dimensions=["OrderMeta.channel"]) - assert "Orders_cte.alt_key" in sql + assert "Orders_cte.alt_key" in sql.replace('"', "") layer.adapter.execute("create table Orders(order_id integer, alt_key varchar)") layer.adapter.execute("create table OrderMeta(meta_key varchar, channel varchar)") layer.adapter.execute("insert into Orders values (1, 'alternate')")