diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 051d188..20ff552 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -46,32 +46,15 @@ jobs: extension_name: quackapi format_checks: "format" - # Own job (not the reusable distribution workflow): needs `services:` for - # Postgres so require-env QUACKAPI_TEST_PG_DSN tests actually run. Linux - # amd64 only — does not publish artifacts or expand the release matrix. - linux-amd64-pg-sqllogic: - name: Linux amd64 SQLLogic with Postgres + # Focused Linux validation for the SQLLogic suite and real HTTP contracts. + # The released extension uses DuckDB SQL only; no external database service + # is required. + linux-amd64-sqllogic: + name: Linux amd64 SQLLogic runs-on: ubuntu-24.04 env: GEN: ninja DUCKDB_GIT_VERSION: v1.5.6 - ENABLE_EXTENSION_AUTOLOADING: "1" - ENABLE_EXTENSION_AUTOINSTALL: "1" - QUACKAPI_TEST_PG_DSN: postgresql://quackapi_pg_cache:qa_pg_cache_test@127.0.0.1:5432/quackapi_pg_cache_test - services: - postgres: - image: postgres:16 - env: - POSTGRES_USER: quackapi_pg_cache - POSTGRES_PASSWORD: qa_pg_cache_test - POSTGRES_DB: quackapi_pg_cache_test - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U quackapi_pg_cache -d quackapi_pg_cache_test" - --health-interval 5s - --health-timeout 5s - --health-retries 10 steps: - name: Checkout uses: actions/checkout@v4 @@ -92,20 +75,14 @@ jobs: - name: Install build deps run: | sudo apt-get update - sudo apt-get install -y ninja-build ccache libpq-dev postgresql-client + sudo apt-get install -y ninja-build ccache libssl-dev libcurl4-openssl-dev - name: Setup Ccache uses: hendrikmuhs/ccache-action@main continue-on-error: true with: - key: ccache-quackapi-pg-sqllogic-linux-amd64-v1.5.6 - restore-keys: ccache-quackapi-pg-sqllogic-linux-amd64- - - - name: Wait for Postgres - run: | - until pg_isready -h 127.0.0.1 -p 5432 -U quackapi_pg_cache -d quackapi_pg_cache_test; do - sleep 1 - done + key: ccache-quackapi-sqllogic-linux-amd64-v1.5.6 + restore-keys: ccache-quackapi-sqllogic-linux-amd64- - name: Build extension run: make release -j"$(nproc)" @@ -113,19 +90,129 @@ jobs: - name: Run quackapi SQLLogic run: ./build/release/test/unittest "test/sql/quackapi*" - - name: Verify outbound worker cleanup + - name: Run BODY SCHEMA SQLLogic coverage with local json_schema run: | - c++ -std=c++17 -pthread -Wall -Wextra -Werror -Isrc/include test/cpp/quackapi_thread_join_test.cpp -o /tmp/quackapi-thread-join-test - /tmp/quackapi-thread-join-test - - - name: Verify build without libpq - run: c++ -std=c++17 -fsyntax-only -isystem duckdb/src/include -Isrc/include src/quackapi_pg.cpp + set -euo pipefail + extension_repo="$PWD/build/release/repository" + community_extension_dir="$RUNNER_TEMP/quackapi-community-extensions" + test_extension_dir="$RUNNER_TEMP/quackapi-test-extensions" + test_config="$RUNNER_TEMP/quackapi-test-config.json" + mkdir -p "$community_extension_dir" "$test_extension_dir" "$extension_repo/v1.5.6/linux_amd64" + ./build/release/duckdb -no-stdin -c "SET extension_directory='$community_extension_dir'; INSTALL json_schema FROM community;" + cp "$community_extension_dir/v1.5.6/linux_amd64/json_schema.duckdb_extension" \ + "$extension_repo/v1.5.6/linux_amd64/" + cat > "$test_config" <&1 | tee "$log_file" + if grep -Eiq "skip.*require json_schema|require json_schema.*skip" "$log_file"; then + echo "BODY SCHEMA coverage was skipped because json_schema was unavailable" >&2 + exit 1 + fi + if ! grep -Eq "16 assertions" "$log_file"; then + echo "Expected all 16 BODY SCHEMA assertions to run" >&2 + exit 1 + fi - name: Verify request database lifetime run: | c++ -std=c++17 -pthread -Wall -Wextra -Werror -isystem duckdb/src/include test/cpp/quackapi_lifetime_test.cpp -Lbuild/release/src -lduckdb -Wl,-rpath,"$PWD/build/release/src" -o /tmp/quackapi-lifetime-test /tmp/quackapi-lifetime-test + - name: Build clean pinned core for loadable extension contracts + run: | + cmake -S duckdb -B build/post-clean-core \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_MAIN_DUCKDB_LIBRARY=ON \ + -DBUILD_SHELL=OFF \ + -DBUILD_UNITTESTS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DEXTENSION_STATIC_BUILD=FALSE \ + -DDUCKDB_EXTENSION_CONFIGS= \ + -DDISABLE_BUILTIN_EXTENSIONS=TRUE \ + -DDISABLE_EXTENSION_LOAD=FALSE \ + -DOVERRIDE_GIT_DESCRIBE=v1.5.6 + cmake --build build/post-clean-core --target duckdb -j"$(nproc)" + test -f build/post-clean-core/src/libduckdb.so + + - name: Build pinned curl_httpfs POST provider + uses: actions/checkout@v4 + with: + repository: asubbarao-ifr/duckdb-curl-filesystem + ref: a48f83c6198a0252672f6f696e446e1525745a8a + path: provider-curl-httpfs + submodules: recursive + + - name: Build pinned HTTPFS-Curl POST provider + uses: actions/checkout@v4 + with: + repository: asubbarao-ifr/duckdb-httpfs + ref: 748f5ac8dddee531384d9901387d43b0f6a2db43 + path: provider-httpfs-curl + submodules: recursive + + - name: Build provider artifacts against the tested DuckDB core + run: | + duckdb_core="$PWD/build/release/src/libduckdb_static.a" + test -f "$duckdb_core" + test "$(git -C provider-curl-httpfs rev-parse HEAD)" = a48f83c6198a0252672f6f696e446e1525745a8a + test "$(git -C provider-httpfs-curl rev-parse HEAD)" = 748f5ac8dddee531384d9901387d43b0f6a2db43 + test "$(git -C provider-curl-httpfs/duckdb rev-parse HEAD)" = 069cc9f9b5be802405797faecc284961b07c70ef + test "$(git -C provider-httpfs-curl/duckdb rev-parse HEAD)" = 069cc9f9b5be802405797faecc284961b07c70ef + make -C provider-curl-httpfs release -j"$(nproc)" \ + EXTENSION_STATIC_BUILD=1 \ + EXT_FLAGS="-DPREBUILT_BINARY=$duckdb_core" \ + OVERRIDE_GIT_DESCRIBE=v1.5.6 + make -C provider-httpfs-curl release -j"$(nproc)" \ + EXTENSION_STATIC_BUILD=1 \ + EXT_FLAGS="-DPREBUILT_BINARY=$duckdb_core" \ + OVERRIDE_GIT_DESCRIBE=v1.5.6 + + - name: Verify outbound POST over real HTTP with both providers + run: | + quackapi_artifact="$PWD/build/release/extension/quackapi/quackapi.duckdb_extension" + quackapi_version="$(git rev-parse --short HEAD)" + duckdb_test_core="$PWD/build/post-clean-core/src/libduckdb.so" + curl_provider="$PWD/provider-curl-httpfs/build/release/extension/curl_httpfs/curl_httpfs.duckdb_extension" + httpfs_provider="$PWD/provider-httpfs-curl/build/release/extension/httpfs/httpfs.duckdb_extension" + cmake -S test/cpp -B build/post_http_test_curl \ + -DQUACKAPI_EXTENSION="$quackapi_artifact" \ + -DQUACKAPI_VERSION="$quackapi_version" \ + -DDUCKDB_LIBRARY="$duckdb_test_core" \ + -DPOST_PROVIDER_EXTENSION="$curl_provider" \ + -DPOST_PROVIDER_NAME=curl_httpfs \ + -DPOST_PROVIDER_VERSION=v0.4.5 \ + -DENABLE_PROVIDER_REPLACEMENT=ON \ + -DCURL_HTTPFS_EXTENSION="$curl_provider" \ + -DCURL_HTTPFS_VERSION=v0.4.5 \ + -DHTTPFS_EXTENSION="$httpfs_provider" \ + -DHTTPFS_VERSION=748f5ac + cmake --build build/post_http_test_curl -j"$(nproc)" + echo "DuckDB CLI identity:" + ./build/release/duckdb -c "PRAGMA version" + for test_binary in \ + build/post_http_test_curl/quackapi_post_http_test \ + build/post_http_test_curl/quackapi_post_tls_test \ + build/post_http_test_curl/quackapi_provider_replacement_test; do + echo "Runtime dependencies for $test_binary:" + readelf -d "$test_binary" | grep -E 'NEEDED|RPATH|RUNPATH' || true + ldd "$test_binary" | grep -E 'duckdb|not found' || true + done + ctest --test-dir build/post_http_test_curl --output-on-failure + cmake -S test/cpp -B build/post_http_test_httpfs \ + -DQUACKAPI_EXTENSION="$quackapi_artifact" \ + -DQUACKAPI_VERSION="$quackapi_version" \ + -DDUCKDB_LIBRARY="$duckdb_test_core" \ + -DPOST_PROVIDER_EXTENSION="$httpfs_provider" \ + -DPOST_PROVIDER_NAME=httpfs \ + -DPOST_PROVIDER_VERSION=748f5ac + cmake --build build/post_http_test_httpfs -j"$(nproc)" + ctest --test-dir build/post_http_test_httpfs --output-on-failure + - name: Verify benchmark correctness gates run: | python3 bench/test_bench_config.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c96f34..f4aaac9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,22 +19,21 @@ set(EXTENSION_SOURCES src/quackapi_extension.cpp src/quackapi_registry.cpp src/quackapi_server.cpp + src/quackapi_request_binding.cpp + src/quackapi_response_encoding.cpp + src/quackapi_logging.cpp src/quackapi_validation.cpp src/quackapi_batteries.cpp src/quackapi_ddl.cpp src/quackapi_auth.cpp - src/quackapi_http_fetch.cpp + src/quackapi_post_result.cpp + src/quackapi_post.cpp + src/quackapi_post_executor.cpp + src/quackapi_post_provider.cpp src/quackapi_limits.cpp src/quackapi_middleware.cpp - src/quackapi_table_api.cpp src/quackapi_openapi.cpp - src/quackapi_graphql.cpp - src/quackapi_queue.cpp - src/quackapi_policy.cpp - src/quackapi_stream.cpp - src/quackapi_from_x.cpp src/quackapi_util.cpp - src/quackapi_pg.cpp ) build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) @@ -50,7 +49,7 @@ target_include_directories(${EXTENSION_NAME} BEFORE PRIVATE ${CMAKE_CURRENT_SOUR target_include_directories(${LOADABLE_EXTENSION_NAME} BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/third_party/httplib) # The vendored httplib default backlog is 5. That drops loopback connections -# when the outbound pool fans out to its 32-client cap before the server can +# when outbound clients fan out to configured concurrency before the server can # accept them. Use the platform's full listen backlog for both extension forms. target_compile_definitions(${EXTENSION_NAME} PRIVATE CPPHTTPLIB_LISTEN_BACKLOG=SOMAXCONN) target_compile_definitions(${LOADABLE_EXTENSION_NAME} PRIVATE CPPHTTPLIB_LISTEN_BACKLOG=SOMAXCONN) @@ -81,42 +80,6 @@ target_include_directories(${LOADABLE_EXTENSION_NAME} PRIVATE target_link_libraries(${EXTENSION_NAME} duckdb_mbedtls) target_link_libraries(${LOADABLE_EXTENSION_NAME} duckdb_mbedtls) -# Optional libpq — pure-HTTP Postgres path (same idea as FastAPI+psycopg). -# When pg_dsn is set on quackapi_serve, simple routes run via PQexecParams -# instead of DuckDB ATTACH. Soft: build still succeeds without libpq. -# -# Never link a host-only Homebrew dylib into a cross-arch build (CI osx_amd64 -# on arm64 runners finds /opt/homebrew arm64 libpq → link fails). -set(QUACKAPI_TRY_LIBPQ ON) -if (APPLE AND CMAKE_OSX_ARCHITECTURES) - if (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" AND - NOT CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|i386") - set(QUACKAPI_TRY_LIBPQ OFF) - message(STATUS "quackapi: skip libpq (target x86_64, host ${CMAKE_HOST_SYSTEM_PROCESSOR})") - elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" AND - NOT CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64|aarch64") - set(QUACKAPI_TRY_LIBPQ OFF) - message(STATUS "quackapi: skip libpq (target arm64, host ${CMAKE_HOST_SYSTEM_PROCESSOR})") - endif() -endif() -if (QUACKAPI_TRY_LIBPQ) - find_path(QUACKAPI_LIBPQ_INCLUDE libpq-fe.h - HINTS /opt/homebrew/opt/libpq/include /usr/local/opt/libpq/include /usr/include/postgresql) - find_library(QUACKAPI_LIBPQ_LIB pq - HINTS /opt/homebrew/opt/libpq/lib /usr/local/opt/libpq/lib) -endif() -if (QUACKAPI_TRY_LIBPQ AND QUACKAPI_LIBPQ_INCLUDE AND QUACKAPI_LIBPQ_LIB) - message(STATUS "quackapi: libpq found — native Postgres path enabled") - target_include_directories(${EXTENSION_NAME} PRIVATE ${QUACKAPI_LIBPQ_INCLUDE}) - target_include_directories(${LOADABLE_EXTENSION_NAME} PRIVATE ${QUACKAPI_LIBPQ_INCLUDE}) - target_link_libraries(${EXTENSION_NAME} ${QUACKAPI_LIBPQ_LIB}) - target_link_libraries(${LOADABLE_EXTENSION_NAME} ${QUACKAPI_LIBPQ_LIB}) - target_compile_definitions(${EXTENSION_NAME} PRIVATE QUACKAPI_HAS_LIBPQ=1) - target_compile_definitions(${LOADABLE_EXTENSION_NAME} PRIVATE QUACKAPI_HAS_LIBPQ=1) -else() - message(STATUS "quackapi: libpq not found — native Postgres path disabled") -endif() - # Response compression: DuckDB-bundled zstd + miniz (gzip). NO new dependencies. # Citations (verified in this tree / upstream): # duckdb/third_party/zstd/CMakeLists.txt → add_library(duckdb_zstd STATIC …) diff --git a/README.md b/README.md index e3fbc03..83034cc 100644 --- a/README.md +++ b/README.md @@ -1,559 +1,149 @@ # quackapi -**A FastAPI-class web framework that lives inside DuckDB.** - -Routes are DDL. Handlers are SQL. Request validation is the database type system. -One process is the database, the HTTP server, and — when you load companion -extensions — the PDF engine, HTML renderer, crawler, and fake-data generator. - -``` -browser ──► DuckDB [ quackapi · pdf · tera · fakeit · webbed · … ] -``` - -There is no app server, no ORM, and no schema re-declared in a second language. -The query already types the response. - -> **Thesis:** the backend *is* the database. -> Prior art: PostgREST, Datasette, Hasura, Oracle APEX. The novel part is one -> **embedded OLAP** process as DB + HTTP framework + (optional) PDF/renderer, -> driven by `CREATE ROUTE` DDL. -> -> **Inspiration:** quackapi was also inspired by the Query.Farm [`httpserver`](https://github.com/Query-farm/httpserver) DuckDB community extension. - ---- - -## Five-line quickstart - -Requires **DuckDB v1.5.6** (linux, macOS, windows_amd64). +QuackAPI serves HTTP routes whose handlers are DuckDB SQL. Request values bind +as prepared parameters, DuckDB types validate them, and query results become +HTTP responses. ```sql INSTALL quackapi FROM community; +LOAD json; LOAD quackapi; -CREATE ROUTE hello GET '/hello' AS SELECT 'world' AS msg; -CREATE ROUTE item GET '/items/:id' AS SELECT $id::INTEGER AS id; +CREATE ROUTE hello GET '/hello' AS SELECT 'world' AS message; +CREATE ROUTE item GET '/items/:id' AS SELECT $id::INTEGER AS id; SELECT * FROM quackapi_serve(8000); --- returns immediately; pass block := true to hold until stop / SIGINT / SIGTERM - --- Readiness (replaces sleep/lsof/curl loops): --- SELECT ready, listen_url FROM quackapi_wait(8000, 5000); ``` ```sh curl http://127.0.0.1:8000/hello -# [{"msg":"world"}] +# [{"message":"world"}] curl http://127.0.0.1:8000/items/42 # [{"id":42}] - -curl http://127.0.0.1:8000/items/abc -# {"detail":[{"loc":["path","id"],"msg":"…","type":"type_error"}]} -# HTTP 422 — FastAPI-shaped validation error -``` - -> **Parser note:** `CREATE ROUTE` is registered when you `LOAD quackapi`. Feed -> `LOAD` first in a sequential session (interactive shell or stdin), then the -> DDL. Putting `LOAD` and `CREATE ROUTE` in one `duckdb -c "…"` string can parse -> the route before the extension is loaded. - ---- - -## CREATE ROUTE — the model - -```sql -CREATE [OR REPLACE] ROUTE '' - [STATUS ] - [REQUIRE ] - [SCHEDULE '<6-field cron expression, seconds first>'] - [GROUP | IN GROUP ] - [BODY SCHEMA ''] - [BODY TYPE ''] - [PARAM [] [HEADER|COOKIE|QUERY [wire-name]] - [DEFAULT ] [GE|GT|LE|LT ] [MIN_LENGTH|MAX_LENGTH ] …] - AS ; - - - Methods: GET, POST, PUT, DELETE, PATCH, HEAD. - - Path segments `:id` / `{id}` and query/body fields bind to `$id`. - - Cast/constraint failures return **422** with FastAPI-shaped - `{"detail":[{"loc":[...],"msg":...,"type":...}]}`. - - Default JSON response is an **array of row objects** (SQL result-set - semantics). A single column named `html` or `text` returns raw HTML/text; - `location` / `set_cookie` set response headers. - - `quackapi_serve([port], host := …, static_dir := …, cors_origins := …, memory_limit := …)` - uses DuckDB's bundled httplib (background listener + workers). - - Built-in OpenAPI: `GET /openapi.json`, `GET /docs`, `GET /redoc`. - - ## Other DDL / functions - - - **Auth:** `CREATE AUTH … AS API_KEY | JWT (SECRET … [, ALGORITHM HS256])`, - `REQUIRE` on routes, `quackapi_add_api_key`, `quackapi_auths`, claims as - `$claims_*`. JWT verify is self-contained (DuckDB-bundled mbedtls + - `json_*` claims) — does not depend on community `crypto` (that extension - is hash/HMAC only, not JWT parse/exp/nbf/claims). - - **Groups:** `CREATE GROUP … WITH (prefix=…, auth=…, tags=…)` — path prefix - + auth inheritance (`quackapi_groups()`). - - **Table API:** `CREATE API FOR TABLE t [AT '/path'] [KEY 'id']` → GET list - + GET by key only. - - **Queue:** `CREATE QUEUE`, `quackapi_enqueue/dequeue/ack/nack`, table - `quackapi_jobs` (broker-less; compose `cronjob` for workers). - - **SSE:** `CREATE STREAM … GET '/path' AS ; - -DROP ROUTE ; -``` - -| Element | Detail | -|---------|--------| -| Method | `GET` `POST` `PUT` `DELETE` `PATCH` `HEAD` | -| Pattern | Absolute path starting with `/` (or relative when using `GROUP`) | -| Path params | `:id` or `{id}` → `$id` | -| Query / body / form / multipart | Same `$name` binding; `$body` for raw payload when declared | -| STATUS | Default 200 | -| REQUIRE | Named scheme from `CREATE AUTH` | -| PARAM | Optional types, defaults, numeric/string constraints, header/cookie sources | -| BODY SCHEMA | Optional JSON Schema string for request body validation | -| Handler | Prepared at CREATE; invalid SQL fails CREATE (not first request) | - -Routes update **live** while the server is running. - -### Response modes - -| Handler shape | HTTP body | -|---------------|-----------| -| Multiple columns / ordinary names | `application/json` — **array of row objects** | -| Single column `html` | `text/html; charset=utf-8` | -| Single column `text` | `text/plain; charset=utf-8` | -| Column `location` | sets `Location` (pair with `STATUS 3xx`) | -| Column `set_cookie` / `set-cookie` | sets `Set-Cookie` (stripped from JSON) | - -JSON types follow DuckDB column types (numbers, booleans, nulls, nested lists/structs). - -### Error shapes (verified) - -| Situation | Status | -|-----------|--------| -| Type / constraint / missing required param | **422** `{"detail":[{loc,msg,type},…]}` | -| Unknown path | **404** | -| Wrong method | **405** + `Allow` | -| Auth failure | **401** | -| Body too large (> 8 MiB) | **413** | -| Handler internal error | **500** `{"detail":"Internal Server Error"}` (SQL text not leaked) | - ---- - -## Examples - -### JSON route - -```sql -CREATE ROUTE get_doc GET '/api/documents/:id' AS -SELECT * FROM documents WHERE id = $id::INTEGER; -``` - -### HTML route - -```sql -CREATE ROUTE home GET '/' AS -SELECT '

quackapi

' AS html; - --- With community tera: --- SELECT tera_render(template, ctx) AS html FROM …; -``` - -### Static directory + CORS - -```sql -SET quackapi_cors_origins = 'https://app.example'; -SELECT * FROM quackapi_serve( - 8000, - host := '0.0.0.0', - static_dir := './static', - cors_origins := '*' -- named param overrides SET -); -``` - -### Typed params + 422 - -```sql -CREATE ROUTE search GET '/search' - PARAM limit INTEGER DEFAULT 10 LE 100 - AS -SELECT id, name FROM users -WHERE name ILIKE $q::VARCHAR || '%' -LIMIT $limit::INTEGER; -``` - -`limit=101` → 422 `less_than_equal`; `limit=abc` → 422 `type_error` with -`loc=["query","limit"]`. - -### POST mutation - -```sql -CREATE ROUTE decide POST '/api/suggestions/:id/decision' STATUS 201 - PARAM status VARCHAR - PARAM actor VARCHAR DEFAULT 'reviewer' - AS -INSERT INTO decisions BY NAME -SELECT $id::INTEGER AS suggestion_id, $status AS status, $actor AS actor -RETURNING *; -``` - -```sh -curl -X POST http://127.0.0.1:8000/api/suggestions/1/decision \ - -H 'Content-Type: application/json' \ - -d '{"status":"accepted"}' -``` -### Auth - -```sql -CREATE AUTH site AS API_KEY; -- default header X-API-Key -SELECT * FROM quackapi_add_api_key('site', 'super-secret', 'alice'); - -CREATE ROUTE me GET '/me' REQUIRE site AS -SELECT $claims_sub AS user; -``` - -```sql -CREATE AUTH jwt_auth AS JWT ( SECRET 'test-secret', ALGORITHM HS256 ); -``` - -### Serve / inspect / stop - -```sql +CREATE ROUTE hello GET '/hello' AS SELECT 'world' AS message; SELECT * FROM quackapi_serve(8000); -SELECT * FROM quackapi_routes(); -SELECT * FROM quackapi_servers(); -SELECT * FROM quackapi_stop(8000); ``` -Built-in (not in `quackapi_routes()`): `/openapi.json`, `/docs`, `/redoc`. - ---- - -## Function table - -Surfaces registered by the extension (source: `src/quackapi_extension.cpp` and -related modules). Auto-detection on the community site will list overloads after -`LOAD`; this table is the human guide. - -### HTTP lifecycle - -| Name | Kind | Purpose | -|------|------|---------| -| `quackapi_serve([port], host, static_dir, cors_origins, memory_limit, log_level, access_log, enable_logging, health_routes, threads, preserve_insertion_order, enable_http_metadata_cache, worker_threads, keep_alive_max_count, keep_alive_timeout_sec, read_timeout_sec, write_timeout_sec, compression, compression_min_bytes, http_client, pg_dsn, block, query_timeout_ms, max_response_bytes, max_pending_requests)` | table | Start listener (default `127.0.0.1:8000`) | -| `quackapi_wait(port [, timeout_ms], host := …)` | table | TCP readiness probe → `ready`, `listen_url` | -| `quackapi_stop([port])` | table | Stop one or all servers | -| `quackapi_routes()` | table | Registry: name, method, pattern, status, handler, require_auth, group_name, tags, format, envelope, empty_status, timeout_sec | -| `quackapi_servers()` | table | host, port, listen_url, http_client, http_client_reason | -| `quackapi_cors_origins` | setting | CORS allow-list (`*` or CSV); empty = off | -| `quackapi_compression` / `quackapi_compression_min_bytes` | settings | Response coding (`auto`, `gzip`, `zstd`, `off`) and 1024-byte default threshold | - -### Auth - -| Name | Kind | Purpose | -|------|------|---------| -| `CREATE` / `DROP AUTH` | DDL | API_KEY or JWT HS256 schemes | -| `quackapi_add_api_key(auth, key, subject)` | table | Store SHA-256 hash of key | -| `quackapi_auths()` | table | name, kind, header (no secrets) | -| `quackapi_verify_auth(scheme, auth_string)` | scalar | Policy engine probe | -| `quackapi_authentication` / `quackapi_authorization` | scalar | Bridges for core quack RPC settings | - -### Groups - -| Name | Kind | Purpose | -|------|------|---------| -| `CREATE` / `DROP GROUP` | DDL | Prefix + default auth + OpenAPI tags | -| `quackapi_groups()` | table | name, prefix, require_auth, tags, members | - -### Table API - -| Name | Kind | Purpose | -|------|------|---------| -| `CREATE API FOR TABLE …` | DDL | Registers GET list + GET by key only | - -### Queue - -| Name | Kind | Purpose | -|------|------|---------| -| `CREATE` / `DROP QUEUE` | DDL | Register queue options | -| `quackapi_enqueue` | scalar | Enqueue JSON/text payload → job_id | -| `quackapi_dequeue` | table | Claim up to n jobs | -| `quackapi_ack` / `quackapi_nack` | scalar | Complete / retry / dead-letter | -| `quackapi_queues()` | table | depth, in_flight, dead, options | -| `quackapi_jobs` | table (catalog) | Durable job rows | - -### Streams & policies - -| Name | Kind | Purpose | -|------|------|---------| -| `CREATE` / `DROP STREAM` | DDL | SSE (`GET` only); WS rejected | -| `quackapi_streams()` | table | Inspect streams | -| `CREATE ROW ACCESS POLICY` / `CREATE MASKING POLICY` | DDL | Claims-oriented policies | -| `ALTER TABLE …` policy bind | DDL | Attach/detach | -| `quackapi_policies()` | table | Inspect policies | - -### Other - -| Name | Kind | Purpose | -|------|------|---------| -| `quackapi_fetch(url[, headers])` | scalar | Pooled outbound GET → `STRUCT(status, reason, body, headers, error, reused_connection)` | -| `quackapi_post(url, body[, ct[, headers]])` | scalar | Pooled outbound POST, same result shape | -| `quackapi_http_pool()` | table | Outbound pool per origin: `host, idle, dialed, reused` | -| `quackapi_http_util_name()` | scalar | Active outbound HTTPUtil name | -| `quackapi_request(method, path [, body], …)` | table | In-process request → status, BLOB body, content type, headers | -| `quackapi_middlewares()` | table | Inspect SQL middleware definitions | - -Internal apply helpers (`quackapi_apply_route`, `quackapi_apply_auth`, …) exist -for the planner and are not part of the public app API. - ---- - -## Platforms & signed-build feasibility - -| Item | Value | -|------|--------| -| Target DuckDB | **v1.5.6** (CI + submodule pin) | -| Language / build | C++11 / **cmake** | -| Extra toolchains | **None** (no vcpkg, no Rust, no Python at build) | -| Linked deps | DuckDB **bundled httplib** + **mbedtls** only | -| Outbound HTTPS | Core `HTTPUtil`; optional `LOAD curl_httpfs` upgrades process-wide client | - -| DuckDB arch | Community CI intent | -|-------------|---------------------| -| `linux_amd64`, `linux_arm64` | Build & sign | -| `osx_amd64`, `osx_arm64` | Build & sign | -| `wasm_mvp`, `wasm_eh`, `wasm_threads` | **Excluded** — no server sockets | -| `windows_amd64` | Build & sign | -| `windows_amd64_mingw`, `windows_amd64_rtools`, `windows_arm64` | **Excluded** — not enabled by the repository descriptor | - -Repo workflow: `duckdb/extension-ci-tools` `@v1.5.6`, -`duckdb_version: v1.5.6`, same `exclude_archs` as `description.yml`. - -**Feasibility:** high for the non-excluded platforms — pure CMake extension -template shape, no exotic system libraries. Blocker before first signed publish: -clean release tag with **no merge-conflict markers** in `src/`, green -`make release` + `test/http/run_all.sh` on at least one host arch. +Route handlers use prepared parameters. Authentication controls access to a +route; it does not add database-wide row filtering or rewrite arbitrary SQL by +tenant. Write tenant predicates into handler SQL or use explicitly scoped +views. ---- +## Outbound POST -## Honest limits +`quackapi_post(url, body[, content_type[, headers]])` performs volatile HTTP +and HTTPS POSTs through DuckDB's HTTPUtil provider. `curl_httpfs` is +recommended and `httpfs` is also supported. Load a provider explicitly; the +extension does not install one. Results preserve input row alignment and +contain status, reason, body, headers, transport error, and connection-reuse +metadata. `quackapi_post_concurrency` bounds simultaneous requests for a +database instance. -1. **Single-writer OLTP** — DuckDB file locking; suitable for small concurrent - reviewer sets, not high-write multi-tenant SaaS. -2. **Instance-scoped registry** — routes/auth/groups/streams/queue *options* - die with the process; re-run DDL on boot. Job **rows** in `quackapi_jobs` - persist. -3. **Serve memory guard** — `quackapi_serve` applies a 256MB default only when - no operator or serve memory limit is configured; use `memory_limit := '4GB'` - or `SET quackapi_memory_limit` for large PDF/HTML work. -4. **Body size** — 8 MiB max payload. -5. **JWT** — HS256 only. -6. **Table API** — read-only GET scaffold. -7. **No WebSocket** on the HTTP sidecar (SSE via `CREATE STREAM` only). -8. **Unsigned until accepted** — community `INSTALL` path goes live after the - community-extensions PR and signing pipeline succeed. +For GET requests, use DuckDB's HTTP reader functions. QuackAPI has no public +fetch or pool-diagnostics API. ---- +## Scope -## Related links +The HTTP response formats are JSON, NDJSON, and CSV. QuackAPI does not provide +the former experimental queue, row-access or masking policy, SSE, GraphQL, +generated table/source-import, native PostgreSQL, Arrow, or Parquet surfaces. +`radio` is a separate message bus; it does not provide the removed queue's +lease, acknowledgement, or retry semantics. Arrow Flight and ADBC are separate +client/protocol choices and are not QuackAPI runtime dependencies. DuckDB's +ordinary SQL transaction and database-file concurrency behavior still applies. -- Source: [github.com/asubbarao/quackapi](https://github.com/asubbarao/quackapi) -- FastAPI parity scorecard: [`docs/FASTAPI_PARITY.md`](FASTAPI_PARITY.md) -- Queue semantics: [`docs/QUEUE.md`](QUEUE.md) -- Outbound HTTP: [`docs/curl_httpfs.md`](curl_httpfs.md) -- Submission YAML: [`description.yml`](../description.yml) +Build target: DuckDB v1.5.6. The extension declares DuckDB JSON as a build +dependency; load JSON before serving routes that use it. `BODY SCHEMA` also +requires `json_schema` to be available locally. Neither is installed by +QuackAPI as a side effect. diff --git a/docs/curl_httpfs.md b/docs/curl_httpfs.md index 781dd59..323ade8 100644 --- a/docs/curl_httpfs.md +++ b/docs/curl_httpfs.md @@ -1,160 +1,80 @@ -# Outbound HTTP and curl_httpfs - -quackapi’s **inbound** server uses DuckDB’s bundled httplib. That is intentional -and unchanged by this document. - -**Outbound** HTTP (anything a route handler fetches via `read_text` / -`read_json` / `read_parquet` / `read_csv` over `https://`, and C++ token-exchange -via `QuackapiHttpFetch`) goes through DuckDB’s shared `HTTPUtil` layer — never -the `curl` CLI. - -## Batteries default: `curl_httpfs` - -On every `quackapi_serve()`, batteries **prefer** the community -[curl_httpfs](https://github.com/dentiny/duckdb-curl-filesystem) extension as -the outbound HTTP client: - -- libcurl **connection pooling** -- **HTTP/2** -- **async** network IO -- **100% compatible** with httpfs (`SET httpfs_client_implementation=…`) - -Flow (same pattern as auto-LOAD companions): - -1. Best-effort `LOAD` / `INSTALL` **httpfs** -2. `INSTALL curl_httpfs FROM community` + `LOAD curl_httpfs` (if not already loaded) -3. `SET httpfs_client_implementation = 'curl'` -4. Log `quackapi.http_client=curl` and record `http_client=curl` on `/healthz` - and `quackapi_servers()` - -A web-framework server that fetches remote data needs a production-grade pooled -client; without it, throughput collapses under concurrency on DuckDB’s default -per-request httplib client. - -### Platform coverage - -Community `description.yml` for curl_httpfs excludes: - -| Excluded | Available (ships binaries) | -|---|---| -| `wasm_mvp`, `wasm_eh`, `wasm_threads` | `linux_amd64`, `linux_arm64`, … | -| `windows_amd64`, `windows_amd64_mingw`, `windows_amd64_rtools` | `osx_amd64`, `osx_arm64` | - -### Preference semantics (`http_client`) - -| Surface | Values | Default | -|---|---|---| -| `quackapi_serve(…, http_client := '…')` | `auto` \| `curl` \| `httplib` | `auto` | -| `SET quackapi_http_client = '…'` | same | `auto` | - -Named param wins over the SET. - -| Value | Behavior | -|---|---| -| **`auto`** | Prefer curl_httpfs. If INSTALL/LOAD fails, **fall back to httplib** and report **loudly**: stderr line, `/healthz` `http_client` + `http_client_reason`, `quackapi_servers()`. | -| **`curl`** | **Require** curl_httpfs. If INSTALL/LOAD fails, **serve fails** (no silent fallback). Production guarantee. | -| **`httplib`** | Skip curl_httpfs install; force stock client. Logs `reason=operator_forced`. | - -```sql --- Default (prefer curl_httpfs; loud fallback if unavailable) -SELECT * FROM quackapi_serve(8000); - --- Production: require curl_httpfs or refuse to serve -SELECT * FROM quackapi_serve(8000, http_client := 'curl'); - --- Explicit stock client (dev / unsupported platform) -SELECT * FROM quackapi_serve(8000, http_client := 'httplib'); - -SET quackapi_http_client = 'curl'; -SELECT * FROM quackapi_serve(8001); -``` - -### auto fallback (loud, never silent) - -When `http_client` is `auto` and curl_httpfs cannot install/load: - -```text -quackapi.http_client=httplib reason=curl_httpfs_unavailable WARN=auto_fallback … -``` - -`/healthz` then reports: - -```json -{"status":"ok", …, "http_client":"httplib", "http_client_reason":"curl_httpfs_unavailable"} -``` - -`quackapi_servers()` has the same `http_client` / `http_client_reason` columns. - -### Production recommendation - -On **linux** and **osx** (platforms where community curl_httpfs ships), production -should either: - -1. **`http_client := 'curl'`** (or `SET quackapi_http_client = 'curl'`) so serve - fails closed if the pooled client is missing, **or** -2. Keep `auto` but **alert on** `/healthz` when - `http_client_reason == "curl_httpfs_unavailable"`. - -Do not ship concurrent outbound https under silent httplib fallback. - -### Confirm the active client +# Outbound HTTP providers + +`quackapi_post` sends outbound HTTP and HTTPS requests through DuckDB's +`HTTPUtil` provider API. Load a supported provider explicitly before the first +call. QuackAPI does not install or select a provider automatically. + +The `curl_httpfs` and HTTPFS-Curl provider changes needed for this contract are +under upstream review. Do not treat released community binaries as supported +providers yet. Compatibility verification uses the pinned provider builds +below. `curl_httpfs` may provide connection reuse and HTTP/2 according to its +build and configuration; exact behavior depends on provider version and +platform. + +The current integration workflow builds both providers against DuckDB +`069cc9f9b5be802405797faecc284961b07c70ef` and runs the same real-HTTP POST +contract tests against each artifact. The `curl_httpfs` source pin is +[`a48f83c`](https://github.com/asubbarao-ifr/duckdb-curl-filesystem/tree/a48f83c6198a0252672f6f696e446e1525745a8a), +which includes provider base `f253956`, timeout fix `e10d372`, POST +Content-Type fix `185b967`, and MultiCurl handle reuse fix `c51455d`. The +HTTPFS-Curl source pin is +[`748f5ac`](https://github.com/asubbarao-ifr/duckdb-httpfs/tree/748f5ac8dddee531384d9901387d43b0f6a2db43), +based on `b26737e`. These pins are test inputs; they do not indicate released +provider support. The integration checks exercise each provider independently +and replace HTTPFS-Curl with MultiCurl while an HTTPFS request is active. ```sql --- After serve -SELECT host, port, listen_url, http_client, http_client_reason -FROM quackapi_servers(); - --- Readiness JSON includes the same fields --- GET /healthz → {"status":"ok", …, "http_client":"curl", "http_client_reason":""} - --- Active HTTPUtil name (MultiCurl / HTTPFS-Curl after curl_httpfs, Built-In otherwise) -SELECT quackapi_http_util_name(); -``` - -### Manual load (optional) - -Batteries already do this on serve. Manual load is still fine for non-serve -sessions: - -```sql -INSTALL curl_httpfs FROM community; LOAD curl_httpfs; -SET httpfs_client_implementation = 'curl'; -- or leave default MultiCurl LOAD quackapi; -``` -### Transparent acceleration for SQL handlers +SELECT (quackapi_post( + 'https://api.example.test/items', + '{"name":"new item"}', + 'application/json', + MAP {'Authorization': 'Bearer …'} +)).status; +``` -Route handlers are ordinary SELECTs. If they call httpfs surfaces, the active -client speeds them up with no quackapi change: +To use `httpfs`, load it instead: ```sql -CREATE ROUTE proxy GET '/proxy' AS -SELECT content FROM read_text('https://example.com/data.json'); +LOAD httpfs; +LOAD quackapi; ``` -See `examples/proxy_curl_httpfs.sql` if present. +If neither supported HTTPUtil provider is loaded, `quackapi_post` returns a +configuration error for each non-NULL input row. Each input row retains its +position and produces one aligned result. Requests are not retried after +transport errors. -### C++ outbound (OAuth/OIDC) +Sub-second request deadlines use DuckDB's `HTTPParams::timeout_usec` field. +The `curl_httpfs` fixes are under review in +[PR #75](https://github.com/dentiny/duckdb-curl-filesystem/pull/75) (fractional +timeouts), [PR #76](https://github.com/dentiny/duckdb-curl-filesystem/pull/76) +(Content-Type), and [PR #77](https://github.com/dentiny/duckdb-curl-filesystem/pull/77) +(reusable MultiCurl clients). The HTTPFS-Curl timeout change is under review in +[`duckdb/duckdb-httpfs#490`](https://github.com/duckdb/duckdb-httpfs/pull/490). +Until those changes and the provider-specific POST fixes are merged, treat +these pinned source builds as compatibility-test inputs, not released support. -Use `QuackapiHttpFetch` (`src/quackapi_http_fetch.{hpp,cpp}`). It only depends -on core `HTTPUtil` headers. When curl_httpfs is loaded (including via -batteries), token-exchange POSTs use the curl client automatically. When only -the built-in util is active, POST fails fast with a message to load curl_httpfs. +## GET requests -## What not to do +For GET requests inside DuckDB SQL, use DuckDB's reader functions such as +`read_text`, `read_json`, and `read_csv` after loading the extension that +provides the reader. QuackAPI has no public `quackapi_fetch` function. -| Anti-pattern | Why | -|---|---| -| Hard-link libcurl / curl_httpfs into quackapi | Separate extension; composition is `SetHTTPUtil` | -| `system("curl ...")` / subprocess | No secrets integration, no pool, unsafe | -| Replace the **inbound** httplib server with curl_httpfs | curl_httpfs is the **client** layer only | -| Rely on silent httplib when you need production pool/HTTP2 | Use `http_client := 'curl'` or alert on `http_client_reason` | +```sql +LOAD httpfs; -## Load order +SELECT content +FROM read_text('https://example.com/data.txt'); +``` -1. `LOAD quackapi` then `quackapi_serve` (batteries prefer curl_httpfs for you), **or** -2. `LOAD curl_httpfs` then `LOAD quackapi` then serve +QuackAPI's inbound server is separate from outbound provider selection. It +continues to use DuckDB's bundled HTTP server implementation. The provider +does not replace the server and does not add TLS termination for inbound +connections. -Either order works for SQL handlers as long as the curl client is active before -the request that performs outbound I/O. +See the [POST function reference](reference/functions.md#outbound-post) and +[migration guide](MIGRATION.md) for the retained SQL contract and removed +transport APIs. diff --git a/docs/fastapi-parity.md b/docs/fastapi-parity.md index e730b32..949ef5f 100644 --- a/docs/fastapi-parity.md +++ b/docs/fastapi-parity.md @@ -1,5 +1,8 @@ # FastAPI → quackapi parity +> Historical scorecard. Its feature list predates the current API surface; see +> the [current migration guide](MIGRATION.md). + **Headline:** the versioned HTTP conformance harness scores **89 / 89 (100%)** against FastAPI-shaped expectations. | | | diff --git a/docs/from-fastapi.md b/docs/from-fastapi.md deleted file mode 100644 index 43e94f4..0000000 --- a/docs/from-fastapi.md +++ /dev/null @@ -1,156 +0,0 @@ -# Coming from FastAPI — `quack_from_X` - -You already have routes, models, and maybe a monorepo of services. **`quack_from_X`** is the bridge family that **reads an existing app** and emits quackapi DDL (`CREATE ROUTE`, `BODY SCHEMA`, …) so you can serve a large fraction of the surface from DuckDB. - -This page is honest about what translates automatically and what remains a human/escape-hatch step. - ---- - -## Corpus (why this is real) - -Across public frameworks, a multi-language IR corpus was extracted: - -| Language | Routes (IR) | Model fields (IR) | -|----------|------------:|------------------:| -| Python | 990 | 2908 | -| Ruby | 52 | 22 | -| Go | 352 | 239 | -| Node | 182 | 1035 | -| **Total** | **1576** | **4204** | - -- **36** top-level repos cloned under the corpus tree -- Python tags include FastAPI (**519**), DRF viewsets (300), Flask (41), … -- Source reports: corpus `PYTHON.md` / `RUBY.md` / `GO.md` / `NODE.md` (see also [FEATURE_STATUS §3](FEATURE_STATUS.md)) - ---- - -## What `quack_from_fastapi` does - -**Status:** partial one-caller (shell + pure SQL). Not yet an in-tree C++ table function named `quack_from_fastapi(path)`. - -Operational shape: - -```text -/path/to/fastapi/app - │ - ▼ -quack_from_fastapi.sh [port] [serve|gen-only] - │ - ├─ read Python AST (route decorators, include_router prefixes) - ├─ walk Pydantic / SQLModel fields → JSON Schema - ├─ emit CREATE OR REPLACE ROUTE … [BODY SCHEMA …] AS … - ├─ FIFO session: LOAD quackapi; register routes; quackapi_serve - ▼ -live HTTP (path params + BODY SCHEMA validation) -``` - -Proven end-to-end (reports under the from-fastapi workdir): - -| App | Result | -|-----|--------| -| fastapi-realworld | pass=10 fail=0 · 19 routes registered · 25 models | -| locus ROH sample | pass=10 fail=0 | - -**Handler A/B/C split** over 519 FastAPI-tagged corpus routes: - -| Bucket | Meaning | Share | -|--------|---------|------:| -| **A** | Pure CRUD / declarative → `CREATE ROUTE AS SELECT/DML` | 96.0% | -| **B** | Side effects covered by existing extensions (crypto/JWT/http client/…) | 3.1% | -| **C** | Imperative residue (e.g. WebSocket demos in that corpus) | 1.0% | -| **A+B auto-carry** | | **99.0%** | - -Realistic product prior for a typical CRUD FastAPI app: roughly **85–95%** of routes+validation can land without new C++. - ---- - -## How to point it at a repo - -> The one-caller currently lives as a **companion script + SQL** outside the extension binary. Paths below match the proven driver layout; promote into-repo `examples/` / `bridges/` after community packaging if not already present. - -```sh -# Conceptual invocation (driver shell) -quack_from_fastapi.sh /path/to/your/fastapi/project 8000 serve -``` - -What you need on the machine: - -1. Built `duckdb -unsigned` with `LOAD quackapi` -2. AST reader extension used by the bridge SQL (`sitting_duck` / `read_ast`) -3. Python sources with recognizable `@app.get` / `APIRouter` / Pydantic models - -Modes: - -| Mode | Behavior | -|------|----------| -| `gen-only` | Emit `routes.sql` only — review before serving | -| `serve` | Register + `quackapi_serve(port)` | - -Inspect afterward: - -```sql -SELECT name, method, pattern FROM quackapi_routes(); -``` - -```sh -curl http://127.0.0.1:8000/docs -``` - ---- - -## Honest boundary - -| Translates well | Escape hatch | -|-----------------|--------------| -| Path + methods + router prefixes | Imperative Python bodies with control flow, external SDKs, threads | -| Query/path types + Field constraints → `PARAM` / schema keywords | Custom dependency graphs that are not auth headers | -| Pydantic models → `BODY SCHEMA` (~81% feature map today) | ~19% needs binder polish (optional/null multi-error, strict format: email/uri/…) | -| Auth *shape* (API key / bearer) as `CREATE AUTH` sketches | Full OIDC login redirects, session cookies | -| CRUD SELECT/INSERT RETURNING façades | Arbitrary business logic — rewrite as SQL or call out to a function you load | - -**Rule of thumb:** - -- **Routing + validation** → automated -- **Handler body** → SQL rewrite or keep a thin external worker and use [CREATE QUEUE](guide/queue.md) / HTTP clients from SQL - -WebSocket handlers will not translate to HTTP Upgrade routes (use [SSE](guide/stream.md)). - ---- - -## Sibling bridges - -| Bridge | Status | -|--------|--------| -| `quack_from_fastapi` | Partial one-caller; e2e green on realworld samples | -| `quack_from_rails` | Proven one-caller (partial runtime fidelity — SQL façades, not Ruby) | -| `quack_from_openapi` / JSON Schema | Designed + fixtures — highest-leverage multi-stack path | -| Express / Nest / Gin / DRF / … | IR extracted; dedicated one-callers not all shipped | - -OpenAPI-universal path is often the best way in when you already export a spec: - -```text -openapi.yaml → CREATE ROUTE + BODY SCHEMA → quackapi_serve -``` - ---- - -## Pydantic → BODY SCHEMA fidelity (today) - -| Band | Coverage | -|------|----------| -| Real now (~81%) | scalars, required/default, lists, nested, Literal/enum via schema, Field min/max/length/pattern, inheritance flatten | -| Needs C++ (~19%) | field-level body `loc` parity, optional/null body defaults, multi-error aggregation, strict `format:` checkers | - -Compose email validation with existing tabular/regex tools rather than reimplementing in the extension core. - ---- - -## Next steps for a FastAPI team - -1. Run the [five-line hello world](index.md) so `CREATE ROUTE` is muscle memory. -2. Map your top 10 endpoints by hand using [routes](guide/routes-and-params.md) + [bodies](guide/request-bodies.md). -3. Point `quack_from_fastapi` at a branch for bulk emit; review `routes.sql`. -4. Replace imperative bodies with SQL or queue workers. -5. Add [policies](guide/policies.md) where you previously had ad-hoc tenant filters in Depends. - -Parity details: [fastapi-parity.md](fastapi-parity.md). diff --git a/docs/gemini-ext-catalog-map.md b/docs/gemini-ext-catalog-map.md index 618ee29..2dbe736 100644 --- a/docs/gemini-ext-catalog-map.md +++ b/docs/gemini-ext-catalog-map.md @@ -1,5 +1,8 @@ # Gemini brainstorm × `ext_catalog_clean` — composition map +> Historical catalog research. Its QuackAPI feature claims predate the API +> cleanup; see the [current migration guide](MIGRATION.md). + **Source:** live quack `:9494` `ext_catalog_clean` (2026-07-29). **Method:** `ILIKE` on `extension_name` / `content`; samples only from `query IS NOT NULL` or README blocks — no invented signatures. **Thesis:** compose first. Prefer a **recipe** (LOAD + `CREATE ROUTE` / `CREATE STREAM` / `CREATE QUEUE`) over new C++. Thin C++ only when the wire format or registry lives inside quackapi. diff --git a/docs/guide/auth.md b/docs/guide/auth.md index 505edd1..1a167fb 100644 --- a/docs/guide/auth.md +++ b/docs/guide/auth.md @@ -118,7 +118,9 @@ CREATE ROUTE me GET '/me' REQUIRE site AS SELECT $claims_sub AS user; ``` -Claims power [row access & masking policies](policies.md). +Claims are available to the authenticated handler as SQL parameters. Route +authentication does not filter rows or rewrite arbitrary SQL by tenant; add +tenant predicates to the handler or an explicitly scoped view. --- @@ -164,4 +166,4 @@ SELECT (quackapi_verify_auth('site', 'wrong')).ok; -- false ## Next - [CREATE GROUP](groups.md) — shared auth on a prefix -- [Policies](policies.md) — claims-keyed row filters and masks +- [Migration guide](../MIGRATION.md) — removed policy APIs and the current tenant-filter boundary diff --git a/docs/guide/coming-soon.md b/docs/guide/coming-soon.md index 342f6a5..c5014ff 100644 --- a/docs/guide/coming-soon.md +++ b/docs/guide/coming-soon.md @@ -1,14 +1,16 @@ -# Coming / in progress +# Scope boundaries -These features are **not** documented as done. They are actively designed or building. Do not rely on them in production recipes yet. +This page records areas outside the current QuackAPI HTTP API. These are +not advertised as implemented or as scheduled replacements. -| Feature | Intent | Status | -|---------|--------|--------| -| **WebSocket routes** | Browser RFC6455 duplex | **Blocked** on transport (HTTP library has no Upgrade API). Use [CREATE STREAM (SSE)](stream.md) instead | -| **OIDC / OAuth2 browser SSO** | `CREATE AUTH … OIDC` | Designed — JWT/API_KEY only today | -| **Signed cookie sessions + CSRF** | Browser session cookies | Designed | -| **Static URL prefix** | Mount `static_dir` under `/assets` | Partial — root `static_dir` only | +| Surface | Current boundary | +|---|---| +| Browser WebSocket upgrade | QuackAPI does not provide an HTTP WebSocket endpoint. `radio` is a separate extension and does not add WebSocket support to QuackAPI. | +| SSE and streaming responses | QuackAPI returns bounded JSON, NDJSON, or CSV responses; it does not provide the removed `CREATE STREAM`/SSE surface. | +| Row filtering and column masking | Route auth protects endpoints but does not rewrite arbitrary SQL or enforce tenant row policies. Put tenant filters in handler SQL or scoped views. | +| Durable job queue | No built-in queue with lease, acknowledgement, visibility-timeout, and retry guarantees remains. A `radio` message bus does not supply those job semantics. | +| GraphQL, generated table APIs, and source import | Define explicit SQL handlers with `CREATE ROUTE`; these former experimental APIs are not included. | +| Arrow Flight | A separate protocol and deployment choice; it is not hosted by QuackAPI or a runtime dependency. | -Authoritative ledger: [FEATURE_STATUS.md](../FEATURE_STATUS.md) sections 2.5 and 4. - -When a row moves to “BUILT & MERGED,” this page will shrink and a full guide will land under `docs/guide/`. +See the [migration guide](../MIGRATION.md) for removed SQL names and current +alternatives. diff --git a/docs/guide/extension-composition.md b/docs/guide/extension-composition.md index 2e3c9e9..5abb493 100644 --- a/docs/guide/extension-composition.md +++ b/docs/guide/extension-composition.md @@ -1,273 +1,77 @@ -# Extension composition — handlers are SQL +# Compose SQL handlers with DuckDB extensions -**The whole product idea:** `CREATE ROUTE` / `CREATE STREAM` handlers are ordinary SQL. -Whatever you can `LOAD` and call from a `SELECT`, a request can call. - -No C++ rewrite per companion. No second process for “the PDF service.” Same DuckDB session, -same address space, same type system. +QuackAPI routes run ordinary SQL in DuckDB. A handler can use tables, views, +scalar functions, and table functions already available in that database. +Loading a companion extension is an explicit deployment choice. ```text -browser ──► quackapi (SQL handlers + thin GraphQL) - │ - ├── local tables / views - ├── CREATE ROUTE / API FOR TABLE (REST → your SQL) - ├── POST /graphql | GRAPHQL ROUTE (GQL → SELECT …) - ├── curl_httpfs / httpfs (read_text, read_json, …) - ├── http_client (http_get, http_post, …) - ├── sitting_duck (AST / quack_from_*; optional GQL parse recipe) - ├── quack (quack_query / ATTACH) - └── pdf / tera / … (community companions) -``` - -**Dual surface:** GraphQL + ordinary HTTP on the same session — both resolve to SQL ([graphql-v0.md](graphql-v0.md)). Companions run *inside* handlers; they are not a second API product. - -Full DDL and function tables stay in the [README](../../README.md) and -[function reference](../reference/functions.md). This page is **recipes only**. - ---- - -## Recipe 1 — Outbound gateway - -Join remote HTTP with local tables inside one route. Prefer DuckDB’s shared -HTTP stack so `quackapi_serve` batteries (`curl_httpfs` when available) speed -you up for free — see [curl_httpfs](../curl_httpfs.md). - -### GET proxy via `read_text` / httpfs - -Runnable shape in-tree: [`examples/proxy_curl_httpfs.sql`](../../examples/proxy_curl_httpfs.sql). - -```sql -LOAD curl_httpfs; -- optional; batteries also try this on serve -LOAD quackapi; - -CREATE OR REPLACE ROUTE proxy GET '/proxy/:url' AS -SELECT - $url AS requested_url, - content AS body, - length(content) AS bytes -FROM read_text($url); - -CREATE OR REPLACE ROUTE http_util GET '/http_util' AS -SELECT quackapi_http_util_name() AS http_util; - -SELECT * FROM quackapi_serve(18080); -``` - -```sh -# percent-encode the remote URL into :url -curl -sS "http://127.0.0.1:18080/http_util" -# [{"http_util":"MultiCurl"}] # when curl_httpfs is active -``` - -There is **no** public SQL `quackapi_fetch` / `quackapi_post`. C++ `QuackapiHttpFetch` -is an internal HTTPUtil helper (OAuth/OIDC wave). Route authors use **SQL readers** -or a community HTTP extension. - -### POST / partner call via community `http_client` - -When you need explicit POST + headers (not just GET of a URL): - -```sql -LOAD http_client; -LOAD quackapi; - -CREATE OR REPLACE ROUTE proxy_score POST '/score' AS -SELECT http_post( - 'https://api.partner.com/v1/score', - map { - 'Authorization': 'Bearer ' || getenv('PARTNER_TOKEN'), - 'Content-Type': 'application/json' - }, - $body::JSON -) AS partner_response; -``` - -(Shape from the [handler bridge notes](../../bridges/from_x/docs/handler_bridge.md). -Confirm `http_post` arity against the `http_client` extension you load.) - -Join local state the same way you always would: - -```sql -CREATE ROUTE enrich GET '/orders/:id' AS -SELECT o.*, r.content AS remote_blob -FROM orders o -CROSS JOIN read_text(o.partner_url) r -WHERE o.id = $id::INTEGER; -``` - ---- - -## Recipe 2 — SSE live (honest transport) - -Push rows as Server-Sent Events. Full guide: [CREATE STREAM](stream.md). - -```sql -LOAD quackapi; - -CREATE STREAM ticks GET '/ticks' AS -SELECT i AS id, 'tick' AS msg FROM range(3) t(i); - --- long-lived poll -CREATE OR REPLACE STREAM live GET '/live' WITH (interval='1s') AS -SELECT now() AS ts; - -SELECT * FROM quackapi_serve(8000); -``` - -```sh -curl -N http://127.0.0.1:8000/ticks -# Content-Type: text/event-stream -# id: 0 -# data: {"id":0,"msg":"tick"} -``` - -**Honest limits (do not plan around these):** - -| Want | Reality | -|------|---------| -| Browser `EventSource` / SSE | **Yes** — `CREATE STREAM … GET` | -| WebSocket Upgrade | **No** — bundled httplib has no Upgrade API; `CREATE STREAM … WS` errors | -| Streams in `quackapi_routes()` | **No** — use `quackapi_streams()` (`transport` is always `'sse'` today) | -| `REQUIRE` auth on streams | Not in the current version | - -A bus extension (e.g. community pub/sub) can feed the `SELECT` behind the stream -when LOADed; quackapi only owns the SSE wire format. - ---- - -## Recipe 3 — `sitting_duck` / `quack_from_x` - -Point at an existing app tree; get **route + model IR rows**. Native table functions -ship in quackapi and **auto-INSTALL/LOAD `sitting_duck` FROM community** on first use. - -```sql -LOAD quackapi; - --- Routes: method, path, handler_name, file, … -SELECT method, path, handler_name -FROM quack_from_fastapi('bridges/from_x/fixtures/fastapi_mini') -ORDER BY method, path; --- GET /articles/{slug} get_article --- POST /login login - --- Models → field IR (then BODY SCHEMA by hand or generator) -SELECT model_name, field_name, field_type, is_required -FROM quack_from_fastapi_models('bridges/from_x/fixtures/fastapi_mini'); +HTTP client → QuackAPI route → prepared DuckDB SQL → response + │ + ├── tables and views + └── explicitly loaded extensions ``` -Siblings: `quack_from_rails`, `quack_from_express`, `quack_from_gin` (+ `_models`). +## Read upstream data -**What is automatic vs not:** - -| Automatic | Escape hatch | -|-----------|----------------| -| Decorators / DSL → route IR | Imperative handler bodies (never transpiled) | -| Pydantic / validates / tags → field IR | Rewrite body as SQL, macro, or [queue](queue.md) worker | -| Review IR → write `CREATE ROUTE` | Keep a thin external service and call it (recipe 1) | - -More narrative, corpus numbers, and migration tips: [Coming from FastAPI](../from-fastapi.md). -Bridge layout and tests: [`bridges/from_x/`](../../bridges/from_x/README.md). - ---- - -## Recipe 4 — Quack mesh - -**Composition, not a second server inside quackapi.** When the community **`quack`** -extension is LOADed, a handler can fan out to another DuckDB process over the -quack remote protocol — same pattern as any other TVF. +Use DuckDB's HTTP readers inside a route after loading a reader provider such +as `httpfs` or `curl_httpfs`: ```sql -LOAD quack; -- separate extension; not bundled into quackapi +LOAD json; +LOAD httpfs; LOAD quackapi; --- Illustrative shape — use the quack version you actually install. --- Common forms in the wild: --- quack_query('quack:host:9494', 'SELECT …', token => '…') --- ATTACH 'quack:host:9494' AS edge (DISABLE_SSL true); - -CREATE ROUTE edge_health GET '/mesh/health' AS -SELECT * -FROM quack_query( - 'quack:localhost:9494', - 'SELECT 1 AS ok', - token := 'your-token' -); - -CREATE ROUTE edge_table GET '/mesh/items' AS -SELECT * -FROM quack_query( - 'quack:localhost:9494', - 'SELECT * FROM items LIMIT 100', - token := 'your-token' -); -``` - -Auth bridge scalars on the **quackapi** side for RPC-style checks (used by -quack-compatible setups; no extra process): - -```sql -SELECT (quackapi_verify_auth('site', 'k-secret')).ok; -SELECT quackapi_authentication('sess', 'token', 'token'); -SELECT quackapi_authorization('sess', 'SELECT 1'); +CREATE ROUTE remote_status GET '/remote-status' AS +SELECT * FROM read_json('https://api.example.test/status'); ``` -quackapi still only listens with `quackapi_serve`. Mesh nodes are whatever quack -servers you already run. - ---- +QuackAPI does not install HTTP providers when a server starts. Reader behavior +and supported schemes come from the loaded DuckDB extension. -## Recipe 5 — PDF companion (one process) +## Send an outbound POST -README showcase: Closure-style PDF review is **one DuckDB process** — routes + -community `pdf` + optional `tera` + tables. Function names come from the **`pdf`** -extension you load (not from quackapi). +`quackapi_post` uses DuckDB's HTTPUtil provider and requires a POST-capable +provider to be explicitly loaded. The patched `curl_httpfs` and HTTPFS-Curl +providers are under upstream review; released community binaries are not yet +supported provider builds for this contract. See +[Outbound POST providers](../curl_httpfs.md) for pinned revisions. ```sql -INSTALL pdf FROM community; -LOAD pdf; +LOAD json; +LOAD curl_httpfs; LOAD quackapi; --- Typical community surfaces (names as used in product docs): --- read_pdf_words, pdf_redact, … --- Confirm against the pdf extension README for your DuckDB version. - -CREATE ROUTE doc_words GET '/docs/:id/words' AS -SELECT * -FROM read_pdf_words(d.path) -JOIN documents d ON d.id = $id::INTEGER; - --- HTML page if tera is loaded: --- LOAD tera; --- CREATE ROUTE case_page GET '/cases/:id' AS --- SELECT tera_render(template, ctx) AS html --- FROM app_templates, … WHERE name = 'case.html'; - -SELECT * FROM quackapi_serve(8000, memory_limit := '4GB'); -- headroom for PDF work +CREATE ROUTE submit POST '/submit' AS +SELECT (quackapi_post( + 'https://api.example.test/submit', + json_object('id', $id::INTEGER)::VARCHAR, + 'application/json' +)).status AS upstream_status; ``` -Calling the “PDF service” is a function call in the same address space — not an RPC. - ---- - -## Non-goals (explicit) - -These are **out of scope** for quackapi composition stories and this guide: +`quackapi_post` keeps one result row per input row, preserves its NULL and +volatile-function behavior, and reports status, response body, headers, and +transport errors. Configure `quackapi_post_concurrency` to bound simultaneous +requests. See [Outbound POST providers](../curl_httpfs.md). -| Non-goal | Why | -|----------|-----| -| GraphQL | Not a quackapi transport; stay REST + SQL | -| Asio / custom async server rewrite | Inbound remains httplib | -| Browser WebSocket Upgrade | Use SSE (`CREATE STREAM … GET`) | -| `WITH HISTORY` / time-travel as a product feature | Ordinary DuckDB time-travel if *you* enable it; not a quackapi API | +## Keep route security explicit -Also not invented here: public `quackapi_fetch` / `quackapi_post` SQL TVFs. +`CREATE AUTH` and `REQUIRE` decide whether a caller may invoke a route. They do +not apply row filters to arbitrary SQL. Put tenant constraints in the handler +query or a scoped view. There is no built-in row-access or masking-policy +rewrite. ---- +## Protocol boundaries -## Next +QuackAPI currently serves ordinary bounded HTTP route responses in JSON, +NDJSON, and CSV. The extension does not host GraphQL, SSE, WebSocket, Arrow +Flight, or PostgreSQL protocols. DuckDB's optional `duckflight` extension +serves Flight SQL as a separate protocol; it is not a QuackAPI dependency. +ADBC clients can connect directly to DuckDB and run ordinary SQL, but do not +add an ADBC endpoint to QuackAPI. The `radio` message-bus extension can be +composed separately, but it does not add protocols to the QuackAPI listener. +In particular, `radio` does not implement +the durable queue lease, acknowledgement, and retry behavior of the removed +QuackAPI queue prototype. -1. [Routes & params](routes-and-params.md) — muscle memory for `$name::TYPE` -2. [SSE streams](stream.md) — full `CREATE STREAM` grammar -3. [Outbound curl_httpfs](../curl_httpfs.md) — client selection + `/healthz` -4. [from-fastapi](../from-fastapi.md) — migration narrative -5. [Function reference](../reference/functions.md) — every `quackapi_*` +See the [migration guide](../MIGRATION.md) for removed function and DDL names. diff --git a/docs/guide/graphql-v0.md b/docs/guide/graphql-v0.md deleted file mode 100644 index 1a1dd2e..0000000 --- a/docs/guide/graphql-v0.md +++ /dev/null @@ -1,247 +0,0 @@ -# Thin GraphQL v0 (catalog-only) - -Owner-driven, **not** months of PostGraphile. Built-in while `quackapi_serve` is running (also via `quackapi_request`); no `CREATE ROUTE` required for the global endpoint. - -**Not DuckGQL.** [DuckGQL](https://duckgql.com/) is ISO GQL (graph analytics: `MATCH`, CSR, …). This surface is **GraphQL-over-HTTP** for tabular reads. - -## Dual surface: GraphQL and HTTP at once - -The database does not care which wire the client used. **Both resolve to SQL** in the same DuckDB session: - -```text - Client (GraphQL document) Client (REST path/body) - │ │ - ▼ ▼ - POST /graphql | GRAPHQL ROUTE CREATE ROUTE / API FOR TABLE - │ │ - ▼ ▼ - parse → table/cols bind $params - │ │ - └──────────► SQL ◄───────────────┘ - │ - same catalog, same auth primitives, - same LOAD companions (tera, sitting_duck, …) -``` - -| Wire | You declare | Engine does | -|------|-------------|-------------| -| **HTTP** | `CREATE ROUTE … AS SELECT …` | Run that handler SQL | -| **GraphQL** | tables allowed (`FOR TABLE` / `GRAPHQL ROUTE … FROM`) | Map `{ users { id } }` → `SELECT "id" FROM "users" LIMIT n` | - -Same tables can expose **both** in one serve — that is intentional, not a conflict: - -```sql -CREATE TABLE users AS SELECT 1 AS id, 'ada' AS name; - -CREATE ROUTE list_users GET '/users' AS - SELECT id, name FROM users; - -CREATE GRAPHQL ROUTE app POST '/gql' FROM users; - --- GET /users --- POST /gql {"query":"{ users { id name } }"} -``` - -Demo: [`examples/dual_surface.sql`](../../examples/dual_surface.sql). -Optional AST→SQL recipe (sitting_duck, not the serve hot path): [`examples/graphql_ast_parse.sql`](../../examples/graphql_ast_parse.sql). - -## What ships - -| Surface | Behavior | -|---------|----------| -| **`POST /graphql`** | JSON body `{"query":"…"}` → GraphQL-ish result | -| **`GET /graphql/schema`** | Main-schema tables → column names (not full `__schema`) | -| **`CREATE GRAPHQL FOR TABLE`** | Explicit allowlist for which tables the built-in endpoint exposes | -| **`CREATE GRAPHQL ROUTE`** | Named path mounts with per-route tables, optional auth + LIMIT | -| Auth | Built-in `/graphql` is public only for explicitly registered, unprotected tables; named routes may `REQUIRE` a `CREATE AUTH` scheme and apply the same row/masking policies as REST | -| Schema source | Current database + `main` catalog identity, including tables and views | - -## Minimal query language - -Only this shape (optional `query` keyword and optional operation name): - -```graphql -query { - tableName { - col1 - col2 - } -} -``` - -or bare: - -```graphql -{ - tableName { - col1 - col2 - } -} -``` - -Multiple root fields are allowed (each becomes its own `SELECT`): - -```graphql -{ - users { id name } - posts { id title } -} -``` - -Each root field maps to: - -```sql -SELECT "col1", "col2" FROM "tableName" LIMIT 100 -``` - -Default row cap: **100** (`QUACKAPI_GRAPHQL_DEFAULT_LIMIT`). - -### Response shape - -Success: - -```json -{ - "data": { - "tableName": [ - {"col1": 1, "col2": "x"} - ] - } -} -``` - -Errors (GraphQL-ish; still HTTP 200 for document/execution issues; HTTP 400 only for unusable JSON body): - -```json -{ - "errors": [ - {"message": "unknown table 'nope' (main schema catalog only)"} - ] -} -``` - -### Explicitly **not** in v0 - -- Mutations / subscriptions -- Nested joins / relation fields -- Arguments, aliases, fragments, variables -- Full GraphQL grammar -- `__schema` / `__type` introspection (use `GET /graphql/schema`) - -## Call it - -```sql -CREATE TABLE cases AS -SELECT 1 AS id, '24-000117' AS case_no; - -CREATE GRAPHQL FOR TABLE cases; - -SELECT * FROM quackapi_serve(8000); -``` - -```sh -curl -sS -X POST http://127.0.0.1:8000/graphql \ - -H 'Content-Type: application/json' \ - -d '{"query":"{ cases { id case_no } }"}' -# {"data":{"cases":[{"id":1,"case_no":"24-000117"}]}} - -curl -sS http://127.0.0.1:8000/graphql/schema -# {"mode":"allowlist","tables":{"cases":["id","case_no"]}, "note":"…"} -``` - -In-process (no TCP): - -```sql -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ cases { id case_no } }"}' -); -``` - -## Allowlist: `CREATE GRAPHQL FOR TABLE` - -By default the built-in endpoint is **closed**: no table is selectable or listed until it is registered. Register one or more tables to expose only those names through `POST /graphql` and `GET /graphql/schema`: - -```sql -CREATE GRAPHQL FOR TABLE users, posts; --- CREATE OR REPLACE GRAPHQL FOR TABLE users; -- idempotent - -SELECT * FROM quackapi_graphql_tables(); --- mode=allowlist, table_name=posts|users - -DROP GRAPHQL FOR TABLE posts; -DROP GRAPHQL ALL; -- clear allowlist → closed mode again -``` - -| Mode | When | `POST /graphql` | `GET /graphql/schema` | -|------|------|-----------------|------------------------| -| **disabled** | allowlist empty (default) | no tables | no tables | -| **allowlist** | ≥1 `CREATE GRAPHQL FOR TABLE` | registered only | registered only | -| **open** | `SET GLOBAL quackapi_graphql_allow_all = true` | all main tables | all main tables | - -`SET GLOBAL quackapi_graphql_allow_all = true` restores the old catalog-wide behavior for a deliberate migration only. It is database-wide and should not be used as an authorization boundary. `CREATE GRAPHQL FOR TABLE` does **not** mount a new path — it configures the global built-in `POST /graphql`. Tables must exist in the current database's `main` schema at CREATE time. Lifecycle matches other quackapi DDL (in-memory registry; re-declare after reopen). - -### How this differs from the bare built-in - -| | Built-in open | After `CREATE GRAPHQL FOR TABLE` | -|--|---------------|----------------------------------| -| Path | `/graphql` | same `/graphql` | -| Catalog | whole main schema | allowlist only | -| REST | — | still use `CREATE ROUTE` / `CREATE API FOR TABLE` for REST | - -## Named mounts: `CREATE GRAPHQL ROUTE` - -First-class **named** GraphQL mounts (like `CREATE ROUTE`) for a private path, auth, or a subset of tables **without** switching the global endpoint: - -```sql -CREATE GRAPHQL ROUTE public_api POST '/api/gql' - FROM users, posts - REQUIRE site - LIMIT 50; - -SELECT * FROM quackapi_graphql_routes(); --- name, method, path, tables, require_auth, limit - --- Query: --- POST /api/gql body {"query":"{ users { id } }"} --- GET /api/gql/schema tables = FROM list only (mode=route) - -DROP GRAPHQL ROUTE public_api; -``` - -| Piece | Rules | -|-------|--------| -| **name** | Registry id; unique; `OR REPLACE` overwrites | -| **METHOD** | **POST only** (v0) | -| **path** | Absolute `/…`; not `/graphql` or `/graphql/schema`; unique among GraphQL routes | -| **FROM tables** | Required ≥1; must exist in the current database's `main` schema at CREATE; **per-route only** (does not flip global allowlist) | -| **REQUIRE** | Optional `CREATE AUTH` scheme (auth scheme must exist at CREATE); required for row/masking-protected tables | -| **LIMIT** | Optional row cap (default 100; range 1..100000) | - -Independence: - -- Built-in `POST /graphql` stays available but catalog-closed until tables are registered; global allowlist via `CREATE GRAPHQL FOR TABLE` is separate. -- A GraphQL route never mutates `quackapi_graphql_tables()`. -- Unregistered path → normal 404. - -**What stays v0** on any GraphQL surface until product need forces growth: - -- No mutations / subscriptions -- No nested joins / relation fields -- No arguments, aliases, fragments, variables -- No full GraphQL grammar / `__schema` introspection - -Alternative spelling rejected: `CREATE ROUTE x POST '/gql' AS GRAPHQL FROM …` (overloads ROUTE AS beyond SELECT). Dedicated noun matches `CREATE API FOR TABLE`. - -## Future schema feeds - -v0 discovers tables/columns from the **live DuckDB catalog**. **sitting_duck** already parses GraphQL (and app languages) as AST tables — use that offline to invent allowlists or richer types from source trees; wire into the thin execute path only when product needs it. **parser_tools** stays for SQL hygiene on generated SELECTs. Intentional debt: not a blocker for catalog-backed read APIs. - -## Design stance - -- **Dual surface:** REST and thin GraphQL side by side; both chew into queries. -- Prefer `CREATE ROUTE` / `CREATE API FOR TABLE` when you own the SQL shape. -- GraphQL v0 is a **read convenience** (column pick + multi-root) over the same tables. -- **Allowlist** + **named routes** lock down exposure without closing open `/graphql` by default. -- Grow grammar only when a concrete client needs args/filters — still as “→ one SELECT,” not Hasura-in-C++. diff --git a/docs/guide/headers-cookies-redirects.md b/docs/guide/headers-cookies-redirects.md index fa54a4e..5ede8f1 100644 --- a/docs/guide/headers-cookies-redirects.md +++ b/docs/guide/headers-cookies-redirects.md @@ -32,44 +32,12 @@ curl -i http://127.0.0.1:8000/echo-rid -H 'X-Request-ID: client-rid-deadbeef' # [{"id":"client-rid-deadbeef"}] ``` -Structured access log lines (when `access_log:=true`, the default) include -`request_id` on stderr as JSON (`"type":"access",…`). - -### Table-backed access logs - -`access_log` also accepts `false` to disable logging or a table name to append -one row per request to an operator-created table. The table writer recognizes -these columns by name: `request_id`, `received_at`, `method`, `path`, -`route_name`, `status`, `duration_ms`, `bytes_out`, `client_ip`, and -`user_agent`. Other columns are left to their defaults. `route_name` is NULL -for a 404 and `static` for a response served by `static_dir`. - -```sql -CREATE TABLE http_access_log ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - route_name VARCHAR, - status INTEGER, - duration_ms DOUBLE, - bytes_out BIGINT, - client_ip VARCHAR, - user_agent VARCHAR, - operator_note VARCHAR DEFAULT 'keep' -); - -SELECT * FROM quackapi_serve(8000, access_log := 'http_access_log'); -``` - -Appends are batched at 100 rows or one second. The pending queue is capped at -10,000 entries; overflow is written to stderr and counted in `/healthz` as -`access_log_overflow_count`. A missing or broken table never changes the -response: quackapi emits one warning for the failure episode, falls back to -the stderr JSON log, and retries with backoff from one second to 60 seconds. A -successful retry emits a recovery line and resumes table logging. The in-process -test client accepts the same option, for example -`quackapi_request('GET', '/health', access_log := 'http_access_log')`. +Request observations use DuckDB's native logging and can be filtered through +DuckDB's log level. Access logging is enabled by default and can be disabled +with `access_log := false`. Header and query logging are opt-in: configure +`log_headers` with an explicit header allowlist and enable `log_query` only +when needed. Sensitive values are redacted before logging. QuackAPI does not +provide a request-ring table or a table-backed logging sink. --- diff --git a/docs/guide/policies.md b/docs/guide/policies.md deleted file mode 100644 index 9a80262..0000000 --- a/docs/guide/policies.md +++ /dev/null @@ -1,141 +0,0 @@ -# Row access & masking policies - -Policies are a security leapfrog over vanilla FastAPI: **claims-keyed row filters** and **column masks** expressed in SQL, bound to tables, enforced on HTTP handlers that touch those tables. - -All examples run against `build/release/duckdb -unsigned` with `LOAD quackapi;`. - ---- - -## The idea - -1. Authenticate with JWT (or API key). -2. Claims bind as `$claims_tenant`, `$claims_role`, `$claims_sub`, … -3. A **row access policy** is a boolean expression over table columns + claims. -4. A **masking policy** rewrites a column value (placeholder `val` = original). -5. `ALTER TABLE` binds policies to tables/columns. -6. Routes that `SELECT` from bound tables get filters/masks automatically. - -Unauthenticated access to a policied table fails **closed** (HTTP **403**). - ---- - -## Row access policy - -```sql -CREATE TABLE pol_orders ( - id INTEGER, - tenant_id VARCHAR, - amount INTEGER -); - -INSERT INTO pol_orders VALUES - (1, 'acme', 100), - (2, 'beta', 200), - (3, 'acme', 150); - -CREATE AUTH jwt_pol AS JWT ( SECRET 'policy-http-secret' ); - -CREATE ROW ACCESS POLICY tenant_isolation - AS (tenant_id VARCHAR) RETURNS BOOLEAN - USING (tenant_id = $claims_tenant OR $claims_role = 'admin'); - -ALTER TABLE pol_orders - ADD ROW ACCESS POLICY tenant_isolation ON (tenant_id); - -CREATE ROUTE list_orders GET '/orders' REQUIRE jwt_pol AS -SELECT id, tenant_id, amount FROM pol_orders ORDER BY id; -``` - -JWT payload for tenant `acme`, role `user` → only acme rows: - -```sh -curl http://127.0.0.1:8000/orders -H "Authorization: Bearer $JWT_ACME" -# [{"id":1,"tenant_id":"acme","amount":100},{"id":3,"tenant_id":"acme","amount":150}] -# HTTP 200 -``` - -Admin role (`$claims_role = 'admin'`) sees all tenants. -Missing JWT on a `REQUIRE` route → **401** (auth runs before policy). -Public route on a policied table without claims → **403** “Policy denies…”. - ---- - -## Masking policy - -```sql -CREATE TABLE pol_users ( - id INTEGER, - email VARCHAR, - tenant_id VARCHAR -); - -CREATE MASKING POLICY mask_email ON VARCHAR - USING (CASE WHEN $claims_role = 'admin' THEN val ELSE '***' END); - -ALTER TABLE pol_users - ADD ROW ACCESS POLICY tenant_isolation ON (tenant_id); - -ALTER TABLE pol_users - MODIFY COLUMN email SET MASKING POLICY mask_email; - -CREATE ROUTE list_users GET '/users' REQUIRE jwt_pol AS -SELECT id, email, tenant_id FROM pol_users ORDER BY id; -``` - -| Role | `email` column | -|------|----------------| -| `user` | `***` | -| `admin` | real address | - -`val` is the original column value inside the masking expression. - ---- - -## Inspect - -```sql -SELECT name, kind, signature, expression, bound_table, bound_columns -FROM quackapi_policies(); -``` - -| kind | Meaning | -|------|---------| -| `ROW_ACCESS` | Row filter | -| `MASKING` | Column rewrite | - ---- - -## Grammar - -```sql -CREATE [OR REPLACE] ROW ACCESS POLICY - AS ( [] [, ...]) RETURNS BOOLEAN - USING (); - -CREATE [OR REPLACE] MASKING POLICY - ON - USING (); - -ALTER TABLE ADD ROW ACCESS POLICY ON (); -ALTER TABLE MODIFY COLUMN SET MASKING POLICY ; - -DROP ROW ACCESS POLICY ; -DROP MASKING POLICY ; -``` - -Names must be unique across both policy kinds. - ---- - -## Design notes - -- Policies are **helpers**, not a full multi-tenant RBAC product. -- Prefer JWT claims you control (`tenant`, `role`) over trusting query params. -- Combine with [CREATE GROUP](groups.md) so every `/api/v1/*` route requires the same auth scheme. - ---- - -## Next - -- [Static files](static-files.md) -- [OpenAPI](openapi.md) diff --git a/docs/guide/queue.md b/docs/guide/queue.md deleted file mode 100644 index 418c4ed..0000000 --- a/docs/guide/queue.md +++ /dev/null @@ -1,155 +0,0 @@ -# CREATE QUEUE — background jobs without Redis - -Jobs live in an ordinary DuckDB table: **`quackapi_jobs`**. No external broker. Enqueue from a route, drain with a worker route (or scheduled SQL). - -All examples run against `build/release/duckdb -unsigned` with `LOAD quackapi;`. - ---- - -## Create a queue - -```sql -CREATE QUEUE default WITH ( - max_attempts=3, - visibility_timeout='30s', - backoff_base_seconds=2 -); -``` - -| Option | Default | Meaning | -|--------|---------|---------| -| `max_attempts` | 3 | After this many claims, nack → `dead` | -| `visibility_timeout` | 30s | Lease length while `running` (`'30s'`, `'5m'`, or bare seconds) | -| `backoff_base_seconds` | 2 | Nack delay ≈ base^attempts (`0` = immediate) | - -```sql -SELECT name, depth, in_flight, dead, - max_attempts, visibility_timeout_sec, backoff_base_sec -FROM quackapi_queues(); -``` - ---- - -## HTTP: enqueue + drain worker - -```sql -CREATE QUEUE default WITH (max_attempts=3, visibility_timeout='30s'); - -CREATE ROUTE enqueue POST '/jobs' STATUS 201 - PARAM payload VARCHAR - AS -SELECT quackapi_enqueue('default', $payload::VARCHAR) AS job_id; - --- Worker tick: claim up to 10 jobs and ack each in one SELECT -CREATE ROUTE drain POST '/drain' AS -SELECT id AS job_id, - 'processed:' || payload AS result, - quackapi_ack('default', id, delivery_generation) AS acked -FROM quackapi_dequeue('default', 10); - -CREATE ROUTE results GET '/results' AS -SELECT id AS job_id, payload, status -FROM quackapi_jobs -WHERE status = 'done' -ORDER BY id; - -CREATE ROUTE stats GET '/stats' AS -SELECT name, depth, in_flight, dead FROM quackapi_queues(); -``` - -Live run: - -```sh -curl -X POST http://127.0.0.1:8000/jobs \ - -H 'Content-Type: application/json' \ - -d '{"payload":"{\"task\":\"email\"}"}' -# [{"job_id":1}] -# HTTP 201 - -curl http://127.0.0.1:8000/stats -# [{"name":"default","depth":1,"in_flight":0,"dead":0}] - -# httplib wants a body on POST — send empty JSON -curl -X POST http://127.0.0.1:8000/drain \ - -H 'Content-Type: application/json' -d '{}' -# [{"job_id":1,"result":"processed:{\"task\":\"email\"}","acked":true}] - -curl http://127.0.0.1:8000/results -# [{"job_id":1,"payload":"{\"task\":\"email\"}","status":"done"}] -``` - ---- - -## SQL API (no HTTP) - -```sql -SELECT quackapi_enqueue('default', '{"task":"send_email"}'); --- → job_id (BIGINT) - -SELECT id, payload, status, attempts -FROM quackapi_dequeue('default', 10); --- claims: status=running, attempts++, visibility lease - -SELECT quackapi_ack('default', 1, 1); --- true if job was running with delivery_generation 1 - -SELECT quackapi_nack('default', 1, 1, true, 'try_again'); --- 'pending' (retry) or 'dead' at max_attempts - -SELECT quackapi_nack('default', 1, 1, false, 'no_retry'); --- straight to 'dead' - -SELECT quackapi_renew('default', 1, 1); --- true if the active lease was renewed -``` - -Payload is stored as **VARCHAR** (JSON text). You may also pass `JSON` when the json extension is loaded. - -Per-job max attempts override: - -```sql -SELECT quackapi_enqueue('default', '{"z":1}', 1); -``` - ---- - -## Semantics - -| Op | Behavior | -|----|----------| -| enqueue | `pending`, visible now | -| dequeue | Exclusive claim → `running` + lease | -| redelivery | `running` rows with expired lease are claimable again | -| ack | `done` | -| nack (requeue) | `pending` + backoff, or `dead` at max_attempts | -| nack (no requeue) | `dead` immediately | - -**Lifecycle note:** queue *options* live on the database instance (re-run `CREATE QUEUE` after reopen). Job *rows* in `quackapi_jobs` are durable in your `.db` file. - -Concurrency: DuckDB is single-writer per file. Claims serialize on the write lock — correct for one process. For multi-node workers, use an external broker. - ---- - -## Worker options - -1. **HTTP drain route** (above) — cron or a second process POSTs `/drain`. -2. **Scheduled SQL** — community `cronjob` extension (or any external scheduler) running the same `SELECT … FROM quackapi_dequeue … ack`. -3. **Interactive shell** — run dequeue/ack by hand while developing. - -There is no built-in C++ thread pool. The queue is storage + claim primitives; **you** schedule the worker. - ---- - -## Drop - -```sql -DROP QUEUE default; --- registry entry removed; quackapi_jobs rows for that queue remain on disk -``` - ---- - -## Next - -- [CREATE STREAM](stream.md) — push events while jobs run -- Deeper notes: [QUEUE.md](../QUEUE.md) diff --git a/docs/guide/request-bodies.md b/docs/guide/request-bodies.md index b7f4013..d21c3ed 100644 --- a/docs/guide/request-bodies.md +++ b/docs/guide/request-bodies.md @@ -138,7 +138,7 @@ SELECT $title::VARCHAR AS title, ## BODY SCHEMA (JSON Schema validation) -Add `BODY SCHEMA ''` to validate the JSON object **before** the handler runs. This uses DuckDB’s `json_schema` machinery under the hood. The `json_schema` extension is loaded when the route is created (and installed from community if needed), so request handling never performs an installation. +Add `BODY SCHEMA ''` to validate the JSON object **before** the handler runs. This uses DuckDB’s `json_schema` extension. Make both `json` and `json_schema` available locally before creating the route; QuackAPI never installs either extension as a side effect. ```sql CREATE ROUTE create_user POST '/users' STATUS 201 diff --git a/docs/guide/stream.md b/docs/guide/stream.md deleted file mode 100644 index 919bc42..0000000 --- a/docs/guide/stream.md +++ /dev/null @@ -1,98 +0,0 @@ -# CREATE STREAM — Server-Sent Events (SSE) - -Push a live event stream over HTTP with `text/event-stream`. Each result row becomes one SSE event (`id:` + `data:` JSON). - -All examples run against `build/release/duckdb -unsigned` with `LOAD quackapi;`. - ---- - -## Basic stream - -```sql -CREATE STREAM ticks GET '/ticks' AS -SELECT i AS id, 'tick' AS msg FROM range(3) t(i); -``` - -```sh -curl -N http://127.0.0.1:8000/ticks -``` - -Live output: - -```text -Content-Type: text/event-stream - -id: 0 -data: {"id":0,"msg":"tick"} - -id: 1 -data: {"id":1,"msg":"tick"} - -id: 2 -data: {"id":2,"msg":"tick"} -``` - -Multiple rows (including `UNION ALL`) each become an event: - -```sql -CREATE STREAM once GET '/once' AS -SELECT 1 AS id, 'a' AS msg -UNION ALL -SELECT 2 AS id, 'b' AS msg; -``` - ---- - -## Polling interval - -For long-lived “keep selecting” streams, set an interval: - -```sql -CREATE STREAM live GET '/live' WITH (interval='1s') AS -SELECT now() AS ts; - --- bare number = seconds -CREATE OR REPLACE STREAM live GET '/live' WITH (interval=2) AS -SELECT 1 AS n; - --- milliseconds -CREATE OR REPLACE STREAM live GET '/live' WITH (interval='250ms') AS -SELECT 1 AS n; -``` - -```sql -SELECT name, method, pattern, transport, interval_ms, handler -FROM quackapi_streams(); --- transport is always 'sse' today -``` - ---- - -## WebSocket — not supported (use SSE) - -```sql -CREATE STREAM chat WS '/ws' AS SELECT 1; --- error: WebSocket is not supported (bundled HTTP library has no Upgrade API) -``` - -Use **`CREATE STREAM … GET`** for browser EventSource / SSE clients. See also [coming soon](coming-soon.md). - ---- - -## Notes - -- Streams are **not** listed in `quackapi_routes()` — use `quackapi_streams()`. -- Path must start with `/`. -- Streams do not take `REQUIRE` auth in the current version. -- Ordinary routes win if both match the same path. - -```sql -DROP STREAM ticks; -``` - ---- - -## Next - -- [Policies](policies.md) -- [Static files](static-files.md) diff --git a/docs/guide/table-api.md b/docs/guide/table-api.md deleted file mode 100644 index a32dc14..0000000 --- a/docs/guide/table-api.md +++ /dev/null @@ -1,90 +0,0 @@ -# CREATE API FOR TABLE — instant read CRUD - -One statement expands a table (or view) into **two GET routes**: list all rows, and get one row by key. - -Read-only in this version — no POST/PUT/DELETE scaffold. Generated routes are ordinary registry rows (`DROP ROUTE` works on each). - -All examples run against `build/release/duckdb -unsigned` with `LOAD quackapi;`. - ---- - -## Minimal - -```sql -CREATE TABLE cases AS -SELECT 1 AS id, '24-000117' AS case_no; - -CREATE API FOR TABLE cases; --- registers: --- GET /cases → SELECT * FROM "cases" --- GET /cases/:id → SELECT * FROM "cases" WHERE "id" = $id -``` - -```sh -curl http://127.0.0.1:8000/cases -# [{"id":1,"case_no":"24-000117"}] - -curl http://127.0.0.1:8000/cases/1 -# [{"id":1,"case_no":"24-000117"}] -``` - -Inspect: - -```sql -SELECT name, method, pattern FROM quackapi_routes() ORDER BY name; --- cases_get GET /cases/:id --- cases_list GET /cases -``` - ---- - -## Custom path and key - -```sql -CREATE TABLE documents AS -SELECT 1 AS doc_id, 'report.pdf' AS filename; - -CREATE API FOR TABLE documents AT '/api/documents' KEY 'doc_id'; --- GET /api/documents --- GET /api/documents/:doc_id -``` - -Defaults: - -| Clause | Default | -|--------|---------| -| `AT` | `'/'` | -| `KEY` | `'id'` | - -`AT` must be a quoted path starting with `/`. -`KEY` must be a bare identifier (letters, digits, underscore) — it becomes the path param name. - ---- - -## OR REPLACE and DROP - -```sql -CREATE OR REPLACE API FOR TABLE documents AT '/api/documents' KEY 'doc_id'; - -DROP ROUTE documents_list; -DROP ROUTE documents_get; -``` - ---- - -## When to use this vs hand-written routes - -| Use `CREATE API FOR TABLE` when… | Write `CREATE ROUTE` when… | -|----------------------------------|----------------------------| -| You want a quick list + get | You need filters, joins, writes | -| Columns are already the API | You need auth, body, custom status | -| Prototyping a table | Production shape differs from the table | - -You can mix both: scaffold with `CREATE API FOR TABLE`, then add custom routes alongside. - ---- - -## Next - -- [CREATE GROUP](groups.md) — put table routes under `/api/v1` with auth -- [Policies](policies.md) — tenant filters on the same tables diff --git a/docs/index.md b/docs/index.md index f97fbbb..6268e2e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,226 +1,36 @@ -# quackapi - -**A FastAPI-class web framework that lives inside DuckDB.** - -You write routes. The database is the type system, the request validator, and the response encoder. One process serves HTTP and runs SQL. - -``` -browser ──► DuckDB [ quackapi ] -``` - -If you already know “routes,” you know enough to start. - ---- - -## Five-line hello world - -Build (once): - -```sh -git clone --recurse-submodules https://github.com/asubbarao/quackapi -cd quackapi -GEN=ninja make release -./build/release/duckdb -unsigned -``` - -In the DuckDB shell: - -```sql -LOAD quackapi; - -CREATE ROUTE hello GET '/hello' AS SELECT 'world' AS msg; -CREATE ROUTE item GET '/items/:id' AS SELECT $id::INTEGER AS id; - -SELECT * FROM quackapi_serve(8000); -``` - -In another terminal (outputs below are from a live run against `build/release/duckdb -unsigned`): - -```sh -curl http://127.0.0.1:8000/hello -# [{"msg":"world"}] - -curl http://127.0.0.1:8000/items/42 -# [{"id":42}] - -curl http://127.0.0.1:8000/items/abc -# {"detail":[{"loc":["path","id"],"msg":"Input should be a valid integer","type":"type_error"}]} -# HTTP 422 -``` - -That is the whole product idea: - -1. **`CREATE ROUTE`** registers an HTTP endpoint whose handler is SQL. -2. **`$id::INTEGER`** (and friends) type-check request values. -3. Failures return **FastAPI-shaped 422** JSON: `{ "detail": [ { "loc", "msg", "type" } ] }`. -4. Success returns a **JSON array of row objects** by default — your `SELECT` list *is* the response model. - -> **Parser tip:** run `LOAD quackapi;` first in a sequential session (interactive shell or FIFO stdin). Putting `LOAD` and `CREATE ROUTE` in one `duckdb -c "…"` string can parse the route before the extension is loaded. - ---- - -## You write routes. Everything else is included. - -| You need | You write | Built-in | -|----------|-----------|----------| -| HTTP methods | `GET` / `POST` / `PUT` / `PATCH` / `DELETE` / `HEAD` | 405 + `Allow`, auto-HEAD for GET | -| Path / query params | `$name::TYPE` or `PARAM name TYPE …` | 422 validation | -| JSON / form / multipart bodies | same `$name` namespace | body binder | -| Auth | `CREATE AUTH` + `REQUIRE` | API key + JWT HS256 | -| Instant list/get CRUD | `CREATE API FOR TABLE` | expands to two GET routes | -| Versioned prefix + shared auth | `CREATE GROUP` | APIRouter-style | -| Background jobs | `CREATE QUEUE` + enqueue/dequeue/ack | durable `quackapi_jobs` table | -| Live push | `CREATE STREAM … GET` | Server-Sent Events | -| Row security | `CREATE ROW ACCESS POLICY` | claims-keyed filters | -| Column redaction | `CREATE MASKING POLICY` | claims-keyed masks | -| Static files | `static_dir := '…'` on serve | unrouted GETs | -| OpenAPI | open the browser | `/openapi.json`, `/docs`, `/redoc` | -| Thin GraphQL (v0) + dual surface | REST + GraphQL both → SQL; `FOR TABLE` + `GRAPHQL ROUTE` | [graphql-v0.md](guide/graphql-v0.md) | - -Start the server with: - -```sql -SELECT * FROM quackapi_serve(8000); --- → http://127.0.0.1:8000 -``` - -Stop with: - -```sql -SELECT * FROM quackapi_stop(); --- or SELECT * FROM quackapi_stop(8000); -``` - ---- - -## Mental model (one page) - -### Request → bind → SQL → response - -1. HTTP request hits a registered pattern (`/items/:id`). -2. Path, query, header, cookie, and body values land in the same **`$name`** parameter namespace. -3. Handler SQL runs with those parameters prepared. -4. Result rows become JSON (or `html` / `text` / redirect / Set-Cookie — see [headers, cookies, redirects](guide/headers-cookies-redirects.md)). - -### Response envelope - -| Handler shape | Response | -|---------------|----------| -| Multiple columns (default) | `application/json` **array of objects** | -| `ENVELOPE object` (FORMAT json) | bare **JSON object** (exactly one row); 0 rows → `null` unless `EMPTY STATUS` | -| `EMPTY STATUS [BODY '…']` | 0-row result → that status (default body `{"detail":"Not Found"}`) | -| Single column named `html` | `text/html; charset=utf-8` raw body | -| Single column named `text` | `text/plain; charset=utf-8` raw body | -| Column `location` (+ `STATUS 3xx`) | redirect (`Location` header) | -| Column `set_cookie` / `set-cookie` | `Set-Cookie` header (stripped from JSON) | - -Example of typed JSON (bool / number / null preserved): - -```sql -CREATE ROUTE page_json GET '/json' AS -SELECT 'world' AS msg, 42 AS n, true AS ok, NULL::INTEGER AS missing; -``` - -```sh -curl http://127.0.0.1:8000/json -# [{"msg":"world","n":42,"ok":true,"missing":null}] -``` - -Single-row object (opt-in; array remains the default): - -```sql -CREATE ROUTE health GET '/api/health' ENVELOPE object AS -SELECT 'ok' AS status; --- → {"status":"ok"} -``` - -### Live registry - -Routes created **after** serve is already running are live immediately. Inspect them: - -```sql -SELECT name, method, pattern, status, require_auth -FROM quackapi_routes(); -``` - -Handler SQL is validated at `CREATE` time. Broken SQL fails at create — not on first request. - ---- - -## Guided tour (read top to bottom) - -### Task guides - -1. [Routes, typed params, and 422](guide/routes-and-params.md) -2. [Request bodies (JSON, form, multipart, BODY SCHEMA)](guide/request-bodies.md) -3. [Headers, cookies, redirects, status codes, content types](guide/headers-cookies-redirects.md) -4. [Auth (API key + JWT) and REQUIRE](guide/auth.md) -5. [CREATE API FOR TABLE (instant CRUD reads)](guide/table-api.md) -6. [CREATE GROUP (prefix + shared auth = versioning)](guide/groups.md) -7. [CREATE QUEUE (background jobs + worker)](guide/queue.md) -8. [CREATE STREAM (SSE)](guide/stream.md) -9. [Extension composition (fetch, SSE, from_x, mesh, PDF)](guide/extension-composition.md) -10. [Row access & masking policies](guide/policies.md) -11. [Static files (`static_dir`)](guide/static-files.md) -12. [OpenAPI, Swagger UI, ReDoc](guide/openapi.md) -13. [Thin GraphQL v0 (catalog tables)](guide/graphql-v0.md) -14. [Response compression](guide/compression.md) -15. [SQL middleware](MIDDLEWARE.md) - -### Reference - -- [DDL grammar (every CREATE noun)](reference/ddl.md) -- [SQL functions (`quackapi_*`)](reference/functions.md) - -### FastAPI readers - -- [FastAPI parity map (89/89)](fastapi-parity.md) -- [Coming from FastAPI (`quack_from_X`)](from-fastapi.md) - -### Feature ledger - -Authoritative built-vs-not list: [FEATURE_STATUS.md](FEATURE_STATUS.md). - ---- - -## Install & load - -### From source (today) - -```sh -GEN=ninja make release -./build/release/duckdb -unsigned -``` - -```sql -LOAD quackapi; --- or absolute path: --- LOAD '/path/to/build/release/extension/quackapi/quackapi.duckdb_extension'; -``` - -### Community - -```sql -INSTALL quackapi FROM community; -LOAD quackapi; -``` - -**Target DuckDB:** v1.5.6. Platforms: Linux/macOS amd64+arm64 and Windows -amd64. wasm, Windows MinGW/rtools, and Windows arm64 are excluded. - ---- - -## Honest limits (so you plan correctly) - -- **Single-writer DuckDB** — fine for many concurrent readers and light writes; not a pooled OLTP app tier. -- **JWT** — HS256 only today (no RS256 / OIDC browser flow yet). -- **`CREATE API FOR TABLE`** — read routes only (list + get by key). -- **WebSocket** — not supported on the HTTP transport; use [SSE streams](guide/stream.md). -- **Response gzip / zstd** — [compression guide](guide/compression.md). -- Registry (routes, auth, groups, queues, streams) lives on the **database instance**. Re-run DDL after reopen. Queue **jobs** (`quackapi_jobs`) are normal tables and survive restart. - ---- - -## Next step - -Open [Routes, typed params, and 422](guide/routes-and-params.md) and run every `curl` against your own `quackapi_serve` process. +# QuackAPI documentation + +QuackAPI registers HTTP routes in DuckDB and executes each handler as SQL. +Start with the [README quickstart](../README.md), then use these guides for the +retained API. + +## Getting started + +- [Routes and typed parameters](guide/routes-and-params.md) +- [Request bodies](guide/request-bodies.md) +- [Authentication](guide/auth.md) +- [Groups](guide/groups.md) +- [Static files](guide/static-files.md) +- [Compression](guide/compression.md) +- [OpenAPI](guide/openapi.md) + +## Reference + +- [Function reference](reference/functions.md) +- [DDL reference](reference/ddl.md) +- [Middleware](MIDDLEWARE.md) +- [Request budgets and limits](RESOURCE_LIMITS.md) +- [Outbound POST providers](curl_httpfs.md) +- [Removed APIs and migration](MIGRATION.md) + +## Scope and operating limits + +The current HTTP response formats are JSON, NDJSON, and CSV. Route +authentication does not enforce row-level policies on arbitrary SQL. DuckDB's +ordinary SQL and database-file concurrency rules apply to requests; the HTTP +server is not a multi-writer OLTP tier. Arrow Flight remains a separate +protocol. + +Historical feature research in this repository predates the API cleanup. Use +the [migration guide](MIGRATION.md) and current references above for supported +behavior. diff --git a/docs/reference/ddl.md b/docs/reference/ddl.md index 931402c..db2184d 100644 --- a/docs/reference/ddl.md +++ b/docs/reference/ddl.md @@ -1,367 +1,125 @@ -# DDL reference — every CREATE noun +# DDL reference -Grammar below is taken from the parser extensions in `src/`. Examples were run against `build/release/duckdb -unsigned` with `LOAD quackapi;`. +Load QuackAPI before executing its DDL so DuckDB has registered the parser +extensions. Load DuckDB's JSON extension before QuackAPI when the database will +serve JSON routes or use JSON body validation: -Drop forms exist for each noun unless noted. - ---- +```sql +LOAD json; +LOAD quackapi; +``` -## 1. CREATE ROUTE +## `CREATE ROUTE` ```sql -CREATE [OR REPLACE] ROUTE '' - [STATUS ] - [REQUIRE ] - [RATE LIMIT PER [BY ip|token|key]] - [FORMAT json|ndjson|csv|parquet|arrow] - [ENVELOPE array|object] - [EMPTY STATUS [BODY '']] - [TIMEOUT |'30s'|'5m'|'1h'] - [GROUP | IN GROUP ] +CREATE [OR REPLACE] ROUTE '' + [STATUS ] + [FORMAT json | ndjson | csv] + [EMPTY STATUS ] + [REQUIRE ] + [GROUP ] + [SCHEDULE ''] [BODY SCHEMA ''] - [BODY TYPE ''] - [PARAM [] [HEADER|COOKIE|QUERY [wire-name]] - [DEFAULT ] [GE|GT|LE|LT ] [MIN_LENGTH|MAX_LENGTH ] …] - [WITH (timeout_sec [=|:=] |'30s')] - AS ; + [BODY TYPE ] + [PARAM ] +AS ; DROP ROUTE ; ``` -| Piece | Rules | -|-------|--------| -| **METHOD** | `GET`, `POST`, `PUT`, `DELETE`, `PATCH`, `HEAD` | -| **pattern** | Quoted. Must start with `/` unless the route is in a GROUP (relative join allowed). Captures: `:id` or `{id}` → `$id` | -| **STATUS** | Integer 100–599. Default 200 | -| **REQUIRE** | Auth scheme name (checked at request time) | -| **RATE LIMIT** | Fixed-window limit; `BY ip` is the default, and `BY token` / `BY key` use the authenticated subject or credential digest. Exceeding the limit returns `429` with `Retry-After`. | -| **FORMAT** | Response body for row data: `json` (default, array of objects), `ndjson` (`application/x-ndjson`), `csv` (`text/csv`), `parquet` (`application/vnd.apache.parquet` file bytes, magic `PAR1`), `arrow` (Arrow IPC stream via community `nanoarrow` `FORMAT ARROWS`, `application/vnd.apache.arrow.stream`, magic `0xFFFFFFFF`). Explicit `ndjson`/`csv`/`parquet`/`arrow` win over `Accept`; default/`json` allows Accept negotiation (`application/x-ndjson`, `application/jsonl`, `text/csv`, `application/vnd.apache.parquet`, `application/parquet`, `application/vnd.apache.arrow.stream`, `application/vnd.apache.arrow.file`). Column modes `html`/`text` still win. | -| **ENVELOPE** | JSON shape for `FORMAT json`: `array` (default, `[{…}]`) or `object` (exactly one row → `{…}`; 0 rows → `null` unless `EMPTY STATUS`; >1 rows → 500). Only valid with `FORMAT json`. OpenAPI advertises `type: object` when set. | -| **EMPTY STATUS** | When the handler returns **0 rows**, respond with this HTTP status instead of success `STATUS` + `[]`/`null`. Optional `BODY ''` (default `{"detail":"Not Found"}`). Works with array or object envelope. Listed in OpenAPI responses. | -| **TIMEOUT** / **WITH (timeout_sec)** | Per-request httplib socket read/write deadline in seconds (also `'30s'` / `'5m'` / `'1h'`, max 24h). Default omit/`0` = serve defaults (`quackapi_serve` `read_timeout_sec`/`write_timeout_sec`, usually 30). When set, the matched request extends `SocketStream` select timeouts and `SO_RCVTIMEO`/`SO_SNDTIMEO` on that connection through handler + response write. Synonyms: `TIMEOUT 180` or `WITH (timeout_sec := 180)`. | -| **GROUP / IN GROUP** | Join group prefix + inherit auth/tags | -| **BODY SCHEMA** | Quoted JSON Schema string; the `json_schema` extension is loaded when the route is created (and may be installed from community); may appear before or after PARAM | -| **BODY TYPE** | Quoted DuckDB `json_transform` structure; native typed body validation; may appear before or after PARAM | -| **PARAM** | Zero or more. Types: INTEGER/INT, BIGINT, VARCHAR/TEXT/STRING, BOOLEAN/BOOL, DOUBLE, FLOAT/REAL, HUGEINT, UBIGINT, UINTEGER | -| **AS** | Any SQL returning a result; validated at CREATE time | - -**Example:** - -```sql -CREATE ROUTE item GET '/items/:id' - PARAM pretty BOOLEAN DEFAULT false - AS -SELECT $id::INTEGER AS id, $pretty AS pretty; -``` +Supported methods are GET, POST, PUT, PATCH, DELETE, and HEAD. Paths begin +with `/`; named path components such as `'/items/:id'` bind to `$id` in the +handler. Query parameters, selected headers/cookies, and request-body values +can also be declared as handler parameters. Values are bound as prepared SQL +parameters. See the [route guide](../guide/routes-and-params.md) for binding +details. -**Object envelope + empty → 404:** +The default response is JSON. `FORMAT` supports only `json`, `ndjson`, and +`csv`; it does not encode Arrow or Parquet. `STATUS` selects the response +status for a successful handler. `EMPTY STATUS` selects the status used when +the handler returns no rows. `BODY SCHEMA` validates a JSON request body and +requires the DuckDB `json` and `json_schema` extensions to be available; route +DDL does not install them. `BODY TYPE` and `PARAM` declare typed inputs. ```sql -CREATE ROUTE health GET '/api/health' - ENVELOPE object - AS SELECT 'ok' AS status, 'sd-shelf' AS service; --- GET → {"status":"ok","service":"sd-shelf"} +CREATE ROUTE item GET '/items/:id' STATUS 200 AS + SELECT $id::INTEGER AS id; -CREATE ROUTE get_doc GET '/documents/:id' - ENVELOPE object - EMPTY STATUS 404 BODY '{"detail":"Not Found"}' - AS SELECT * FROM documents WHERE id = $id::INTEGER; --- 1 row → {…}; 0 rows → 404 {"detail":"Not Found"} +CREATE ROUTE create_item POST '/items' + BODY TYPE JSON + BODY SCHEMA '{"type":"object","required":["name"]}' +AS + SELECT json_extract_string($body, '$.name') AS name; ``` -**Long-running handler (extends socket IO past the serve default):** +`SCHEDULE` accepts a six-field cron expression with seconds first. Scheduled +routes cannot contain path parameters. -```sql -CREATE ROUTE chrome_exhaust POST '/chrome/exhaust' - PARAM url_prefix VARCHAR - WITH (timeout_sec := 180) - AS SELECT chrome_exhaust($url_prefix) AS result; --- equivalent: … TIMEOUT 180 AS SELECT … -``` - -**Response control columns:** `html`, `text`, `location`, `set_cookie` / `set-cookie` — see [headers guide](../guide/headers-cookies-redirects.md). - ---- - -## 2. CREATE AUTH +## `CREATE AUTH` ```sql -CREATE [OR REPLACE] AUTH AS API_KEY [ ( HEADER '' ) ]; +CREATE [OR REPLACE] AUTH AS API_KEY + [(HEADER '')]; -CREATE [OR REPLACE] AUTH AS JWT ( - SECRET '' - [, ALGORITHM HS256] -); +CREATE [OR REPLACE] AUTH AS JWT + (SECRET '' [, ALGORITHM HS256]); DROP AUTH ; ``` -| Kind | Default header | Notes | -|------|----------------|-------| -| `API_KEY` | `X-API-Key` | Keys added via `quackapi_add_api_key` (SHA-256 stored) | -| `JWT` | `Authorization` (Bearer) | **HS256 only** | - -**Example:** - -```sql -CREATE AUTH site AS API_KEY; -CREATE AUTH jwt_auth AS JWT ( SECRET 'conformance-secret' ); -``` - ---- - -## 3. CREATE GROUP / CREATE API GROUP - -```sql -CREATE [OR REPLACE] [API] GROUP - WITH ( - prefix='/absolute/path' - [, auth= | require=] - [, tags='csv,tags'] - ); - --- Keyword form (no WITH): -CREATE [OR REPLACE] API GROUP - PREFIX '/p' - [TAGS 't'] - [REQUIRE ]; - -DROP [API] GROUP ; -``` - -`CREATE GROUP` and `CREATE API GROUP` are synonyms. `auth=` and `require=` are synonyms. - -**Example:** - -```sql -CREATE GROUP v1 WITH (prefix='/api/v1', auth=api, tags='items,v1'); -CREATE ROUTE items_list GET '/items' GROUP v1 AS SELECT 1 AS id; -``` - ---- - -## 4. CREATE API FOR TABLE - -```sql -CREATE [OR REPLACE] API FOR TABLE
- [AT ''] - [KEY '']; -``` - -| Clause | Default | Constraint | -|--------|---------|------------| -| table | — | Bare id or `"quoted""ident"` | -| `AT` | `'/
'` | Quoted path starting with `/` | -| `KEY` | `'id'` | Bare identifier → path param | - -Expands to: - -- `GET ` → `SELECT * FROM
` -- `GET /:` → `SELECT * FROM
WHERE = $` - -Read-only. Generated names: `
_list`, `
_get`. - -**Example:** - -```sql -CREATE API FOR TABLE documents AT '/api/documents' KEY 'doc_id'; -``` - ---- - -## 5. CREATE GRAPHQL FOR TABLE - -Optional allowlist for the built-in thin GraphQL endpoint (`POST /graphql`, `GET /graphql/schema`). Empty allowlist = **open** mode (all main-schema tables). See [graphql-v0.md](../guide/graphql-v0.md). - -```sql -CREATE [OR REPLACE] GRAPHQL FOR TABLE
[,
...]; - -DROP GRAPHQL FOR TABLE
[,
...]; - -DROP GRAPHQL ALL; -- clear allowlist → open mode -``` - -| Piece | Rules | -|-------|--------| -| **table** | Bare id or `"quoted""ident"`; must exist in schema `main` at CREATE | -| **OR REPLACE** | Idempotent re-register (no error if already listed) | -| **DROP … ALL** | Clears entire allowlist | - -Inspect: - -```sql -SELECT * FROM quackapi_graphql_tables(); --- open mode: one row mode='open', table_name NULL --- allowlist: mode='allowlist', one row per table_name -``` - -**Example:** +API keys default to the `X-API-Key` header. Add keys with +`quackapi_add_api_key`; key inspection never returns the secret value. JWT +authentication supports HS256. Apply an auth definition with `REQUIRE` on a +route or as a group's inherited auth. ```sql -CREATE GRAPHQL FOR TABLE users, posts; +CREATE AUTH api AS API_KEY; +SELECT * FROM quackapi_add_api_key('api', 'secret-key', 'user-42'); +CREATE ROUTE me GET '/me' REQUIRE api AS SELECT $claims_sub AS subject; ``` -### CREATE GRAPHQL ROUTE +Route auth decides whether a request may reach its handler. It does not filter +rows or rewrite arbitrary SQL by tenant. Put tenant predicates directly in +the handler or use a scoped view. -Named path mounts independent of the built-in `/graphql` allowlist. See [graphql-v0.md](../guide/graphql-v0.md). +## `CREATE GROUP` ```sql -CREATE [OR REPLACE] GRAPHQL ROUTE - POST '' - FROM
[,
...] - [REQUIRE ] - [LIMIT ]; +CREATE [OR REPLACE] GROUP + WITH (prefix = '' [, auth = ] + [, tags = '[, ...]']); -DROP GRAPHQL ROUTE ; +DROP GROUP ; ``` -| Piece | Rules | -|-------|--------| -| **path** | Absolute; not reserved (`/graphql`, `/graphql/schema`); unique among GraphQL routes | -| **FROM** | Required; tables must exist in `main` | -| **REQUIRE** | Optional; auth scheme must already exist | -| **LIMIT** | Optional 1..100000 (default 100) | - -Inspect: - -```sql -SELECT * FROM quackapi_graphql_routes(); --- name, method, path, tables, require_auth, limit -``` - -HTTP: `POST ` (query body) and `GET /schema`. - ---- - -## 6. CREATE QUEUE - -```sql -CREATE [OR REPLACE] QUEUE - [WITH ( - max_attempts=, -- default 3; range 1..1000 - visibility_timeout='30s'|30, -- s/m/h suffix or seconds - backoff_base_seconds= -- default 2; 0 = immediate - )]; - -DROP QUEUE ; -``` - -Queue name: `[A-Za-z0-9_-]+`. - -Creates durable table **`quackapi_jobs`** on first use. - -**Example:** +A group prefixes route paths and can provide default auth and OpenAPI tags. +A route's explicit `REQUIRE` takes precedence over inherited group auth. ```sql -CREATE QUEUE emails WITH (max_attempts=5, visibility_timeout='30s'); +CREATE GROUP api_v1 WITH (prefix = '/api/v1', auth = api, tags = 'v1,public'); +CREATE ROUTE status GET '/status' GROUP api_v1 AS SELECT 'ok' AS status; ``` ---- - -## 7. CREATE STREAM - -```sql -CREATE [OR REPLACE] STREAM GET '' - [WITH (interval='1s'|2|'250ms')] - AS
- ADD ROW ACCESS POLICY ON ([, ...]); - -DROP ROW ACCESS POLICY ; -``` - -Expression may reference policy columns and `$claims_*`. - -**Example:** - -```sql -CREATE ROW ACCESS POLICY tenant_isolation - AS (tenant_id VARCHAR) RETURNS BOOLEAN - USING (tenant_id = $claims_tenant OR $claims_role = 'admin'); - -ALTER TABLE pol_orders - ADD ROW ACCESS POLICY tenant_isolation ON (tenant_id); -``` - ---- - -## 9. CREATE MASKING POLICY - -```sql -CREATE [OR REPLACE] MASKING POLICY - ON - USING (); - -ALTER TABLE
- MODIFY COLUMN SET MASKING POLICY ; - -DROP MASKING POLICY ; -``` - -Inside `USING`, **`val`** is the original column value. Claims bind as `$claims_*`. - -**Example:** - -```sql -CREATE MASKING POLICY mask_email ON VARCHAR - USING (CASE WHEN $claims_role = 'admin' THEN val ELSE '***' END); - -ALTER TABLE pol_users - MODIFY COLUMN email SET MASKING POLICY mask_email; -``` - ---- - -## 10. CREATE MIDDLEWARE +## `CREATE MIDDLEWARE` ```sql CREATE [OR REPLACE] MIDDLEWARE - BEFORE|AFTER [GROUP ] - AS ; - - - Methods: GET, POST, PUT, DELETE, PATCH, HEAD. - - Path segments `:id` / `{id}` and query/body fields bind to `$id`. - - Cast/constraint failures return **422** with FastAPI-shaped - `{"detail":[{"loc":[...],"msg":...,"type":...}]}`. - - Default JSON response is an **array of row objects** (SQL result-set - semantics). A single column named `html` or `text` returns raw HTML/text; - `location` / `set_cookie` set response headers. - - `quackapi_serve([port], host := …, static_dir := …, cors_origins := …, memory_limit := …)` - uses DuckDB's bundled httplib (background listener + workers). - - Built-in OpenAPI: `GET /openapi.json`, `GET /docs`, `GET /redoc`. - - ## Other DDL / functions - - - **Auth:** `CREATE AUTH … AS API_KEY | JWT (SECRET … [, ALGORITHM HS256])`, - `REQUIRE` on routes, `quackapi_add_api_key`, `quackapi_auths`, claims as - `$claims_*`. JWT verify is self-contained (DuckDB-bundled mbedtls + - `json_*` claims) — does not depend on community `crypto` (that extension - is hash/HMAC only, not JWT parse/exp/nbf/claims). - - **Groups:** `CREATE GROUP … WITH (prefix=…, auth=…, tags=…)` — path prefix - + auth inheritance (`quackapi_groups()`). - - **Table API:** `CREATE API FOR TABLE t [AT '/path'] [KEY 'id']` → GET list - + GET by key only. - - **Queue:** `CREATE QUEUE`, `quackapi_enqueue/dequeue/ack/nack`, table - `quackapi_jobs` (broker-less; compose `cronjob` for workers). - - **SSE:** `CREATE STREAM … GET '/path' AS
[, …]` -//! `DROP GRAPHQL FOR TABLE
[, …]` -//! `DROP GRAPHQL ALL` -//! `CREATE [OR REPLACE] GRAPHQL ROUTE POST '' FROM … [REQUIRE …] [LIMIT n]` -//! `DROP GRAPHQL ROUTE ` -class GraphqlDdlParserExtension : public ParserExtension { -public: - GraphqlDdlParserExtension(); -}; - -//! Register quackapi_graphql_tables() + quackapi_graphql_routes() inspection TFs. -void RegisterQuackapiGraphqlFunctions(ExtensionLoader &loader); - -} // namespace duckdb diff --git a/src/include/quackapi_http_fetch.hpp b/src/include/quackapi_http_fetch.hpp deleted file mode 100644 index 1642b05..0000000 --- a/src/include/quackapi_http_fetch.hpp +++ /dev/null @@ -1,90 +0,0 @@ -//===----------------------------------------------------------------------===// -// quackapi_http_fetch.hpp -// -// Outbound HTTP for quackapi. Two paths, one rule: the TCP connection MUST -// survive across requests. -// -// 1. http:// → the VENDORED httplib client, checked out of a per-host -// free-list pool (QuackapiHttpPool). Keep-alive + TCP_NODELAY. -// Full method support, no companion extension required. -// 2. https:// → DuckDB's HTTPUtil, so `LOAD curl_httpfs` / `LOAD httpfs` -// transparently supplies TLS. Pooled the same way, via -// HTTPUtil::Request(request, client) which reuses the client -// we hand it instead of building a fresh one. -// -// Why the pool exists: HTTPUtil::Request(request) — the single-argument form — -// declares a local `unique_ptr client;` and lets it die at the end -// of the call (duckdb/src/main/http/http_util.cpp:128). Every request therefore -// pays a fresh DNS + TCP + (TLS) handshake. The community `http_client` -// extension has the same shape, which is why a route calling http_post() once -// per request measured ~40ms against an upstream whose own floor was 13ms, -// flat across worker_threads — while ten calls inside ONE query amortised to -// 9.2ms. Connections were only ever reused WITHIN a query execution. A route -// handler makes exactly one call per request: the worst case. -// -// Never shell out to the curl binary. Never link libcurl into quackapi. -//===----------------------------------------------------------------------===// -#pragma once - -#include "duckdb/common/http_util.hpp" -#include "duckdb/common/string.hpp" -#include "duckdb/common/types.hpp" -#include "duckdb/common/unordered_map.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/main/extension/extension_loader.hpp" - -namespace duckdb { - -//! Result of an in-database outbound HTTP call. -struct QuackapiHttpFetchResult { - HTTPStatusCode status = HTTPStatusCode::INVALID; - string body; - string reason; - bool success = false; - //! Empty unless the transport failed before a response (DNS, connect, …). - string request_error; - HTTPHeaders headers; - //! True when this call reused a pooled connection rather than dialling. - bool reused_connection = false; - - bool Ok() const { - return success && request_error.empty(); - } -}; - -//! Snapshot of the outbound connection pool, exposed as quackapi_http_pool(). -struct QuackapiHttpPoolStats { - string host; - idx_t idle = 0; - idx_t dialed = 0; - idx_t reused = 0; -}; - -struct QuackapiHttpFetch { - //! Active util name: "MultiCurl", "HTTPFS-Curl", "HTTPFS", "Built-In", … - static string ActiveHttpUtilName(DatabaseInstance &db); - - //! GET url. headers are optional extra request headers (e.g. Authorization). - //! stall_ms > 0 (plain http:// only): after sending the request, wait that - //! many milliseconds before reading the response. Test seam for server - //! write-timeout / per-route timeout_sec — a compute-bound handler cannot - //! trip socket IO deadlines. - static QuackapiHttpFetchResult Get(DatabaseInstance &db, const string &url, - const unordered_map &extra_headers = {}, int32_t stall_ms = 0); - - //! POST url with a raw body and Content-Type. - //! http:// needs nothing loaded; https:// requires httpfs / curl_httpfs. - static QuackapiHttpFetchResult Post(DatabaseInstance &db, const string &url, const string &body, - const string &content_type = "application/x-www-form-urlencoded", - const unordered_map &extra_headers = {}); - - //! Per-host pool counters, for tests and quackapi_http_pool(). - static vector PoolStats(); - //! Drop every idle connection (test isolation; not needed in production). - static void ResetPool(); -}; - -//! quackapi_fetch / quackapi_post / quackapi_http_pool / quackapi_parallel_fetch. -void RegisterQuackapiHttpFetchFunctions(ExtensionLoader &loader); - -} // namespace duckdb diff --git a/src/include/quackapi_limits.hpp b/src/include/quackapi_limits.hpp index c6d2a17..6e6c3e3 100644 --- a/src/include/quackapi_limits.hpp +++ b/src/include/quackapi_limits.hpp @@ -6,6 +6,7 @@ namespace duckdb { class Connection; +class ClientContext; struct QuackapiDeadlineState; //! Shared watchdog, not a new OS thread for every request. The guard must die @@ -20,11 +21,15 @@ class QuackapiQueryDeadline { private: std::shared_ptr state; + ClientContext *context; std::chrono::steady_clock::time_point previous; }; //! Outbound clients inherit the request's remaining execution budget. std::chrono::steady_clock::time_point QuackapiCurrentDeadline(); +//! Get the active deadline for a ClientContext, including work scheduled on a +//! DuckDB task thread where thread-local state is not inherited. +std::chrono::steady_clock::time_point QuackapiCurrentDeadline(ClientContext &context); void QuackapiSetCurrentDeadline(std::chrono::steady_clock::time_point deadline); int64_t QuackapiRemainingTimeoutMillis(int64_t fallback_ms); } // namespace duckdb diff --git a/src/include/quackapi_logging.hpp b/src/include/quackapi_logging.hpp new file mode 100644 index 0000000..e08b1f4 --- /dev/null +++ b/src/include/quackapi_logging.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include "duckdb/logging/logging.hpp" + +namespace duckdb { + +class DatabaseInstance; +struct QuackapiRequestRecord; + +void EmitQuackapiAccessLog(DatabaseInstance &db, const QuackapiRequestRecord &entry, LogLevel level); + +} // namespace duckdb diff --git a/src/include/quackapi_pg.hpp b/src/include/quackapi_pg.hpp deleted file mode 100644 index 36ffd83..0000000 --- a/src/include/quackapi_pg.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include "duckdb/common/string.hpp" -#include "duckdb/common/types/value.hpp" -#include "duckdb/common/case_insensitive_map.hpp" - -#include -#include - -namespace duckdb { - -//! Thin libpq path for pure-HTTP routes that hit Postgres — same shape as -//! FastAPI+psycopg: bind params, fresh PQexecParams, JSON rows. Bypasses DuckDB -//! ATTACH/scanner for that request (optional; when pg_dsn is set on serve or -//! quackapi_request / SET quackapi_pg_dsn). -//! -//! Whether the native libpq execution path completed, is proven inapplicable, -//! or failed after PostgreSQL transport/execution began. Callers may fall back -//! to DuckDB only for NOT_APPLICABLE: replaying a failed mutating PG command -//! against another backend is unsafe. -enum class QuackapiPgNativeResult : uint8_t { - NOT_APPLICABLE, - SUCCESS, - FAILED, -}; - -//! A bounded native libpq execution. `max_response_bytes` caps JSON assembly -//! while rows arrive in single-row mode. On FAILED, err_out is a sanitized -//! stable category; "PostgreSQL deadline exceeded" maps to HTTP 504. -QuackapiPgNativeResult QuackapiTryPgNative(const string &dsn, const string &handler_sql, - const case_insensitive_map_t> &provided, - idx_t max_response_bytes, string &json_body, string &err_out); - -} // namespace duckdb diff --git a/src/include/quackapi_policy.hpp b/src/include/quackapi_policy.hpp deleted file mode 100644 index 17e7542..0000000 --- a/src/include/quackapi_policy.hpp +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once - -#include "duckdb/function/table_function.hpp" -#include "duckdb/parser/parser_extension.hpp" - -namespace duckdb { - -class DatabaseInstance; -class ExtensionLoader; - -//! CREATE / DROP ROW ACCESS POLICY, CREATE / DROP MASKING POLICY, -//! ALTER TABLE … ADD/DROP ROW ACCESS POLICY, ALTER TABLE … SET/UNSET MASKING POLICY. -class PolicyDdlParserExtension : public ParserExtension { -public: - PolicyDdlParserExtension(); -}; - -TableFunction GetApplyPolicyFunction(); -TableFunction GetQuackapiPoliciesFunction(); - -//! Rewrite handler SQL so parsed references to policy-bound base tables become -//! secure subqueries (row filter + column masks). Uses policy expressions as-is -//! ($claims_* remain named parameters for the caller's verified-claims binder). -//! -//! When policies are configured, sources whose base-table identity cannot be -//! safely rewritten (for example views and table macros) fail closed. For an -//! unauthenticated access `deny_unauthenticated` is set; otherwise -//! `policy_error` describes the rejected source for server-side logging only. -string RewriteHandlerWithPolicies(DatabaseInstance &db, const string &handler_sql, bool authenticated, - bool &deny_unauthenticated, string &policy_error); - -//! Compatibility wrapper for callers that only need the unauthenticated result. -inline string RewriteHandlerWithPolicies(DatabaseInstance &db, const string &handler_sql, bool authenticated, - bool &deny_unauthenticated) { - string policy_error; - return RewriteHandlerWithPolicies(db, handler_sql, authenticated, deny_unauthenticated, policy_error); -} - -//! True when parsed handler SQL references a policy-bound table or an indirect -//! source that must be rejected while policies are active. -bool HandlerTouchesPoliciedTable(DatabaseInstance &db, const string &handler_sql); - -} // namespace duckdb diff --git a/src/include/quackapi_post.hpp b/src/include/quackapi_post.hpp new file mode 100644 index 0000000..d2ef6d3 --- /dev/null +++ b/src/include/quackapi_post.hpp @@ -0,0 +1,14 @@ +//===----------------------------------------------------------------------===// +// quackapi_post.hpp +// +// SQL registration boundary for quackapi_post. +//===----------------------------------------------------------------------===// +#pragma once + +#include "duckdb/main/extension/extension_loader.hpp" + +namespace duckdb { + +void RegisterQuackapiPostFunctions(ExtensionLoader &loader); + +} // namespace duckdb diff --git a/src/include/quackapi_post_executor.hpp b/src/include/quackapi_post_executor.hpp new file mode 100644 index 0000000..2fd4821 --- /dev/null +++ b/src/include/quackapi_post_executor.hpp @@ -0,0 +1,47 @@ +//===----------------------------------------------------------------------===// +// quackapi_post_executor.hpp +// +// Database-owned bounded execution for outbound POST requests. +//===----------------------------------------------------------------------===// +#pragma once + +#include "quackapi_post_result.hpp" +#include "duckdb/storage/object_cache.hpp" + +#include +#include + +namespace duckdb { + +struct QuackapiPostWorkerState { + virtual ~QuackapiPostWorkerState() = default; +}; + +struct QuackapiPostRequest { + string url; + string body; + string content_type; + unordered_map headers; + shared_ptr> cancelled; +}; + +class ClientContext; +class DatabaseInstance; + +class QuackapiPostExecutor : public ObjectCacheEntry { +public: + explicit QuackapiPostExecutor(DatabaseInstance &db); + ~QuackapiPostExecutor() override; + string GetObjectType() override; + optional_idx GetEstimatedCacheMemory() const override; + static vector Execute(ClientContext &context, vector requests, + const vector &skip); + static idx_t EffectiveConcurrency(ClientContext &context); + static string ObjectType(); + +private: + class Impl; + unique_ptr impl; +}; + +} // namespace duckdb diff --git a/src/include/quackapi_post_provider.hpp b/src/include/quackapi_post_provider.hpp new file mode 100644 index 0000000..63567bf --- /dev/null +++ b/src/include/quackapi_post_provider.hpp @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// quackapi_post_provider.hpp +// +// HTTPUtil-backed outbound POST provider. Each executor worker owns its +// provider client state; clients are never shared across workers. +//===----------------------------------------------------------------------===// +#pragma once + +#include "quackapi_post_result.hpp" + +namespace duckdb { + +struct QuackapiPostWorkerState; +struct QuackapiPostRequest; + +struct QuackapiPostProvider { + static unique_ptr CreateWorkerState(DatabaseInstance &db); + static QuackapiHttpFetchResult Post(DatabaseInstance &db, QuackapiPostWorkerState &state, + const QuackapiPostRequest &request); +}; + +} // namespace duckdb diff --git a/src/include/quackapi_post_result.hpp b/src/include/quackapi_post_result.hpp new file mode 100644 index 0000000..74ae80b --- /dev/null +++ b/src/include/quackapi_post_result.hpp @@ -0,0 +1,36 @@ +//===----------------------------------------------------------------------===// +// quackapi_post_result.hpp +// +// Shared response type and SQL conversion helpers for quackapi_post. +//===----------------------------------------------------------------------===// +#pragma once + +#include "duckdb/common/http_util.hpp" +#include "duckdb/common/string.hpp" +#include "duckdb/common/types.hpp" +#include "duckdb/common/unordered_map.hpp" + +namespace duckdb { + +//! Result of an outbound POST call. +struct QuackapiHttpFetchResult { + HTTPStatusCode status = HTTPStatusCode::INVALID; + string body; + string reason; + bool success = false; + //! Empty unless the transport failed before a response (DNS, connect, …). + string request_error; + HTTPHeaders headers; + //! True when this call reused a provider connection. + bool reused_connection = false; + + bool Ok() const { + return success && request_error.empty(); + } +}; + +LogicalType QuackapiHttpFetchResultType(); +Value QuackapiHttpFetchResultValue(const QuackapiHttpFetchResult &result); +unordered_map QuackapiHttpHeadersFromValue(const Value &value); + +} // namespace duckdb diff --git a/src/include/quackapi_queue.hpp b/src/include/quackapi_queue.hpp deleted file mode 100644 index 72b8ab5..0000000 --- a/src/include/quackapi_queue.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/extension/extension_loader.hpp" -#include "duckdb/parser/parser_extension.hpp" - -namespace duckdb { - -//! CREATE [OR REPLACE] QUEUE / DROP QUEUE syntax. -//! Grammar: -//! CREATE [OR REPLACE] QUEUE -//! [WITH (max_attempts=, visibility_timeout='30s'|30, backoff_base_seconds=)] -//! DROP QUEUE -class QueueDdlParserExtension : public ParserExtension { -public: - QueueDdlParserExtension(); -}; - -//! Internal apply target for planned CREATE/DROP QUEUE. -TableFunction GetApplyQueueFunction(); - -//! Register quackapi_enqueue / dequeue / ack / nack / queues functions. -void RegisterQuackapiQueueFunctions(ExtensionLoader &loader); - -//! Ensure the durable quackapi_jobs table + sequence exist (idempotent). -void EnsureQuackapiJobsTable(DatabaseInstance &db); - -} // namespace duckdb diff --git a/src/include/quackapi_request_binding.hpp b/src/include/quackapi_request_binding.hpp new file mode 100644 index 0000000..9b1a354 --- /dev/null +++ b/src/include/quackapi_request_binding.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include "duckdb/common/case_insensitive_map.hpp" +#include "duckdb/common/types.hpp" +#include "httplib.hpp" + +namespace duckdb { + +//! Preserve whether a JSON member was an actual JSON null. A string value of +//! "null" remains a string and must not silently become SQL NULL. +struct JsonBodyField { + string value; + bool explicit_null; + + JsonBodyField() : explicit_null(false) { + } + JsonBodyField(string value_p, bool explicit_null_p) : value(std::move(value_p)), explicit_null(explicit_null_p) { + } +}; + +enum class JsonBodyParseResult : uint8_t { SUCCESS, INVALID_JSON, NOT_OBJECT }; + +bool IsJsonMediaType(const string &media); +bool IsFormUrlEncodedMediaType(const string &media); +bool IsMultipartMediaType(const string &media); +bool IsBodyMethod(const string &method); +void ParseFormUrlEncoded(const string &body, case_insensitive_map_t &out); +JsonBodyParseResult ParseJsonBodyFields(const string &raw_body, case_insensitive_map_t &fields); +void ParseQueryStringIntoParams(const string &query, duckdb_httplib::Params ¶ms); + +} // namespace duckdb diff --git a/src/include/quackapi_response_encoding.hpp b/src/include/quackapi_response_encoding.hpp new file mode 100644 index 0000000..80ba57a --- /dev/null +++ b/src/include/quackapi_response_encoding.hpp @@ -0,0 +1,85 @@ +#pragma once + +#include "duckdb/common/types.hpp" +#include "httplib.hpp" +#include + +namespace duckdb { + +class Connection; +struct QuackapiRoute; +class DataChunk; + +enum class ResponseMode { JSON, HTML, TEXT }; +enum class BodyFormat { JSON, NDJSON, CSV }; +enum class JsonRowMode { ARRAY, OBJECT, NDJSON }; + +//! Bounded accumulator shared by streaming encoders. Each append is atomic: +//! once the next append would exceed the limit, it is rejected and Data() is +//! empty so callers cannot accidentally send a truncated response. +class QuackapiResponseWriter { +public: + explicit QuackapiResponseWriter(idx_t max_bytes); + bool Append(const char *data, idx_t length); + bool Append(const string &data); + idx_t BytesWritten() const; + bool ExceededLimit() const; + const string &Data() const; + string TakeData(); + +private: + idx_t max_bytes; + idx_t bytes_written; + bool exceeded_limit; + string data; +}; + +//! Vectorized JSON row encoder. A native DuckDB json_object expression is +//! bound once for the result schema and evaluated against each result chunk. +class QuackapiJsonChunkEncoder { +public: + QuackapiJsonChunkEncoder(Connection &con, const vector &names, const vector &types, + const vector &data_cols, JsonRowMode mode, QuackapiResponseWriter &writer); + ~QuackapiJsonChunkEncoder(); + QuackapiJsonChunkEncoder(QuackapiJsonChunkEncoder &&) noexcept; + QuackapiJsonChunkEncoder &operator=(QuackapiJsonChunkEncoder &&) noexcept; + QuackapiJsonChunkEncoder(const QuackapiJsonChunkEncoder &) = delete; + QuackapiJsonChunkEncoder &operator=(const QuackapiJsonChunkEncoder &) = delete; + + void AppendChunk(DataChunk &chunk); + idx_t RowCount() const; + void Finish(); + +private: + class State; + unique_ptr state; +}; + +//! DuckDB-native CSV writer for result chunks. All output, including the +//! header and row terminators, flows through the same bounded response writer. +class QuackapiCsvChunkEncoder { +public: + QuackapiCsvChunkEncoder(Connection &con, const vector &names, const vector &types, + const vector &data_cols, QuackapiResponseWriter &writer); + ~QuackapiCsvChunkEncoder(); + QuackapiCsvChunkEncoder(QuackapiCsvChunkEncoder &&) noexcept; + QuackapiCsvChunkEncoder &operator=(QuackapiCsvChunkEncoder &&) noexcept; + QuackapiCsvChunkEncoder(const QuackapiCsvChunkEncoder &) = delete; + QuackapiCsvChunkEncoder &operator=(const QuackapiCsvChunkEncoder &) = delete; + + void AppendChunk(DataChunk &chunk); + void Finish(); + +private: + class State; + unique_ptr state; +}; + +bool IsSpecialResponseColumn(const string &name); +void SetJson(duckdb_httplib::Response &res, int status, const string &body); +void SetInternalError(duckdb_httplib::Response &res, const string &server_side_detail); +ResponseMode ResponseModeFor(const vector &names); +BodyFormat ResolveBodyFormat(const QuackapiRoute &route, const duckdb_httplib::Request &req); +string EmptyResultBody(const QuackapiRoute &route); + +} // namespace duckdb diff --git a/src/include/quackapi_server.hpp b/src/include/quackapi_server.hpp index 82b940e..0866fe7 100644 --- a/src/include/quackapi_server.hpp +++ b/src/include/quackapi_server.hpp @@ -2,7 +2,6 @@ #include #include -#include #include #include @@ -21,7 +20,6 @@ namespace duckdb { class ClientContext; class DatabaseInstance; -class Value; //! Max request body accepted by quackapi (8 MiB). Larger bodies get 413. static constexpr size_t QUACKAPI_PAYLOAD_MAX_LENGTH = 8ull * 1024ull * 1024ull; @@ -47,12 +45,8 @@ enum class QuackapiLogLevel : uint8_t { //! Response compression policy. AUTO negotiates zstd/gzip from Accept-Encoding. enum class QuackapiCompressionMode : uint8_t { OFF = 0, GZIP = 1, ZSTD = 2, AUTO = 3 }; -//! Serve options (static files, CORS, batteries-included server defaults, -//! response compression). Defaults are correct-by-default for a server -//! process: logging on, health routes on, throughput-oriented DuckDB SETs -//! applied at serve time, and Accept-Encoding compression on (zstd preferred, -//! then gzip). CORS stays off (browser cross-origin blocked) until the -//! operator opts in with cors_origins. +//! Serve options for static files, CORS, HTTP limits, logging, and compression. +//! CORS stays off until the operator opts in with cors_origins. struct QuackapiServeOptions { string static_dir; //! Empty = CORS disabled. "*" = reflect any Origin (or * when no Origin). @@ -64,25 +58,12 @@ struct QuackapiServeOptions { QuackapiLogLevel log_level = QuackapiLogLevel::INFO; //! Requests at or above this duration are included at WARN level. int64_t slow_request_ms = 1000; - //! Emit one structured access-log line per request (stderr, JSON). Default true. + //! Emit one structured request event through DuckDB's native logger. Default true. bool access_log = true; - //! When non-empty, append access-log rows to this operator-created table instead - //! of stderr. The writer falls back to stderr if the table cannot be used. - string access_log_table; - //! Header names permitted in the access-log map; empty avoids retaining headers. + //! Header names permitted in the structured access log; empty omits headers. string log_headers; //! Preserve the raw query string in the access log after key-based redaction. bool log_query = false; - //! Enable DuckDB built-in QueryLog at serve (CALL enable_logging). Default - //! **false** when DuckDB logging was not already enabled — per-handler QueryLog - //! to stdout destroys HTTP throughput. Preserve prior operator logging unless - //! explicitly overridden; use access_log for ops. - bool enable_logging = false; - //! True when the operator supplied the enable_logging named parameter. - bool enable_logging_explicit = false; - //! Number of completed TCP requests retained for quackapi_requests(). - //! Zero disables the ring and its allocation. - int64_t request_ring = 10000; // --- Batteries: health routes (ON by default) --- //! Auto-register GET /health + GET /healthz. Default true. @@ -104,30 +85,6 @@ struct QuackapiServeOptions { int64_t max_response_bytes = 16 * 1024 * 1024; int32_t max_pending_requests = 256; - // --- Batteries: DuckDB SETs applied at serve (overridable) --- - //! Empty = apply non-clobber memory guard (256MB when still at system default). - string memory_limit; - //! Empty = leave DuckDB threads at system default (all cores). Else e.g. "8". - string threads; - //! When true (default), SET preserve_insertion_order=false for throughput. - bool preserve_insertion_order = false; - //! When true (default), SET enable_http_metadata_cache=true for outbound HTTP. - bool enable_http_metadata_cache = true; - - // --- Batteries: outbound HTTP client (curl_httpfs preferred) --- - //! Preference: "auto" (default — prefer curl_httpfs, fall back to httplib with - //! loud reason), "curl" (require curl_httpfs — fail serve if unavailable), or - //! "httplib" (skip curl_httpfs). Named param / SET quackapi_http_client. - //! Does NOT touch the inbound httplib SERVER — only the client used by - //! httpfs / read_* over https. - string http_client = "auto"; - //! Filled at serve after probe: "curl" or "httplib". - string http_client_active; - //! Why active is what it is. Empty when curl is active after a successful - //! probe. "operator_forced" when operator chose httplib. - //! "curl_httpfs_unavailable" when auto fell back (never silent). - string http_client_reason; - //! Request-id source for X-Request-ID. Always **uuidv7** (C++ core, no SQL) //! so the hot path never pays a SELECT per request. Probe may still record //! "tsid" only if an operator forces a future path; default is uuidv7. @@ -138,15 +95,9 @@ struct QuackapiServeOptions { QuackapiCompressionMode compression = QuackapiCompressionMode::AUTO; //! Bodies smaller than this many bytes are left uncompressed (default 1024). idx_t compression_min_bytes = 1024; - - // --- Native Postgres (optional; same shape as FastAPI+psycopg) --- - //! When non-empty, simple routes execute via libpq (thread-local conn, - //! fresh PQexecParams per request) instead of DuckDB ATTACH. - //! Empty = DuckDB path only. Example: postgresql://user:pass@127.0.0.1:5432/db - string pg_dsn; }; -//! One completed request, shared by the stderr and table access-log sinks. +//! One completed request, emitted through DuckDB's native logger. struct QuackapiRequestRecord { string request_id; int64_t received_at_micros; @@ -176,70 +127,6 @@ struct QuackapiRequestRecord { string query; }; -struct QuackapiTelemetryStatus { - string sink; - string target; - idx_t queued = 0; - idx_t exported_total = 0; - idx_t dropped_total = 0; - string last_error; - int64_t last_export_age_ms = -1; -}; - -class QuackapiTelemetrySink { -public: - virtual ~QuackapiTelemetrySink() = default; - - virtual void Enqueue(const QuackapiRequestRecord &record) = 0; - virtual void FlushPending() = 0; - virtual void Flush() = 0; - virtual QuackapiTelemetryStatus Status() const = 0; -}; - -//! Convert the opt-in request fields to the access-log MAP/VARCHAR values. -Value QuackapiRequestHeadersValue(const QuackapiRequestRecord &entry); -Value QuackapiRequestQueryValue(const QuackapiRequestRecord &entry); - -//! Fixed storage keeps request recording bounded while the mutex makes snapshots -//! safe without holding the request path behind a reader's work. -class QuackapiRequestRing { -public: - explicit QuackapiRequestRing(size_t capacity) : records(capacity) { - } - - void Push(const QuackapiRequestRecord &record) { - std::lock_guard lock(mutex); - if (records.empty()) { - return; - } - records[head] = record; - head = (head + 1) % records.size(); - if (count < records.size()) { - count++; - } - } - - std::vector Snapshot() const { - std::lock_guard lock(mutex); - std::vector result; - result.reserve(count); - if (records.empty()) { - return result; - } - const auto first = count == records.size() ? head : 0; - for (size_t i = 0; i < count; i++) { - result.push_back(records[(first + i) % records.size()]); - } - return result; - } - -private: - std::vector records; - mutable std::mutex mutex; - size_t head = 0; - size_t count = 0; -}; - //! Parse log_level named param / setting. Accepts silent|error|warn|info|debug //! (case-insensitive). Unknown → INFO. QuackapiLogLevel ParseQuackapiLogLevel(const string &raw); @@ -288,14 +175,6 @@ class QuackapiHttpServer { const QuackapiServeOptions &Options() const { return options; } - //! Copy the retained records oldest-first while holding only the ring lock. - std::vector SnapshotRequests() const; - //! Keep per-sink counters queryable without exposing sink ownership to the registry. - std::vector SnapshotTelemetryStatus() const; - //! Flush queued telemetry without stopping the server. - void FlushTelemetryPending(); - //! Compatibility counter used by /healthz. - idx_t AccessLogOverflowCount() const; //! True while the TCP listener thread is alive (false after StopAccepting). bool IsRunning() const { return is_running.load(); @@ -324,8 +203,6 @@ class QuackapiHttpServer { unique_ptr server; std::vector listen_threads; std::atomic is_running {false}; - unique_ptr request_ring; - std::vector> telemetry_sinks; }; //! In-process HTTP-shape invoke (no TCP). Builds a Request, runs Dispatch, returns @@ -333,19 +210,12 @@ class QuackapiHttpServer { //! Optional body for POST/PUT/PATCH (Content-Type application/json when non-empty). //! req_headers: optional request headers (e.g. X-Request-ID, Accept, Authorization). //! headers_out: response header map (first value per name; case as httplib stores it). -//! Optional pg_dsn: when non-empty, in-process dispatch uses the native libpq -//! path (same as quackapi_serve(..., pg_dsn := ...)). Empty keeps DuckDB-only. void QuackapiInProcessRequest(DatabaseInstance &db, const string &method, const string &path, const string &body, int &status_out, string &body_out, string &content_type_out, const unordered_map *req_headers = nullptr, - unordered_map *headers_out = nullptr, const string &pg_dsn = string(), + unordered_map *headers_out = nullptr, const QuackapiServeOptions *request_options = nullptr); -//! Apply batteries-included DuckDB SETs / logging at quackapi_serve() time. -//! Overridable via QuackapiServeOptions; never disables safety features. -//! Returns a human-readable summary of what was applied (for docs / debugging). -string ApplyQuackapiServerDefaults(ClientContext &context, QuackapiServeOptions &opts); - //! Auto-register GET /health + GET /healthz into the route registry (OR REPLACE //! reserved names). When opts.health_routes is false, drops those reserved //! routes so a prior serve(true) → stop → serve(false) does not leave them. diff --git a/src/include/quackapi_state.hpp b/src/include/quackapi_state.hpp index 9bd4eeb..f2c2909 100644 --- a/src/include/quackapi_state.hpp +++ b/src/include/quackapi_state.hpp @@ -3,7 +3,6 @@ #include #include #include -#include #include "duckdb/common/optional_idx.hpp" #include "duckdb/common/shared_ptr.hpp" @@ -76,20 +75,6 @@ struct QuackapiParamSpec { idx_t max_length = 0; }; -//! One durable job queue registered via CREATE QUEUE. -//! Jobs live in the shared `quackapi_jobs` table (survives restart in the .db -//! file). Queue options live on the DatabaseInstance (like routes) and must be -//! re-declared after reopen — same lifecycle as CREATE ROUTE. -struct QuackapiQueue { - string name; - //! Attempts before a job is moved to status='dead' (dead-letter). - int32_t max_attempts = 3; - //! Lease duration in seconds after a successful claim (visibility timeout). - int32_t visibility_timeout_sec = 30; - //! Base delay in seconds for exponential nack backoff: base^min(attempts,6). - int32_t backoff_base_sec = 2; -}; - //! One registered route group (FastAPI APIRouter-style prefix + default auth). //! Expanded into routes at CREATE ROUTE time — runtime registry stays flat. struct QuackapiGroup { @@ -100,24 +85,6 @@ struct QuackapiGroup { string require_auth; //! OpenAPI tags CSV (e.g. "items,v1"). Inherited by routes that join the group. string tags; - //! Seam for future shared policy / middleware name. Unused in v1 — do not depend. - string policy; -}; - -//! Named GraphQL-over-HTTP mount (CREATE GRAPHQL ROUTE). Independent of the -//! built-in POST /graphql global allowlist. -struct QuackapiGraphqlRoute { - string name; - //! Upper-case HTTP method; v0 only POST for the query mount. - string method = "POST"; - //! Absolute path, trailing slash stripped (except "/"). Schema is GET path+"/schema". - string path; - //! Per-route table allowlist (required, ≥1). Does not mutate global graphql_tables. - vector tables; - //! Empty = public. Non-empty = CREATE AUTH scheme name. - string require_auth; - //! Row cap for each root field SELECT; 0 means default at execute time. - idx_t limit = 100; }; //! One registered route. Immutable once snapshotted; the registry replaces @@ -150,7 +117,7 @@ struct QuackapiRoute { int rate_limit_per_sec = 0; //! "ip" (default), "token" (Authorization / X-API-Key header raw), "key" alias of token. string rate_limit_by; - //! Default response body format: "json" (default), "ndjson", "csv", "parquet", or "arrow". + //! Default response body format: "json" (default), "ndjson", or "csv". //! Explicit non-json wins over Accept negotiation; json allows Accept override. string response_format = "json"; //! JSON shape for FORMAT json: "array" (default, `[{…}]`) or "object" (single row → `{…}`). @@ -171,61 +138,6 @@ struct QuackapiRoute { string schedule; }; -//! Row-access policy: predicate over table columns + $claims_* (JWT/auth claims). -//! CREATE ROW ACCESS POLICY name AS (col [type], ...) RETURNS BOOLEAN USING (expr) -struct QuackapiRowAccessPolicy { - string name; - //! Argument column names from the AS (...) signature (order preserved). - vector arg_columns; - //! Optional type names parallel to arg_columns (may be empty strings). - vector arg_types; - //! Boolean SQL expression; may reference arg column names and $claims_*. - string expression; -}; - -//! Dynamic data masking policy: rewrite a column value from claims context. -//! CREATE MASKING POLICY name ON USING (expr) — `val` is the column value. -struct QuackapiMaskingPolicy { - string name; - string value_type; // e.g. VARCHAR, INTEGER - //! SQL expression returning the same type; use identifier `val` for the raw value. - string expression; -}; - -//! Table binding for a row-access policy (ALTER TABLE … ADD ROW ACCESS POLICY). -struct QuackapiRowAccessBinding { - string table_name; - string policy_name; - //! Table columns mapped onto the policy signature (same arity as policy args). - vector columns; -}; - -//! Column binding for a masking policy (ALTER TABLE … SET MASKING POLICY). -struct QuackapiMaskingBinding { - string table_name; - string column_name; - string policy_name; -}; - -//! Transport for CREATE STREAM. SSE is first-class on cpp-httplib; WebSocket is -//! not available on the bundled httplib (no Upgrade/WS API) — WS is rejected at DDL. -enum class QuackapiStreamTransport : uint8_t { - SSE = 0, -}; - -//! One SSE push stream registered via CREATE STREAM. -//! Emits text/event-stream (one event per row). Optional interval re-runs the -//! SELECT for polling/tailing — no separate thread pool (blocks the httplib worker). -struct QuackapiStream { - string name; - string method = "GET"; // SSE is GET; WS deferred - string pattern; // e.g. /events - string handler_sql; // AS -//! DROP STREAM -//! -//! SSE only (text/event-stream). WebSocket is not supported on bundled -//! cpp-httplib — CREATE STREAM … WS is rejected with a clear error. -class StreamDdlParserExtension : public ParserExtension { -public: - StreamDdlParserExtension(); -}; - -TableFunction GetApplyStreamFunction(); - -//! Register quackapi_streams() inspection table function. -void RegisterQuackapiStreamFunctions(ExtensionLoader &loader); - -} // namespace duckdb diff --git a/src/include/quackapi_table_api.hpp b/src/include/quackapi_table_api.hpp deleted file mode 100644 index c63277e..0000000 --- a/src/include/quackapi_table_api.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include "duckdb/parser/parser_extension.hpp" - -namespace duckdb { - -//! `CREATE API FOR TABLE
[AT ''] [KEY '']` -//! -//! `
` is a bare identifier or a SQL double-quoted identifier -//! (`"weird""name"` for a name that embeds `"`). KEY must be a bare identifier -//! (letters/digits/underscore) because it becomes the path param `$key`. -//! -//! Generates read routes from a table/view at execution time (PostgREST-style): -//! GET -> SELECT * FROM
-//! GET /: -> SELECT * FROM
WHERE = $ -//! -//! base_path defaults to '/
', key defaults to 'id'. Read-only in this -//! version; the generated routes are ordinary registry rows, so `quackapi_routes()` -//! shows them and `DROP ROUTE` removes them individually. -class TableApiDdlParserExtension : public ParserExtension { -public: - TableApiDdlParserExtension(); -}; - -} // namespace duckdb diff --git a/src/include/quackapi_thread_join.hpp b/src/include/quackapi_thread_join.hpp deleted file mode 100644 index c47d233..0000000 --- a/src/include/quackapi_thread_join.hpp +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -namespace duckdb { - -// Join before captured state is destroyed, including when a later thread fails -// to start. Destroying an already-joinable std::thread would terminate the host. -template -struct JoinOutboundThreads { - THREADS &threads; - ~JoinOutboundThreads() { - for (auto &thread : threads) { - if (thread.joinable()) { - thread.join(); - } - } - } -}; - -} // namespace duckdb diff --git a/src/include/quackapi_validation.hpp b/src/include/quackapi_validation.hpp index a42bf17..4ebebfc 100644 --- a/src/include/quackapi_validation.hpp +++ b/src/include/quackapi_validation.hpp @@ -4,6 +4,8 @@ namespace duckdb { +class Connection; + //! One FastAPI-compatible request-validation issue. loc_json is deliberately a //! JSON array so body paths may contain array indexes as numbers. struct QuackapiValidationIssue { @@ -20,11 +22,11 @@ string QuackapiValidationErrorsJson(const vector &issue //! body location, e.g. /items/0/sku -> ["body","items",0,"sku"]. json_body is //! the validated request payload and distinguishes a numeric object key from //! an array index. -string QuackapiValidationBodyPointerLoc(const string &pointer, const string &json_body); +string QuackapiValidationBodyPointerLoc(Connection &con, const string &pointer, const string &json_body); //! Convert DuckDB's json_transform structure syntax into the compatible //! OpenAPI schema fragment. The declaration remains the runtime source of //! truth; this only projects its native STRUCT/LIST/scalar shape for clients. -string QuackapiDuckdbTransformToOpenApi(const string &body_type); +string QuackapiDuckdbTransformToOpenApi(Connection &con, const string &body_type); } // namespace duckdb diff --git a/src/quackapi_batteries.cpp b/src/quackapi_batteries.cpp index 164ae0e..d25f223 100644 --- a/src/quackapi_batteries.cpp +++ b/src/quackapi_batteries.cpp @@ -1,13 +1,7 @@ #include "quackapi_server.hpp" -#include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" -#include "duckdb/common/types/uuid.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/main/config.hpp" -#include "duckdb/main/connection.hpp" #include "duckdb/main/database.hpp" -#include "duckdb/main/query_result.hpp" #include "quackapi_state.hpp" @@ -32,381 +26,6 @@ QuackapiLogLevel ParseQuackapiLogLevel(const string &raw) { return QuackapiLogLevel::INFO; } -static const char *LogLevelDuckDBName(QuackapiLogLevel level) { - switch (level) { - case QuackapiLogLevel::SILENT: - case QuackapiLogLevel::ERROR_LEVEL: - return "ERROR"; - case QuackapiLogLevel::WARN: - return "WARNING"; - case QuackapiLogLevel::DEBUG_LEVEL: - return "DEBUG"; - case QuackapiLogLevel::INFO: - default: - return "INFO"; - } -} - -//! Safe serve default when nothing was configured (valsafe HARDENING P1-2). -static constexpr const char *SERVE_DEFAULT_MEMORY_LIMIT = "256MB"; - -static bool IsAtSystemDefaultMemoryLimit(DatabaseInstance &db) { - auto &config = DBConfig::GetConfig(db); - if (!config.file_system) { - return false; - } - auto available = DBConfig::GetSystemAvailableMemory(*config.file_system); - idx_t system_default; - if (available == DBConfigOptions().maximum_memory) { - system_default = available; - } else { - // Match DBConfig::SetDefaultMaxMemory(). - system_default = available * 8 / 10; - } - return config.options.maximum_memory == system_default; -} - -static bool RunSet(Connection &con, const string &sql, string &err_out) { - auto res = con.Query(sql); - if (res->HasError()) { - err_out = res->GetError(); - return false; - } - return true; -} - -string ApplyQuackapiServerDefaults(ClientContext &context, QuackapiServeOptions &opts) { - // Correct-by-default SETs/PRAGMAs for a long-lived HTTP server process. - // Each SET is documented (WHY) and overridable via serve() named params. - // NEVER disables safety (no allow_unsigned_extensions, no disabled checks). - vector applied; - // Build stamp so operators/agents can confirm the loaded extension matches the tree. - applied.push_back(StringUtil::Format("quackapi_request_path=perf (enable_logging=%s access_log=%s) " - "(WHY: per-request Connection reset + uuidv7 ids)", - opts.enable_logging ? "true" : "false", opts.access_log ? "true" : "false")); - Connection con(*context.db); - string err; - - // --- memory_limit --- - // WHY: unbounded RAM is the #1 footgun for multi-tenant HTTP handlers; a - // conservative ceiling prevents a single query from OOMing the host. - // Non-clobber: never overwrite an operator-set DuckDB memory_limit. - { - string limit_to_apply; - if (!opts.memory_limit.empty()) { - limit_to_apply = opts.memory_limit; - } else if (IsAtSystemDefaultMemoryLimit(*context.db)) { - limit_to_apply = SERVE_DEFAULT_MEMORY_LIMIT; - } - if (!limit_to_apply.empty()) { - try { - DBConfig::ParseMemoryLimit(limit_to_apply); - } catch (std::exception &ex) { - throw InvalidInputException("quackapi_serve: invalid memory_limit '%s': %s", limit_to_apply, ex.what()); - } - auto escaped = StringUtil::Replace(limit_to_apply, "'", "''"); - if (RunSet(con, StringUtil::Format("SET memory_limit TO '%s'", escaped), err)) { - applied.push_back( - StringUtil::Format("memory_limit=%s (WHY: RAM guardrail for multi-query HTTP workers; " - "prevents one handler from OOMing the process)", - limit_to_apply)); - } else { - fprintf(stderr, "quackapi: could not SET memory_limit: %s\n", err.c_str()); - } - } else { - applied.push_back("memory_limit= (WHY: non-clobber; left alone)"); - } - } - - // --- preserve_insertion_order --- - // WHY: insertion-order preservation serializes some pipelines; servers almost - // never need row order unless ORDER BY is present — false raises throughput. - { - const char *pio = opts.preserve_insertion_order ? "true" : "false"; - if (RunSet(con, StringUtil::Format("SET preserve_insertion_order = %s", pio), err)) { - applied.push_back(StringUtil::Format("preserve_insertion_order=%s (WHY: %s)", pio, - opts.preserve_insertion_order - ? "operator requested stable scan order" - : "server throughput — allow reordering when no ORDER BY")); - } else { - fprintf(stderr, "quackapi: could not SET preserve_insertion_order: %s\n", err.c_str()); - } - } - - // --- postgres attach OLTP: disable ctid parallel page scan --- - // WHY: with pg_use_ctid_scan=true (DuckDB default), point lookups like - // SELECT … FROM pg.t WHERE id = $id - // profile as POSTGRES_SCAN cumulative_rows_scanned ≈ whole table (~100k+), - // not an index probe. false forces COPY (SELECT … WHERE id=…) so Postgres - // uses the PK. Best-effort: no-op if postgres extension not loaded yet. - if (RunSet(con, "SET pg_use_ctid_scan = false", err)) { - applied.push_back("pg_use_ctid_scan=false (WHY: ATTACH point lookups use PG index via " - "WHERE pushdown, not full ctid range scan)"); - } - - // --- enable_http_metadata_cache --- - // WHY: outbound HTTP (httpfs / curl_httpfs companions) reuses ETag / - // Last-Modified; cuts origin load for repeated remote reads from handlers. - // (enable_object_cache is a DuckDB no-op placeholder — intentionally skipped.) - { - const char *v = opts.enable_http_metadata_cache ? "true" : "false"; - if (RunSet(con, StringUtil::Format("SET enable_http_metadata_cache = %s", v), err)) { - applied.push_back( - StringUtil::Format("enable_http_metadata_cache=%s (WHY: cache HTTP ETag/Last-Modified for " - "outbound companion fetches)", - v)); - } else { - fprintf(stderr, "quackapi: could not SET enable_http_metadata_cache: %s\n", err.c_str()); - } - } - - // --- threads --- - // WHY: default leaves DuckDB at all-cores (correct for a dedicated server). - // Override only when the operator passes threads:= so multi-tenant hosts can - // cap CPU without editing duckdb config files. - if (!opts.threads.empty()) { - auto escaped = StringUtil::Replace(opts.threads, "'", "''"); - // Accept bare int or quoted. - string sql; - bool all_digit = !opts.threads.empty(); - for (char c : opts.threads) { - if (c < '0' || c > '9') { - all_digit = false; - break; - } - } - if (all_digit) { - sql = StringUtil::Format("SET threads = %s", opts.threads); - } else { - sql = StringUtil::Format("SET threads TO '%s'", escaped); - } - if (RunSet(con, sql, err)) { - applied.push_back(StringUtil::Format("threads=%s (WHY: operator-capped worker pool for multi-tenant hosts)", - opts.threads)); - } else { - fprintf(stderr, "quackapi: could not SET threads: %s\n", err.c_str()); - } - } else { - applied.push_back("threads= (WHY: max parallel query work for server)"); - } - - // --- DuckDB built-in logging (OFF by default unless already enabled) --- - // WHY: QueryLog-per-handler-SQL to stdout is a multi-ms tax under load and - // serializes workers. HTTP ops use access_log (structured stderr). Opt in - // with enable_logging:=true when debugging query plans / errors; preserve an - // operator-enabled logger unless enable_logging:=false was explicit. - const auto log_config = context.db->GetLogManager().GetConfig(); - if (opts.enable_logging && opts.log_level != QuackapiLogLevel::SILENT) { - const char *level = LogLevelDuckDBName(opts.log_level); - // Prefer CALL enable_logging (current DuckDB API) — sets storage + level. - auto call = con.Query(StringUtil::Format("CALL enable_logging(level:='%s', storage:='stdout')", level)); - if (call->HasError()) { - // Fall back to SET surface if CALL signature differs. - RunSet(con, "SET enable_logging = true", err); - RunSet(con, StringUtil::Format("SET logging_level = '%s'", level), err); - RunSet(con, "SET logging_storage = 'stdout'", err); - applied.push_back(StringUtil::Format("enable_logging=true logging_level=%s logging_storage=stdout " - "(WHY: built-in query/error capture for server ops; CALL failed: %s)", - level, call->GetError())); - } else { - applied.push_back(StringUtil::Format("CALL enable_logging(level:='%s', storage:='stdout') " - "(WHY: DuckDB built-in logger ON — queries + errors to stdout)", - level)); - } - // Deprecated but still present: keep HTTP client logging on (already true - // by default on v1.5.4 — re-assert for older forks). - if (RunSet(con, "SET enable_http_logging = true", err)) { - applied.push_back("enable_http_logging=true (WHY: outbound HTTP client request log; " - "deprecated DuckDB setting, still effective)"); - } - } else if (log_config.enabled && !opts.enable_logging_explicit) { - applied.push_back("enable_logging= (WHY: non-clobber; left alone)"); - if (log_config.enabled_log_types.find("QueryLog") != log_config.enabled_log_types.end()) { - fprintf(stderr, - "quackapi: WARN DuckDB QueryLog is enabled; per-request logging can reduce HTTP throughput\n"); - } - } else { - // Force OFF when logging was not already enabled, or when the operator - // explicitly requested false; QueryLog serializes every handler SQL. - if (RunSet(con, "SET enable_logging = false", err)) { - applied.push_back("enable_logging=false (WHY: default — QueryLog-per-request kills HTTP RPS; " - "opt in with enable_logging:=true; use access_log for ops)"); - } else { - applied.push_back(StringUtil::Format("enable_logging=", err)); - } - RunSet(con, "SET enable_http_logging = false", err); - } - - // --- Compose quack transport log if the option exists --- - // WHY: when duckdb-quack is loaded, share one log surface with REST. - { - auto &config = DBConfig::GetConfig(*context.db); - Value existing; - if (config.TryGetCurrentSetting("quack_log_level", existing) || - config.TryGetCurrentSetting("quack_logging", existing)) { - // Best-effort: set to a verbose-enough value when present. - if (config.TryGetCurrentSetting("quack_log_level", existing)) { - RunSet(con, "SET quack_log_level = 'info'", err); - applied.push_back("quack_log_level=info (WHY: wire quack transport log when present)"); - } - if (config.TryGetCurrentSetting("quack_logging", existing)) { - RunSet(con, "SET quack_logging = true", err); - applied.push_back("quack_logging=true (WHY: wire quack transport log when present)"); - } - } else { - applied.push_back("quack transport log= (WHY: quack not loaded; no settings to wire)"); - } - } - - // --- Outbound HTTP client: prefer curl_httpfs (pooled libcurl + HTTP/2 + async) --- - // WHY: routes that read remote https (read_json/read_parquet/read_csv/read_text) - // or proxy upstream APIs collapse under concurrency on DuckDB's default - // per-request httplib client. curl_httpfs is a drop-in httpfs client layer. - // CLIENT only — does not replace the inbound httplib SERVER. - // Platform matrix (community description.yml excluded_platforms): unavailable on - // wasm_* and windows_*; available on linux_* and osx_*. - // - // Preference semantics: - // auto — prefer curl_httpfs; fall back to httplib with LOUD reason on - // healthz / quackapi_servers() / stderr (never silent) - // curl — REQUIRE curl_httpfs; fail serve if INSTALL/LOAD fails - // httplib — operator forced stock client (no curl_httpfs install) - { - string pref = StringUtil::Lower(opts.http_client); - StringUtil::Trim(pref); - if (pref.empty()) { - pref = "auto"; - } - if (pref != "auto" && pref != "curl" && pref != "httplib") { - throw InvalidInputException("quackapi_serve: http_client must be 'auto', 'curl', or 'httplib' (got '%s')", - opts.http_client); - } - - if (pref == "httplib") { - // Operator forced stock client — do not INSTALL/LOAD curl_httpfs. - // If curl_httpfs was already LOADed earlier in the process, flip the - // backend back to httplib so this serve matches the knob. - string set_err; - if (RunSet(con, "SET httpfs_client_implementation = 'httplib'", set_err)) { - // ok - } else { - // Setting may not exist until httpfs/curl_httpfs is loaded — fine. - (void)set_err; - } - opts.http_client_active = "httplib"; - opts.http_client_reason = "operator_forced"; - fprintf(stderr, "quackapi.http_client=httplib reason=operator_forced\n"); - applied.push_back("http_client=httplib (WHY: operator forced stock httplib client; " - "no curl_httpfs install)"); - } else { - // auto | curl — probe curl_httpfs. auto falls back; curl fails hard. - string fail_detail; - bool loaded = false; - - // Best-effort: ensure core httpfs first (curl_httpfs is 100% compatible - // with it and usually loads it, but explicit order matches the README). - auto httpfs_load = con.Query("LOAD httpfs"); - if (httpfs_load->HasError()) { - auto httpfs_inst = con.Query("INSTALL httpfs"); - if (!httpfs_inst->HasError()) { - httpfs_load = con.Query("LOAD httpfs"); - } - } - // httpfs failure is non-fatal here — curl_httpfs may still provide the layer. - - auto curl_load = con.Query("LOAD curl_httpfs"); - if (curl_load->HasError()) { - auto curl_inst = con.Query("INSTALL curl_httpfs FROM community"); - if (curl_inst->HasError()) { - fail_detail = curl_inst->GetError(); - } else { - curl_load = con.Query("LOAD curl_httpfs"); - if (curl_load->HasError()) { - fail_detail = curl_load->GetError(); - } else { - loaded = true; - } - } - } else { - loaded = true; - } - - if (loaded) { - // Toggle the shared httpfs client backend. 'curl' selects the - // curl-based implementation (MultiCurl / HTTPFS-Curl depending on - // curl_httpfs version); leave multi_curl default if SET fails. - string set_err; - if (!RunSet(con, "SET httpfs_client_implementation = 'curl'", set_err)) { - // Some builds expose only curl_httpfs_client_implementation. - if (!RunSet(con, "SET curl_httpfs_client_implementation = 'curl'", set_err) && - !RunSet(con, "SET curl_httpfs_client_implementation = 'multi_curl'", set_err)) { - // Still loaded — default after LOAD is already MultiCurl. - (void)set_err; - } - } - opts.http_client_active = "curl"; - opts.http_client_reason.clear(); - fprintf(stderr, "quackapi.http_client=curl\n"); - applied.push_back("http_client=curl (WHY: curl_httpfs — libcurl pool + HTTP/2 + async IO for " - "outbound https reads from handlers; 100% httpfs-compatible)"); - } else if (pref == "curl") { - // Production guarantee: no silent httplib when operator required curl. - const string detail = - fail_detail.empty() ? string("unknown") : StringUtil::Replace(fail_detail, "\n", " "); - fprintf(stderr, "quackapi.http_client=curl FAILED reason=curl_httpfs_unavailable detail=%s\n", - detail.c_str()); - throw InvalidInputException( - "quackapi_serve: http_client='curl' requires curl_httpfs but INSTALL/LOAD failed " - "(platform excluded, offline catalog, or load error). " - "Use http_client:='auto' for graceful fallback, or install curl_httpfs. detail=%s", - detail); - } else { - // auto — graceful but LOUD fallback (healthz + servers + stderr carry reason). - opts.http_client_active = "httplib"; - opts.http_client_reason = "curl_httpfs_unavailable"; - const string detail = - fail_detail.empty() ? string("unknown") : StringUtil::Replace(fail_detail, "\n", " "); - fprintf(stderr, - "quackapi.http_client=httplib reason=curl_httpfs_unavailable " - "WARN=auto_fallback production_should_force_curl_or_ensure_curl_httpfs " - "detail=%s\n", - detail.c_str()); - applied.push_back(StringUtil::Format( - "http_client=httplib reason=curl_httpfs_unavailable (WHY: auto fallback — " - "curl_httpfs not installable/loadable; /healthz + quackapi_servers report reason; " - "production: http_client:='curl' or ensure curl_httpfs; detail=%s)", - detail)); - } - } - } - - // Transport knobs are applied in QuackapiHttpServer ctor (httplib SERVER). - applied.push_back(StringUtil::Format("http keep_alive_max_count=%d keep_alive_timeout_sec=%d " - "(WHY: connection reuse cuts TCP/TLS handshake cost)", - opts.keep_alive_max_count, opts.keep_alive_timeout_sec)); - applied.push_back(StringUtil::Format("http read_timeout_sec=%d write_timeout_sec=%d " - "(WHY: bound stalled clients so workers are not pinned forever)", - opts.read_timeout_sec, opts.write_timeout_sec)); - applied.push_back(StringUtil::Format("http worker_threads=%d (WHY: concurrent request handlers; cap prevents " - "unbounded thread spawn under load)", - opts.worker_threads)); - applied.push_back(StringUtil::Format("http payload_max_length=%llu (WHY: body size DoS guard — 413 above cap)", - (unsigned long long)QUACKAPI_PAYLOAD_MAX_LENGTH)); - const string access_log_destination = - opts.access_log_table.empty() ? (opts.access_log ? "stderr" : "off") : opts.access_log_table; - applied.push_back(StringUtil::Format("access_log=%s log_level=%s (WHY: every request → structured access log " - "for correlation with X-Request-ID)", - access_log_destination, LogLevelDuckDBName(opts.log_level))); - applied.push_back(StringUtil::Format("health_routes=%s (WHY: /health liveness + /healthz readiness out of the box)", - opts.health_routes ? "true" : "false")); - - string summary = StringUtil::Join(applied, "\n"); - if (opts.log_level >= QuackapiLogLevel::INFO) { - fprintf(stderr, "quackapi: server defaults applied:\n%s\n", summary.c_str()); - } - return summary; -} - void ProbeQuackapiRequestIdSource(DatabaseInstance &db, QuackapiServeOptions &opts) { // Request IDs are always core uuidv7 generated in C++ (see NextRequestId). // Never INSTALL/LOAD/SELECT tsid at serve or per request — that path added diff --git a/src/quackapi_ddl.cpp b/src/quackapi_ddl.cpp index ee89bcc..f0866a5 100644 --- a/src/quackapi_ddl.cpp +++ b/src/quackapi_ddl.cpp @@ -4,6 +4,7 @@ #include "duckdb/main/client_context.hpp" #include "duckdb/main/connection.hpp" #include "duckdb/main/database.hpp" +#include "duckdb/main/extension_helper.hpp" #include "duckdb/main/prepared_statement.hpp" #include "duckdb/parser/parser_extension.hpp" @@ -316,7 +317,7 @@ bool ParseRouteTimeoutSec(const string &raw, int32_t &out_sec, string &err) { //! Grammar: //! CREATE [OR REPLACE] ROUTE '' -//! [STATUS ] [REQUIRE ] [FORMAT json|ndjson|csv|parquet|arrow] +//! [STATUS ] [REQUIRE ] [FORMAT json|ndjson|csv] //! [ENVELOPE array|object] [EMPTY STATUS [BODY '']] //! [TIMEOUT |'30s'|'5m'] [SCHEDULE '<6-field cron, seconds first>'] //! [GROUP | IN GROUP ] @@ -552,21 +553,17 @@ ParserExtensionParseResult RouteDdlParse(ParserExtensionInfo *, const string &qu rest = QuackapiTrim(rest.substr(token_end)); continue; } - // [FORMAT json|ndjson|csv|parquet|arrow] + // [FORMAT json|ndjson|csv] if (StringUtil::StartsWith(rest_upper, "FORMAT") && (rest.size() == 6 || StringUtil::CharacterIsSpace(rest[6]))) { rest = QuackapiTrim(rest.substr(6)); auto token_end = NextTokenEnd(rest); if (token_end == 0) { - return ParserExtensionParseResult("FORMAT expects json, ndjson, csv, parquet, or arrow"); + return ParserExtensionParseResult("FORMAT expects json, ndjson, or csv"); } auto fmt = StringUtil::Lower(rest.substr(0, token_end)); - // "arrows" is a friendly alias of the nanoarrow COPY format name. - if (fmt == "arrows") { - fmt = "arrow"; - } - if (fmt != "json" && fmt != "ndjson" && fmt != "csv" && fmt != "parquet" && fmt != "arrow") { - return ParserExtensionParseResult("FORMAT must be json, ndjson, csv, parquet, or arrow"); + if (fmt != "json" && fmt != "ndjson" && fmt != "csv") { + return ParserExtensionParseResult("FORMAT must be json, ndjson, or csv"); } response_format = fmt; rest = QuackapiTrim(rest.substr(token_end)); @@ -1209,7 +1206,6 @@ void ApplyRouteExec(ClientContext &context, TableFunctionInput &data_p, DataChun if (bind_data.route.tags.empty() && !group.tags.empty()) { bind_data.route.tags = group.tags; } - // policy seam: group.policy reserved for future shared policy; unused in v1. } else if (bind_data.route.pattern.empty() || bind_data.route.pattern[0] != '/') { throw InvalidInputException("Route pattern must start with '/'"); } @@ -1223,28 +1219,21 @@ void ApplyRouteExec(ClientContext &context, TableFunctionInput &data_p, DataChun { Connection con(*context.db); if (!bind_data.route.body_schema.empty()) { - // BODY SCHEMA depends on the community json_schema extension. Resolve it - // while creating the route, outside any request deadline, so the first - // request cannot unexpectedly pay the installation/network cost. - auto load = con.Query("LOAD json_schema"); - if (load->HasError()) { - auto install = con.Query("INSTALL json_schema FROM community"); - if (install->HasError()) { - throw InvalidInputException("BODY SCHEMA requires the json_schema extension: %s", - install->GetError()); - } - load = con.Query("LOAD json_schema"); + // Load dependencies only when already available locally; route DDL must + // never install an extension as a side effect. + if (!ExtensionHelper::TryAutoLoadAvailableExtension(*context.db, "json")) { + throw InvalidInputException( + "BODY SCHEMA requires the DuckDB JSON extension; load json before creating " + "this route"); } - if (load->HasError()) { - throw InvalidInputException("BODY SCHEMA requires the json_schema extension: %s", load->GetError()); + if (!ExtensionHelper::TryAutoLoadAvailableExtension(*context.db, "json_schema")) { + throw InvalidInputException("BODY SCHEMA requires the json_schema extension to be available and " + "loadable; make it available locally before creating this route"); } // Catch malformed schema text at CREATE time rather than on the first // request. TRY_CAST returns NULL for invalid JSON without throwing. - auto json_load = con.Query("LOAD json"); - if (json_load->HasError()) { - throw InvalidInputException("BODY SCHEMA requires the json extension: %s", json_load->GetError()); - } + // The core json extension was loaded above without an install fallback. auto schema_json = con.Query("SELECT TRY_CAST(? AS JSON) IS NOT NULL", Value(bind_data.route.body_schema)); if (schema_json->HasError()) { @@ -1258,19 +1247,8 @@ void ApplyRouteExec(ClientContext &context, TableFunctionInput &data_p, DataChun } auto prepared = con.Prepare(bind_data.route.handler_sql); if (prepared->HasError()) { - // Native libpq path (pg_dsn): handler SQL is Postgres dialect and may - // not prepare under DuckDB (domains, DO, INSERT…RETURNING, …). Allow - // CREATE when pg_dsn is configured; request time uses libpq, else 500. - auto err = prepared->GetError(); - auto el = StringUtil::Lower(err); - bool pg_returning = StringUtil::Contains(el, "returning clause not yet supported") && - StringUtil::Contains(el, "postgres"); - Value pg_dsn_setting; - bool has_pg_dsn = context.TryGetCurrentSetting("quackapi_pg_dsn", pg_dsn_setting) && - !pg_dsn_setting.IsNull() && !pg_dsn_setting.GetValue().empty(); - if (!pg_returning && !has_pg_dsn) { - throw InvalidInputException("Invalid handler SQL for route \"%s\": %s", bind_data.route.name, err); - } + throw InvalidInputException("Invalid handler SQL for route \"%s\": %s", bind_data.route.name, + prepared->GetError()); } } state.AddRoute(bind_data.route, bind_data.or_replace); @@ -1377,7 +1355,7 @@ bool ParseQuotedString(const string &s, idx_t start, string &out, idx_t &end) { //! Grammar: //! CREATE [OR REPLACE] [API] GROUP WITH ( -//! prefix='/abs', [auth=|require=], [tags='csv'], [policy=] +//! prefix='/abs', [auth=|require=], [tags='csv'] //! ) //! DROP [API] GROUP //! @@ -1537,12 +1515,8 @@ ParserExtensionParseResult GroupDdlParse(ParserExtensionInfo *, const string &qu group.require_auth = value; } else if (key == "tags") { group.tags = value; - } else if (key == "policy") { - // Seam for future shared policy; stored, unused at request time in v1. - group.policy = value; } else { - return ParserExtensionParseResult("Unknown GROUP option \"" + key + - "\" — expected prefix, auth, tags, policy"); + return ParserExtensionParseResult("Unknown GROUP option \"" + key + "\" — expected prefix, auth, tags"); } } if (!have_prefix) { @@ -1617,7 +1591,6 @@ unique_ptr ApplyGroupBind(ClientContext &, TableFunctionBindInput bind_data->group.prefix = input.inputs[3].GetValue(); bind_data->group.require_auth = input.inputs[4].GetValue(); bind_data->group.tags = input.inputs[5].GetValue(); - bind_data->group.policy = input.inputs[6].GetValue(); BindStatusColumn(return_types, names); return std::move(bind_data); } @@ -1648,7 +1621,7 @@ void ApplyGroupExec(ClientContext &context, TableFunctionInput &data_p, DataChun TableFunction MakeApplyGroupFunction() { return MakeApplyDdlFunction("quackapi_apply_group", {LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, + LogicalType::VARCHAR, LogicalType::VARCHAR}, ApplyGroupExec, ApplyGroupBind); } @@ -1663,7 +1636,6 @@ ParserExtensionPlanResult GroupDdlPlan(ParserExtensionInfo *, ClientContext &, result.parameters.push_back(Value(data.group.prefix)); result.parameters.push_back(Value(data.group.require_auth)); result.parameters.push_back(Value(data.group.tags)); - result.parameters.push_back(Value(data.group.policy)); FinishDdlPlan(result); return result; } diff --git a/src/quackapi_extension.cpp b/src/quackapi_extension.cpp index 3dc3146..7ec5987 100644 --- a/src/quackapi_extension.cpp +++ b/src/quackapi_extension.cpp @@ -21,15 +21,9 @@ #include "quackapi_auth.hpp" #include "quackapi_ddl.hpp" -#include "quackapi_from_x.hpp" -#include "quackapi_graphql.hpp" -#include "quackapi_http_fetch.hpp" -#include "quackapi_queue.hpp" -#include "quackapi_policy.hpp" +#include "quackapi_post.hpp" #include "quackapi_server.hpp" #include "quackapi_state.hpp" -#include "quackapi_stream.hpp" -#include "quackapi_table_api.hpp" namespace duckdb { @@ -100,16 +94,9 @@ struct ServeBindData : public TableFunctionData { string static_dir; //! Empty (default) = CORS off. Pass '*' or a comma-separated origin list. string cors_origins; - //! Empty = not provided by operator. Named param wins over SET quackapi_memory_limit. - string memory_limit; - //! Batteries defaults — all correct-by-default for servers; overridable. string log_level = "info"; - string access_log = "true"; - bool enable_logging = false; + bool access_log = true; bool health_routes = true; - string threads; - bool preserve_insertion_order = false; - bool enable_http_metadata_cache = true; int32_t worker_threads = static_cast(QUACKAPI_DEFAULT_WORKER_THREADS); int32_t keep_alive_max_count = static_cast(QUACKAPI_DEFAULT_KEEP_ALIVE_MAX); int32_t keep_alive_timeout_sec = static_cast(QUACKAPI_DEFAULT_KEEP_ALIVE_TIMEOUT_SEC); @@ -119,43 +106,32 @@ struct ServeBindData : public TableFunctionData { string compression = "auto"; //! Min body size in bytes before compression. Default 1024. idx_t compression_min_bytes = 1024; - //! Outbound HTTP client preference: auto|curl|httplib (default auto). - string http_client = "auto"; - //! Optional libpq DSN for native Postgres execute (bypass ATTACH). - string pg_dsn; //! When true, hold the query open after listen_url until stop / SIGINT / SIGTERM. //! Default false — serve still returns immediately (backward compatible). bool block = false; //! First Exec emitted listen_url; second Exec blocks when block=true. bool started = false; bool finished = false; - int64_t request_ring = 10000; }; -static void ParseAccessLogString(const string &raw, bool &enabled, string &table) { - auto value = StringUtil::Lower(raw); - if (value == "true") { - enabled = true; - table.clear(); - } else if (value == "false") { - enabled = false; - table.clear(); - } else if (!raw.empty()) { - enabled = true; - table = raw; - } else { - throw InvalidInputException("quackapi_serve: access_log must be true, false, or a table name"); - } -} - -static string AccessLogValueString(const Value &value) { +static bool AccessLogValue(const Value &value) { if (value.IsNull()) { - throw InvalidInputException("quackapi_serve: access_log must not be NULL"); + throw InvalidInputException("access_log must not be NULL"); + } + if (value.type().id() == LogicalTypeId::BOOLEAN) { + return value.GetValue(); } if (value.type().id() == LogicalTypeId::VARCHAR) { - return value.GetValue(); + auto raw = StringUtil::Lower(value.GetValue()); + StringUtil::Trim(raw); + if (raw == "true") { + return true; + } + if (raw == "false") { + return false; + } } - return value.ToString(); + throw InvalidInputException("access_log must be true or false; table sinks are no longer supported"); } static void BindResourceLimits(ClientContext &context, TableFunctionBindInput &input, QuackapiServeOptions &opts) { @@ -238,17 +214,6 @@ static unique_ptr ServeBind(ClientContext &context, TableFunctionB bind_data->cors_origins = setting.GetValue(); } } - // memory_limit named param wins; else fall back to SET quackapi_memory_limit. - // Empty means "not provided" — ApplyQuackapiServerDefaults uses non-clobber logic. - auto mem_entry = input.named_parameters.find("memory_limit"); - if (mem_entry != input.named_parameters.end()) { - bind_data->memory_limit = mem_entry->second.GetValue(); - } else { - Value setting; - if (context.TryGetCurrentSetting("quackapi_memory_limit", setting) && !setting.IsNull()) { - bind_data->memory_limit = setting.GetValue(); - } - } // log_level named param wins; else SET quackapi_log_level. auto log_entry = input.named_parameters.find("log_level"); if (log_entry != input.named_parameters.end()) { @@ -262,53 +227,14 @@ static unique_ptr ServeBind(ClientContext &context, TableFunctionB } } } - // request_ring named param wins; else SET quackapi_request_ring. - auto ring_entry = input.named_parameters.find("request_ring"); - if (ring_entry != input.named_parameters.end()) { - bind_data->request_ring = ring_entry->second.GetValue(); - } else { - Value setting; - if (context.TryGetCurrentSetting("quackapi_request_ring", setting) && !setting.IsNull()) { - bind_data->request_ring = setting.GetValue(); - } - } - // The ring is allocated up front at serve time, so bound it to keep a typo from exhausting memory. - if (bind_data->request_ring < 0 || bind_data->request_ring > 1000000) { - throw InvalidInputException("quackapi_serve: request_ring must be between 0 and 1000000"); - } auto access_entry = input.named_parameters.find("access_log"); if (access_entry != input.named_parameters.end()) { - bind_data->access_log = AccessLogValueString(access_entry->second); - bool enabled; - string table; - ParseAccessLogString(bind_data->access_log, enabled, table); - } - auto enlog_entry = input.named_parameters.find("enable_logging"); - if (enlog_entry != input.named_parameters.end()) { - bind_data->enable_logging = enlog_entry->second.GetValue(); - bind_data->limits.enable_logging_explicit = true; + bind_data->access_log = AccessLogValue(access_entry->second); } auto health_entry = input.named_parameters.find("health_routes"); if (health_entry != input.named_parameters.end()) { bind_data->health_routes = health_entry->second.GetValue(); } - auto threads_entry = input.named_parameters.find("threads"); - if (threads_entry != input.named_parameters.end()) { - // Accept INTEGER or VARCHAR. - if (threads_entry->second.type().id() == LogicalTypeId::VARCHAR) { - bind_data->threads = threads_entry->second.GetValue(); - } else { - bind_data->threads = threads_entry->second.ToString(); - } - } - auto pio_entry = input.named_parameters.find("preserve_insertion_order"); - if (pio_entry != input.named_parameters.end()) { - bind_data->preserve_insertion_order = pio_entry->second.GetValue(); - } - auto http_meta_entry = input.named_parameters.find("enable_http_metadata_cache"); - if (http_meta_entry != input.named_parameters.end()) { - bind_data->enable_http_metadata_cache = http_meta_entry->second.GetValue(); - } auto wt_entry = input.named_parameters.find("worker_threads"); if (wt_entry != input.named_parameters.end()) { bind_data->worker_threads = wt_entry->second.GetValue(); @@ -345,16 +271,6 @@ static unique_ptr ServeBind(ClientContext &context, TableFunctionB } } // compression_min_bytes named param wins; else SET (default 1024). - // pg_dsn named param wins; else SET quackapi_pg_dsn. - auto pg_dsn_entry = input.named_parameters.find("pg_dsn"); - if (pg_dsn_entry != input.named_parameters.end()) { - bind_data->pg_dsn = pg_dsn_entry->second.GetValue(); - } else { - Value setting; - if (context.TryGetCurrentSetting("quackapi_pg_dsn", setting) && !setting.IsNull()) { - bind_data->pg_dsn = setting.GetValue(); - } - } auto min_entry = input.named_parameters.find("compression_min_bytes"); if (min_entry != input.named_parameters.end()) { auto v = min_entry->second.GetValue(); @@ -372,19 +288,6 @@ static unique_ptr ServeBind(ClientContext &context, TableFunctionB bind_data->compression_min_bytes = static_cast(v); } } - // http_client named param wins; else SET quackapi_http_client (default auto). - auto hc_entry = input.named_parameters.find("http_client"); - if (hc_entry != input.named_parameters.end()) { - bind_data->http_client = hc_entry->second.GetValue(); - } else { - Value setting; - if (context.TryGetCurrentSetting("quackapi_http_client", setting) && !setting.IsNull()) { - auto s = setting.GetValue(); - if (!s.empty()) { - bind_data->http_client = s; - } - } - } auto block_entry = input.named_parameters.find("block"); if (block_entry != input.named_parameters.end()) { bind_data->block = block_entry->second.GetValue(); @@ -436,30 +339,18 @@ static void ServeExec(ClientContext &context, TableFunctionInput &data_p, DataCh // Compose with quack's auth settings when present (no-op if quack unloaded). ComposeQuackAuthSettings(context); - // Batteries-included serve options (all ON / server-optimal by default). QuackapiServeOptions opts = bind_data.limits; opts.static_dir = bind_data.static_dir; opts.cors_origins = bind_data.cors_origins; - opts.memory_limit = bind_data.memory_limit; opts.log_level = ParseQuackapiLogLevel(bind_data.log_level); - opts.request_ring = bind_data.request_ring; - ParseAccessLogString(bind_data.access_log, opts.access_log, opts.access_log_table); - opts.enable_logging = bind_data.enable_logging; + opts.access_log = bind_data.access_log; opts.health_routes = bind_data.health_routes; - opts.threads = bind_data.threads; - opts.preserve_insertion_order = bind_data.preserve_insertion_order; - opts.enable_http_metadata_cache = bind_data.enable_http_metadata_cache; opts.worker_threads = bind_data.worker_threads; opts.keep_alive_max_count = bind_data.keep_alive_max_count; opts.keep_alive_timeout_sec = bind_data.keep_alive_timeout_sec; opts.read_timeout_sec = bind_data.read_timeout_sec; opts.write_timeout_sec = bind_data.write_timeout_sec; - opts.http_client = bind_data.http_client; - opts.pg_dsn = bind_data.pg_dsn; - // Apply DuckDB SETs / logging / resource guards (overridable, never unsafe). - // Outbound client: auto prefers curl_httpfs (loud fallback); curl requires it. - ApplyQuackapiServerDefaults(context, opts); // Compose request_id source: community tsid if LOADable, else core uuidv7. ProbeQuackapiRequestIdSource(*context.db, opts); // Auto-register /health + /healthz into the route registry (listed by routes()). @@ -634,10 +525,8 @@ struct RequestBindData : public TableFunctionData { string path; string body; unordered_map req_headers; - //! Optional libpq DSN for this request (named param wins over SET quackapi_pg_dsn). - string pg_dsn; - //! Empty (default) keeps in-process requests quiet. Otherwise true, false, or a table name. - string access_log; + //! In-process requests are quiet by default; access_log := true opts into native logging. + bool access_log = false; string log_level = "info"; bool finished = false; }; @@ -645,9 +534,8 @@ struct RequestBindData : public TableFunctionData { static unique_ptr RequestBind(ClientContext &context, TableFunctionBindInput &input, vector &return_types, vector &names) { if (input.inputs.size() < 2 || input.inputs.size() > 3) { - throw InvalidInputException( - "quackapi_request(method, path [, body] [, headers := MAP] [, pg_dsn := VARCHAR]) expects 2 or 3 " - "positional args"); + throw InvalidInputException("quackapi_request(method, path [, body] [, headers := MAP]) expects 2 or 3 " + "positional args"); } auto bind_data = make_uniq(); BindResourceLimits(context, input, bind_data->limits); @@ -670,22 +558,9 @@ static unique_ptr RequestBind(ClientContext &context, TableFunctio } } } - // pg_dsn named param wins; else SET quackapi_pg_dsn (empty = DuckDB path). - auto pg_dsn_it = input.named_parameters.find("pg_dsn"); - if (pg_dsn_it != input.named_parameters.end() && !pg_dsn_it->second.IsNull()) { - bind_data->pg_dsn = pg_dsn_it->second.GetValue(); - } else { - Value setting; - if (context.TryGetCurrentSetting("quackapi_pg_dsn", setting) && !setting.IsNull()) { - bind_data->pg_dsn = setting.GetValue(); - } - } auto access_entry = input.named_parameters.find("access_log"); if (access_entry != input.named_parameters.end()) { - bind_data->access_log = AccessLogValueString(access_entry->second); - bool enabled; - string table; - ParseAccessLogString(bind_data->access_log, enabled, table); + bind_data->access_log = AccessLogValue(access_entry->second); } // log_level named param wins; otherwise use SET quackapi_log_level. auto log_entry = input.named_parameters.find("log_level"); @@ -724,12 +599,9 @@ static void RequestExec(ClientContext &context, TableFunctionInput &data_p, Data unordered_map resp_headers; QuackapiServeOptions opts = bind_data.limits; opts.log_level = ParseQuackapiLogLevel(bind_data.log_level); - opts.access_log = false; - if (!bind_data.access_log.empty()) { - ParseAccessLogString(bind_data.access_log, opts.access_log, opts.access_log_table); - } + opts.access_log = bind_data.access_log; QuackapiInProcessRequest(*context.db, bind_data.method, bind_data.path, bind_data.body, status, body, content_type, - &bind_data.req_headers, &resp_headers, bind_data.pg_dsn, &opts); + &bind_data.req_headers, &resp_headers, &opts); output.SetValue(0, 0, Value::INTEGER(status)); output.SetValue(1, 0, Value::BLOB(const_data_ptr_cast(body.data()), body.size())); output.SetValue(2, 0, Value(content_type)); @@ -825,7 +697,7 @@ static void RoutesExec(ClientContext &, TableFunctionInput &data_p, DataChunk &o struct ServersBindData : public TableFunctionData {}; struct ServersGlobalState : public GlobalTableFunctionState { - vector> servers; + vector> servers; idx_t offset = 0; }; @@ -837,10 +709,6 @@ static unique_ptr ServersBind(ClientContext &, TableFunctionBindIn names.emplace_back("port"); return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("listen_url"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("http_client"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("http_client_reason"); return make_uniq(); } @@ -857,279 +725,15 @@ static void ServersExec(ClientContext &, TableFunctionInput &data_p, DataChunk & auto &server = state.servers[state.offset]; const auto &host = std::get<0>(server); const auto port = std::get<1>(server); - const auto &http_client = std::get<2>(server); - const auto &http_client_reason = std::get<3>(server); output.SetValue(0, row, Value(host)); output.SetValue(1, row, Value::INTEGER(port)); output.SetValue(2, row, Value(StringUtil::Format("http://%s:%d", host, port))); - output.SetValue(3, row, Value(http_client)); - output.SetValue(4, row, Value(http_client_reason)); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -//===--------------------------------------------------------------------===// -// quackapi_requests([port]) — recent TCP requests from the in-memory ring -//===--------------------------------------------------------------------===// - -struct RequestsBindData : public TableFunctionData { - int32_t port = 0; -}; - -struct RequestsGlobalState : public GlobalTableFunctionState { - vector records; - idx_t offset = 0; -}; - -static Value RequestsValue(const QuackapiRequestRecord &record, idx_t column) { - switch (column) { - case 0: - return Value(record.request_id); - case 1: - return Value::TIMESTAMP(timestamp_t(record.received_at_micros)); - case 2: - return Value(record.method); - case 3: - return Value(record.path); - case 4: - return record.route_name.empty() ? Value() : Value(record.route_name); - case 5: - return record.route_path.empty() ? Value() : Value(record.route_path); - case 6: - return Value::INTEGER(record.status); - case 7: - return Value::DOUBLE(record.duration_ms); - case 8: - return record.sql_prepare_ms < 0 ? Value() : Value::DOUBLE(record.sql_prepare_ms); - case 9: - return record.sql_execute_ms < 0 ? Value() : Value::DOUBLE(record.sql_execute_ms); - case 10: - return record.rows_out < 0 ? Value() : Value::BIGINT(record.rows_out); - case 11: - return Value::BIGINT(record.bytes_in); - case 12: - return Value::BIGINT(record.bytes_out); - case 13: - return Value(record.client_ip); - case 14: - return Value(record.user_agent); - case 15: - return record.http_version.empty() ? Value() : Value(record.http_version); - case 16: - return record.error_type.empty() ? Value() : Value(record.error_type); - case 17: - return record.error_message.empty() ? Value() : Value(record.error_message); - case 18: - return record.trace_id.empty() ? Value() : Value(record.trace_id); - case 19: - return record.span_id.empty() ? Value() : Value(record.span_id); - case 20: - return record.parent_span_id.empty() ? Value() : Value(record.parent_span_id); - case 21: - return Value::BOOLEAN(record.sampled); - case 22: - return QuackapiRequestHeadersValue(record); - case 23: - return QuackapiRequestQueryValue(record); - default: - throw InternalException("unknown quackapi request column"); - } -} - -static unique_ptr RequestsBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - const bool has_port = !input.inputs.empty() && !input.inputs[0].IsNull(); - if (has_port) { - bind_data->port = input.inputs[0].GetValue(); - } - if (has_port && (bind_data->port < 1 || bind_data->port > 65535)) { - throw InvalidInputException("quackapi_requests: port must be between 1 and 65535"); - } - const LogicalType types[] = {LogicalType::VARCHAR, - LogicalType::TIMESTAMP, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::INTEGER, - LogicalType::DOUBLE, - LogicalType::DOUBLE, - LogicalType::DOUBLE, - LogicalType::BIGINT, - LogicalType::BIGINT, - LogicalType::BIGINT, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::BOOLEAN, - LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR), - LogicalType::VARCHAR}; - const char *const columns[] = {"request_id", "received_at", "method", "path", "route_name", - "route_path", "status", "duration_ms", "sql_prepare_ms", "sql_execute_ms", - "rows_out", "bytes_in", "bytes_out", "client_ip", "user_agent", - "http_version", "error_type", "error_message", "trace_id", "span_id", - "parent_span_id", "sampled", "headers", "query"}; - for (idx_t i = 0; i < 24; i++) { - return_types.emplace_back(types[i]); - names.emplace_back(columns[i]); - } - return std::move(bind_data); -} - -static unique_ptr RequestsInit(ClientContext &context, TableFunctionInitInput &input) { - auto state = make_uniq(); - auto &bind_data = input.bind_data->Cast(); - state->records = QuackapiState::Get(*context.db).SnapshotRequests(bind_data.port); - return std::move(state); -} - -static void RequestsExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.records.size() && row < STANDARD_VECTOR_SIZE) { - auto &record = state.records[state.offset]; - for (idx_t column = 0; column < 24; column++) { - output.SetValue(column, row, RequestsValue(record, column)); - } row++; state.offset++; } output.SetCardinality(row); } -//===--------------------------------------------------------------------===// -// Keep sink health queryable without making the writer implementation public. -//===--------------------------------------------------------------------===// - -struct TelemetryStatusBindData : public TableFunctionData { - int32_t port = 0; -}; - -struct TelemetryStatusGlobalState : public GlobalTableFunctionState { - vector statuses; - idx_t offset = 0; -}; - -static unique_ptr TelemetryStatusBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - const bool has_port = !input.inputs.empty() && !input.inputs[0].IsNull(); - if (has_port) { - bind_data->port = input.inputs[0].GetValue(); - } - if (has_port && (bind_data->port < 1 || bind_data->port > 65535)) { - throw InvalidInputException("quackapi_telemetry_status: port must be between 1 and 65535"); - } - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("sink"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("target"); - return_types.emplace_back(LogicalType::BIGINT); - names.emplace_back("queued"); - return_types.emplace_back(LogicalType::BIGINT); - names.emplace_back("exported_total"); - return_types.emplace_back(LogicalType::BIGINT); - names.emplace_back("dropped_total"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("last_error"); - return_types.emplace_back(LogicalType::BIGINT); - names.emplace_back("last_export_age_ms"); - return std::move(bind_data); -} - -static unique_ptr TelemetryStatusInit(ClientContext &context, TableFunctionInitInput &input) { - auto state = make_uniq(); - auto &bind_data = input.bind_data->Cast(); - state->statuses = QuackapiState::Get(*context.db).SnapshotTelemetryStatus(bind_data.port); - return std::move(state); -} - -static void TelemetryStatusExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.statuses.size() && row < STANDARD_VECTOR_SIZE) { - auto &status = state.statuses[state.offset]; - output.SetValue(0, row, Value(status.sink)); - output.SetValue(1, row, Value(status.target)); - output.SetValue(2, row, Value::BIGINT(static_cast(status.queued))); - output.SetValue(3, row, Value::BIGINT(static_cast(status.exported_total))); - output.SetValue(4, row, Value::BIGINT(static_cast(status.dropped_total))); - output.SetValue(5, row, Value(status.last_error)); - output.SetValue(6, row, Value::BIGINT(status.last_export_age_ms)); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -struct TelemetryFlushBindData : public TableFunctionData { - int32_t port = 0; - bool finished = false; -}; - -static unique_ptr TelemetryFlushBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - const bool has_port = !input.inputs.empty() && !input.inputs[0].IsNull(); - if (has_port) { - bind_data->port = input.inputs[0].GetValue(); - } - if (has_port && (bind_data->port < 1 || bind_data->port > 65535)) { - throw InvalidInputException("quackapi_telemetry_flush: port must be between 1 and 65535"); - } - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("status"); - return std::move(bind_data); -} - -static void TelemetryFlushExec(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &bind_data = data_p.bind_data->CastNoConst(); - if (bind_data.finished) { - return; - } - if (!QuackapiState::Get(*context.db).FlushTelemetry(bind_data.port)) { - output.SetValue(0, 0, Value("No quackapi server to flush")); - } else if (bind_data.port == 0) { - output.SetValue(0, 0, Value("Flushed telemetry for all quackapi servers")); - } else { - output.SetValue(0, 0, - Value(StringUtil::Format("Flushed telemetry for quackapi server on port %d", bind_data.port))); - } - output.SetCardinality(1); - bind_data.finished = true; -} - -//===--------------------------------------------------------------------===// -// quackapi_http_util_name() — active outbound HTTPUtil (no curl_httpfs dep) -//===--------------------------------------------------------------------===// -// When curl_httpfs is LOADed this is typically "MultiCurl". Same underlying -// DBConfig::GetHTTPUtil().GetName() that curl_httpfs_http_util_name() reads. - -static void HttpUtilNameFunction(DataChunk &, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - result.Reference(Value(QuackapiHttpFetch::ActiveHttpUtilName(db))); -} - -//===--------------------------------------------------------------------===// -// quackapi_last_write_timeout_sec() — effective write deadline of last request -//===--------------------------------------------------------------------===// -// Reports what HandleRequest / ApplyRouteIoTimeout actually applied — serve -// write_timeout_sec when no route override ran, or the route timeout_sec when -// ApplyRouteIoTimeout extended the socket deadlines. Not a registry lookup. - -static void LastWriteTimeoutSecFunction(DataChunk &, ExpressionState &state, Vector &result) { - auto &qa = QuackapiState::Get(*state.GetContext().db); - result.Reference(Value::INTEGER(qa.GetLastEffectiveWriteTimeoutSec())); -} - //===--------------------------------------------------------------------===// // Load //===--------------------------------------------------------------------===// @@ -1143,15 +747,6 @@ static void LoadInternal(ExtensionLoader &loader) { "CORS allowed origins for quackapi_serve (* or comma-separated list). " "Empty (default) disables CORS. Overridden by cors_origins named parameter.", LogicalType::VARCHAR, Value("")); - // SET quackapi_memory_limit = '4GB' | '512MB' | … - // Empty (default) = leave DuckDB memory_limit alone if already non-default; - // otherwise apply the 256MB serve guard. Serve-time memory_limit := '…' wins. - config.AddExtensionOption("quackapi_memory_limit", - "Memory limit applied by quackapi_serve (e.g. '4GB', '512MB'). " - "Empty (default): do not clobber a non-default DuckDB memory_limit; " - "only apply the 256MB serve default when nothing was configured. " - "Overridden by memory_limit named parameter.", - LogicalType::VARCHAR, Value("")); // SET quackapi_log_level = 'info' | 'debug' | 'warn' | 'error' | 'silent' // Default info — informative access + DuckDB logs, not silent. config.AddExtensionOption("quackapi_log_level", @@ -1162,19 +757,15 @@ static void LoadInternal(ExtensionLoader &loader) { "Requests at or above this duration are logged at warn level. " "Default 1000ms. Overridden by slow_request_ms named parameter.", LogicalType::BIGINT, Value::BIGINT(1000)); - config.AddExtensionOption("quackapi_request_ring", - "Number of recent TCP requests retained by quackapi_requests(). " - "Default 10000; zero disables the in-memory ring. Overridden by request_ring named " - "parameter.", - LogicalType::BIGINT, Value::BIGINT(10000)); config.AddExtensionOption("quackapi_log_headers", - "Comma-separated request-header allowlist for access-log maps; empty (default) " - "avoids retaining headers. Overridden by log_headers named parameter.", + "Comma-separated request-header allowlist for native access logs; empty (default) " + "omits headers. Overridden by log_headers named parameter.", LogicalType::VARCHAR, Value("")); - config.AddExtensionOption("quackapi_log_query", - "Preserve a redacted raw query string in access logs; default false. Overridden by " - "log_query named parameter.", - LogicalType::BOOLEAN, Value::BOOLEAN(false)); + config.AddExtensionOption( + "quackapi_log_query", + "Preserve a redacted raw query string in native access logs; default false. Overridden by " + "log_query named parameter.", + LogicalType::BOOLEAN, Value::BOOLEAN(false)); // SET quackapi_compression = 'auto'|'gzip'|'zstd'|'off'. // Default auto. Legacy true/false values map to auto/off. config.AddExtensionOption("quackapi_compression", @@ -1187,54 +778,26 @@ static void LoadInternal(ExtensionLoader &loader) { "Minimum response body size (bytes) before compression. " "Default 1024. Overridden by compression_min_bytes named parameter.", LogicalType::BIGINT, Value::BIGINT(1024)); - // SET quackapi_http_client = 'auto' | 'curl' | 'httplib' - // Default auto: INSTALL+LOAD curl_httpfs; fall back to httplib with loud reason. - // curl: REQUIRE curl_httpfs — fail serve if INSTALL/LOAD fails (no silent fallback). - // httplib: force stock client. Serve-time http_client := '…' wins. - // Inbound server remains httplib regardless. - config.AddExtensionOption("quackapi_http_client", - "Outbound HTTP client for httpfs/route fetches: auto|curl|httplib. " - "Default auto prefers curl_httpfs (pool, HTTP/2, async) and falls back " - "to httplib with http_client_reason on /healthz when unavailable. " - "curl fails serve if curl_httpfs cannot INSTALL/LOAD. Overridden by " - "http_client named parameter. Does not change the inbound HTTP server.", - LogicalType::VARCHAR, Value("auto")); - // SET quackapi_pg_dsn = 'postgresql://…' — native libpq path for serve + request. - // Empty (default) = DuckDB handler path only. Overridden by pg_dsn named parameter. - config.AddExtensionOption("quackapi_pg_dsn", - "Postgres DSN for the native libpq handler path (thread-local PGconn). " - "Empty (default) keeps DuckDB-only execution. Overridden by pg_dsn named " - "parameter on quackapi_serve / quackapi_request.", - LogicalType::VARCHAR, Value("")); - config.AddExtensionOption("quackapi_query_timeout_ms", "Maximum query execution time in milliseconds", LogicalType::BIGINT, Value::BIGINT(30000)); config.AddExtensionOption("quackapi_max_response_bytes", "Maximum uncompressed response bytes", LogicalType::BIGINT, Value::BIGINT(16 * 1024 * 1024)); config.AddExtensionOption("quackapi_max_pending_requests", "Maximum queued HTTP connections", LogicalType::BIGINT, Value::BIGINT(256)); - config.AddExtensionOption("quackapi_graphql_allow_all", "Explicit legacy opt-in to public GraphQL catalog exposure", - LogicalType::BOOLEAN, Value::BOOLEAN(false)); // quackapi_serve() / quackapi_serve(port) with batteries-included options. // All logging / health / server SETs ON by default; every knob overridable. - // Also carries compression + compression_min_bytes + http_client. + // Also carries compression + compression_min_bytes. TableFunctionSet serve_set("quackapi_serve"); TableFunction serve("quackapi_serve", {LogicalType::INTEGER}, ServeExec, ServeBind); serve.named_parameters["host"] = LogicalType::VARCHAR; serve.named_parameters["static_dir"] = LogicalType::VARCHAR; serve.named_parameters["cors_origins"] = LogicalType::VARCHAR; - serve.named_parameters["memory_limit"] = LogicalType::VARCHAR; serve.named_parameters["log_level"] = LogicalType::VARCHAR; serve.named_parameters["slow_request_ms"] = LogicalType::BIGINT; - serve.named_parameters["request_ring"] = LogicalType::BIGINT; serve.named_parameters["log_headers"] = LogicalType::VARCHAR; serve.named_parameters["log_query"] = LogicalType::BOOLEAN; serve.named_parameters["access_log"] = LogicalType::ANY; - serve.named_parameters["enable_logging"] = LogicalType::BOOLEAN; serve.named_parameters["health_routes"] = LogicalType::BOOLEAN; - serve.named_parameters["threads"] = LogicalType::VARCHAR; - serve.named_parameters["preserve_insertion_order"] = LogicalType::BOOLEAN; - serve.named_parameters["enable_http_metadata_cache"] = LogicalType::BOOLEAN; serve.named_parameters["worker_threads"] = LogicalType::INTEGER; serve.named_parameters["keep_alive_max_count"] = LogicalType::INTEGER; serve.named_parameters["keep_alive_timeout_sec"] = LogicalType::INTEGER; @@ -1242,8 +805,6 @@ static void LoadInternal(ExtensionLoader &loader) { serve.named_parameters["write_timeout_sec"] = LogicalType::INTEGER; serve.named_parameters["compression"] = LogicalType::ANY; serve.named_parameters["compression_min_bytes"] = LogicalType::BIGINT; - serve.named_parameters["http_client"] = LogicalType::VARCHAR; - serve.named_parameters["pg_dsn"] = LogicalType::VARCHAR; serve.named_parameters["block"] = LogicalType::BOOLEAN; serve.named_parameters["query_timeout_ms"] = LogicalType::BIGINT; serve.named_parameters["max_response_bytes"] = LogicalType::BIGINT; @@ -1282,7 +843,6 @@ static void LoadInternal(ExtensionLoader &loader) { request2.named_parameters["slow_request_ms"] = LogicalType::BIGINT; request2.named_parameters["log_headers"] = LogicalType::VARCHAR; request2.named_parameters["log_query"] = LogicalType::BOOLEAN; - request2.named_parameters["pg_dsn"] = LogicalType::VARCHAR; request2.named_parameters["query_timeout_ms"] = LogicalType::BIGINT; request2.named_parameters["max_response_bytes"] = LogicalType::BIGINT; request_set.AddFunction(request2); @@ -1294,7 +854,6 @@ static void LoadInternal(ExtensionLoader &loader) { request3.named_parameters["slow_request_ms"] = LogicalType::BIGINT; request3.named_parameters["log_headers"] = LogicalType::VARCHAR; request3.named_parameters["log_query"] = LogicalType::BOOLEAN; - request3.named_parameters["pg_dsn"] = LogicalType::VARCHAR; request3.named_parameters["query_timeout_ms"] = LogicalType::BIGINT; request3.named_parameters["max_response_bytes"] = LogicalType::BIGINT; request_set.AddFunction(request3); @@ -1302,84 +861,27 @@ static void LoadInternal(ExtensionLoader &loader) { loader.RegisterFunction(TableFunction("quackapi_routes", {}, RoutesExec, RoutesBind, RoutesInit)); loader.RegisterFunction(TableFunction("quackapi_servers", {}, ServersExec, ServersBind, ServersInit)); - TableFunctionSet requests_set("quackapi_requests"); - TableFunction requests1("quackapi_requests", {LogicalType::INTEGER}, RequestsExec, RequestsBind, RequestsInit); - requests_set.AddFunction(requests1); - requests1.arguments.clear(); - requests_set.AddFunction(requests1); - loader.RegisterFunction(requests_set); - TableFunctionSet telemetry_status_set("quackapi_telemetry_status"); - TableFunction telemetry_status1("quackapi_telemetry_status", {LogicalType::INTEGER}, TelemetryStatusExec, - TelemetryStatusBind, TelemetryStatusInit); - telemetry_status_set.AddFunction(telemetry_status1); - telemetry_status1.arguments.clear(); - telemetry_status_set.AddFunction(telemetry_status1); - loader.RegisterFunction(telemetry_status_set); - TableFunctionSet telemetry_flush_set("quackapi_telemetry_flush"); - TableFunction telemetry_flush1("quackapi_telemetry_flush", {LogicalType::INTEGER}, TelemetryFlushExec, - TelemetryFlushBind); - telemetry_flush_set.AddFunction(telemetry_flush1); - telemetry_flush1.arguments.clear(); - telemetry_flush_set.AddFunction(telemetry_flush1); - loader.RegisterFunction(telemetry_flush_set); loader.RegisterFunction(GetQuackapiGroupsFunction()); // Auth inspection + API key management (secrets/hashes never exposed). loader.RegisterFunction(GetQuackapiAuthsFunction()); loader.RegisterFunction(GetQuackapiAddApiKeyFunction()); - // Row-access + masking policy inspection (claims-keyed, not DB roles). - loader.RegisterFunction(GetQuackapiPoliciesFunction()); - // quack auth bridge: quackapi_authentication / quackapi_authorization / // quackapi_verify_auth — same signatures as quack_check_token / // quack_nop_authorization so they can be SET as quack_*_function. RegisterQuackAuthBridgeFunctions(loader); - // CREATE STREAM + quackapi_streams() (SSE push; WS deferred on httplib). - RegisterQuackapiStreamFunctions(loader); - - // Outbound client diagnostic — works with Built-In, HTTPFS, MultiCurl, … - // Does NOT auto-LOAD curl_httpfs; missing companion must not fail LOAD quackapi. - loader.RegisterFunction(ScalarFunction("quackapi_http_util_name", {}, LogicalType::VARCHAR, HttpUtilNameFunction)); + RegisterQuackapiPostFunctions(loader); - // Effective write deadline applied to the most recent route request - // (ApplyRouteIoTimeout / serve write_timeout_sec). Test/introspection seam. - loader.RegisterFunction( - ScalarFunction("quackapi_last_write_timeout_sec", {}, LogicalType::INTEGER, LastWriteTimeoutSecFunction)); - - // Outbound HTTP with a connection pool that survives ACROSS requests: - // quackapi_fetch / quackapi_post / quackapi_http_pool. See the header of - // quackapi_http_fetch.hpp for why the un-pooled path costs ~27ms a call. - RegisterQuackapiHttpFetchFunctions(loader); - - // Durable broker-less job queue (CREATE QUEUE + enqueue/dequeue/ack/nack). - // Backing store is the plain quackapi_jobs table; worker = compose cronjob. - RegisterQuackapiQueueFunctions(loader); loader.RegisterFunction(GetApplyMiddlewareFunction()); loader.RegisterFunction(GetQuackapiMiddlewaresFunction()); - // quack_from_{fastapi,rails,express,gin}[+_models]: sitting_duck extractors - // that turn a real web-app source tree into route/model IR rows. - RegisterQuackapiFromXFunctions(loader); - - // Thin GraphQL allowlist + named route inspection (CREATE GRAPHQL … DDL). - RegisterQuackapiGraphqlFunctions(loader); - - // CREATE ROUTE / GROUP / AUTH / QUEUE / POLICY / STREAM / API FOR TABLE / - // GRAPHQL FOR TABLE / GRAPHQL ROUTE — all first-class nouns registered. + // CREATE ROUTE / GROUP / AUTH and middleware DDL. ExtensionCallbackManager::Get(db).Register(RouteDdlParserExtension()); ExtensionCallbackManager::Get(db).Register(GroupDdlParserExtension()); ExtensionCallbackManager::Get(db).Register(AuthDdlParserExtension()); - ExtensionCallbackManager::Get(db).Register(TableApiDdlParserExtension()); - ExtensionCallbackManager::Get(db).Register(QueueDdlParserExtension()); ExtensionCallbackManager::Get(db).Register(MiddlewareDdlParserExtension()); - // CREATE ROW ACCESS / MASKING POLICY + ALTER TABLE policy bind - ExtensionCallbackManager::Get(db).Register(PolicyDdlParserExtension()); - // CREATE STREAM (SSE) - ExtensionCallbackManager::Get(db).Register(StreamDdlParserExtension()); - // CREATE / DROP GRAPHQL FOR TABLE (+ DROP GRAPHQL ALL) + GRAPHQL ROUTE - ExtensionCallbackManager::Get(db).Register(GraphqlDdlParserExtension()); } void QuackapiExtension::Load(ExtensionLoader &loader) { diff --git a/src/quackapi_from_x.cpp b/src/quackapi_from_x.cpp deleted file mode 100644 index d0b9a80..0000000 --- a/src/quackapi_from_x.cpp +++ /dev/null @@ -1,335 +0,0 @@ -#include "quackapi_from_x.hpp" -#include "quackapi_from_x_sql.hpp" - -#include "duckdb/common/exception.hpp" -#include "duckdb/common/string_util.hpp" -#include "duckdb/common/vector_operations/binary_executor.hpp" -#include "duckdb/function/scalar_function.hpp" -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/main/connection.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/main/extension/extension_loader.hpp" -#include "duckdb/main/query_result.hpp" - -namespace duckdb { - -namespace { - -//===--------------------------------------------------------------------===// -// sitting_duck bootstrap (INSTALL FROM community + LOAD; idempotent) -//===--------------------------------------------------------------------===// - -void EnsureSittingDuck(Connection &con) { - // LOAD is cheap and idempotent when already present. - auto load = con.Query("LOAD sitting_duck"); - if (!load->HasError()) { - return; - } - auto inst = con.Query("INSTALL sitting_duck FROM community"); - if (inst->HasError()) { - throw InvalidInputException("quack_from_x: could not INSTALL sitting_duck FROM community: %s\n" - "sitting_duck is a runtime dependency of quack_from_* extractors.", - inst->GetError().c_str()); - } - load = con.Query("LOAD sitting_duck"); - if (load->HasError()) { - throw InvalidInputException("quack_from_x: could not LOAD sitting_duck: %s", load->GetError().c_str()); - } -} - -//===--------------------------------------------------------------------===// -// Path expansion: bare directory → framework-specific globs for __REPO__ -//===--------------------------------------------------------------------===// - -bool LooksLikeGlob(const string &path) { - return path.find('*') != string::npos || path.find('?') != string::npos; -} - -string RtrimSlashes(string p) { - while (p.size() > 1 && (p.back() == '/' || p.back() == '\\')) { - p.pop_back(); - } - return p; -} - -//! Escape a path for embedding inside a single-quoted SQL string literal. -string EscapeSqlString(const string &s) { - string out; - out.reserve(s.size() + 8); - for (char c : s) { - if (c == '\'') { - out += "''"; - } else { - out += c; - } - } - return out; -} - -//! Expand a user path argument into the value substituted for __REPO__. -//! framework/kind control default globs when the user passes a bare directory. -string ExpandRepoPath(const string &raw_path, const string &framework, const string &kind) { - if (raw_path.empty()) { - throw InvalidInputException("quack_from_x: path must be non-empty"); - } - string path = RtrimSlashes(raw_path); - - // Caller already supplied a glob or a specific source file — use as-is. - if (LooksLikeGlob(path)) { - return path; - } - // Specific source file extensions: use as-is. - auto lower = StringUtil::Lower(path); - if (StringUtil::EndsWith(lower, ".py") || StringUtil::EndsWith(lower, ".rb") || - StringUtil::EndsWith(lower, ".ts") || StringUtil::EndsWith(lower, ".js") || - StringUtil::EndsWith(lower, ".go")) { - return path; - } - - // Bare directory defaults. - if (framework == "fastapi") { - return path + "/**/*.py"; - } - if (framework == "rails") { - if (kind == "routes") { - return path + "/config/routes.rb"; - } - // models SQL embeds '__REPO__/app/models/**/*.rb' itself — pass bare root. - return path; - } - if (framework == "express") { - // express SQL embeds '__REPO__/**/*.{ts,js}' itself — pass bare root. - return path; - } - if (framework == "gin") { - return path + "/**/*.go"; - } - return path; -} - -string SubstituteRepo(const string &sql_template, const string &repo_path) { - // SQL files use the literal token __REPO__ inside string literals. - return StringUtil::Replace(sql_template, "__REPO__", EscapeSqlString(repo_path)); -} - -//===--------------------------------------------------------------------===// -// Materialized extraction runner -//===--------------------------------------------------------------------===// - -struct FromXFunctionInfo : public TableFunctionInfo { - string framework; - string kind; // "routes" | "models" - FromXFunctionInfo(string framework_p, string kind_p) : framework(std::move(framework_p)), kind(std::move(kind_p)) { - } -}; - -struct FromXBindData : public TableFunctionData { - string framework; - string kind; // "routes" | "models" - string path; - bool is_routes = true; -}; - -struct FromXGlobalState : public GlobalTableFunctionState { - vector> rows; - idx_t offset = 0; - - idx_t MaxThreads() const override { - return 1; - } -}; - -void SetRouteReturnTypes(vector &return_types, vector &names) { - return_types = {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::VARCHAR, LogicalType::UINTEGER, LogicalType::VARCHAR}; - names = {"method", "path", "handler_name", "file", "start_line", "evidence"}; -} - -void SetModelReturnTypes(vector &return_types, vector &names) { - return_types = {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::BOOLEAN, LogicalType::BOOLEAN, LogicalType::BOOLEAN, - LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::UINTEGER}; - names = {"model_name", "field_name", "field_type", "is_required", "is_optional", - "has_default", "default_expr", "file", "field_line"}; -} - -unique_ptr FromXBind(ClientContext &, TableFunctionBindInput &input, vector &return_types, - vector &names) { - auto &info = input.info->Cast(); - auto data = make_uniq(); - data->framework = info.framework; - data->kind = info.kind; - data->is_routes = (info.kind == "routes"); - data->path = input.inputs[0].GetValue(); - if (data->is_routes) { - SetRouteReturnTypes(return_types, names); - } else { - SetModelReturnTypes(return_types, names); - } - return std::move(data); -} - -Value CoerceColumn(const Value &v, const LogicalType &want) { - if (v.IsNull()) { - return Value(want); - } - if (v.type() == want) { - return v; - } - // Best-effort: cast via string/int paths we control. - try { - if (want.id() == LogicalTypeId::VARCHAR) { - return Value(v.ToString()); - } - if (want.id() == LogicalTypeId::BOOLEAN) { - if (v.type().id() == LogicalTypeId::BOOLEAN) { - return v; - } - return Value::BOOLEAN(v.GetValue()); - } - if (want.id() == LogicalTypeId::UINTEGER) { - if (v.type().IsIntegral()) { - return Value::UINTEGER(static_cast(v.GetValue())); - } - return Value::UINTEGER(static_cast(std::stoul(v.ToString()))); - } - } catch (...) { - return Value(want); - } - return Value(want); -} - -unique_ptr FromXInit(ClientContext &context, TableFunctionInitInput &input) { - auto &bind = input.bind_data->Cast(); - auto state = make_uniq(); - - const char *sql_tmpl = quackapi_from_x_sql::Lookup(bind.framework.c_str(), bind.kind.c_str()); - if (!sql_tmpl) { - throw InternalException("quack_from_x: missing embedded SQL for %s/%s", bind.framework, bind.kind); - } - - string repo = ExpandRepoPath(bind.path, bind.framework, bind.kind); - string sql = SubstituteRepo(sql_tmpl, repo); - - Connection con(*context.db); - EnsureSittingDuck(con); - - auto result = con.Query(sql); - if (result->HasError()) { - throw InvalidInputException("quack_from_%s%s(%s): extraction failed: %s", bind.framework, - bind.is_routes ? "" : "_models", bind.path, result->GetError().c_str()); - } - - // Expected column counts for coercion. - const idx_t ncol = bind.is_routes ? 6 : 9; - vector want_types; - vector want_names; - if (bind.is_routes) { - SetRouteReturnTypes(want_types, want_names); - } else { - SetModelReturnTypes(want_types, want_names); - } - - while (true) { - auto chunk = result->Fetch(); - if (!chunk || chunk->size() == 0) { - break; - } - for (idx_t r = 0; r < chunk->size(); r++) { - vector row; - row.reserve(ncol); - for (idx_t c = 0; c < ncol; c++) { - if (c < chunk->ColumnCount()) { - row.push_back(CoerceColumn(chunk->GetValue(c, r), want_types[c])); - } else { - row.push_back(Value(want_types[c])); - } - } - state->rows.push_back(std::move(row)); - } - } - return std::move(state); -} - -void FromXExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - if (state.offset >= state.rows.size()) { - return; - } - idx_t count = 0; - const idx_t ncol = output.ColumnCount(); - while (state.offset < state.rows.size() && count < STANDARD_VECTOR_SIZE) { - const auto &row = state.rows[state.offset]; - for (idx_t c = 0; c < ncol && c < row.size(); c++) { - output.SetValue(c, count, row[c]); - } - state.offset++; - count++; - } - output.SetCardinality(count); -} - -TableFunction MakeFromXFunction(const string &name, const string &framework, const string &kind) { - TableFunction tf(name, {LogicalType::VARCHAR}, FromXExec, FromXBind, FromXInit); - tf.function_info = make_shared_ptr(framework, kind); - return tf; -} - -//===--------------------------------------------------------------------===// -// quack_from_x_sql(framework, kind) — expose embedded SQL for drift tests -//===--------------------------------------------------------------------===// - -void FromXSqlScalar(DataChunk &args, ExpressionState &, Vector &result) { - BinaryExecutor::Execute( - args.data[0], args.data[1], result, args.size(), [&](string_t fw, string_t kind) { - auto *sql = quackapi_from_x_sql::Lookup(fw.GetString().c_str(), kind.GetString().c_str()); - if (!sql) { - throw InvalidInputException("quack_from_x_sql: unknown framework/kind '%s'/'%s' " - "(frameworks: fastapi|rails|express|gin; kind: routes|models)", - fw.GetString(), kind.GetString()); - } - return StringVector::AddString(result, sql); - }); -} - -//===--------------------------------------------------------------------===// -// quack_from_x_sql_relpath(framework, kind) — disk path for drift tests -//===--------------------------------------------------------------------===// - -void FromXSqlRelpathScalar(DataChunk &args, ExpressionState &, Vector &result) { - BinaryExecutor::Execute( - args.data[0], args.data[1], result, args.size(), [&](string_t fw, string_t kind) { - for (const auto &e : quackapi_from_x_sql::All()) { - if (std::string(e.framework) == fw.GetString() && std::string(e.kind) == kind.GetString()) { - return StringVector::AddString(result, e.relpath); - } - } - throw InvalidInputException("quack_from_x_sql_relpath: unknown '%s'/'%s'", fw.GetString(), - kind.GetString()); - }); -} - -} // namespace - -void RegisterQuackapiFromXFunctions(ExtensionLoader &loader) { - // Route extractors - loader.RegisterFunction(MakeFromXFunction("quack_from_fastapi", "fastapi", "routes")); - loader.RegisterFunction(MakeFromXFunction("quack_from_rails", "rails", "routes")); - loader.RegisterFunction(MakeFromXFunction("quack_from_express", "express", "routes")); - loader.RegisterFunction(MakeFromXFunction("quack_from_gin", "gin", "routes")); - - // Model extractors - loader.RegisterFunction(MakeFromXFunction("quack_from_fastapi_models", "fastapi", "models")); - loader.RegisterFunction(MakeFromXFunction("quack_from_rails_models", "rails", "models")); - loader.RegisterFunction(MakeFromXFunction("quack_from_express_models", "express", "models")); - loader.RegisterFunction(MakeFromXFunction("quack_from_gin_models", "gin", "models")); - - // Embed drift helpers - loader.RegisterFunction(ScalarFunction("quack_from_x_sql", {LogicalType::VARCHAR, LogicalType::VARCHAR}, - LogicalType::VARCHAR, FromXSqlScalar)); - loader.RegisterFunction(ScalarFunction("quack_from_x_sql_relpath", {LogicalType::VARCHAR, LogicalType::VARCHAR}, - LogicalType::VARCHAR, FromXSqlRelpathScalar)); -} - -} // namespace duckdb diff --git a/src/quackapi_graphql.cpp b/src/quackapi_graphql.cpp deleted file mode 100644 index c4a13ce..0000000 --- a/src/quackapi_graphql.cpp +++ /dev/null @@ -1,1308 +0,0 @@ -#include "quackapi_graphql.hpp" - -#include - -#include "duckdb/common/exception.hpp" -#include "duckdb/common/string_util.hpp" -#include "duckdb/common/types/value.hpp" -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/main/connection.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/main/prepared_statement.hpp" -#include "duckdb/main/query_result.hpp" -#include "duckdb/parser/parser_extension.hpp" - -#include "quackapi_limits.hpp" -#include "quackapi_policy.hpp" -#include "quackapi_state.hpp" -#include "quackapi_util.hpp" - -namespace duckdb { - -namespace { - -string GraphqlError(const string &message) { - return "{\"errors\":[{\"message\":\"" + QuackapiJsonEscape(message) + "\"}]}"; -} - -string QuoteIdent(const string &ident) { - string out = "\""; - for (char c : ident) { - if (c == '"') { - out += "\"\""; - } else { - out += c; - } - } - out += "\""; - return out; -} - -//! GraphQL Name: [_A-Za-z][_0-9A-Za-z]* — character scan, no regex. -bool IsGraphqlNameStart(char c) { - return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; -} -bool IsGraphqlNameCont(char c) { - return IsGraphqlNameStart(c) || (c >= '0' && c <= '9'); -} - -void SkipWs(const string &s, idx_t &i) { - while (i < s.size()) { - char c = s[i]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == ',') { - // GraphQL allows commas as insignificant separators. - i++; - continue; - } - // Line comments: # … EOL - if (c == '#') { - while (i < s.size() && s[i] != '\n') { - i++; - } - continue; - } - break; - } -} - -bool MatchChar(const string &s, idx_t &i, char expected) { - SkipWs(s, i); - if (i < s.size() && s[i] == expected) { - i++; - return true; - } - return false; -} - -bool ReadName(const string &s, idx_t &i, string &out) { - SkipWs(s, i); - if (i >= s.size() || !IsGraphqlNameStart(s[i])) { - return false; - } - idx_t start = i; - i++; - while (i < s.size() && IsGraphqlNameCont(s[i])) { - i++; - } - out = s.substr(start, i - start); - return true; -} - -struct GraphqlField { - string name; - vector columns; // empty = invalid for v0 (need selection set) -}; - -struct GraphqlDocument { - vector roots; - string error; -}; - -bool ContainsGraphqlName(const vector &names, const string &name) { - for (auto &existing : names) { - if (existing == name) { - return true; - } - } - return false; -} - -//! Parse only: [query [Name]] { Field { Name+ }+ } -//! Field must have a nested selection set of leaf column names (no deeper nest). -GraphqlDocument ParseThinGraphql(const string &doc) { - GraphqlDocument out; - if (doc.size() > QUACKAPI_GRAPHQL_MAX_DOCUMENT_BYTES) { - out.error = "GraphQL document exceeds the 64 KiB limit"; - return out; - } - idx_t i = 0; - SkipWs(doc, i); - if (i >= doc.size()) { - out.error = "empty GraphQL document"; - return out; - } - - // Optional operation: query [Name] - string first; - idx_t save = i; - if (ReadName(doc, i, first)) { - if (first == "query") { - // Optional operation name — skip one Name if present. - string op_name; - idx_t before = i; - if (ReadName(doc, i, op_name)) { - // If next significant char is '{', op_name was the operation name. - // If we mis-consumed a field name (no op name, bare `{ field {…} }` - // after a mistaken "query" keyword path), we only enter this branch - // when the keyword was literally "query". - SkipWs(doc, i); - if (i < doc.size() && doc[i] == '{') { - // op_name was optional name — fine, leave i at '{' - } else { - // Not a selection set after the name — treat as error. - out.error = "expected selection set '{' after query"; - return out; - } - } else { - i = before; - } - } else if (first == "mutation" || first == "subscription") { - out.error = "mutations and subscriptions are not supported in GraphQL v0"; - return out; - } else { - // Leading name without 'query' keyword — not valid unless we rewind - // and expect '{'. Rewind: document must start with '{'. - i = save; - } - } - - if (!MatchChar(doc, i, '{')) { - out.error = "expected selection set '{'"; - return out; - } - - while (true) { - SkipWs(doc, i); - if (i < doc.size() && doc[i] == '}') { - i++; - break; - } - string field_name; - if (!ReadName(doc, i, field_name)) { - out.error = "expected field name in root selection set"; - return out; - } - if (field_name == "__schema" || field_name == "__type") { - out.error = "introspection fields are not supported in v0; use GET /graphql/schema"; - return out; - } - if (!MatchChar(doc, i, '{')) { - out.error = "v0 requires a column selection set: { " + field_name + " { col1 col2 } }"; - return out; - } - GraphqlField field; - field.name = field_name; - while (true) { - SkipWs(doc, i); - if (i < doc.size() && doc[i] == '}') { - i++; - break; - } - string col; - if (!ReadName(doc, i, col)) { - out.error = "expected column name inside " + field_name; - return out; - } - // No nested selection under columns in v0. - SkipWs(doc, i); - if (i < doc.size() && doc[i] == '{') { - out.error = "nested selections are not supported in GraphQL v0"; - return out; - } - if (i < doc.size() && doc[i] == '(') { - out.error = "field arguments are not supported in GraphQL v0"; - return out; - } - if (ContainsGraphqlName(field.columns, col)) { - out.error = "duplicate column '" + col + "' in " + field_name; - return out; - } - field.columns.push_back(col); - if (field.columns.size() > QUACKAPI_GRAPHQL_MAX_COLUMNS_PER_FIELD) { - out.error = "selection for " + field_name + " exceeds the column limit"; - return out; - } - } - if (field.columns.empty()) { - out.error = "selection set for " + field_name + " must list at least one column"; - return out; - } - for (auto &existing : out.roots) { - if (existing.name == field.name) { - out.error = "duplicate root field '" + field.name + "' is not supported in GraphQL v0"; - return out; - } - } - out.roots.push_back(std::move(field)); - if (out.roots.size() > QUACKAPI_GRAPHQL_MAX_ROOT_FIELDS) { - out.error = "GraphQL document exceeds the root-field limit"; - return out; - } - idx_t total_fields = 0; - for (auto &root : out.roots) { - total_fields += root.columns.size(); - } - if (total_fields > QUACKAPI_GRAPHQL_MAX_TOTAL_FIELDS) { - out.error = "GraphQL document exceeds the total-field limit"; - return out; - } - } - - SkipWs(doc, i); - if (i < doc.size()) { - out.error = "unexpected trailing content after selection set"; - return out; - } - if (out.roots.empty()) { - out.error = "selection set is empty"; - return out; - } - return out; -} - -struct GraphqlCatalogObject { - string database_name; - string schema_name; - string object_name; - vector columns; -}; - -//! Built-in GraphQL is deliberately scoped to the current database's main -//! schema. The catalog identity is kept with every object and reused in the -//! generated SELECT, so an attached catalog or a same-named temporary object -//! cannot be selected after validation. -bool LoadGraphqlCatalog(Connection &con, unordered_map &objects, string &err) { - objects.clear(); - auto res = con.Query( - "WITH objects AS (" - " SELECT database_name, schema_name, table_name AS object_name FROM duckdb_tables() " - " WHERE database_name = current_database() AND schema_name = 'main' AND NOT internal AND NOT temporary " - " UNION ALL " - " SELECT database_name, schema_name, view_name AS object_name FROM duckdb_views() " - " WHERE database_name = current_database() AND schema_name = 'main' AND NOT internal AND NOT temporary" - "), cols AS (" - " SELECT database_name, schema_name, table_name, column_name, column_index FROM duckdb_columns() " - " WHERE database_name = current_database() AND schema_name = 'main' AND NOT internal" - ") " - "SELECT o.database_name, o.schema_name, o.object_name, " - " coalesce(list(c.column_name ORDER BY c.column_index), []) AS columns " - "FROM objects o LEFT JOIN cols c " - " ON c.database_name = o.database_name AND c.schema_name = o.schema_name AND c.table_name = o.object_name " - "GROUP BY ALL ORDER BY o.object_name"); - if (res->HasError()) { - err = res->GetError(); - return false; - } - while (true) { - auto chunk = res->Fetch(); - if (!chunk || chunk->size() == 0) { - break; - } - for (idx_t row = 0; row < chunk->size(); row++) { - GraphqlCatalogObject object; - object.database_name = chunk->GetValue(0, row).ToString(); - object.schema_name = chunk->GetValue(1, row).ToString(); - object.object_name = chunk->GetValue(2, row).ToString(); - auto cols = chunk->GetValue(3, row); - if (!cols.IsNull() && cols.type().id() == LogicalTypeId::LIST) { - for (auto &column : ListValue::GetChildren(cols)) { - if (!column.IsNull()) { - object.columns.push_back(column.ToString()); - } - } - } - objects.emplace(object.object_name, std::move(object)); - } - } - return true; -} - -bool TableExists(Connection &con, const string &table) { - unordered_map objects; - string err; - return LoadGraphqlCatalog(con, objects, err) && objects.find(table) != objects.end(); -} - -bool ColumnExists(Connection &con, const string &table, const string &column) { - unordered_map objects; - string err; - if (!LoadGraphqlCatalog(con, objects, err)) { - return false; - } - auto object = objects.find(table); - return object != objects.end() && ContainsGraphqlName(object->second.columns, column); -} - -string QuoteGraphqlObject(const GraphqlCatalogObject &object) { - return QuoteIdent(object.database_name) + "." + QuoteIdent(object.schema_name) + "." + - QuoteIdent(object.object_name); -} - -bool GraphqlLegacyOpenEnabled(Connection &con) { - auto result = con.Query("SELECT current_setting('quackapi_graphql_allow_all')"); - if (result->HasError()) { - return false; - } - auto chunk = result->Fetch(); - if (!chunk || chunk->size() == 0 || chunk->GetValue(0, 0).IsNull()) { - return false; - } - return StringUtil::Lower(chunk->GetValue(0, 0).ToString()) == "true"; -} - -//! Execute a prevalidated source SELECT as a JSON array. Claims are only bound -//! to named parameters generated by the policy rewriter; GraphQL input never -//! becomes SQL text other than quoted catalog identifiers from the snapshot. -string SelectTableJson(Connection &con, const string &source_sql, const vector &columns, - const GraphqlExecOptions &options, string &err) { - string select_list; - for (idx_t c = 0; c < columns.size(); c++) { - if (c > 0) { - select_list += ", "; - } - select_list += QuoteIdent(columns[c]); - } - // Build json_object('col', "col", …) so types follow DuckDB → JSON. - string json_obj = "json_object("; - for (idx_t c = 0; c < columns.size(); c++) { - if (c > 0) { - json_obj += ", "; - } - json_obj += "'" + StringUtil::Replace(columns[c], "'", "''") + "', " + QuoteIdent(columns[c]); - } - json_obj += ")"; - - string sql = - "SELECT coalesce(json_group_array(" + json_obj + "), '[]'::JSON)::VARCHAR FROM (" + source_sql + ") _gql"; - QuackapiQueryDeadline deadline(con, options.query_timeout_ms); - auto prepared = con.Prepare(sql); - if (prepared->HasError()) { - err = prepared->GetError(); - return {}; - } - case_insensitive_map_t values; - for (auto &entry : prepared->named_param_map) { - auto param = entry.first; - if (!StringUtil::StartsWith(StringUtil::Lower(param), "claims_")) { - err = "unexpected parameter in GraphQL query"; - return {}; - } - string claim_name = param.substr(7); - if (!options.claims) { - values[param] = BoundParameterData(Value()); - continue; - } - auto claim = options.claims->find(claim_name); - values[param] = - claim == options.claims->end() ? BoundParameterData(Value()) : BoundParameterData(Value(claim->second)); - } - auto res = prepared->Execute(values); - if (res->HasError()) { - err = deadline.Expired() ? "query deadline exceeded" : res->GetError(); - return {}; - } - auto chunk = res->Fetch(); - if (!chunk || chunk->size() == 0 || chunk->GetValue(0, 0).IsNull()) { - return "[]"; - } - auto json = chunk->GetValue(0, 0).ToString(); - if (json.size() > options.max_response_bytes) { - err = "GraphQL response exceeds configured byte limit"; - return {}; - } - return json; -} - -//===--------------------------------------------------------------------===// -// CREATE / DROP GRAPHQL FOR TABLE (allowlist for built-in /graphql) -//===--------------------------------------------------------------------===// - -//! Parse a SQL identifier: bare token OR double-quoted with "" escapes. -bool ParseIdent(string &rest, string &out) { - rest = QuackapiTrim(rest); - if (rest.empty()) { - return false; - } - if (rest[0] == '"') { - string result; - idx_t i = 1; - while (i < rest.size()) { - if (rest[i] == '"') { - if (i + 1 < rest.size() && rest[i + 1] == '"') { - result += '"'; - i += 2; - continue; - } - out = result; - rest = QuackapiTrim(rest.substr(i + 1)); - return !out.empty(); - } - result += rest[i]; - i++; - } - return false; // unterminated - } - // Bare identifier: up to whitespace or comma. - idx_t i = 0; - while (i < rest.size() && !StringUtil::CharacterIsSpace(rest[i]) && rest[i] != ',') { - if (rest[i] == '"') { - return false; - } - i++; - } - if (i == 0) { - return false; - } - out = rest.substr(0, i); - rest = QuackapiTrim(rest.substr(i)); - return true; -} - -bool ParseTableList(string &rest, vector &tables, string &err) { - tables.clear(); - while (true) { - string name; - if (!ParseIdent(rest, name)) { - err = "invalid table name — use a bare identifier or \"quoted\"\"ident\""; - return false; - } - if (name.find('\'') != string::npos) { - err = "invalid table name"; - return false; - } - tables.push_back(name); - rest = QuackapiTrim(rest); - if (rest.empty()) { - return true; - } - if (rest[0] == ',') { - rest = QuackapiTrim(rest.substr(1)); - if (rest.empty()) { - err = "trailing comma in table list"; - return false; - } - continue; - } - err = "unexpected trailing content after table list"; - return false; - } -} - -struct GraphqlDdlParseData : public ParserExtensionParseData { - //! "CREATE" | "DROP" | "CLEAR" | "CREATE_ROUTE" | "DROP_ROUTE" - string action; - bool or_replace = false; - vector tables; - //! CREATE/DROP GRAPHQL ROUTE fields - string route_name; - string method = "POST"; - string path; - string require_auth; - idx_t limit = QUACKAPI_GRAPHQL_DEFAULT_LIMIT; - bool has_limit = false; - - unique_ptr Copy() const override { - auto copy = make_uniq(); - copy->action = action; - copy->or_replace = or_replace; - copy->tables = tables; - copy->route_name = route_name; - copy->method = method; - copy->path = path; - copy->require_auth = require_auth; - copy->limit = limit; - copy->has_limit = has_limit; - return std::move(copy); - } - string ToString() const override { - if (action == "CLEAR") { - return "DROP GRAPHQL ALL"; - } - if (action == "DROP_ROUTE") { - return "DROP GRAPHQL ROUTE " + route_name; - } - if (action == "CREATE_ROUTE") { - string s = string(or_replace ? "CREATE OR REPLACE" : "CREATE") + " GRAPHQL ROUTE " + route_name + " " + - method + " '" + path + "' FROM"; - for (idx_t i = 0; i < tables.size(); i++) { - s += (i == 0 ? " " : ", ") + tables[i]; - } - if (!require_auth.empty()) { - s += " REQUIRE " + require_auth; - } - if (has_limit) { - s += " LIMIT " + std::to_string(limit); - } - return s; - } - string s = action + " GRAPHQL FOR TABLE"; - for (idx_t i = 0; i < tables.size(); i++) { - s += (i == 0 ? " " : ", ") + tables[i]; - } - return s; - } -}; - -//! Parse table list that may be followed by REQUIRE / LIMIT keywords. -bool ParseTableListUntilKeywords(string &rest, vector &tables, string &err) { - tables.clear(); - while (true) { - rest = QuackapiTrim(rest); - if (rest.empty()) { - if (tables.empty()) { - err = "expected at least one table name"; - return false; - } - return true; - } - auto upper = StringUtil::Upper(rest); - // Stop before optional clauses. - if ((StringUtil::StartsWith(upper, "REQUIRE") && (rest.size() == 7 || StringUtil::CharacterIsSpace(rest[7]))) || - (StringUtil::StartsWith(upper, "LIMIT") && (rest.size() == 5 || StringUtil::CharacterIsSpace(rest[5])))) { - if (tables.empty()) { - err = "expected at least one table name before " + rest.substr(0, 6); - return false; - } - return true; - } - string name; - if (!ParseIdent(rest, name)) { - err = "invalid table name — use a bare identifier or \"quoted\"\"ident\""; - return false; - } - if (name.find('\'') != string::npos) { - err = "invalid table name"; - return false; - } - tables.push_back(name); - rest = QuackapiTrim(rest); - if (rest.empty()) { - return true; - } - if (rest[0] == ',') { - rest = QuackapiTrim(rest.substr(1)); - if (rest.empty()) { - err = "trailing comma in table list"; - return false; - } - continue; - } - // Next token may be REQUIRE/LIMIT — loop will stop. - upper = StringUtil::Upper(rest); - if ((StringUtil::StartsWith(upper, "REQUIRE") && (rest.size() == 7 || StringUtil::CharacterIsSpace(rest[7]))) || - (StringUtil::StartsWith(upper, "LIMIT") && (rest.size() == 5 || StringUtil::CharacterIsSpace(rest[5])))) { - return true; - } - err = "unexpected trailing content after table list"; - return false; - } -} - -//! Grammar: -//! CREATE [OR REPLACE] GRAPHQL FOR TABLE
[,
...] -//! DROP GRAPHQL FOR TABLE
[,
...] -//! DROP GRAPHQL ALL -//! CREATE [OR REPLACE] GRAPHQL ROUTE POST '' FROM -//! [REQUIRE ] [LIMIT ] -//! DROP GRAPHQL ROUTE -ParserExtensionParseResult GraphqlDdlParse(ParserExtensionInfo *, const string &query) { - auto q = QuackapiTrim(query); - auto upper = StringUtil::Upper(q); - - // DROP GRAPHQL ALL — token boundary so "DROP GRAPHQL ALLOWED" is not ours. - if (StringUtil::StartsWith(upper, "DROP GRAPHQL ALL") && - (upper.size() == 16 || (!StringUtil::CharacterIsAlphaNumeric(upper[16]) && upper[16] != '_'))) { - auto tail = QuackapiTrim(q.substr(16)); - if (!tail.empty()) { - return ParserExtensionParseResult("DROP GRAPHQL ALL takes no arguments"); - } - auto data = make_uniq(); - data->action = "CLEAR"; - return ParserExtensionParseResult(std::move(data)); - } - - // DROP GRAPHQL ROUTE - if (StringUtil::StartsWith(upper, "DROP GRAPHQL ROUTE ")) { - auto name = QuackapiTrim(q.substr(19)); - if (name.empty() || name.find(' ') != string::npos) { - return ParserExtensionParseResult("DROP GRAPHQL ROUTE expects a single route name"); - } - auto data = make_uniq(); - data->action = "DROP_ROUTE"; - data->route_name = name; - return ParserExtensionParseResult(std::move(data)); - } - - // CREATE [OR REPLACE] GRAPHQL ROUTE … - bool route_or_replace = false; - idx_t route_pos = 0; - if (StringUtil::StartsWith(upper, "CREATE GRAPHQL ROUTE ")) { - route_pos = 21; - } else if (StringUtil::StartsWith(upper, "CREATE OR REPLACE GRAPHQL ROUTE ")) { - route_pos = 32; - route_or_replace = true; - } - if (route_pos > 0) { - auto rest = QuackapiTrim(q.substr(route_pos)); - // - auto sp = rest.find(' '); - if (sp == string::npos) { - return ParserExtensionParseResult("CREATE GRAPHQL ROUTE POST '' FROM
[, …]"); - } - auto name = rest.substr(0, sp); - rest = QuackapiTrim(rest.substr(sp)); - // POST - sp = rest.find(' '); - if (sp == string::npos) { - return ParserExtensionParseResult("Expected POST '' after GRAPHQL ROUTE name"); - } - auto method = StringUtil::Upper(rest.substr(0, sp)); - if (method != "POST") { - return ParserExtensionParseResult("CREATE GRAPHQL ROUTE only supports POST in v0 (got " + method + ")"); - } - rest = QuackapiTrim(rest.substr(sp)); - // '' - if (rest.empty() || rest[0] != '\'') { - return ParserExtensionParseResult("Expected quoted '' after POST"); - } - auto path_end = rest.find('\'', 1); - if (path_end == string::npos) { - return ParserExtensionParseResult("Unterminated GraphQL route path"); - } - auto path = rest.substr(1, path_end - 1); - if (path.empty() || path[0] != '/') { - return ParserExtensionParseResult("GraphQL route path must be absolute (start with '/')"); - } - rest = QuackapiTrim(rest.substr(path_end + 1)); - // FROM - auto rest_u = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(rest_u, "FROM") || !(rest.size() == 4 || StringUtil::CharacterIsSpace(rest[4]))) { - return ParserExtensionParseResult("Expected FROM
[, …] after path"); - } - rest = QuackapiTrim(rest.substr(4)); - vector tables; - string err; - if (!ParseTableListUntilKeywords(rest, tables, err)) { - return ParserExtensionParseResult("CREATE GRAPHQL ROUTE: " + err); - } - string require_auth; - idx_t limit = QUACKAPI_GRAPHQL_DEFAULT_LIMIT; - bool has_limit = false; - for (int round = 0; round < 4; round++) { - rest = QuackapiTrim(rest); - if (rest.empty()) { - break; - } - rest_u = StringUtil::Upper(rest); - if (StringUtil::StartsWith(rest_u, "REQUIRE") && - (rest.size() == 7 || StringUtil::CharacterIsSpace(rest[7]))) { - rest = QuackapiTrim(rest.substr(7)); - idx_t te = 0; - while (te < rest.size() && !StringUtil::CharacterIsSpace(rest[te])) { - te++; - } - if (te == 0) { - return ParserExtensionParseResult("REQUIRE expects an auth name"); - } - require_auth = rest.substr(0, te); - rest = QuackapiTrim(rest.substr(te)); - continue; - } - if (StringUtil::StartsWith(rest_u, "LIMIT") && - (rest.size() == 5 || StringUtil::CharacterIsSpace(rest[5]))) { - rest = QuackapiTrim(rest.substr(5)); - idx_t te = 0; - while (te < rest.size() && !StringUtil::CharacterIsSpace(rest[te])) { - te++; - } - if (te == 0) { - return ParserExtensionParseResult("LIMIT expects a positive integer"); - } - auto lim_s = rest.substr(0, te); - for (char c : lim_s) { - if (c < '0' || c > '9') { - return ParserExtensionParseResult("LIMIT expects a positive integer"); - } - } - try { - limit = (idx_t)std::stoull(lim_s); - } catch (...) { - return ParserExtensionParseResult("LIMIT expects a positive integer"); - } - if (limit == 0 || limit > 100000) { - return ParserExtensionParseResult("LIMIT must be between 1 and 100000"); - } - has_limit = true; - rest = QuackapiTrim(rest.substr(te)); - continue; - } - return ParserExtensionParseResult("Unexpected trailing content after GRAPHQL ROUTE: " + rest); - } - auto data = make_uniq(); - data->action = "CREATE_ROUTE"; - data->or_replace = route_or_replace; - data->route_name = name; - data->method = method; - data->path = path; - data->tables = std::move(tables); - data->require_auth = require_auth; - data->limit = limit; - data->has_limit = has_limit; - return ParserExtensionParseResult(std::move(data)); - } - - bool or_replace = false; - bool is_drop = false; - idx_t pos = 0; - if (StringUtil::StartsWith(upper, "CREATE GRAPHQL FOR TABLE ")) { - pos = 25; - } else if (StringUtil::StartsWith(upper, "CREATE OR REPLACE GRAPHQL FOR TABLE ")) { - pos = 36; - or_replace = true; - } else if (StringUtil::StartsWith(upper, "DROP GRAPHQL FOR TABLE ")) { - pos = 23; - is_drop = true; - } else { - // Not ours — let the core parser (or another extension) handle it. - return ParserExtensionParseResult(); - } - - auto rest = QuackapiTrim(q.substr(pos)); - if (rest.empty()) { - return ParserExtensionParseResult(is_drop ? "DROP GRAPHQL FOR TABLE expects at least one table name" - : "CREATE GRAPHQL FOR TABLE expects at least one table name"); - } - - vector tables; - string err; - if (!ParseTableList(rest, tables, err)) { - return ParserExtensionParseResult(string(is_drop ? "DROP" : "CREATE") + " GRAPHQL FOR TABLE: " + err); - } - - auto data = make_uniq(); - data->action = is_drop ? "DROP" : "CREATE"; - data->or_replace = or_replace; - data->tables = std::move(tables); - return ParserExtensionParseResult(std::move(data)); -} - -struct ApplyGraphqlBindData : public TableFunctionData { - string action; - bool or_replace = false; - vector tables; - string route_name; - string method = "POST"; - string path; - string require_auth; - idx_t limit = QUACKAPI_GRAPHQL_DEFAULT_LIMIT; - bool finished = false; -}; - -unique_ptr ApplyGraphqlBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - bind_data->action = input.inputs[0].GetValue(); - bind_data->or_replace = input.inputs[1].GetValue(); - // tables as VARCHAR[] (LIST) - if (!input.inputs[2].IsNull() && input.inputs[2].type().id() == LogicalTypeId::LIST) { - for (auto &child : ListValue::GetChildren(input.inputs[2])) { - if (!child.IsNull()) { - bind_data->tables.push_back(child.ToString()); - } - } - } - if (input.inputs.size() > 3 && !input.inputs[3].IsNull()) { - bind_data->route_name = input.inputs[3].GetValue(); - } - if (input.inputs.size() > 4 && !input.inputs[4].IsNull()) { - bind_data->method = input.inputs[4].GetValue(); - } - if (input.inputs.size() > 5 && !input.inputs[5].IsNull()) { - bind_data->path = input.inputs[5].GetValue(); - } - if (input.inputs.size() > 6 && !input.inputs[6].IsNull()) { - bind_data->require_auth = input.inputs[6].GetValue(); - } - if (input.inputs.size() > 7 && !input.inputs[7].IsNull()) { - bind_data->limit = (idx_t)input.inputs[7].GetValue(); - } - BindStatusColumn(return_types, names); - return std::move(bind_data); -} - -void ApplyGraphqlExec(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &bind_data = data_p.bind_data->CastNoConst(); - if (bind_data.finished) { - return; - } - auto &state = QuackapiState::Get(*context.db); - Connection con(*context.db); - - if (bind_data.action == "CLEAR") { - state.ClearGraphqlTables(); - EmitOneShotStatus(output, bind_data.finished, "GraphQL allowlist cleared (catalog closed)"); - return; - } - - if (bind_data.action == "CREATE_ROUTE") { - for (auto &table : bind_data.tables) { - if (!TableExists(con, table)) { - throw InvalidInputException("CREATE GRAPHQL ROUTE: table or view \"%s\" not found in schema main", - table); - } - } - if (!bind_data.require_auth.empty()) { - QuackapiAuth auth; - if (!state.GetAuth(bind_data.require_auth, auth)) { - throw InvalidInputException("CREATE GRAPHQL ROUTE: auth scheme \"%s\" does not exist", - bind_data.require_auth); - } - } - QuackapiGraphqlRoute route; - route.name = bind_data.route_name; - route.method = bind_data.method; - route.path = bind_data.path; - route.tables = bind_data.tables; - route.require_auth = bind_data.require_auth; - route.limit = bind_data.limit == 0 ? QUACKAPI_GRAPHQL_DEFAULT_LIMIT : bind_data.limit; - state.AddGraphqlRoute(route, bind_data.or_replace); - string msg = "GraphQL route " + route.name + ": " + route.method + " " + route.path + " FROM "; - for (idx_t i = 0; i < route.tables.size(); i++) { - if (i > 0) { - msg += ", "; - } - msg += route.tables[i]; - } - if (!route.require_auth.empty()) { - msg += " REQUIRE " + route.require_auth; - } - msg += " LIMIT " + std::to_string(route.limit); - EmitOneShotStatus(output, bind_data.finished, msg); - return; - } - - if (bind_data.action == "DROP_ROUTE") { - if (state.DropGraphqlRoute(bind_data.route_name)) { - EmitOneShotStatus(output, bind_data.finished, "Dropped GraphQL route " + bind_data.route_name); - } else { - throw InvalidInputException("GraphQL route \"%s\" does not exist", bind_data.route_name); - } - return; - } - - if (bind_data.action == "CREATE") { - // Validate the full table list before mutating so a mid-list miss/duplicate - // cannot half-apply earlier names (CREATE GRAPHQL ROUTE already does this). - for (auto &table : bind_data.tables) { - if (!TableExists(con, table)) { - throw InvalidInputException("CREATE GRAPHQL FOR TABLE: table or view \"%s\" not found in schema main", - table); - } - } - if (!bind_data.or_replace) { - auto existing = state.SnapshotGraphqlTables(); - for (auto &table : bind_data.tables) { - for (auto &e : existing) { - if (e == table) { - throw InvalidInputException( - "GraphQL table \"%s\" already registered — use CREATE OR REPLACE GRAPHQL FOR TABLE", table); - } - } - } - } - for (auto &table : bind_data.tables) { - state.AddGraphqlTable(table, bind_data.or_replace); - } - string msg = "GraphQL registered: "; - for (idx_t i = 0; i < bind_data.tables.size(); i++) { - if (i > 0) { - msg += ", "; - } - msg += bind_data.tables[i]; - } - msg += " (allowlist mode on POST /graphql)"; - EmitOneShotStatus(output, bind_data.finished, msg); - return; - } - - if (bind_data.action == "DROP") { - vector dropped; - vector missing; - for (auto &table : bind_data.tables) { - if (state.DropGraphqlTable(table)) { - dropped.push_back(table); - } else { - missing.push_back(table); - } - } - if (dropped.empty()) { - throw InvalidInputException("DROP GRAPHQL FOR TABLE: no matching registered tables"); - } - string msg = "GraphQL unregistered: "; - for (idx_t i = 0; i < dropped.size(); i++) { - if (i > 0) { - msg += ", "; - } - msg += dropped[i]; - } - if (state.SnapshotGraphqlTables().empty()) { - msg += " (allowlist empty → catalog closed)"; - } - if (!missing.empty()) { - msg += "; not registered: "; - for (idx_t i = 0; i < missing.size(); i++) { - if (i > 0) { - msg += ", "; - } - msg += missing[i]; - } - } - EmitOneShotStatus(output, bind_data.finished, msg); - return; - } - - throw InvalidInputException("internal: unknown GraphQL DDL action \"%s\"", bind_data.action); -} - -TableFunction MakeApplyGraphqlFunction() { - return MakeApplyDdlFunction("quackapi_apply_graphql", - {LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::LIST(LogicalType::VARCHAR), - LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::BIGINT}, - ApplyGraphqlExec, ApplyGraphqlBind); -} - -ParserExtensionPlanResult GraphqlDdlPlan(ParserExtensionInfo *, ClientContext &, - unique_ptr parse_data) { - auto &data = static_cast(*parse_data); - ParserExtensionPlanResult result; - result.function = MakeApplyGraphqlFunction(); - result.parameters.push_back(Value(data.action)); - result.parameters.push_back(Value::BOOLEAN(data.or_replace)); - vector table_vals; - for (auto &t : data.tables) { - table_vals.emplace_back(t); - } - result.parameters.push_back(Value::LIST(LogicalType::VARCHAR, table_vals)); - result.parameters.push_back(Value(data.route_name)); - result.parameters.push_back(Value(data.method)); - result.parameters.push_back(Value(data.path)); - result.parameters.push_back(Value(data.require_auth)); - result.parameters.push_back(Value::BIGINT((int64_t)data.limit)); - FinishDdlPlan(result); - return result; -} - -//===--------------------------------------------------------------------===// -// quackapi_graphql_tables() — inspect allowlist -//===--------------------------------------------------------------------===// - -struct GraphqlTablesBindData : public TableFunctionData {}; - -struct GraphqlTablesGlobalState : public GlobalTableFunctionState { - vector tables; - bool allowlist_active = false; - bool legacy_open = false; - idx_t offset = 0; - bool emitted_mode = false; -}; - -unique_ptr GraphqlTablesBind(ClientContext &, TableFunctionBindInput &, vector &return_types, - vector &names) { - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("mode"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("table_name"); - return make_uniq(); -} - -unique_ptr GraphqlTablesInit(ClientContext &context, TableFunctionInitInput &) { - auto state = make_uniq(); - auto &api = QuackapiState::Get(*context.db); - state->tables = api.SnapshotGraphqlTables(); - Connection con(*context.db); - state->legacy_open = GraphqlLegacyOpenEnabled(con); - state->allowlist_active = !state->legacy_open && !state->tables.empty(); - return std::move(state); -} - -void GraphqlTablesExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - if (!state.allowlist_active) { - // One informational row: explicit legacy open or secure closed mode. - if (state.emitted_mode) { - return; - } - output.SetValue(0, 0, Value(state.legacy_open ? "open" : "disabled")); - output.SetValue(1, 0, Value()); - output.SetCardinality(1); - state.emitted_mode = true; - return; - } - idx_t row = 0; - while (state.offset < state.tables.size() && row < STANDARD_VECTOR_SIZE) { - output.SetValue(0, row, Value("allowlist")); - output.SetValue(1, row, Value(state.tables[state.offset])); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -} // namespace - -bool GraphqlExtractQuery(DatabaseInstance &db, const string &raw_body, string &query_out, string &error_json) { - if (raw_body.empty()) { - error_json = GraphqlError("request body is empty; expected JSON {\"query\":\"...\"}"); - return false; - } - Connection con(db); - auto res = con.Query("SELECT CASE" - " WHEN TRY_CAST(? AS JSON) IS NULL THEN NULL" - " WHEN json_type(?::JSON) != 'OBJECT' THEN NULL" - " ELSE json_extract_string(?::JSON, '$.query')" - " END", - Value(raw_body), Value(raw_body), Value(raw_body)); - if (res->HasError()) { - error_json = GraphqlError("invalid JSON body"); - return false; - } - auto chunk = res->Fetch(); - if (!chunk || chunk->size() == 0 || chunk->GetValue(0, 0).IsNull()) { - error_json = GraphqlError("JSON body must be an object with a non-null \"query\" string"); - return false; - } - query_out = chunk->GetValue(0, 0).ToString(); - if (query_out.empty()) { - error_json = GraphqlError("\"query\" must be a non-empty string"); - return false; - } - return true; -} - -bool TableInList(const vector &tables, const string &name) { - for (auto &t : tables) { - if (t == name) { - return true; - } - } - return false; -} - -string ExecuteGraphqlQuery(DatabaseInstance &db, const string &query, const GraphqlExecOptions &options) { - auto doc = ParseThinGraphql(query); - if (!doc.error.empty()) { - return GraphqlError(doc.error); - } - idx_t limit = options.limit; - if (limit == 0) { - limit = QUACKAPI_GRAPHQL_DEFAULT_LIMIT; - } - limit = std::min(limit, QUACKAPI_GRAPHQL_MAX_TOTAL_ROWS / doc.roots.size()); - - auto &state = QuackapiState::Get(db); - const bool route_mode = options.allowed_tables != nullptr; - Connection con(db); - const bool legacy_open = !route_mode && GraphqlLegacyOpenEnabled(con); - const auto global_tables = state.SnapshotGraphqlTables(); - const bool global_allowlist = !route_mode && !legacy_open; - unordered_map catalog; - string catalog_error; - QuackapiQueryDeadline catalog_deadline(con, options.query_timeout_ms); - if (!LoadGraphqlCatalog(con, catalog, catalog_error)) { - return GraphqlError("schema catalog query failed: " + catalog_error); - } - if (catalog_deadline.Expired()) { - return GraphqlError("schema catalog query deadline exceeded"); - } - string data = "{"; - bool first = true; - for (auto &field : doc.roots) { - if (route_mode) { - if (!TableInList(*options.allowed_tables, field.name)) { - return GraphqlError("table '" + field.name + - "' is not on this GraphQL route — CREATE GRAPHQL ROUTE … FROM " + field.name); - } - } else if (global_allowlist && !TableInList(global_tables, field.name)) { - return GraphqlError("table '" + field.name + "' is not registered for GraphQL — CREATE GRAPHQL FOR TABLE " + - field.name); - } - auto object = catalog.find(field.name); - if (object == catalog.end()) { - return GraphqlError("unknown table '" + field.name + "' (main schema catalog only)"); - } - for (auto &col : field.columns) { - if (!ContainsGraphqlName(object->second.columns, col)) { - return GraphqlError("unknown column '" + col + "' on table '" + field.name + "'"); - } - } - string source_sql = "SELECT "; - for (idx_t c = 0; c < field.columns.size(); c++) { - if (c > 0) { - source_sql += ", "; - } - source_sql += QuoteIdent(field.columns[c]); - } - source_sql += " FROM " + QuoteGraphqlObject(object->second) + " LIMIT " + std::to_string(limit); - bool deny_unauthenticated = false; - string policy_error; - source_sql = - RewriteHandlerWithPolicies(db, source_sql, options.authenticated, deny_unauthenticated, policy_error); - if (deny_unauthenticated || !policy_error.empty()) { - return GraphqlError("access denied by table policy"); - } - string err; - string rows_json = SelectTableJson(con, source_sql, field.columns, options, err); - if (!err.empty()) { - return GraphqlError("query failed for '" + field.name + "': " + err); - } - if (!first) { - data += ","; - } - first = false; - data += "\"" + QuackapiJsonEscape(field.name) + "\":" + rows_json; - if (data.size() > options.max_response_bytes) { - return GraphqlError("GraphQL response exceeds configured byte limit"); - } - } - data += "}"; - return "{\"data\":" + data + "}"; -} - -string GraphqlSchemaNote() { - return "GraphQL v0 catalog-only schema. Full __schema introspection not implemented. " - "CREATE GRAPHQL FOR TABLE registers an allowlist for POST /graphql; empty = closed. " - "SET GLOBAL quackapi_graphql_allow_all = true restores legacy open catalog mode. " - "CREATE GRAPHQL ROUTE mounts named paths with per-route tables."; -} - -string BuildGraphqlSchema(DatabaseInstance &db, const GraphqlExecOptions &options) { - Connection con(db); - auto &state = QuackapiState::Get(db); - const bool route_mode = options.allowed_tables != nullptr; - const bool legacy_open = !route_mode && GraphqlLegacyOpenEnabled(con); - const bool allowlist = !route_mode && !legacy_open; - vector allowed; - if (route_mode) { - allowed = *options.allowed_tables; - } else if (allowlist) { - allowed = state.SnapshotGraphqlTables(); - } - - unordered_map catalog; - string catalog_error; - QuackapiQueryDeadline catalog_deadline(con, options.query_timeout_ms); - if (!LoadGraphqlCatalog(con, catalog, catalog_error)) { - return GraphqlError("schema catalog query failed: " + catalog_error); - } - if (catalog_deadline.Expired()) { - return GraphqlError("schema catalog query deadline exceeded"); - } - - string mode; - if (route_mode) { - mode = options.mode.empty() ? "route" : options.mode; - } else { - mode = legacy_open ? "open" : (allowed.empty() ? "disabled" : "allowlist"); - } - - string body = "{\"mode\":\"" + mode + "\""; - if (route_mode && !options.route_name.empty()) { - body += ",\"route\":\"" + QuackapiJsonEscape(options.route_name) + "\""; - } - body += ",\"tables\":{"; - bool first_table = true; - vector names; - for (auto &entry : catalog) { - names.push_back(entry.first); - } - std::sort(names.begin(), names.end()); - for (auto &table : names) { - if ((route_mode || allowlist) && !TableInList(allowed, table)) { - continue; - } - // Schema discovery follows the same policy gate as execution. A public - // GraphQL endpoint must not reveal a protected table or view name. - if (!options.authenticated && - HandlerTouchesPoliciedTable(db, "SELECT * FROM " + QuoteGraphqlObject(catalog[table]))) { - continue; - } - if (!first_table) { - body += ","; - } - first_table = false; - body += "\"" + QuackapiJsonEscape(table) + "\":["; - for (idx_t c = 0; c < catalog[table].columns.size(); c++) { - if (c > 0) { - body += ","; - } - body += "\"" + QuackapiJsonEscape(catalog[table].columns[c]) + "\""; - } - body += "]"; - if (body.size() > options.max_response_bytes) { - return GraphqlError("GraphQL response exceeds configured byte limit"); - } - } - body += "},\"note\":\"" + QuackapiJsonEscape(GraphqlSchemaNote()) + "\"}"; - return body; -} - -GraphqlDdlParserExtension::GraphqlDdlParserExtension() { - parse_function = GraphqlDdlParse; - plan_function = GraphqlDdlPlan; -} - -//===--------------------------------------------------------------------===// -// quackapi_graphql_routes() — inspect named mounts -//===--------------------------------------------------------------------===// - -struct GraphqlRoutesBindData : public TableFunctionData {}; - -struct GraphqlRoutesGlobalState : public GlobalTableFunctionState { - vector routes; - idx_t offset = 0; -}; - -unique_ptr GraphqlRoutesBind(ClientContext &, TableFunctionBindInput &, vector &return_types, - vector &names) { - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("name"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("method"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("path"); - return_types.emplace_back(LogicalType::LIST(LogicalType::VARCHAR)); - names.emplace_back("tables"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("require_auth"); - return_types.emplace_back(LogicalType::BIGINT); - names.emplace_back("limit"); - return make_uniq(); -} - -unique_ptr GraphqlRoutesInit(ClientContext &context, TableFunctionInitInput &) { - auto state = make_uniq(); - state->routes = QuackapiState::Get(*context.db).SnapshotGraphqlRoutes(); - return std::move(state); -} - -void GraphqlRoutesExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.routes.size() && row < STANDARD_VECTOR_SIZE) { - auto &r = state.routes[state.offset]; - output.SetValue(0, row, Value(r.name)); - output.SetValue(1, row, Value(r.method)); - output.SetValue(2, row, Value(r.path)); - vector table_vals; - for (auto &t : r.tables) { - table_vals.emplace_back(t); - } - output.SetValue(3, row, Value::LIST(LogicalType::VARCHAR, table_vals)); - output.SetValue(4, row, r.require_auth.empty() ? Value() : Value(r.require_auth)); - output.SetValue(5, row, Value::BIGINT((int64_t)r.limit)); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -void RegisterQuackapiGraphqlFunctions(ExtensionLoader &loader) { - TableFunction tf("quackapi_graphql_tables", {}, GraphqlTablesExec, GraphqlTablesBind, GraphqlTablesInit); - loader.RegisterFunction(tf); - TableFunction routes("quackapi_graphql_routes", {}, GraphqlRoutesExec, GraphqlRoutesBind, GraphqlRoutesInit); - loader.RegisterFunction(routes); -} - -} // namespace duckdb diff --git a/src/quackapi_http_fetch.cpp b/src/quackapi_http_fetch.cpp deleted file mode 100644 index 78628f2..0000000 --- a/src/quackapi_http_fetch.cpp +++ /dev/null @@ -1,954 +0,0 @@ -#include "quackapi_http_fetch.hpp" -#include "quackapi_thread_join.hpp" -#include "quackapi_limits.hpp" - -#include -#include -#include -#include - -#include "duckdb/common/exception.hpp" -#include "duckdb/common/exception/http_exception.hpp" -#include "duckdb/common/string_util.hpp" -#include "duckdb/function/scalar_function.hpp" -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/config.hpp" -#include "duckdb/main/client_context.hpp" - -#include "httplib.hpp" - -namespace duckdb { - -namespace { - -constexpr idx_t MAX_IDLE_PER_HOST = 64; -// Concurrency cap when one chunk carries many independent calls. Kept at or below -// MAX_IDLE_PER_HOST so a fan-out cannot dial more sockets than the pool will keep, -// which would turn every burst into a churn of one-shot connections. -constexpr idx_t MAX_OUTBOUND_FANOUT = 32; -// Read/write: LLM upstreams are slow; the caller cancels. Connect stays short so -// a dead peer (or CI loopback after stop) fails fast instead of parking a worker. -constexpr time_t OUTBOUND_TIMEOUT_SECONDS = 600; -constexpr time_t OUTBOUND_CONNECT_TIMEOUT_SECONDS = 5; - -void InsertExtraHeaders(HTTPHeaders &headers, const unordered_map &extra) { - for (auto &kv : extra) { - headers.Insert(kv.first, kv.second); - } -} - -QuackapiHttpFetchResult FromResponse(unique_ptr response) { - QuackapiHttpFetchResult out; - if (!response) { - out.request_error = "null HTTP response"; - return out; - } - out.status = response->status; - out.body = std::move(response->body); - out.reason = std::move(response->reason); - out.success = response->Success(); - out.request_error = response->GetRequestError(); - out.headers = std::move(response->headers); - return out; -} - -//===--------------------------------------------------------------------===// -// Connection pool -//===--------------------------------------------------------------------===// -// Free-list per proto_host_port. Acquire pops (or dials); Release pushes back. -// A pool rather than one shared client per host on purpose: httplib::Client -// serialises requests on its socket, so a single shared client would trade the -// handshake cost for a concurrency ceiling of 1. N idle clients give both reuse -// and parallelism — the same shape as httpx's pool on the FastAPI side. - -struct PooledPlain { - unique_ptr client; -}; - -struct PooledUtil { - unique_ptr client; -}; - -struct HostPool { - vector plain_idle; - vector util_idle; - idx_t dialed = 0; - idx_t reused = 0; -}; - -struct ConnectionPool { - static ConnectionPool &Get() { - static ConnectionPool instance; - return instance; - } - - std::mutex lock; - unordered_map hosts; -}; - -//! Split "http://host:port/a/b?c" into origin + path. Mirrors HTTPUtil::DecomposeURL. -void SplitURL(const string &url, string &origin, string &path) { - HTTPUtil::DecomposeURL(url, path, origin); - if (path.empty()) { - path = "/"; - } -} - -bool IsPlainHTTP(const string &url) { - return StringUtil::StartsWith(StringUtil::Lower(url), "http://"); -} - -//! Split "http://host:port" (from SplitURL origin) into host + port. -void SplitOriginHostPort(const string &origin, string &host, int &port) { - host.clear(); - port = 80; - string rest = origin; - if (StringUtil::StartsWith(StringUtil::Lower(rest), "http://")) { - rest = rest.substr(7); - } else if (StringUtil::StartsWith(StringUtil::Lower(rest), "https://")) { - rest = rest.substr(8); - port = 443; - } - // Strip any leftover path (should not be present on an origin). - auto slash = rest.find('/'); - if (slash != string::npos) { - rest = rest.substr(0, slash); - } - auto colon = rest.rfind(':'); - if (colon != string::npos && rest.find(']') == string::npos) { - // hostname:port (skip IPv6 bracket form for this test seam) - host = rest.substr(0, colon); - port = std::stoi(rest.substr(colon + 1)); - } else if (colon != string::npos && rest.front() == '[') { - // [ipv6]:port - auto br = rest.rfind(']'); - if (br != string::npos && br + 1 < rest.size() && rest[br + 1] == ':') { - host = rest.substr(1, br - 1); - port = std::stoi(rest.substr(br + 2)); - } else { - host = rest; - } - } else { - host = rest; - } -} - -//! Parse a raw HTTP/1.x response into status + body. Tolerates truncation. -void ParseRawHttpResponse(const string &raw, QuackapiHttpFetchResult &out) { - auto header_end = raw.find("\r\n\r\n"); - string header_block; - if (header_end == string::npos) { - header_block = raw; - out.body.clear(); - } else { - header_block = raw.substr(0, header_end); - out.body = raw.substr(header_end + 4); - } - auto line_end = header_block.find("\r\n"); - string status_line = line_end == string::npos ? header_block : header_block.substr(0, line_end); - // "HTTP/1.1 200 OK" - auto sp1 = status_line.find(' '); - if (sp1 != string::npos) { - auto sp2 = status_line.find(' ', sp1 + 1); - string code = sp2 == string::npos ? status_line.substr(sp1 + 1) : status_line.substr(sp1 + 1, sp2 - sp1 - 1); - try { - int status = std::stoi(code); - out.status = HTTPUtil::ToStatusCode(status); - out.success = status >= 200 && status < 400; - if (sp2 != string::npos) { - out.reason = status_line.substr(sp2 + 1); - } - } catch (...) { - out.request_error = "failed to parse HTTP status line"; - } - } else if (!raw.empty()) { - out.request_error = "malformed HTTP response"; - } - if (line_end != string::npos) { - idx_t pos = line_end + 2; - while (pos < header_block.size()) { - auto next = header_block.find("\r\n", pos); - string line = next == string::npos ? header_block.substr(pos) : header_block.substr(pos, next - pos); - auto colon = line.find(':'); - if (colon != string::npos) { - string key = line.substr(0, colon); - string val = line.substr(colon + 1); - StringUtil::Trim(key); - StringUtil::Trim(val); - out.headers.Insert(key, val); - } - if (next == string::npos) { - break; - } - pos = next + 2; - } - } -} - -//! Plain HTTP GET that sends the request, waits stall_ms, then reads. Matches -//! a raw socket that stops reading — the only client shape that trips httplib -//! server write_timeout / ApplyRouteIoTimeout. -QuackapiHttpFetchResult PlainGetWithStall(const string &url, const unordered_map &extra_headers, - int32_t stall_ms) { - QuackapiHttpFetchResult out; - string origin, path; - SplitURL(url, origin, path); - string host; - int port = 80; - SplitOriginHostPort(origin, host, port); - if (host.empty()) { - out.request_error = "stall_ms: could not parse host from URL"; - return out; - } - - duckdb_httplib::Error conn_error = duckdb_httplib::Error::Success; - auto sock = duckdb_httplib::detail::create_client_socket( - host, "", port, AF_UNSPEC, /*tcp_nodelay=*/true, /*ipv6_v6only=*/false, duckdb_httplib::default_socket_options, - OUTBOUND_CONNECT_TIMEOUT_SECONDS, 0, OUTBOUND_TIMEOUT_SECONDS, 0, OUTBOUND_TIMEOUT_SECONDS, 0, /*intf=*/"", - conn_error); - if (sock == INVALID_SOCKET) { - out.request_error = "stall_ms connect: " + duckdb_httplib::to_string(conn_error); - return out; - } - - string req = "GET " + path + " HTTP/1.1\r\nHost: " + host; - if (port != 80) { - req += ":" + std::to_string(port); - } - req += "\r\nConnection: close\r\n"; - for (auto &kv : extra_headers) { - req += kv.first + ": " + kv.second + "\r\n"; - } - req += "\r\n"; - - const char *send_ptr = req.data(); - size_t send_left = req.size(); - while (send_left > 0) { - auto n = duckdb_httplib::detail::send_socket(sock, send_ptr, send_left, CPPHTTPLIB_SEND_FLAGS); - if (n <= 0) { - duckdb_httplib::detail::close_socket(sock); - out.request_error = "stall_ms: send failed"; - return out; - } - send_ptr += n; - send_left -= static_cast(n); - } - - if (stall_ms > 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(stall_ms)); - } - - string raw; - char buf[16384]; - while (true) { - auto n = duckdb_httplib::detail::read_socket(sock, buf, sizeof(buf), CPPHTTPLIB_RECV_FLAGS); - if (n <= 0) { - break; - } - raw.append(buf, static_cast(n)); - } - duckdb_httplib::detail::close_socket(sock); - - if (raw.empty()) { - out.request_error = "stall_ms: empty response (peer closed during stall)"; - return out; - } - ParseRawHttpResponse(raw, out); - return out; -} - -unique_ptr DialPlain(const string &origin) { - auto client = make_uniq(origin); - client->set_keep_alive(true); - // Same Nagle/delayed-ACK deadlock the server side hit (quackapi_server.cpp:917), - // just on the other end of the socket: without this a small POST body waits on - // the peer's delayed ACK before the request is even complete. - client->set_tcp_nodelay(true); - client->set_follow_location(true); - client->set_decompress(true); - client->set_read_timeout(OUTBOUND_TIMEOUT_SECONDS, 0); - client->set_write_timeout(OUTBOUND_TIMEOUT_SECONDS, 0); - client->set_connection_timeout(OUTBOUND_CONNECT_TIMEOUT_SECONDS, 0); - return client; -} - -duckdb_httplib::Headers ToHttplibHeaders(const unordered_map &extra) { - duckdb_httplib::Headers headers; - for (auto &kv : extra) { - headers.emplace(kv.first, kv.second); - } - return headers; -} - -QuackapiHttpFetchResult FromHttplibResult(const duckdb_httplib::Result &res) { - QuackapiHttpFetchResult out; - if (res.error() != duckdb_httplib::Error::Success) { - out.request_error = duckdb_httplib::to_string(res.error()); - return out; - } - auto &response = res.value(); - out.status = HTTPUtil::ToStatusCode(response.status); - out.body = response.body; - out.reason = response.reason; - out.success = response.status >= 200 && response.status < 400; - for (auto &entry : response.headers) { - out.headers.Insert(entry.first, entry.second); - } - return out; -} - -//! Run `call` against a pooled plain-HTTP client for `origin`, then return the -//! client to the pool. The client is only recycled when the transport stayed -//! healthy — a broken socket must not be handed to the next request. -template -QuackapiHttpFetchResult WithPlainClient(const string &origin, CALL &&call) { - unique_ptr client; - bool reused = false; - { - auto &pool = ConnectionPool::Get(); - std::lock_guard guard(pool.lock); - auto &host = pool.hosts[origin]; - if (!host.plain_idle.empty()) { - client = std::move(host.plain_idle.back().client); - host.plain_idle.pop_back(); - host.reused++; - reused = true; - } else { - host.dialed++; - } - } - if (!client) { - client = DialPlain(origin); - } - // Reset on every checkout: a previous request's short deadline must not leak. - const auto timeout_ms = QuackapiRemainingTimeoutMillis(OUTBOUND_TIMEOUT_SECONDS * 1000); - client->set_max_timeout(static_cast(timeout_ms)); - client->set_read_timeout(static_cast(timeout_ms / 1000), static_cast((timeout_ms % 1000) * 1000)); - client->set_write_timeout(static_cast(timeout_ms / 1000), static_cast((timeout_ms % 1000) * 1000)); - const auto connect_ms = std::min(timeout_ms, OUTBOUND_CONNECT_TIMEOUT_SECONDS * 1000); - client->set_connection_timeout(static_cast(connect_ms / 1000), - static_cast((connect_ms % 1000) * 1000)); - - QuackapiHttpFetchResult result; - bool healthy = false; - try { - auto res = call(*client); - healthy = res.error() == duckdb_httplib::Error::Success; - result = FromHttplibResult(res); - } catch (...) { - // Drop the client on the floor; a half-written socket is not reusable. - throw; - } - result.reused_connection = reused; - - if (healthy) { - auto &pool = ConnectionPool::Get(); - std::lock_guard guard(pool.lock); - auto &host = pool.hosts[origin]; - if (host.plain_idle.size() < MAX_IDLE_PER_HOST) { - host.plain_idle.push_back(PooledPlain {std::move(client)}); - } - } - return result; -} - -//! Same checkout/return dance for the HTTPUtil path (https, or a loaded -//! curl_httpfs the operator wants used). The two-argument HTTPUtil::Request -//! reuses the client we pass instead of constructing a throwaway one. -template -QuackapiHttpFetchResult WithUtilClient(DatabaseInstance &db, const string &url, BUILD &&build) { - auto &http_util = HTTPUtil::Get(db); - string origin, path; - SplitURL(url, origin, path); - const auto key = http_util.GetName() + "|" + origin; - - unique_ptr client; - bool reused = false; - { - auto &pool = ConnectionPool::Get(); - std::lock_guard guard(pool.lock); - auto &host = pool.hosts[key]; - if (!host.util_idle.empty()) { - client = std::move(host.util_idle.back().client); - host.util_idle.pop_back(); - host.reused++; - reused = true; - } else { - host.dialed++; - } - } - - auto params = http_util.InitializeParameters(db, url); - params->keep_alive = true; - const auto timeout_ms = QuackapiRemainingTimeoutMillis(OUTBOUND_TIMEOUT_SECONDS * 1000); - params->timeout = timeout_ms / 1000; - params->timeout_usec = (timeout_ms % 1000) * 1000; - auto result = build(http_util, *params, client); - result.reused_connection = reused; - - if (client && result.request_error.empty()) { - auto &pool = ConnectionPool::Get(); - std::lock_guard guard(pool.lock); - auto &host = pool.hosts[key]; - if (host.util_idle.size() < MAX_IDLE_PER_HOST) { - host.util_idle.push_back(PooledUtil {std::move(client)}); - } - } - return result; -} - -} // namespace - -//===--------------------------------------------------------------------===// -// Public API -//===--------------------------------------------------------------------===// - -string QuackapiHttpFetch::ActiveHttpUtilName(DatabaseInstance &db) { - return HTTPUtil::Get(db).GetName(); -} - -QuackapiHttpFetchResult QuackapiHttpFetch::Get(DatabaseInstance &db, const string &url, - const unordered_map &extra_headers, int32_t stall_ms) { - if (stall_ms < 0) { - throw InvalidInputException("quackapi_fetch stall_ms must be >= 0"); - } - if (stall_ms > 0) { - if (!IsPlainHTTP(url)) { - throw InvalidInputException("quackapi_fetch stall_ms requires an http:// URL (plain TCP stall seam)"); - } - return PlainGetWithStall(url, extra_headers, stall_ms); - } - if (IsPlainHTTP(url)) { - string origin, path; - SplitURL(url, origin, path); - auto headers = ToHttplibHeaders(extra_headers); - return WithPlainClient(origin, [&](duckdb_httplib::Client &client) { return client.Get(path, headers); }); - } - - return WithUtilClient(db, url, [&](HTTPUtil &http_util, HTTPParams ¶ms, unique_ptr &client) { - HTTPHeaders headers(db); - InsertExtraHeaders(headers, extra_headers); - GetRequestInfo request(url, headers, params, /*response_handler=*/nullptr, - /*content_handler=*/nullptr); - request.try_request = true; - return FromResponse(http_util.Request(request, client)); - }); -} - -QuackapiHttpFetchResult QuackapiHttpFetch::Post(DatabaseInstance &db, const string &url, const string &body, - const string &content_type, - const unordered_map &extra_headers) { - if (IsPlainHTTP(url)) { - // The vendored httplib implements POST, so plain HTTP needs no companion - // extension at all — unlike the HTTPUtil path below. - string origin, path; - SplitURL(url, origin, path); - auto headers = ToHttplibHeaders(extra_headers); - const auto &ct = content_type.empty() ? string("application/octet-stream") : content_type; - return WithPlainClient(origin, - [&](duckdb_httplib::Client &client) { return client.Post(path, headers, body, ct); }); - } - - // Built-In HTTPLibClient does not implement POST (http_util.cpp). Surface a - // clear error pointing operators at curl_httpfs / httpfs rather than a raw - // NotImplementedException deep in the client. - auto &util = HTTPUtil::Get(db); - const auto util_name = util.GetName(); - if (util_name == "Built-In") { - throw InvalidConfigurationException( - "quackapi outbound POST over https requires an HTTP client with full method support. " - "LOAD curl_httpfs (recommended) or LOAD httpfs, then retry. Active HTTPUtil is '%s'.", - util_name); - } - - return WithUtilClient(db, url, [&](HTTPUtil &http_util, HTTPParams ¶ms, unique_ptr &client) { - HTTPHeaders headers(db); - if (!content_type.empty()) { - headers.Insert("Content-Type", content_type); - } - InsertExtraHeaders(headers, extra_headers); - PostRequestInfo request(url, headers, params, const_data_ptr_cast(body.data()), body.size()); - request.try_request = true; - return FromResponse(http_util.Request(request, client)); - }); -} - -vector QuackapiHttpFetch::PoolStats() { - vector out; - auto &pool = ConnectionPool::Get(); - std::lock_guard guard(pool.lock); - for (auto &entry : pool.hosts) { - QuackapiHttpPoolStats stats; - stats.host = entry.first; - stats.idle = entry.second.plain_idle.size() + entry.second.util_idle.size(); - stats.dialed = entry.second.dialed; - stats.reused = entry.second.reused; - out.push_back(std::move(stats)); - } - return out; -} - -void QuackapiHttpFetch::ResetPool() { - // Drop idle keep-alive sockets *before* httplib Server join. Stop paths - // detach ~QuackapiHttpServer; if pooled clients still hold the peer open, - // worker join can block until read timeout (600s) and CI process exit hangs - // with no log upload (Windows runner "lost communication"). - auto &pool = ConnectionPool::Get(); - std::lock_guard guard(pool.lock); - for (auto &entry : pool.hosts) { - for (auto &plain : entry.second.plain_idle) { - if (plain.client) { - plain.client->stop(); - } - } - // HTTPUtil clients: unique_ptr reset is enough (no keep-alive join path - // into our inbound Server). - entry.second.plain_idle.clear(); - entry.second.util_idle.clear(); - } - pool.hosts.clear(); -} - -//===--------------------------------------------------------------------===// -// SQL surface -//===--------------------------------------------------------------------===// - -namespace { - -LogicalType FetchResultType() { - child_list_t children; - children.emplace_back("status", LogicalType::INTEGER); - children.emplace_back("reason", LogicalType::VARCHAR); - children.emplace_back("body", LogicalType::VARCHAR); - children.emplace_back("headers", LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); - children.emplace_back("error", LogicalType::VARCHAR); - children.emplace_back("reused_connection", LogicalType::BOOLEAN); - return LogicalType::STRUCT(std::move(children)); -} - -Value ToValue(const QuackapiHttpFetchResult &result) { - vector header_keys; - vector header_values; - for (auto &entry : result.headers) { - header_keys.emplace_back(entry.first); - header_values.emplace_back(entry.second); - } - child_list_t children; - children.emplace_back("status", Value::INTEGER(static_cast(result.status))); - children.emplace_back("reason", Value(result.reason)); - children.emplace_back("body", Value(result.body)); - children.emplace_back("headers", Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(header_keys), - std::move(header_values))); - children.emplace_back("error", - result.request_error.empty() ? Value(LogicalType::VARCHAR) : Value(result.request_error)); - children.emplace_back("reused_connection", Value::BOOLEAN(result.reused_connection)); - return Value::STRUCT(std::move(children)); -} - -//! MAP(VARCHAR, VARCHAR) argument → header map. NULL / missing = no extras. -unordered_map HeadersFromValue(const Value &value) { - unordered_map out; - if (value.IsNull()) { - return out; - } - auto &entries = MapValue::GetChildren(value); - for (auto &entry : entries) { - auto &kv = StructValue::GetChildren(entry); - if (kv.size() != 2 || kv[0].IsNull()) { - continue; - } - out[kv[0].ToString()] = kv[1].IsNull() ? string() : kv[1].ToString(); - } - return out; -} - -int32_t StallMsFromArgs(DataChunk &args, idx_t row, idx_t col) { - if (args.ColumnCount() <= col) { - return 0; - } - auto v = args.data[col].GetValue(row); - if (v.IsNull()) { - return 0; - } - return v.GetValue(); -} - -//! Issue one independent outbound call per row of a chunk, CONCURRENTLY. -//! -//! Why this exists: these are scalar functions, so a chunk can carry up to -//! STANDARD_VECTOR_SIZE independent requests. Run serially, N calls against an -//! upstream with latency L cost N*L while the machine idles and the connection -//! pool sits unused. The pool already tolerates concurrent checkout (every path -//! takes ConnectionPool::lock), so fanning out is the whole fix. -//! -//! `perform(i)` must be safe to call from any thread. Exceptions are captured per -//! row rather than allowed to escape: WithPlainClient deliberately rethrows on a -//! half-written socket, and an exception crossing a std::thread boundary would -//! call std::terminate and take the server down with it. Argument validation that -//! must still raise to the caller therefore happens BEFORE the fan-out. -//! -//! The single-request case — what a route handler doing one upstream call per -//! request looks like — stays fully inline. No thread is created for it. -template -void FanOutRequests(idx_t count, const vector &skip, vector &out, PERFORM &&perform) { - const auto deadline = QuackapiCurrentDeadline(); - auto run_one = [&](idx_t i) { - QuackapiSetCurrentDeadline(deadline); - try { - out[i] = perform(i); - } catch (std::exception &e) { - out[i].request_error = e.what(); - } catch (...) { - out[i].request_error = "unknown error during outbound HTTP request"; - } - }; - - idx_t active = 0; - for (idx_t i = 0; i < count; i++) { - if (!skip[i]) { - active++; - } - } - if (active == 0) { - return; - } - if (active == 1) { - for (idx_t i = 0; i < count; i++) { - if (!skip[i]) { - run_one(i); - } - } - return; - } - - const idx_t n_threads = active < MAX_OUTBOUND_FANOUT ? active : MAX_OUTBOUND_FANOUT; - std::atomic cursor {0}; - vector threads; - threads.reserve(n_threads); - JoinOutboundThreads> join_threads {threads}; - for (idx_t t = 0; t < n_threads; t++) { - threads.emplace_back([&]() { - for (;;) { - const idx_t i = cursor.fetch_add(1); - if (i >= count) { - return; - } - if (!skip[i]) { - run_one(i); - } - } - }); - } -} - -void FetchScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - const auto count = args.size(); - result.SetVectorType(VectorType::FLAT_VECTOR); - - // Extract every argument up front, single-threaded: DataChunk/Vector reads are - // not safe to interleave with the fan-out below. stall_ms is validated here too, - // so its InvalidInputException still reaches the caller instead of being captured - // into a per-row error by the fan-out. - vector skip(count, false); - vector urls(count); - vector> headers(count); - vector stalls(count, 0); - for (idx_t i = 0; i < count; i++) { - auto url = args.data[0].GetValue(i); - if (url.IsNull()) { - skip[i] = true; - continue; - } - urls[i] = url.ToString(); - if (args.ColumnCount() == 2) { - // Overload is either (url, headers MAP) or (url, stall_ms INTEGER). - if (args.data[1].GetType().id() == LogicalTypeId::INTEGER || - args.data[1].GetType().id() == LogicalTypeId::BIGINT) { - stalls[i] = StallMsFromArgs(args, i, 1); - } else { - headers[i] = HeadersFromValue(args.data[1].GetValue(i)); - } - } else if (args.ColumnCount() >= 3) { - headers[i] = HeadersFromValue(args.data[1].GetValue(i)); - stalls[i] = StallMsFromArgs(args, i, 2); - } - if (stalls[i] < 0) { - throw InvalidInputException("quackapi_fetch stall_ms must be >= 0"); - } - if (stalls[i] > 0 && !IsPlainHTTP(urls[i])) { - throw InvalidInputException("quackapi_fetch stall_ms requires an http:// URL (plain TCP stall seam)"); - } - } - - vector out(count); - FanOutRequests(count, skip, out, - [&](idx_t i) { return QuackapiHttpFetch::Get(db, urls[i], headers[i], stalls[i]); }); - - for (idx_t i = 0; i < count; i++) { - result.SetValue(i, skip[i] ? Value(FetchResultType()) : ToValue(out[i])); - } -} - -void PostScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - const auto count = args.size(); - result.SetVectorType(VectorType::FLAT_VECTOR); - - vector skip(count, false); - vector urls(count); - vector bodies(count); - vector content_types(count); - vector> headers(count); - for (idx_t i = 0; i < count; i++) { - auto url = args.data[0].GetValue(i); - if (url.IsNull()) { - skip[i] = true; - continue; - } - urls[i] = url.ToString(); - auto body = args.data[1].GetValue(i); - bodies[i] = body.IsNull() ? string() : body.ToString(); - content_types[i] = "application/json"; - if (args.ColumnCount() > 2) { - auto ct = args.data[2].GetValue(i); - if (!ct.IsNull()) { - content_types[i] = ct.ToString(); - } - } - if (args.ColumnCount() > 3) { - headers[i] = HeadersFromValue(args.data[3].GetValue(i)); - } - } - - vector out(count); - FanOutRequests(count, skip, out, [&](idx_t i) { - return QuackapiHttpFetch::Post(db, urls[i], bodies[i], content_types[i], headers[i]); - }); - - for (idx_t i = 0; i < count; i++) { - result.SetValue(i, skip[i] ? Value(FetchResultType()) : ToValue(out[i])); - } -} - -struct PoolGlobalState : public GlobalTableFunctionState { - vector rows; - idx_t offset = 0; -}; - -struct FetchTableBindData : public TableFunctionData { - string url; - unordered_map headers; - int32_t stall_ms = 0; - bool finished = false; -}; - -unique_ptr PoolBind(ClientContext &, TableFunctionBindInput &, vector &return_types, - vector &names) { - names = {"host", "idle", "dialed", "reused"}; - return_types = {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT, LogicalType::BIGINT}; - return nullptr; -} - -unique_ptr PoolInit(ClientContext &, TableFunctionInitInput &) { - auto state = make_uniq(); - state->rows = QuackapiHttpFetch::PoolStats(); - return std::move(state); -} - -void PoolExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.rows.size() && row < STANDARD_VECTOR_SIZE) { - auto &entry = state.rows[state.offset]; - output.SetValue(0, row, Value(entry.host)); - output.SetValue(1, row, Value::BIGINT(NumericCast(entry.idle))); - output.SetValue(2, row, Value::BIGINT(NumericCast(entry.dialed))); - output.SetValue(3, row, Value::BIGINT(NumericCast(entry.reused))); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -//! Concurrent GETs — the seam SQLLogic needs for claim-race / fan-out tests. -//! `FROM range(n)` around scalar quackapi_fetch constant-folds to one request; -//! this table function fires n real threads and returns one row per attempt. -struct ParallelFetchBindData : public TableFunctionData { - string url; - int32_t n = 1; -}; - -struct ParallelFetchRow { - int32_t idx = 0; - int32_t status = 0; - string body; - string error; -}; - -struct ParallelFetchGlobalState : public GlobalTableFunctionState { - vector rows; - idx_t offset = 0; -}; - -unique_ptr ParallelFetchBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind = make_uniq(); - bind->url = input.inputs[0].GetValue(); - bind->n = input.inputs[1].GetValue(); - if (bind->n < 1) { - throw InvalidInputException("quackapi_parallel_fetch: n must be >= 1"); - } - if (bind->n > 256) { - throw InvalidInputException("quackapi_parallel_fetch: n max is 256"); - } - names = {"idx", "status", "body", "error"}; - return_types = {LogicalType::INTEGER, LogicalType::INTEGER, LogicalType::VARCHAR, LogicalType::VARCHAR}; - return std::move(bind); -} - -unique_ptr ParallelFetchInit(ClientContext &context, TableFunctionInitInput &input) { - auto &bind = input.bind_data->Cast(); - auto state = make_uniq(); - state->rows.resize(NumericCast(bind.n)); - const auto deadline = QuackapiCurrentDeadline(); - auto &db = *context.db; - vector threads; - threads.reserve(NumericCast(bind.n)); - JoinOutboundThreads> join_threads {threads}; - for (int32_t i = 0; i < bind.n; i++) { - threads.emplace_back([&db, &bind, &state, i, deadline]() { - QuackapiSetCurrentDeadline(deadline); - auto &row = state->rows[NumericCast(i)]; - row.idx = i; - try { - auto res = QuackapiHttpFetch::Get(db, bind.url); - row.status = static_cast(res.status); - row.body = std::move(res.body); - row.error = std::move(res.request_error); - } catch (std::exception &ex) { - row.error = ex.what(); - } catch (...) { - row.error = "unknown error during outbound HTTP request"; - } - }); - } - for (auto &t : threads) { - t.join(); - } - return std::move(state); -} - -void ParallelFetchExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.rows.size() && row < STANDARD_VECTOR_SIZE) { - auto &r = state.rows[state.offset]; - output.SetValue(0, row, Value::INTEGER(r.idx)); - output.SetValue(1, row, Value::INTEGER(r.status)); - output.SetValue(2, row, Value(r.body)); - output.SetValue(3, row, r.error.empty() ? Value(LogicalType::VARCHAR) : Value(r.error)); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -} // namespace - -void RegisterQuackapiHttpFetchFunctions(ExtensionLoader &loader) { - const auto header_type = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); - const auto result_type = FetchResultType(); - - // VOLATILE on every overload, deliberately. Without it DuckDB constant-folds - // a call whose arguments are all literals — `FROM range(1000)` around a fetch - // with a fixed URL executes ONE request and copies the answer 1000 times. - // That is wrong for a function with side effects and it silently invalidates - // any benchmark built on it. - ScalarFunctionSet fetch_set("quackapi_fetch"); - ScalarFunction fetch1("quackapi_fetch", {LogicalType::VARCHAR}, result_type, FetchScalar); - fetch1.stability = FunctionStability::VOLATILE; - ScalarFunction fetch2("quackapi_fetch", {LogicalType::VARCHAR, header_type}, result_type, FetchScalar); - fetch2.stability = FunctionStability::VOLATILE; - // stall_ms: after send, wait N ms before reading (socket write-timeout test seam). - ScalarFunction fetch_stall("quackapi_fetch", {LogicalType::VARCHAR, LogicalType::INTEGER}, result_type, - FetchScalar); - fetch_stall.stability = FunctionStability::VOLATILE; - ScalarFunction fetch_hdr_stall("quackapi_fetch", {LogicalType::VARCHAR, header_type, LogicalType::INTEGER}, - result_type, FetchScalar); - fetch_hdr_stall.stability = FunctionStability::VOLATILE; - fetch_set.AddFunction(fetch1); - fetch_set.AddFunction(fetch2); - fetch_set.AddFunction(fetch_stall); - fetch_set.AddFunction(fetch_hdr_stall); - loader.RegisterFunction(fetch_set); - - // Table form exposes named stall_ms := N (scalars have no named_parameters). - TableFunction fetch_tf( - "quackapi_fetch", {LogicalType::VARCHAR}, - [](ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &bind = data_p.bind_data->CastNoConst(); - if (bind.finished) { - return; - } - auto result = QuackapiHttpFetch::Get(*context.db, bind.url, bind.headers, bind.stall_ms); - auto value = ToValue(result); - auto &children = StructValue::GetChildren(value); - for (idx_t c = 0; c < children.size(); c++) { - output.SetValue(c, 0, children[c]); - } - output.SetCardinality(1); - bind.finished = true; - }, - [](ClientContext &, TableFunctionBindInput &input, vector &return_types, - vector &names) -> unique_ptr { - auto bind = make_uniq(); - if (input.inputs.empty() || input.inputs[0].IsNull()) { - throw InvalidInputException("quackapi_fetch(url, stall_ms := N): url must be non-NULL"); - } - bind->url = input.inputs[0].ToString(); - auto headers_it = input.named_parameters.find("headers"); - if (headers_it != input.named_parameters.end() && !headers_it->second.IsNull()) { - bind->headers = HeadersFromValue(headers_it->second); - } - auto stall_it = input.named_parameters.find("stall_ms"); - if (stall_it != input.named_parameters.end() && !stall_it->second.IsNull()) { - bind->stall_ms = stall_it->second.GetValue(); - } - auto result_type = FetchResultType(); - for (auto &child : StructType::GetChildTypes(result_type)) { - names.push_back(child.first); - return_types.push_back(child.second); - } - return std::move(bind); - }); - fetch_tf.named_parameters["headers"] = header_type; - fetch_tf.named_parameters["stall_ms"] = LogicalType::INTEGER; - loader.RegisterFunction(fetch_tf); - - ScalarFunctionSet post_set("quackapi_post"); - ScalarFunction post2("quackapi_post", {LogicalType::VARCHAR, LogicalType::VARCHAR}, result_type, PostScalar); - post2.stability = FunctionStability::VOLATILE; - ScalarFunction post2j("quackapi_post", {LogicalType::VARCHAR, LogicalType::JSON()}, result_type, PostScalar); - post2j.stability = FunctionStability::VOLATILE; - ScalarFunction post3("quackapi_post", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, - result_type, PostScalar); - post3.stability = FunctionStability::VOLATILE; - ScalarFunction post4("quackapi_post", - {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, header_type}, result_type, - PostScalar); - post4.stability = FunctionStability::VOLATILE; - post_set.AddFunction(post2); - post_set.AddFunction(post2j); - post_set.AddFunction(post3); - post_set.AddFunction(post4); - loader.RegisterFunction(post_set); - - loader.RegisterFunction(TableFunction("quackapi_http_pool", {}, PoolExec, PoolBind, PoolInit)); - - TableFunction parallel_fetch("quackapi_parallel_fetch", {LogicalType::VARCHAR, LogicalType::INTEGER}, - ParallelFetchExec, ParallelFetchBind, ParallelFetchInit); - loader.RegisterFunction(parallel_fetch); -} - -} // namespace duckdb diff --git a/src/quackapi_limits.cpp b/src/quackapi_limits.cpp index 8a2d812..39b8f2e 100644 --- a/src/quackapi_limits.cpp +++ b/src/quackapi_limits.cpp @@ -1,11 +1,13 @@ #include "quackapi_limits.hpp" #include "duckdb/main/connection.hpp" +#include "duckdb/main/client_context.hpp" #include #include #include #include #include #include +#include namespace duckdb { static thread_local std::chrono::steady_clock::time_point current_deadline = @@ -18,6 +20,41 @@ struct QuackapiDeadlineState { std::atomic expired {false}; }; +static std::mutex deadline_registry_mutex; +static std::unordered_map>> deadline_registry; + +static void RegisterDeadline(ClientContext *context, const std::shared_ptr &state) { + if (!context) { + return; + } + std::lock_guard guard(deadline_registry_mutex); + auto &stack = deadline_registry[context]; + stack.erase(std::remove_if(stack.begin(), stack.end(), + [](const std::weak_ptr &entry) { return entry.expired(); }), + stack.end()); + stack.push_back(state); +} + +static void UnregisterDeadline(ClientContext *context, const std::shared_ptr &state) { + if (!context) { + return; + } + std::lock_guard guard(deadline_registry_mutex); + auto found = deadline_registry.find(context); + if (found == deadline_registry.end()) { + return; + } + auto &stack = found->second; + stack.erase(std::remove_if(stack.begin(), stack.end(), + [&](const std::weak_ptr &entry) { + return entry.expired() || entry.lock() == state; + }), + stack.end()); + if (stack.empty()) { + deadline_registry.erase(found); + } +} + class DeadlineWatchdog { public: DeadlineWatchdog() : worker([this]() { Run(); }) { @@ -74,14 +111,20 @@ static DeadlineWatchdog &Watchdog() { return watchdog; } -QuackapiQueryDeadline::QuackapiQueryDeadline(Connection &connection, int64_t timeout_ms) : previous(current_deadline) { +QuackapiQueryDeadline::QuackapiQueryDeadline(Connection &connection, int64_t timeout_ms) + : context(connection.context.get()), previous(current_deadline) { state = std::make_shared(); state->connection = &connection; - state->deadline = std::min(previous, std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms)); + const auto context_deadline = connection.context ? QuackapiCurrentDeadline(*connection.context) + : std::chrono::steady_clock::time_point::max(); + state->deadline = std::min(std::min(previous, context_deadline), + std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms)); current_deadline = state->deadline; + RegisterDeadline(context, state); Watchdog().Add(state); } QuackapiQueryDeadline::~QuackapiQueryDeadline() { + UnregisterDeadline(context, state); { std::lock_guard guard(state->mutex); state->connection = nullptr; @@ -94,6 +137,23 @@ bool QuackapiQueryDeadline::Expired() const { std::chrono::steady_clock::time_point QuackapiCurrentDeadline() { return current_deadline; } +std::chrono::steady_clock::time_point QuackapiCurrentDeadline(ClientContext &context) { + std::lock_guard guard(deadline_registry_mutex); + auto found = deadline_registry.find(&context); + if (found == deadline_registry.end()) { + return std::chrono::steady_clock::time_point::max(); + } + auto &stack = found->second; + while (!stack.empty()) { + auto state = stack.back().lock(); + if (state) { + return state->deadline; + } + stack.pop_back(); + } + deadline_registry.erase(found); + return std::chrono::steady_clock::time_point::max(); +} void QuackapiSetCurrentDeadline(std::chrono::steady_clock::time_point deadline) { current_deadline = deadline; } @@ -101,8 +161,11 @@ int64_t QuackapiRemainingTimeoutMillis(int64_t fallback_ms) { if (current_deadline == std::chrono::steady_clock::time_point::max()) { return fallback_ms; } - return std::max(1, std::min(fallback_ms, std::chrono::duration_cast( - current_deadline - std::chrono::steady_clock::now()) - .count())); + const auto remaining = current_deadline - std::chrono::steady_clock::now(); + auto remaining_ms = std::chrono::duration_cast(remaining); + if (remaining_ms < remaining) { + remaining_ms += std::chrono::milliseconds(1); + } + return std::max(1, std::min(fallback_ms, remaining_ms.count())); } } // namespace duckdb diff --git a/src/quackapi_logging.cpp b/src/quackapi_logging.cpp new file mode 100644 index 0000000..60861c8 --- /dev/null +++ b/src/quackapi_logging.cpp @@ -0,0 +1,75 @@ +#include "quackapi_logging.hpp" + +#include + +#include "duckdb/common/string_util.hpp" +#include "duckdb/logging/log_type.hpp" +#include "duckdb/logging/logger.hpp" +#include "quackapi_server.hpp" +#include "quackapi_util.hpp" + +namespace duckdb { +namespace { + +string JsonString(const string &value) { + return "\"" + QuackapiJsonEscape(value) + "\""; +} + +string JsonOptional(const string &value) { + return value.empty() ? "null" : JsonString(value); +} + +string JsonOptionalDouble(double value) { + return value < 0 ? "null" : StringUtil::Format("%.3f", value); +} + +string JsonOptionalInteger(int64_t value) { + return value < 0 ? "null" : std::to_string(value); +} + +string BuildAccessLogJson(const QuackapiRequestRecord &entry) { + string result = StringUtil::Format( + "{\"type\":\"access\",\"request_id\":%s,\"received_at_micros\":%lld,\"method\":%s,\"path\":%s," + "\"route_name\":%s,\"route_path\":%s,\"status\":%d,\"duration_ms\":%.3f," + "\"sql_prepare_ms\":%s,\"sql_execute_ms\":%s,\"rows_out\":%s,\"bytes_in\":%lld,\"bytes_out\":%lld," + "\"client_ip\":%s,\"user_agent\":%s,\"http_version\":%s,\"error_type\":%s,\"error_message\":%s," + "\"trace_id\":%s,\"span_id\":%s,\"parent_span_id\":%s,\"sampled\":%s,\"headers\":", + JsonString(entry.request_id), static_cast(entry.received_at_micros), JsonString(entry.method), + JsonString(entry.path), JsonOptional(entry.route_name), JsonOptional(entry.route_path), entry.status, + entry.duration_ms, JsonOptionalDouble(entry.sql_prepare_ms), JsonOptionalDouble(entry.sql_execute_ms), + JsonOptionalInteger(entry.rows_out), static_cast(entry.bytes_in), + static_cast(entry.bytes_out), JsonString(entry.client_ip), JsonString(entry.user_agent), + JsonOptional(entry.http_version), JsonOptional(entry.error_type), JsonOptional(entry.error_message), + JsonOptional(entry.trace_id), JsonOptional(entry.span_id), JsonOptional(entry.parent_span_id), + entry.sampled ? "true" : "false"); + if (!entry.headers_logged) { + result += "null"; + } else { + result += "{"; + bool first = true; + for (auto &header : entry.headers) { + if (!first) { + result += ","; + } + first = false; + result += JsonString(header.first) + ":" + JsonString(header.second); + } + result += "}"; + } + result += ",\"query\":"; + result += entry.query_logged ? JsonString(entry.query) : "null"; + result += "}"; + return result; +} + +} // namespace + +void EmitQuackapiAccessLog(DatabaseInstance &db, const QuackapiRequestRecord &entry, LogLevel level) { + auto &logger = Logger::Get(db); + if (!logger.ShouldLog(DefaultLogType::NAME, level)) { + return; + } + logger.WriteLog(DefaultLogType::NAME, level, BuildAccessLogJson(entry)); +} + +} // namespace duckdb diff --git a/src/quackapi_middleware.cpp b/src/quackapi_middleware.cpp index 8737104..efdf820 100644 --- a/src/quackapi_middleware.cpp +++ b/src/quackapi_middleware.cpp @@ -15,7 +15,6 @@ #include "duckdb/planner/expression/bound_parameter_data.hpp" #include "quackapi_limits.hpp" -#include "quackapi_policy.hpp" #include "quackapi_util.hpp" namespace duckdb { @@ -455,24 +454,11 @@ vector QuackapiMiddlewareRegistry::SnapshotAll() { bool ExecuteQuackapiMiddleware(Connection &connection, DatabaseInstance &db, QuackapiMiddlewarePhase phase, const string &group_name, QuackapiMiddlewareContext &context, bool authenticated, const unordered_map &claims) { + (void)authenticated; try { auto middleware = QuackapiMiddlewareRegistry::Get(db).Snapshot(phase, group_name); for (auto &definition : middleware) { - bool deny_unauthenticated = false; - string policy_error; - auto handler_sql = RewriteHandlerWithPolicies(db, definition.handler_sql, authenticated, - deny_unauthenticated, policy_error); - if (deny_unauthenticated) { - context.status = 401; - context.response_body = "{\"detail\":\"Not authenticated\"}"; - return false; - } - if (!policy_error.empty()) { - SetMiddlewareFailure(context); - return false; - } - - auto prepared = connection.Prepare(handler_sql); + auto prepared = connection.Prepare(definition.handler_sql); if (prepared->HasError()) { SetMiddlewareFailure(context); return false; diff --git a/src/quackapi_openapi.cpp b/src/quackapi_openapi.cpp index a2b3fcc..da6016b 100644 --- a/src/quackapi_openapi.cpp +++ b/src/quackapi_openapi.cpp @@ -314,12 +314,12 @@ string BuildOpenApiDocument(DatabaseInstance &db, const string &server_url) { string body_schema; if (!route.body_schema.empty() && !route.body_type.empty()) { body_schema = "{\"allOf\":[" + route.body_schema + "," + - QuackapiDuckdbTransformToOpenApi(route.body_type) + + QuackapiDuckdbTransformToOpenApi(con, route.body_type) + "],\"x-duckdb-json-transform\":" + JsonString(route.body_type) + "}"; } else if (!route.body_schema.empty()) { body_schema = route.body_schema; } else { - body_schema = QuackapiDuckdbTransformToOpenApi(route.body_type); + body_schema = QuackapiDuckdbTransformToOpenApi(con, route.body_type); if (body_schema.size() > 1 && body_schema.back() == '}') { body_schema.pop_back(); body_schema += ",\"x-duckdb-json-transform\":" + JsonString(route.body_type) + "}"; diff --git a/src/quackapi_pg.cpp b/src/quackapi_pg.cpp deleted file mode 100644 index ff870f8..0000000 --- a/src/quackapi_pg.cpp +++ /dev/null @@ -1,617 +0,0 @@ -#include "quackapi_pg.hpp" - -#include "duckdb/common/string_util.hpp" -#include "quackapi_util.hpp" -#include "quackapi_limits.hpp" -#include "duckdb/parser/parser.hpp" -#include - -#ifndef QUACKAPI_HAS_LIBPQ -// Built without libpq — always fall through to DuckDB. -namespace duckdb { -QuackapiPgNativeResult QuackapiTryPgNative(const string &, const string &, - const case_insensitive_map_t> &, idx_t, string &, - string &) { - return QuackapiPgNativeResult::NOT_APPLICABLE; -} -} // namespace duckdb -#else - -#include - -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#else -#include -#endif - -namespace duckdb { - -namespace { - -// Thread-local connection: one PGconn per HTTP worker thread (psycopg-pool shape). -// -// Do NOT cache PQprepare by SQL alone: the named statement embeds catalog OIDs -// (domains, types, …). DROP + CREATE of the same name leaves a sticky dead OID -// ("cache lookup failed for type") while the SQL text is unchanged. Mirror the -// DuckDB handler path — reuse the connection, execute fresh each request. -struct TlsPg { - string dsn; - PGconn *conn = nullptr; - string err; - //! Cached, rounded-down PostgreSQL statement_timeout. Reusing it avoids a - //! second network round trip on the normal same-budget request path. - int64_t statement_timeout_ms = -1; - ~TlsPg() { - if (conn) { - PQfinish(conn); - conn = nullptr; - } - } - void Reset() { - if (conn) { - PQfinish(conn); - conn = nullptr; - } - statement_timeout_ms = -1; - } - PGconn *Get(const string &want_dsn); -}; - -bool DeadlineExpired() { - auto deadline = QuackapiCurrentDeadline(); - return deadline != std::chrono::steady_clock::time_point::max() && std::chrono::steady_clock::now() >= deadline; -} - -//! Wait on the libpq socket without ever exceeding the HTTP request's shared -//! deadline. This covers connection polling, output flush, and result input. -bool WaitForPgSocket(PGconn *conn, bool want_read, bool want_write, string &err_out) { - const int fd = PQsocket(conn); - if (fd < 0) { - err_out = "PostgreSQL transport failed"; - return false; - } - while (true) { - auto deadline = QuackapiCurrentDeadline(); - int64_t timeout_ms = 30000; -#ifdef _WIN32 - timeval timeout {}; -#endif - if (deadline == std::chrono::steady_clock::time_point::max()) { - // Keep the finite fallback for callers outside an HTTP request. - timeout_ms = 30000; -#ifdef _WIN32 - timeout.tv_sec = 30; -#endif - } else { - auto remaining = - std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()) - .count(); - if (remaining <= 0) { - err_out = "PostgreSQL deadline exceeded"; - return false; - } - timeout_ms = std::max(1, (remaining + 999) / 1000); -#ifdef _WIN32 - timeout.tv_sec = static_cast(remaining / 1000000); - timeout.tv_usec = static_cast(remaining % 1000000); -#endif - } - -#ifdef _WIN32 - fd_set read_fds, write_fds; - FD_ZERO(&read_fds); - FD_ZERO(&write_fds); - if (want_read) { - FD_SET(fd, &read_fds); - } - if (want_write) { - FD_SET(fd, &write_fds); - } - const int rc = select(0, want_read ? &read_fds : nullptr, want_write ? &write_fds : nullptr, nullptr, &timeout); -#else - // poll has no FD_SETSIZE ceiling, unlike select/FD_SET. The HTTP process - // can legitimately hold unrelated descriptors in addition to its fixed - // worker connections, so this must remain safe at high descriptor values. - pollfd poll_fd {}; - poll_fd.fd = fd; - poll_fd.events = static_cast((want_read ? POLLIN : 0) | (want_write ? POLLOUT : 0)); - const int bounded_timeout = static_cast(std::min(timeout_ms, std::numeric_limits::max())); - const int rc = poll(&poll_fd, 1, bounded_timeout); - if (rc > 0 && (poll_fd.revents & (POLLERR | POLLHUP | POLLNVAL))) { - err_out = "PostgreSQL transport failed"; - return false; - } -#endif - if (rc > 0) { - return true; - } - if (rc == 0) { - err_out = "PostgreSQL deadline exceeded"; - return false; - } -#ifndef _WIN32 - if (errno == EINTR) { - continue; - } -#endif - err_out = "PostgreSQL transport failed"; - return false; - } -} - -PGconn *TlsPg::Get(const string &want_dsn) { - if (conn && dsn == want_dsn && PQstatus(conn) == CONNECTION_OK) { - return conn; - } - Reset(); - dsn = want_dsn; - const char *keys[] = {"dbname", nullptr}; - const char *values[] = {dsn.c_str(), nullptr}; - conn = PQconnectStartParams(keys, values, 1); - if (!conn) { - err = "PostgreSQL connection failed"; - return nullptr; - } - while (true) { - auto status = PQconnectPoll(conn); - if (status == PGRES_POLLING_OK) { - if (PQsetnonblocking(conn, 1) == 0) { - return conn; - } - err = "PostgreSQL transport failed"; - Reset(); - return nullptr; - } - if (status == PGRES_POLLING_FAILED) { - err = DeadlineExpired() ? "PostgreSQL deadline exceeded" : "PostgreSQL connection failed"; - Reset(); - return nullptr; - } - const bool wants_read = status == PGRES_POLLING_READING || status == PGRES_POLLING_ACTIVE; - const bool wants_write = status == PGRES_POLLING_WRITING || status == PGRES_POLLING_ACTIVE; - if ((!wants_read && !wants_write) || !WaitForPgSocket(conn, wants_read, wants_write, err)) { - if (err.empty()) { - err = "PostgreSQL transport failed"; - } - Reset(); - return nullptr; - } - } -} - -TlsPg &Tls() { - thread_local TlsPg t; - return t; -} - -// $name or $name::TYPE → collect unique names in order of first appearance. -void CollectNamedParams(const string &sql, const vector &protected_text, vector &names) { - names.clear(); - for (idx_t i = 0; i < sql.size(); i++) { - if (protected_text[i] || sql[i] != '$') { - continue; - } - if (i + 1 >= sql.size() || !((sql[i + 1] >= 'A' && sql[i + 1] <= 'Z') || - (sql[i + 1] >= 'a' && sql[i + 1] <= 'z') || sql[i + 1] == '_')) { - continue; - } - idx_t j = i + 1; - while (j < sql.size() && ((sql[j] >= 'A' && sql[j] <= 'Z') || (sql[j] >= 'a' && sql[j] <= 'z') || - (sql[j] >= '0' && sql[j] <= '9') || sql[j] == '_')) { - j++; - } - string name = sql.substr(i + 1, j - (i + 1)); - // skip ::type - bool seen = false; - for (auto &n : names) { - if (StringUtil::Lower(n) == StringUtil::Lower(name)) { - seen = true; - break; - } - } - if (!seen) { - names.push_back(name); - } - i = j - 1; - } -} - -// DuckDB route SQL → Postgres text suitable for PQexecParams. -// - strip "pg." catalog qualifier -// - $name::TYPE / $name → $1, $2, … (order of first appearance) -// Rejects obvious DuckDB-only surface (postgres_execute, etc.). -bool ToPgSql(const string &in, string &out, vector ¶m_names, string &err) { - // The lexer owns quoting, dollar-quoted bodies and comments. Rewriting - // their contents corrupts SQL literals and can invent phantom parameters. - vector protected_text(in.size(), false); - string executable = in; - auto tokens = Parser::Tokenize(in); - for (idx_t t = 0; t < tokens.size(); t++) { - const auto &token = tokens[t]; - if (token.type == SimplifiedTokenType::SIMPLIFIED_TOKEN_STRING_CONSTANT || - token.type == SimplifiedTokenType::SIMPLIFIED_TOKEN_COMMENT || - (token.start < in.size() && in[token.start] == '"')) { - auto end = t + 1 < tokens.size() ? tokens[t + 1].start : in.size(); - for (idx_t p = token.start; p < end; p++) { - protected_text[p] = true; - executable[p] = ' '; - } - } - } - auto lower = StringUtil::Lower(executable); - if (StringUtil::Contains(lower, "postgres_execute") || StringUtil::Contains(lower, "postgres_query") || - StringUtil::Contains(lower, "postgres_scan")) { - err = "duckdb-only TVF"; - return false; - } - CollectNamedParams(in, protected_text, param_names); - // Map name → $n - case_insensitive_map_t idx; - for (idx_t i = 0; i < param_names.size(); i++) { - idx[param_names[i]] = i + 1; - } - out.clear(); - out.reserve(in.size() + 8); - for (idx_t i = 0; i < in.size(); i++) { - if (protected_text[i]) { - out.push_back(in[i]); - continue; - } - // strip pg. qualifier (case-insensitive) - if ((in[i] == 'p' || in[i] == 'P') && i + 2 < in.size() && (in[i + 1] == 'g' || in[i + 1] == 'G') && - in[i + 2] == '.') { - // word boundary before p - if (i == 0 || !((in[i - 1] >= 'A' && in[i - 1] <= 'Z') || (in[i - 1] >= 'a' && in[i - 1] <= 'z') || - (in[i - 1] >= '0' && in[i - 1] <= '9') || in[i - 1] == '_')) { - i += 2; - continue; - } - } - if (in[i] == '$' && i + 1 < in.size() && - ((in[i + 1] >= 'A' && in[i + 1] <= 'Z') || (in[i + 1] >= 'a' && in[i + 1] <= 'z') || in[i + 1] == '_')) { - idx_t j = i + 1; - while (j < in.size() && ((in[j] >= 'A' && in[j] <= 'Z') || (in[j] >= 'a' && in[j] <= 'z') || - (in[j] >= '0' && in[j] <= '9') || in[j] == '_')) { - j++; - } - string name = in.substr(i + 1, j - (i + 1)); - // Keep explicit casts: they determine parameter and response types. - auto it = idx.find(name); - if (it == idx.end()) { - err = "unmapped param $" + name; - return false; - } - out += "$"; - out += std::to_string(it->second); - i = j - 1; - continue; - } - out.push_back(in[i]); - } - return true; -} - -//! Append while enforcing the configured final JSON body ceiling. This guards -//! each intermediate append instead of discovering an oversized body only -//! after libpq rows have been fully serialized. -bool AppendJson(string &body, const char *data, idx_t length, idx_t max_bytes, string &err_out) { - if (DeadlineExpired()) { - err_out = "PostgreSQL deadline exceeded"; - return false; - } - if (length > max_bytes || body.size() > max_bytes - length) { - err_out = "PostgreSQL response exceeds configured byte limit"; - return false; - } - body.append(data, length); - return true; -} - -bool AppendJsonLiteral(string &body, const char *literal, idx_t max_bytes, string &err_out) { - return AppendJson(body, literal, std::strlen(literal), max_bytes, err_out); -} - -//! Escape directly into the bounded response rather than allocating a second -//! unbounded escaped copy of an individual PostgreSQL field. -bool AppendJsonQuoted(string &body, const char *data, idx_t length, idx_t max_bytes, string &err_out) { - if (!AppendJsonLiteral(body, "\"", max_bytes, err_out)) { - return false; - } - static const char hex[] = "0123456789abcdef"; - for (idx_t i = 0; i < length; i++) { - if ((i & 4095) == 0 && DeadlineExpired()) { - err_out = "PostgreSQL deadline exceeded"; - return false; - } - const auto ch = static_cast(data[i]); - if (ch == '"' || ch == '\\') { - const char escaped[] = {'\\', static_cast(ch)}; - if (!AppendJson(body, escaped, sizeof(escaped), max_bytes, err_out)) { - return false; - } - } else if (ch == '\b') { - if (!AppendJsonLiteral(body, "\\b", max_bytes, err_out)) - return false; - } else if (ch == '\f') { - if (!AppendJsonLiteral(body, "\\f", max_bytes, err_out)) - return false; - } else if (ch == '\n') { - if (!AppendJsonLiteral(body, "\\n", max_bytes, err_out)) - return false; - } else if (ch == '\r') { - if (!AppendJsonLiteral(body, "\\r", max_bytes, err_out)) - return false; - } else if (ch == '\t') { - if (!AppendJsonLiteral(body, "\\t", max_bytes, err_out)) - return false; - } else if (ch < 0x20) { - const char escaped[] = {'\\', 'u', '0', '0', hex[ch >> 4], hex[ch & 0x0F]}; - if (!AppendJson(body, escaped, sizeof(escaped), max_bytes, err_out)) { - return false; - } - } else { - const char plain[] = {static_cast(ch)}; - if (!AppendJson(body, plain, sizeof(plain), max_bytes, err_out)) { - return false; - } - } - } - return AppendJsonLiteral(body, "\"", max_bytes, err_out); -} - -bool EqualsAsciiIgnoreCase(const char *value, idx_t length, const char *expected) { - const auto expected_length = std::strlen(expected); - if (length != expected_length) { - return false; - } - for (idx_t i = 0; i < length; i++) { - auto lhs = static_cast(value[i]); - auto rhs = static_cast(expected[i]); - if (lhs >= 'A' && lhs <= 'Z') - lhs = static_cast(lhs + ('a' - 'A')); - if (rhs >= 'A' && rhs <= 'Z') - rhs = static_cast(rhs + ('a' - 'A')); - if (lhs != rhs) - return false; - } - return true; -} - -bool AppendPgRowsJson(PGresult *res, string &body, bool &first_row, idx_t max_bytes, string &err_out) { - const int rows = PQntuples(res); - const int cols = PQnfields(res); - for (int row = 0; row < rows; row++) { - if (!first_row && !AppendJsonLiteral(body, ",", max_bytes, err_out)) { - return false; - } - first_row = false; - if (!AppendJsonLiteral(body, "{", max_bytes, err_out)) { - return false; - } - for (int col = 0; col < cols; col++) { - if (col && !AppendJsonLiteral(body, ",", max_bytes, err_out)) { - return false; - } - const char *name = PQfname(res, col); - if (!AppendJsonQuoted(body, name ? name : "", name ? std::strlen(name) : 0, max_bytes, err_out) || - !AppendJsonLiteral(body, ":", max_bytes, err_out)) { - return false; - } - if (PQgetisnull(res, row, col)) { - if (!AppendJsonLiteral(body, "null", max_bytes, err_out)) - return false; - continue; - } - const char *value = PQgetvalue(res, row, col); - const idx_t length = static_cast(PQgetlength(res, row, col)); - const auto oid = PQftype(res, col); - const bool numeric = - oid == 20 || oid == 21 || oid == 23 || oid == 26 || oid == 700 || oid == 701 || oid == 1700; - const bool non_finite = EqualsAsciiIgnoreCase(value, length, "nan") || - EqualsAsciiIgnoreCase(value, length, "infinity") || - EqualsAsciiIgnoreCase(value, length, "-infinity"); - if (oid == 16) { - if (!AppendJsonLiteral(body, (length == 1 && (value[0] == 't' || value[0] == 'T')) ? "true" : "false", - max_bytes, err_out)) - return false; - } else if (oid == 114 || oid == 3802 || (numeric && !non_finite)) { - if (!AppendJson(body, value, length, max_bytes, err_out)) - return false; - } else if (!AppendJsonQuoted(body, value, length, max_bytes, err_out)) { - return false; - } - } - if (!AppendJsonLiteral(body, "}", max_bytes, err_out)) { - return false; - } - } - return true; -} - -bool IsPgCancellation(PGresult *result) { - const char *state = PQresultErrorField(result, PG_DIAG_SQLSTATE); - return state && std::strcmp(state, "57014") == 0; -} - -bool FlushPgOutput(PGconn *conn, string &err_out) { - while (true) { - const auto status = PQflush(conn); - if (status == 0) { - return true; - } - if (status < 0) { - err_out = "PostgreSQL transport failed"; - return false; - } - if (!WaitForPgSocket(conn, false, true, err_out)) { - return false; - } - } -} - -//! Consume a single non-row PostgreSQL command without a synchronous PQexec. -bool ReceivePgCommand(PGconn *conn, string &err_out) { - if (!FlushPgOutput(conn, err_out)) { - return false; - } - bool saw_command = false; - while (true) { - if (PQconsumeInput(conn) == 0) { - err_out = "PostgreSQL transport failed"; - return false; - } - while (!PQisBusy(conn)) { - auto *result = PQgetResult(conn); - if (!result) { - if (!saw_command) - err_out = "PostgreSQL execution failed"; - return saw_command; - } - auto status = PQresultStatus(result); - const bool ok = status == PGRES_COMMAND_OK; - const bool canceled = IsPgCancellation(result); - PQclear(result); - if (!ok) { - err_out = - canceled || DeadlineExpired() ? "PostgreSQL deadline exceeded" : "PostgreSQL execution failed"; - return false; - } - saw_command = true; - } - if (!WaitForPgSocket(conn, true, false, err_out)) { - return false; - } - } -} - -bool EnsurePgStatementTimeout(TlsPg &tls, PGconn *conn, string &err_out) { - // Round down so the server-side deadline never extends the shared request - // budget. A stable bucket avoids a SET round trip on common requests. - const auto remaining = QuackapiRemainingTimeoutMillis(30000); - const int64_t target = std::max(1, (remaining / 10) * 10); - if (tls.statement_timeout_ms == target) { - return true; - } - const string sql = "SET statement_timeout = " + std::to_string(target); - if (PQsendQuery(conn, sql.c_str()) == 0 || !ReceivePgCommand(conn, err_out)) { - if (err_out.empty()) - err_out = "PostgreSQL transport failed"; - return false; - } - tls.statement_timeout_ms = target; - return true; -} - -} // namespace - -QuackapiPgNativeResult QuackapiTryPgNative(const string &dsn, const string &handler_sql, - const case_insensitive_map_t> &provided, - idx_t max_response_bytes, string &json_body, string &err_out) { - if (dsn.empty()) { - return QuackapiPgNativeResult::NOT_APPLICABLE; - } - string pg_sql; - vector names; - if (!ToPgSql(handler_sql, pg_sql, names, err_out)) { - // Translation happens before any network operation, so DuckDB may safely - // handle its own syntax/parameter validation when this path cannot. - return QuackapiPgNativeResult::NOT_APPLICABLE; - } - vector vals; - vector storage; - storage.reserve(names.size()); - vals.reserve(names.size()); - for (auto &name : names) { - auto it = provided.find(name); - if (it == provided.end()) { - // Let DuckDB's normal binder produce the usual 422 missing-parameter - // response. PostgreSQL has not observed a command at this point. - return QuackapiPgNativeResult::NOT_APPLICABLE; - } - storage.push_back(it->second.second); - vals.push_back(storage.back().c_str()); - } - auto &tls = Tls(); - PGconn *conn = tls.Get(dsn); - if (!conn) { - err_out = tls.err.empty() ? "PostgreSQL connection failed" : tls.err; - return QuackapiPgNativeResult::FAILED; - } - if (!EnsurePgStatementTimeout(tls, conn, err_out)) { - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - - // Single-row mode means libpq yields each tuple as it arrives. We serialize - // and cap it incrementally instead of allowing PQexecParams to accumulate an - // unbounded full result before the HTTP response limit can be checked. - if (PQsendQueryParams(conn, pg_sql.c_str(), static_cast(vals.size()), nullptr, vals.data(), nullptr, nullptr, - 0) == 0 || - PQsetSingleRowMode(conn) == 0) { - err_out = "PostgreSQL transport failed"; - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - if (!FlushPgOutput(conn, err_out)) { - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - - json_body.clear(); - if (!AppendJsonLiteral(json_body, "[", max_response_bytes, err_out)) { - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - bool first_row = true; - while (true) { - if (PQconsumeInput(conn) == 0) { - err_out = "PostgreSQL transport failed"; - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - while (!PQisBusy(conn)) { - auto *result = PQgetResult(conn); - if (!result) { - if (!AppendJsonLiteral(json_body, "]", max_response_bytes, err_out)) { - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - return QuackapiPgNativeResult::SUCCESS; - } - const auto status = PQresultStatus(result); - if (status == PGRES_SINGLE_TUPLE) { - const bool appended = AppendPgRowsJson(result, json_body, first_row, max_response_bytes, err_out); - PQclear(result); - if (!appended) { - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - continue; - } - const bool ok = status == PGRES_TUPLES_OK || status == PGRES_COMMAND_OK; - const bool canceled = IsPgCancellation(result); - PQclear(result); - if (!ok) { - err_out = - canceled || DeadlineExpired() ? "PostgreSQL deadline exceeded" : "PostgreSQL execution failed"; - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - } - if (!WaitForPgSocket(conn, true, false, err_out)) { - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - } -} - -} // namespace duckdb - -#endif // QUACKAPI_HAS_LIBPQ diff --git a/src/quackapi_policy.cpp b/src/quackapi_policy.cpp deleted file mode 100644 index 35d85da..0000000 --- a/src/quackapi_policy.cpp +++ /dev/null @@ -1,1371 +0,0 @@ -#include "quackapi_policy.hpp" - -#include - -#include "duckdb/common/exception.hpp" -#include "duckdb/common/string_util.hpp" -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/main/connection.hpp" -#include "duckdb/parser/parser.hpp" -#include "duckdb/parser/parser_extension.hpp" -#include "duckdb/parser/parsed_expression_iterator.hpp" -#include "duckdb/parser/expression/subquery_expression.hpp" -#include "duckdb/parser/query_node/select_node.hpp" -#include "duckdb/parser/query_node/set_operation_node.hpp" -#include "duckdb/parser/statement/select_statement.hpp" -#include "duckdb/parser/tableref/list.hpp" - -#include "quackapi_state.hpp" -#include "quackapi_util.hpp" - -namespace duckdb { - -namespace { - -bool IsIdentStart(char c) { - return StringUtil::CharacterIsAlpha(c) || c == '_'; -} - -bool IsIdentChar(char c) { - return StringUtil::CharacterIsAlpha(c) || StringUtil::CharacterIsDigit(c) || c == '_'; -} - -//! First whitespace-delimited token (space/tab/newline all count as boundaries). -string NextToken(const string &s, idx_t &consumed) { - idx_t i = 0; - while (i < s.size() && !StringUtil::CharacterIsSpace(s[i])) { - i++; - } - consumed = i; - return s.substr(0, i); -} - -//! Extract a parenthesized span starting at rest[0] == '('. Balanced parens; -//! ignores parens inside single-quoted strings ('' escape). -bool ExtractParenGroup(const string &rest, idx_t start, string &inner, idx_t &end_out) { - if (start >= rest.size() || rest[start] != '(') { - return false; - } - int depth = 0; - bool in_str = false; - for (idx_t i = start; i < rest.size(); i++) { - char c = rest[i]; - if (in_str) { - if (c == '\'') { - if (i + 1 < rest.size() && rest[i + 1] == '\'') { - i++; // escaped quote - } else { - in_str = false; - } - } - continue; - } - if (c == '\'') { - in_str = true; - continue; - } - if (c == '(') { - depth++; - } else if (c == ')') { - depth--; - if (depth == 0) { - inner = rest.substr(start + 1, i - start - 1); - end_out = i + 1; - return true; - } - } - } - return false; -} - -//! Parse "col [TYPE], col [TYPE], ..." into names + optional types. -bool ParseColList(const string &inner, vector &names, vector &types, string &err) { - names.clear(); - types.clear(); - string s = QuackapiTrim(inner); - if (s.empty()) { - err = "column list must not be empty"; - return false; - } - idx_t i = 0; - while (i < s.size()) { - while (i < s.size() && (StringUtil::CharacterIsSpace(s[i]) || s[i] == ',')) { - i++; - } - if (i >= s.size()) { - break; - } - if (!IsIdentStart(s[i])) { - err = "expected column name in list"; - return false; - } - idx_t start = i; - i++; - while (i < s.size() && IsIdentChar(s[i])) { - i++; - } - string col = s.substr(start, i - start); - while (i < s.size() && StringUtil::CharacterIsSpace(s[i])) { - i++; - } - string typ; - if (i < s.size() && IsIdentStart(s[i])) { - idx_t ts = i; - i++; - while (i < s.size() && IsIdentChar(s[i])) { - i++; - } - typ = StringUtil::Upper(s.substr(ts, i - ts)); - if (typ == "INT") { - typ = "INTEGER"; - } else if (typ == "BOOL") { - typ = "BOOLEAN"; - } else if (typ == "TEXT" || typ == "STRING") { - typ = "VARCHAR"; - } else if (typ == "REAL") { - typ = "FLOAT"; - } - } - names.push_back(col); - types.push_back(typ); - } - if (names.empty()) { - err = "column list must not be empty"; - return false; - } - return true; -} - -string JoinComma(const vector &parts) { - string out; - for (idx_t i = 0; i < parts.size(); i++) { - if (i > 0) { - out += ", "; - } - out += parts[i]; - } - return out; -} - -//! Replace whole-word identifier `val` with `replacement` (masking policy body). -string SubstituteValPlaceholder(const string &expr, const string &replacement) { - string out; - out.reserve(expr.size() + 8); - bool in_str = false; - for (idx_t i = 0; i < expr.size();) { - char c = expr[i]; - if (in_str) { - out += c; - if (c == '\'') { - if (i + 1 < expr.size() && expr[i + 1] == '\'') { - out += expr[i + 1]; - i += 2; - continue; - } - in_str = false; - } - i++; - continue; - } - if (c == '\'') { - in_str = true; - out += c; - i++; - continue; - } - if (IsIdentStart(c)) { - idx_t j = i + 1; - while (j < expr.size() && IsIdentChar(expr[j])) { - j++; - } - string tok = expr.substr(i, j - i); - if (StringUtil::Lower(tok) == "val") { - out += replacement; - } else { - out += tok; - } - i = j; - continue; - } - out += c; - i++; - } - return out; -} - -// `physical_relation` is built from parsed catalog/schema/table identifiers. It -// is deliberately not reconstructed from route text, so an attached catalog or -// quoted identifier cannot silently target a different object. -string BuildSecureSubqueryQuoted(const string &physical_relation, const QuackapiRowAccessPolicy *rap_for_where, - const string &where_expr_or_empty, - const vector> &masked_cols) { - string where_sql; - if (!where_expr_or_empty.empty()) { - where_sql = " WHERE (" + where_expr_or_empty + ")"; - } - (void)rap_for_where; - if (masked_cols.empty()) { - return "SELECT * FROM " + physical_relation + where_sql; - } - string excludes; - string extras; - for (idx_t i = 0; i < masked_cols.size(); i++) { - if (i > 0) { - excludes += ", "; - extras += ", "; - } - // Quote masked column names in EXCLUDE / AS - string col_esc; - for (char c : masked_cols[i].first) { - if (c == '"') { - col_esc += "\"\""; - } else { - col_esc += c; - } - } - string colq = "\"" + col_esc + "\""; - excludes += colq; - extras += "(" + masked_cols[i].second + ") AS " + colq; - } - return "SELECT * EXCLUDE (" + excludes + "), " + extras + " FROM " + physical_relation + where_sql; -} - -//===--------------------------------------------------------------------===// -// DDL parse data -//===--------------------------------------------------------------------===// - -struct PolicyDdlParseData : public ParserExtensionParseData { - //! CREATE_ROW | CREATE_MASK | DROP_ROW | DROP_MASK | BIND_ROW | UNBIND_ROW | BIND_MASK | UNBIND_MASK - string action; - bool or_replace = false; - string name; - string value_type; // masking ON - vector arg_columns; // RAP signature / bind ON cols / mask column - vector arg_types; - string expression; - string table_name; - string column_name; // masking bind - - unique_ptr Copy() const override { - auto copy = make_uniq(); - copy->action = action; - copy->or_replace = or_replace; - copy->name = name; - copy->value_type = value_type; - copy->arg_columns = arg_columns; - copy->arg_types = arg_types; - copy->expression = expression; - copy->table_name = table_name; - copy->column_name = column_name; - return std::move(copy); - } - string ToString() const override { - return action + " POLICY " + name; - } -}; - -ParserExtensionParseResult PolicyDdlParse(ParserExtensionInfo *, const string &query) { - auto q = QuackapiTrim(query); - auto upper = StringUtil::Upper(q); - - // ---- DROP ROW ACCESS POLICY / DROP MASKING POLICY ---- - if (StringUtil::StartsWith(upper, "DROP ROW ACCESS POLICY ")) { - auto name = QuackapiTrim(q.substr(23)); - if (name.empty() || name.find(' ') != string::npos) { - return ParserExtensionParseResult("DROP ROW ACCESS POLICY expects a single policy name"); - } - auto data = make_uniq(); - data->action = "DROP_ROW"; - data->name = name; - return ParserExtensionParseResult(std::move(data)); - } - if (StringUtil::StartsWith(upper, "DROP MASKING POLICY ")) { - auto name = QuackapiTrim(q.substr(20)); - if (name.empty() || name.find(' ') != string::npos) { - return ParserExtensionParseResult("DROP MASKING POLICY expects a single policy name"); - } - auto data = make_uniq(); - data->action = "DROP_MASK"; - data->name = name; - return ParserExtensionParseResult(std::move(data)); - } - - // ---- CREATE [OR REPLACE] ROW ACCESS POLICY ---- - bool or_replace = false; - idx_t pos = 0; - bool is_create_row = false; - bool is_create_mask = false; - if (StringUtil::StartsWith(upper, "CREATE OR REPLACE ROW ACCESS POLICY ")) { - pos = 36; - or_replace = true; - is_create_row = true; - } else if (StringUtil::StartsWith(upper, "CREATE ROW ACCESS POLICY ")) { - pos = 25; - is_create_row = true; - } else if (StringUtil::StartsWith(upper, "CREATE OR REPLACE MASKING POLICY ")) { - pos = 33; - or_replace = true; - is_create_mask = true; - } else if (StringUtil::StartsWith(upper, "CREATE MASKING POLICY ")) { - pos = 22; - is_create_mask = true; - } - - if (is_create_row) { - auto rest = QuackapiTrim(q.substr(pos)); - idx_t name_len = 0; - auto name = NextToken(rest, name_len); - if (name.empty() || name_len >= rest.size()) { - return ParserExtensionParseResult( - "CREATE ROW ACCESS POLICY AS () RETURNS BOOLEAN USING ()"); - } - rest = QuackapiTrim(rest.substr(name_len)); - auto ru = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(ru, "AS")) { - return ParserExtensionParseResult("Expected AS () after policy name"); - } - rest = QuackapiTrim(rest.substr(2)); - if (rest.empty() || rest[0] != '(') { - return ParserExtensionParseResult("Expected AS ()"); - } - string col_inner; - idx_t after_cols = 0; - if (!ExtractParenGroup(rest, 0, col_inner, after_cols)) { - return ParserExtensionParseResult("Unterminated column list in AS (...)"); - } - vector cols, types; - string err; - if (!ParseColList(col_inner, cols, types, err)) { - return ParserExtensionParseResult("ROW ACCESS POLICY AS (...): " + err); - } - rest = QuackapiTrim(rest.substr(after_cols)); - ru = StringUtil::Upper(rest); - // RETURNS BOOLEAN - if (!StringUtil::StartsWith(ru, "RETURNS")) { - return ParserExtensionParseResult("Expected RETURNS BOOLEAN after AS (...)"); - } - rest = QuackapiTrim(rest.substr(7)); - ru = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(ru, "BOOLEAN")) { - return ParserExtensionParseResult("ROW ACCESS POLICY must RETURNS BOOLEAN"); - } - rest = QuackapiTrim(rest.substr(7)); - ru = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(ru, "USING")) { - return ParserExtensionParseResult("Expected USING () after RETURNS BOOLEAN"); - } - rest = QuackapiTrim(rest.substr(5)); - if (rest.empty() || rest[0] != '(') { - return ParserExtensionParseResult("Expected USING ()"); - } - string expr; - idx_t after_expr = 0; - if (!ExtractParenGroup(rest, 0, expr, after_expr)) { - return ParserExtensionParseResult("Unterminated USING (...) expression"); - } - expr = QuackapiTrim(expr); - if (expr.empty()) { - return ParserExtensionParseResult("USING expression must not be empty"); - } - rest = QuackapiTrim(rest.substr(after_expr)); - if (!rest.empty()) { - return ParserExtensionParseResult("Unexpected tokens after USING (...)"); - } - auto data = make_uniq(); - data->action = "CREATE_ROW"; - data->or_replace = or_replace; - data->name = name; - data->arg_columns = std::move(cols); - data->arg_types = std::move(types); - data->expression = expr; - return ParserExtensionParseResult(std::move(data)); - } - - if (is_create_mask) { - auto rest = QuackapiTrim(q.substr(pos)); - idx_t name_len = 0; - auto name = NextToken(rest, name_len); - if (name.empty() || name_len >= rest.size()) { - return ParserExtensionParseResult("CREATE MASKING POLICY ON USING ()"); - } - rest = QuackapiTrim(rest.substr(name_len)); - auto ru = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(ru, "ON")) { - return ParserExtensionParseResult("Expected ON after masking policy name"); - } - // ON may be followed by space or newline - if (rest.size() == 2 || !StringUtil::CharacterIsSpace(rest[2])) { - // "ON" alone without trailing space after token — still need type next - if (rest.size() == 2) { - return ParserExtensionParseResult("Expected ON USING ()"); - } - } - rest = QuackapiTrim(rest.substr(2)); - idx_t type_len = 0; - auto vtype = StringUtil::Upper(NextToken(rest, type_len)); - if (vtype.empty() || type_len >= rest.size()) { - return ParserExtensionParseResult("Expected ON USING ()"); - } - if (vtype == "INT") { - vtype = "INTEGER"; - } else if (vtype == "BOOL") { - vtype = "BOOLEAN"; - } else if (vtype == "TEXT" || vtype == "STRING") { - vtype = "VARCHAR"; - } else if (vtype == "REAL") { - vtype = "FLOAT"; - } - rest = QuackapiTrim(rest.substr(type_len)); - ru = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(ru, "USING")) { - return ParserExtensionParseResult("Expected USING () after ON "); - } - rest = QuackapiTrim(rest.substr(5)); - if (rest.empty() || rest[0] != '(') { - return ParserExtensionParseResult("Expected USING ()"); - } - string expr; - idx_t after_expr = 0; - if (!ExtractParenGroup(rest, 0, expr, after_expr)) { - return ParserExtensionParseResult("Unterminated USING (...) expression"); - } - expr = QuackapiTrim(expr); - if (expr.empty()) { - return ParserExtensionParseResult("USING expression must not be empty"); - } - rest = QuackapiTrim(rest.substr(after_expr)); - if (!rest.empty()) { - return ParserExtensionParseResult("Unexpected tokens after USING (...)"); - } - auto data = make_uniq(); - data->action = "CREATE_MASK"; - data->or_replace = or_replace; - data->name = name; - data->value_type = vtype; - data->expression = expr; - return ParserExtensionParseResult(std::move(data)); - } - - // ---- ALTER TABLE … ---- - if (StringUtil::StartsWith(upper, "ALTER TABLE ")) { - auto rest = QuackapiTrim(q.substr(12)); - // table name - string table; - if (rest.empty()) { - return ParserExtensionParseResult(); - } - if (rest[0] == '"') { - // quoted - idx_t i = 1; - string result; - while (i < rest.size()) { - if (rest[i] == '"') { - if (i + 1 < rest.size() && rest[i + 1] == '"') { - result += '"'; - i += 2; - continue; - } - table = result; - rest = QuackapiTrim(rest.substr(i + 1)); - break; - } - result += rest[i]; - i++; - } - if (table.empty()) { - return ParserExtensionParseResult(); - } - } else { - idx_t i = 0; - while (i < rest.size() && IsIdentChar(rest[i])) { - i++; - } - if (i == 0) { - return ParserExtensionParseResult(); - } - table = rest.substr(0, i); - rest = QuackapiTrim(rest.substr(i)); - } - auto ru = StringUtil::Upper(rest); - - // ADD [OR REPLACE] ROW ACCESS POLICY

ON () - bool add_or_replace = false; - if (StringUtil::StartsWith(ru, "ADD OR REPLACE ROW ACCESS POLICY ")) { - rest = QuackapiTrim(rest.substr(33)); - add_or_replace = true; - } else if (StringUtil::StartsWith(ru, "ADD ROW ACCESS POLICY ")) { - rest = QuackapiTrim(rest.substr(22)); - } else if (StringUtil::StartsWith(ru, "DROP ROW ACCESS POLICY ")) { - auto pname = QuackapiTrim(rest.substr(23)); - if (pname.empty() || pname.find(' ') != string::npos) { - return ParserExtensionParseResult("ALTER TABLE … DROP ROW ACCESS POLICY expects a policy name"); - } - auto data = make_uniq(); - data->action = "UNBIND_ROW"; - data->table_name = table; - data->name = pname; - return ParserExtensionParseResult(std::move(data)); - } else if (StringUtil::StartsWith(ru, "MODIFY COLUMN ") || StringUtil::StartsWith(ru, "ALTER COLUMN ")) { - idx_t skip = StringUtil::StartsWith(ru, "MODIFY COLUMN ") ? 14 : 13; - rest = QuackapiTrim(rest.substr(skip)); - // column name - string col; - idx_t ci = 0; - if (rest.empty()) { - return ParserExtensionParseResult("ALTER TABLE … COLUMN expects a column name"); - } - if (rest[0] == '"') { - return ParserExtensionParseResult("Quoted column names in SET MASKING POLICY not yet supported"); - } - while (ci < rest.size() && IsIdentChar(rest[ci])) { - ci++; - } - col = rest.substr(0, ci); - rest = QuackapiTrim(rest.substr(ci)); - ru = StringUtil::Upper(rest); - if (StringUtil::StartsWith(ru, "SET MASKING POLICY ")) { - auto pname = QuackapiTrim(rest.substr(19)); - if (pname.empty() || pname.find(' ') != string::npos) { - return ParserExtensionParseResult("SET MASKING POLICY expects a policy name"); - } - auto data = make_uniq(); - data->action = "BIND_MASK"; - data->or_replace = true; // SET replaces - data->table_name = table; - data->column_name = col; - data->name = pname; - return ParserExtensionParseResult(std::move(data)); - } - if (StringUtil::StartsWith(ru, "UNSET MASKING POLICY") || - StringUtil::StartsWith(ru, "DROP MASKING POLICY")) { - auto data = make_uniq(); - data->action = "UNBIND_MASK"; - data->table_name = table; - data->column_name = col; - return ParserExtensionParseResult(std::move(data)); - } - // Not our ALTER COLUMN form — leave to core. - return ParserExtensionParseResult(); - } else { - // Not a policy ALTER — leave to DuckDB core. - return ParserExtensionParseResult(); - } - - // ADD ROW ACCESS POLICY path continues - idx_t pname_len = 0; - string pname = NextToken(rest, pname_len); - if (pname.empty()) { - return ParserExtensionParseResult("ADD ROW ACCESS POLICY expects a policy name"); - } - rest = pname_len >= rest.size() ? string() : QuackapiTrim(rest.substr(pname_len)); - ru = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(ru, "ON")) { - return ParserExtensionParseResult("Expected ON () after policy name"); - } - rest = QuackapiTrim(rest.substr(2)); - if (rest.empty() || rest[0] != '(') { - return ParserExtensionParseResult("Expected ON ()"); - } - string col_inner; - idx_t after = 0; - if (!ExtractParenGroup(rest, 0, col_inner, after)) { - return ParserExtensionParseResult("Unterminated ON (...) column list"); - } - vector cols, types; - string err; - if (!ParseColList(col_inner, cols, types, err)) { - return ParserExtensionParseResult("ON (...): " + err); - } - rest = QuackapiTrim(rest.substr(after)); - if (!rest.empty()) { - return ParserExtensionParseResult("Unexpected tokens after ON (...)"); - } - auto data = make_uniq(); - data->action = "BIND_ROW"; - data->or_replace = add_or_replace; - data->table_name = table; - data->name = pname; - data->arg_columns = std::move(cols); - return ParserExtensionParseResult(std::move(data)); - } - - return ParserExtensionParseResult(); -} - -//===--------------------------------------------------------------------===// -// Apply table function -//===--------------------------------------------------------------------===// - -struct ApplyPolicyBindData : public TableFunctionData { - string action; - bool or_replace = false; - string name; - string value_type; - string arg_columns_csv; // comma-separated - string arg_types_csv; - string expression; - string table_name; - string column_name; - bool finished = false; -}; - -vector SplitCsv(const string &csv) { - vector out; - if (csv.empty()) { - return out; - } - idx_t i = 0; - while (i < csv.size()) { - idx_t j = i; - while (j < csv.size() && csv[j] != ',') { - j++; - } - out.push_back(QuackapiTrim(csv.substr(i, j - i))); - i = j < csv.size() ? j + 1 : j; - } - return out; -} - -unique_ptr ApplyPolicyBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - bind_data->action = input.inputs[0].GetValue(); - bind_data->or_replace = input.inputs[1].GetValue(); - bind_data->name = input.inputs[2].GetValue(); - bind_data->value_type = input.inputs[3].GetValue(); - bind_data->arg_columns_csv = input.inputs[4].GetValue(); - bind_data->arg_types_csv = input.inputs[5].GetValue(); - bind_data->expression = input.inputs[6].GetValue(); - bind_data->table_name = input.inputs[7].GetValue(); - bind_data->column_name = input.inputs[8].GetValue(); - BindStatusColumn(return_types, names); - return std::move(bind_data); -} - -void ApplyPolicyExec(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &bind_data = data_p.bind_data->CastNoConst(); - if (bind_data.finished) { - return; - } - auto &state = QuackapiState::Get(*context.db); - string message; - auto cols = SplitCsv(bind_data.arg_columns_csv); - auto types = SplitCsv(bind_data.arg_types_csv); - - if (bind_data.action == "CREATE_ROW") { - QuackapiRowAccessPolicy p; - p.name = bind_data.name; - p.arg_columns = cols; - p.arg_types = types; - while (p.arg_types.size() < p.arg_columns.size()) { - p.arg_types.push_back(""); - } - p.expression = bind_data.expression; - state.AddRowAccessPolicy(p, bind_data.or_replace); - message = StringUtil::Format("Row access policy %s", p.name); - } else if (bind_data.action == "CREATE_MASK") { - QuackapiMaskingPolicy p; - p.name = bind_data.name; - p.value_type = bind_data.value_type; - p.expression = bind_data.expression; - state.AddMaskingPolicy(p, bind_data.or_replace); - message = StringUtil::Format("Masking policy %s ON %s", p.name, p.value_type); - } else if (bind_data.action == "DROP_ROW") { - if (!state.DropRowAccessPolicy(bind_data.name)) { - throw InvalidInputException("Row access policy \"%s\" does not exist", bind_data.name); - } - message = StringUtil::Format("Dropped row access policy %s", bind_data.name); - } else if (bind_data.action == "DROP_MASK") { - if (!state.DropMaskingPolicy(bind_data.name)) { - throw InvalidInputException("Masking policy \"%s\" does not exist", bind_data.name); - } - message = StringUtil::Format("Dropped masking policy %s", bind_data.name); - } else if (bind_data.action == "BIND_ROW") { - QuackapiRowAccessBinding b; - b.table_name = bind_data.table_name; - b.policy_name = bind_data.name; - b.columns = cols; - state.BindRowAccessPolicy(b, bind_data.or_replace); - message = StringUtil::Format("Bound row access policy %s on %s", b.policy_name, b.table_name); - } else if (bind_data.action == "UNBIND_ROW") { - if (!state.UnbindRowAccessPolicy(bind_data.table_name, bind_data.name)) { - throw InvalidInputException("Table \"%s\" has no row access policy \"%s\"", bind_data.table_name, - bind_data.name); - } - message = StringUtil::Format("Dropped row access policy %s from %s", bind_data.name, bind_data.table_name); - } else if (bind_data.action == "BIND_MASK") { - QuackapiMaskingBinding b; - b.table_name = bind_data.table_name; - b.column_name = bind_data.column_name; - b.policy_name = bind_data.name; - state.BindMaskingPolicy(b, bind_data.or_replace); - message = StringUtil::Format("Set masking policy %s on %s.%s", b.policy_name, b.table_name, b.column_name); - } else if (bind_data.action == "UNBIND_MASK") { - if (!state.UnbindMaskingPolicy(bind_data.table_name, bind_data.column_name)) { - throw InvalidInputException("Column \"%s\".\"%s\" has no masking policy", bind_data.table_name, - bind_data.column_name); - } - message = StringUtil::Format("Unset masking policy on %s.%s", bind_data.table_name, bind_data.column_name); - } else { - throw InvalidInputException("Unknown policy action \"%s\"", bind_data.action); - } - - EmitOneShotStatus(output, bind_data.finished, message); -} - -TableFunction MakeApplyPolicyFunction() { - // action, or_replace, name, value_type, arg_columns_csv, arg_types_csv, expression, table, column - return MakeApplyDdlFunction("quackapi_apply_policy", - {LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::VARCHAR}, - ApplyPolicyExec, ApplyPolicyBind); -} - -ParserExtensionPlanResult PolicyDdlPlan(ParserExtensionInfo *, ClientContext &, - unique_ptr parse_data) { - auto &data = static_cast(*parse_data); - ParserExtensionPlanResult result; - result.function = MakeApplyPolicyFunction(); - result.parameters.push_back(Value(data.action)); - result.parameters.push_back(Value::BOOLEAN(data.or_replace)); - result.parameters.push_back(Value(data.name)); - result.parameters.push_back(Value(data.value_type)); - result.parameters.push_back(Value(JoinComma(data.arg_columns))); - result.parameters.push_back(Value(JoinComma(data.arg_types))); - result.parameters.push_back(Value(data.expression)); - result.parameters.push_back(Value(data.table_name)); - result.parameters.push_back(Value(data.column_name)); - FinishDdlPlan(result); - return result; -} - -//===--------------------------------------------------------------------===// -// quackapi_policies() -//===--------------------------------------------------------------------===// - -struct PolicyRow { - string name; - string kind; - string signature; - string expression; - string bound_table; - string bound_columns; -}; - -struct PoliciesBindData : public TableFunctionData {}; - -struct PoliciesGlobalState : public GlobalTableFunctionState { - vector rows; - idx_t offset = 0; -}; - -unique_ptr PoliciesBind(ClientContext &, TableFunctionBindInput &, vector &return_types, - vector &names) { - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("name"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("kind"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("signature"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("expression"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("bound_table"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("bound_columns"); - return make_uniq(); -} - -unique_ptr PoliciesInit(ClientContext &context, TableFunctionInitInput &) { - auto st = make_uniq(); - auto &state = QuackapiState::Get(*context.db); - auto raps = state.SnapshotRowAccessPolicies(); - auto masks = state.SnapshotMaskingPolicies(); - auto rap_binds = state.SnapshotRowAccessBindings(); - auto mask_binds = state.SnapshotMaskingBindings(); - - for (auto &p : raps) { - string sig = "("; - for (idx_t i = 0; i < p.arg_columns.size(); i++) { - if (i > 0) { - sig += ", "; - } - sig += p.arg_columns[i]; - if (i < p.arg_types.size() && !p.arg_types[i].empty()) { - sig += " "; - sig += p.arg_types[i]; - } - } - sig += ") RETURNS BOOLEAN"; - bool any = false; - for (auto &b : rap_binds) { - if (b.policy_name == p.name) { - PolicyRow row; - row.name = p.name; - row.kind = "ROW_ACCESS"; - row.signature = sig; - row.expression = p.expression; - row.bound_table = b.table_name; - row.bound_columns = JoinComma(b.columns); - st->rows.push_back(std::move(row)); - any = true; - } - } - if (!any) { - PolicyRow row; - row.name = p.name; - row.kind = "ROW_ACCESS"; - row.signature = sig; - row.expression = p.expression; - st->rows.push_back(std::move(row)); - } - } - for (auto &p : masks) { - string sig = "ON " + p.value_type; - bool any = false; - for (auto &b : mask_binds) { - if (b.policy_name == p.name) { - PolicyRow row; - row.name = p.name; - row.kind = "MASKING"; - row.signature = sig; - row.expression = p.expression; - row.bound_table = b.table_name; - row.bound_columns = b.column_name; - st->rows.push_back(std::move(row)); - any = true; - } - } - if (!any) { - PolicyRow row; - row.name = p.name; - row.kind = "MASKING"; - row.signature = sig; - row.expression = p.expression; - st->rows.push_back(std::move(row)); - } - } - return std::move(st); -} - -void PoliciesExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.rows.size() && row < STANDARD_VECTOR_SIZE) { - auto &r = state.rows[state.offset]; - output.SetValue(0, row, Value(r.name)); - output.SetValue(1, row, Value(r.kind)); - output.SetValue(2, row, Value(r.signature)); - output.SetValue(3, row, Value(r.expression)); - output.SetValue(4, row, Value(r.bound_table)); - output.SetValue(5, row, Value(r.bound_columns)); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -} // namespace - -//===--------------------------------------------------------------------===// -// Enforcement rewrite -//===--------------------------------------------------------------------===// - -//! Parsed identity of a catalog relation. Policy DDL historically stores a -//! string name, so turn that name into the same AST representation used for a -//! route before comparing it to a route reference. -struct PolicyTableIdentity { - string catalog; - string schema; - string table; -}; - -string QuoteCatalogIdentifier(const string &ident) { - string out = "\""; - for (auto c : ident) { - if (c == '"') { - out += "\"\""; - } else { - out += c; - } - } - return out + "\""; -} - -bool SameCatalogIdentifier(const string &left, const string &right) { - return StringUtil::Lower(left) == StringUtil::Lower(right); -} - -string CurrentCatalog(Connection &con) { - auto result = con.Query("SELECT current_database()"); - if (result->HasError()) { - return string(); - } - auto chunk = result->Fetch(); - if (!chunk || chunk->size() == 0 || chunk->GetValue(0, 0).IsNull()) { - return string(); - } - return chunk->GetValue(0, 0).ToString(); -} - -PolicyTableIdentity IdentityFromBaseRef(const BaseTableRef &ref, const string &default_catalog) { - PolicyTableIdentity identity; - identity.catalog = ref.catalog_name.empty() ? default_catalog : ref.catalog_name; - identity.schema = ref.schema_name.empty() ? "main" : ref.schema_name; - identity.table = ref.table_name; - return identity; -} - -bool ParsePolicyTableIdentity(const string &table_name, const string &default_catalog, PolicyTableIdentity &identity) { - try { - Parser parser; - parser.ParseQuery("SELECT * FROM " + table_name); - if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { - return false; - } - auto &select = parser.statements[0]->Cast(); - if (!select.node || select.node->type != QueryNodeType::SELECT_NODE) { - return false; - } - auto &node = select.node->Cast(); - if (!node.from_table || node.from_table->type != TableReferenceType::BASE_TABLE) { - return false; - } - identity = IdentityFromBaseRef(node.from_table->Cast(), default_catalog); - return !identity.table.empty(); - } catch (...) { - return false; - } -} - -bool SamePolicyTableIdentity(const PolicyTableIdentity &left, const PolicyTableIdentity &right) { - return SameCatalogIdentifier(left.catalog, right.catalog) && SameCatalogIdentifier(left.schema, right.schema) && - SameCatalogIdentifier(left.table, right.table); -} - -string QuotePolicyTableIdentity(const PolicyTableIdentity &identity) { - return QuoteCatalogIdentifier(identity.catalog) + "." + QuoteCatalogIdentifier(identity.schema) + "." + - QuoteCatalogIdentifier(identity.table); -} - -//! Replace a policy signature argument without touching SQL string literals. -string ReplacePolicyIdentifier(const string &expr, const string &from, const string &to) { - string out; - bool in_str = false; - for (idx_t i = 0; i < expr.size();) { - if (in_str) { - out += expr[i]; - if (expr[i] == '\'') { - if (i + 1 < expr.size() && expr[i + 1] == '\'') { - out += expr[i + 1]; - i += 2; - continue; - } - in_str = false; - } - i++; - continue; - } - if (expr[i] == '\'') { - in_str = true; - out += expr[i++]; - continue; - } - if (IsIdentStart(expr[i])) { - idx_t end = i + 1; - while (end < expr.size() && IsIdentChar(expr[end])) { - end++; - } - auto token = expr.substr(i, end - i); - out += SameCatalogIdentifier(token, from) ? to : token; - i = end; - continue; - } - out += expr[i++]; - } - return out; -} - -struct PolicyRewriteContext { - DatabaseInstance &db; - Connection con; - const vector &rap_bindings; - const vector &mask_bindings; - bool authenticated; - string default_catalog; - bool deny_unauthenticated = false; - string error; - unordered_map view_cache; - - PolicyRewriteContext(DatabaseInstance &db_p, const vector &rap_bindings_p, - const vector &mask_bindings_p, bool authenticated_p) - : db(db_p), con(db_p), rap_bindings(rap_bindings_p), mask_bindings(mask_bindings_p), - authenticated(authenticated_p), default_catalog(CurrentCatalog(con)) { - } - - void Reject(const string &reason) { - if (error.empty()) { - error = reason; - } - if (!authenticated) { - deny_unauthenticated = true; - } - } -}; - -bool IsCatalogView(PolicyRewriteContext &ctx, const PolicyTableIdentity &identity, bool &is_view) { - auto key = StringUtil::Lower(identity.catalog) + "\x1f" + StringUtil::Lower(identity.schema) + "\x1f" + - StringUtil::Lower(identity.table); - auto cached = ctx.view_cache.find(key); - if (cached != ctx.view_cache.end()) { - is_view = cached->second; - return true; - } - auto result = ctx.con.Query( - "SELECT 1 FROM duckdb_views() WHERE database_name = ? AND schema_name = ? AND view_name = ? LIMIT 1", - Value(identity.catalog), Value(identity.schema), Value(identity.table)); - if (result->HasError()) { - return false; - } - auto chunk = result->Fetch(); - is_view = chunk && chunk->size() > 0; - ctx.view_cache.emplace(std::move(key), is_view); - return true; -} - -bool ParseSecureSubquery(const string &sql, unique_ptr &out) { - try { - Parser parser; - parser.ParseQuery(sql); - if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { - return false; - } - out.reset(static_cast(parser.statements[0].release())); - return true; - } catch (...) { - return false; - } -} - -bool RewritePolicyQueryNode(QueryNode &node, PolicyRewriteContext &ctx); -bool RewritePolicySelect(SelectStatement &statement, PolicyRewriteContext &ctx); - -//! Policy-bearing relations can occur in scalar/EXISTS/IN subqueries inside a -//! SELECT list, predicate, join condition, or modifier — not only in FROM. -//! Walk parsed expressions recursively so those reads receive the same AST -//! rewrite as top-level relation sources. -bool RewritePolicyExpression(ParsedExpression &expression, PolicyRewriteContext &ctx) { - if (!ctx.error.empty()) { - return false; - } - try { - if (expression.GetExpressionClass() == ExpressionClass::SUBQUERY) { - auto &subquery = expression.Cast(); - if (!subquery.subquery || !RewritePolicySelect(*subquery.subquery, ctx)) { - if (ctx.error.empty()) { - ctx.Reject("policy enforcement could not inspect a nested subquery"); - } - return false; - } - } - bool rewritten = true; - ParsedExpressionIterator::EnumerateChildren(expression, [&](ParsedExpression &child) { - if (rewritten && !RewritePolicyExpression(child, ctx)) { - rewritten = false; - } - }); - return rewritten && ctx.error.empty(); - } catch (...) { - ctx.Reject("policy enforcement could not inspect a nested expression"); - return false; - } -} - -bool RewritePolicyModifiers(QueryNode &node, PolicyRewriteContext &ctx) { - bool rewritten = true; - try { - ParsedExpressionIterator::EnumerateQueryNodeModifiers(node, [&](unique_ptr &expression) { - if (rewritten && expression && !RewritePolicyExpression(*expression, ctx)) { - rewritten = false; - } - }); - } catch (...) { - ctx.Reject("policy enforcement could not inspect a query modifier"); - return false; - } - return rewritten && ctx.error.empty(); -} - -bool RewritePolicySelect(SelectStatement &statement, PolicyRewriteContext &ctx) { - if (!statement.node) { - ctx.Reject("policy enforcement could not inspect an empty SELECT statement"); - return false; - } - return RewritePolicyQueryNode(*statement.node, ctx); -} - -bool RewritePolicyTableRef(unique_ptr &ref, PolicyRewriteContext &ctx) { - if (!ref || !ctx.error.empty()) { - return false; - } - switch (ref->type) { - case TableReferenceType::BASE_TABLE: { - auto &base = ref->Cast(); - auto identity = IdentityFromBaseRef(base, ctx.default_catalog); - bool is_view = false; - if (!IsCatalogView(ctx, identity, is_view)) { - ctx.Reject("policy enforcement could not resolve catalog object identity"); - return false; - } - if (is_view) { - // A view can hide a protected relation. Do not infer dependencies from - // view SQL text: reject the indirect source until it has a bound rewrite. - ctx.Reject("policy enforcement rejects indirect view reads"); - return false; - } - - const QuackapiRowAccessBinding *rap_binding = nullptr; - const QuackapiMaskingBinding *first_mask = nullptr; - for (auto &binding : ctx.rap_bindings) { - PolicyTableIdentity bound; - if (!ParsePolicyTableIdentity(binding.table_name, ctx.default_catalog, bound)) { - ctx.Reject("policy binding has an invalid catalog identity"); - return false; - } - if (SamePolicyTableIdentity(identity, bound)) { - rap_binding = &binding; - break; - } - } - for (auto &binding : ctx.mask_bindings) { - PolicyTableIdentity bound; - if (!ParsePolicyTableIdentity(binding.table_name, ctx.default_catalog, bound)) { - ctx.Reject("policy binding has an invalid catalog identity"); - return false; - } - if (SamePolicyTableIdentity(identity, bound)) { - first_mask = &binding; - break; - } - } - if (!rap_binding && !first_mask) { - return true; - } - if (!ctx.authenticated) { - ctx.deny_unauthenticated = true; - return false; - } - - auto &state = QuackapiState::Get(ctx.db); - QuackapiRowAccessPolicy rap_storage; - const QuackapiRowAccessPolicy *rap = nullptr; - string where_expr; - if (rap_binding) { - if (!state.GetRowAccessPolicy(rap_binding->policy_name, rap_storage)) { - ctx.Reject("row-access policy binding references a missing policy"); - return false; - } - where_expr = rap_storage.expression; - for (idx_t i = 0; i < rap_storage.arg_columns.size() && i < rap_binding->columns.size(); i++) { - if (!SameCatalogIdentifier(rap_storage.arg_columns[i], rap_binding->columns[i])) { - where_expr = - ReplacePolicyIdentifier(where_expr, rap_storage.arg_columns[i], rap_binding->columns[i]); - } - } - rap = &rap_storage; - } - - vector> masked; - for (auto &binding : ctx.mask_bindings) { - PolicyTableIdentity bound; - if (!ParsePolicyTableIdentity(binding.table_name, ctx.default_catalog, bound)) { - ctx.Reject("policy binding has an invalid catalog identity"); - return false; - } - if (!SamePolicyTableIdentity(identity, bound)) { - continue; - } - QuackapiMaskingPolicy mask; - if (!state.GetMaskingPolicy(binding.policy_name, mask)) { - ctx.Reject("masking policy binding references a missing policy"); - return false; - } - masked.emplace_back(binding.column_name, - SubstituteValPlaceholder(mask.expression, QuoteCatalogIdentifier(binding.column_name))); - } - - unique_ptr secure_query; - if (!ParseSecureSubquery(BuildSecureSubqueryQuoted(QuotePolicyTableIdentity(identity), rap, where_expr, masked), - secure_query)) { - ctx.Reject("policy enforcement could not construct a secure subquery"); - return false; - } - string alias = ref->alias.empty() ? base.table_name : ref->alias; - auto replacement = make_uniq(std::move(secure_query), alias); - ref->CopyProperties(*replacement); - replacement->alias = alias; - ref = std::move(replacement); - return true; - } - case TableReferenceType::SUBQUERY: - return RewritePolicySelect(*ref->Cast().subquery, ctx); - case TableReferenceType::JOIN: { - auto &join = ref->Cast(); - if (!RewritePolicyTableRef(join.left, ctx) || !RewritePolicyTableRef(join.right, ctx)) { - return false; - } - return !join.condition || RewritePolicyExpression(*join.condition, ctx); - } - case TableReferenceType::PIVOT: { - auto &pivot = ref->Cast(); - if (!RewritePolicyTableRef(pivot.source, ctx)) { - return false; - } - for (auto &aggregate : pivot.aggregates) { - if (aggregate && !RewritePolicyExpression(*aggregate, ctx)) { - return false; - } - } - return true; - } - case TableReferenceType::TABLE_FUNCTION: - ctx.Reject("policy enforcement rejects table functions and table macros"); - return false; - case TableReferenceType::CTE: - case TableReferenceType::EMPTY_FROM: - return true; - case TableReferenceType::EXPRESSION_LIST: { - auto &values = ref->Cast(); - for (auto &row : values.values) { - for (auto &expression : row) { - if (expression && !RewritePolicyExpression(*expression, ctx)) { - return false; - } - } - } - return true; - } - default: - ctx.Reject("policy enforcement rejects an unsupported relation source"); - return false; - } -} - -bool RewritePolicyQueryNode(QueryNode &node, PolicyRewriteContext &ctx) { - for (auto &entry : node.cte_map.map) { - if (!entry.second || !entry.second->query || !RewritePolicySelect(*entry.second->query, ctx)) { - ctx.Reject("policy enforcement could not inspect a common table expression"); - return false; - } - } - switch (node.type) { - case QueryNodeType::SELECT_NODE: { - auto &select = node.Cast(); - if (select.from_table && !RewritePolicyTableRef(select.from_table, ctx)) { - return false; - } - for (auto &expression : select.select_list) { - if (expression && !RewritePolicyExpression(*expression, ctx)) { - return false; - } - } - for (auto &expression : select.groups.group_expressions) { - if (expression && !RewritePolicyExpression(*expression, ctx)) { - return false; - } - } - if ((select.where_clause && !RewritePolicyExpression(*select.where_clause, ctx)) || - (select.having && !RewritePolicyExpression(*select.having, ctx)) || - (select.qualify && !RewritePolicyExpression(*select.qualify, ctx))) { - return false; - } - return RewritePolicyModifiers(node, ctx); - } - case QueryNodeType::SET_OPERATION_NODE: { - auto &setop = node.Cast(); - for (auto &child : setop.children) { - if (!child || !RewritePolicyQueryNode(*child, ctx)) { - return false; - } - } - return RewritePolicyModifiers(node, ctx); - } - default: - ctx.Reject("policy enforcement rejects an unsupported query form"); - return false; - } -} - -//! Parse and rewrite every supported SELECT shape. Callers receive an explicit -//! error when a protected relation cannot be inspected safely. -bool RewriteHandlerWithPoliciesAst(DatabaseInstance &db, const string &handler_sql, bool authenticated, - bool &deny_unauthenticated, string &policy_error, string &rewritten_sql) { - deny_unauthenticated = false; - policy_error.clear(); - auto &state = QuackapiState::Get(db); - auto rap_bindings = state.SnapshotRowAccessBindings(); - auto mask_bindings = state.SnapshotMaskingBindings(); - if (rap_bindings.empty() && mask_bindings.empty()) { - rewritten_sql = handler_sql; - return true; - } - PolicyRewriteContext ctx(db, rap_bindings, mask_bindings, authenticated); - try { - Parser parser; - parser.ParseQuery(handler_sql); - if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { - ctx.Reject("policy enforcement supports one SELECT statement per handler"); - } else { - auto &select = parser.statements[0]->Cast(); - RewritePolicySelect(select, ctx); - if (ctx.error.empty() && !ctx.deny_unauthenticated) { - rewritten_sql = select.ToString(); - } - } - } catch (...) { - ctx.Reject("policy enforcement could not parse handler SQL"); - } - deny_unauthenticated = ctx.deny_unauthenticated; - policy_error = ctx.error; - if (!policy_error.empty() && !authenticated) { - deny_unauthenticated = true; - } - if (policy_error.empty() && !deny_unauthenticated) { - return true; - } - rewritten_sql = handler_sql; - return true; -} - -bool HandlerTouchesPoliciedTable(DatabaseInstance &db, const string &handler_sql) { - bool deny_unauthenticated = false; - string policy_error; - string rewritten; - RewriteHandlerWithPoliciesAst(db, handler_sql, false, deny_unauthenticated, policy_error, rewritten); - return deny_unauthenticated || !policy_error.empty(); -} - -string RewriteHandlerWithPolicies(DatabaseInstance &db, const string &handler_sql, bool authenticated, - bool &deny_unauthenticated, string &policy_error) { - string parsed_rewrite; - RewriteHandlerWithPoliciesAst(db, handler_sql, authenticated, deny_unauthenticated, policy_error, parsed_rewrite); - return parsed_rewrite; -} - -PolicyDdlParserExtension::PolicyDdlParserExtension() { - parse_function = PolicyDdlParse; - plan_function = PolicyDdlPlan; -} - -TableFunction GetApplyPolicyFunction() { - return MakeApplyPolicyFunction(); -} - -TableFunction GetQuackapiPoliciesFunction() { - return TableFunction("quackapi_policies", {}, PoliciesExec, PoliciesBind, PoliciesInit); -} - -} // namespace duckdb diff --git a/src/quackapi_post.cpp b/src/quackapi_post.cpp new file mode 100644 index 0000000..a0f6862 --- /dev/null +++ b/src/quackapi_post.cpp @@ -0,0 +1,257 @@ +#include "quackapi_post.hpp" +#include "quackapi_post_executor.hpp" +#include "quackapi_post_provider.hpp" +#include "quackapi_post_result.hpp" + +#include "duckdb/function/scalar_function.hpp" +#include "duckdb/function/table_function.hpp" +#include "duckdb/main/config.hpp" +#include "duckdb/main/database.hpp" + +namespace duckdb { + +namespace { + +struct QuackapiPostTableBindData : public TableFunctionData { + idx_t url_idx; + idx_t body_idx; + optional_idx content_type_idx; + optional_idx headers_idx; + vector input_names; + vector input_types; + + unique_ptr Copy() const override { + return make_uniq(*this); + } +}; + +struct QuackapiPostTableGlobalState : public GlobalTableFunctionState { + idx_t MaxThreads() const override { + return 1; + } +}; + +struct QuackapiPostTableLocalState : public LocalTableFunctionState { + idx_t input_offset = 0; +}; + +bool InitializePostRequest(const Value &url, const Value &body, const Value &content_type, const Value &headers, + QuackapiPostRequest &request) { + if (url.IsNull()) { + return false; + } + request.url = url.ToString(); + request.body = body.IsNull() ? string() : body.ToString(); + request.content_type = content_type.IsNull() ? "application/json" : content_type.ToString(); + request.headers = QuackapiHttpHeadersFromValue(headers); + return true; +} + +unique_ptr BindPostTable(ClientContext &, TableFunctionBindInput &input, + vector &return_types, vector &names) { + if (input.input_table_names.size() != input.input_table_types.size()) { + throw BinderException("quackapi_post(table) could not resolve the input table columns"); + } + auto bind_data = make_uniq(); + bind_data->input_names = input.input_table_names; + bind_data->input_types = input.input_table_types; + optional_idx url_idx; + optional_idx body_idx; + for (idx_t i = 0; i < input.input_table_names.size(); i++) { + const auto name = StringUtil::Lower(input.input_table_names[i]); + auto set_column = [&](optional_idx &column_idx) { + if (column_idx.IsValid()) { + throw BinderException( + "quackapi_post(table) requires unique input columns named url, body, content_type, and headers"); + } + column_idx = i; + }; + if (name == "url") { + set_column(url_idx); + } else if (name == "body") { + set_column(body_idx); + } else if (name == "content_type") { + set_column(bind_data->content_type_idx); + } else if (name == "headers") { + set_column(bind_data->headers_idx); + } else if (name == "response") { + throw BinderException( + "quackapi_post(table) cannot append response: the input already has a response column"); + } + } + if (!url_idx.IsValid() || !body_idx.IsValid()) { + throw BinderException("quackapi_post(table) requires input columns named url and body"); + } + if (input.input_table_types[url_idx.GetIndex()] != LogicalType::VARCHAR) { + throw BinderException("quackapi_post(table) requires url to be VARCHAR"); + } + const auto body_type = input.input_table_types[body_idx.GetIndex()]; + if (body_type != LogicalType::VARCHAR && body_type != LogicalType::JSON()) { + throw BinderException("quackapi_post(table) requires body to be VARCHAR or JSON"); + } + if (bind_data->content_type_idx.IsValid() && + input.input_table_types[bind_data->content_type_idx.GetIndex()] != LogicalType::VARCHAR) { + throw BinderException("quackapi_post(table) requires content_type to be VARCHAR"); + } + const auto headers_type = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); + if (bind_data->headers_idx.IsValid() && + input.input_table_types[bind_data->headers_idx.GetIndex()] != headers_type) { + throw BinderException("quackapi_post(table) requires headers to be MAP(VARCHAR, VARCHAR)"); + } + bind_data->url_idx = url_idx.GetIndex(); + bind_data->body_idx = body_idx.GetIndex(); + return_types = bind_data->input_types; + names = bind_data->input_names; + return_types.emplace_back(QuackapiHttpFetchResultType()); + names.emplace_back("response"); + return bind_data; +} + +unique_ptr InitPostTableGlobal(ClientContext &, TableFunctionInitInput &) { + return make_uniq(); +} + +unique_ptr InitPostTableLocal(ExecutionContext &, TableFunctionInitInput &, + GlobalTableFunctionState *) { + return make_uniq(); +} + +OperatorResultType PostTableInOut(ExecutionContext &context, TableFunctionInput &data, DataChunk &input, + DataChunk &output) { + if (context.client.IsInterrupted()) { + throw InterruptException(); + } + if (input.size() == 0) { + output.SetCardinality(0); + return OperatorResultType::NEED_MORE_INPUT; + } + auto &bind_data = data.bind_data->Cast(); + auto &local_state = data.local_state->Cast(); + const auto concurrency = QuackapiPostExecutor::EffectiveConcurrency(context.client); + const auto batch_size = MinValue(concurrency, input.size() - local_state.input_offset); + vector requests(batch_size); + vector skip(batch_size, false); + const Value default_content_type(LogicalType::VARCHAR); + const Value default_headers(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + for (idx_t i = 0; i < batch_size; i++) { + const auto row = local_state.input_offset + i; + const auto url = input.data[bind_data.url_idx].GetValue(row); + const auto body = input.data[bind_data.body_idx].GetValue(row); + Value content_type = default_content_type; + if (bind_data.content_type_idx.IsValid()) { + content_type = input.data[bind_data.content_type_idx.GetIndex()].GetValue(row); + } + Value headers = default_headers; + if (bind_data.headers_idx.IsValid()) { + headers = input.data[bind_data.headers_idx.GetIndex()].GetValue(row); + } + skip[i] = !InitializePostRequest(url, body, content_type, headers, requests[i]); + } + auto responses = QuackapiPostExecutor::Execute(context.client, std::move(requests), skip); + const auto result_column = output.ColumnCount() - 1; + const auto result_type = QuackapiHttpFetchResultType(); + SelectionVector selection(batch_size); + for (idx_t i = 0; i < batch_size; i++) { + selection.set_index(i, local_state.input_offset + i); + } + for (idx_t i = 0; i < batch_size; i++) { + if (skip[i]) { + output.data[result_column].SetValue(i, Value(result_type)); + } else { + auto response = std::move(responses[i]); + output.data[result_column].SetValue(i, QuackapiHttpFetchResultValue(response)); + } + } + for (idx_t column = 0; column < input.ColumnCount(); column++) { + output.data[column].Slice(input.data[column], selection, batch_size); + } + output.SetCardinality(batch_size); + local_state.input_offset += batch_size; + if (local_state.input_offset < input.size()) { + return OperatorResultType::HAVE_MORE_OUTPUT; + } + local_state.input_offset = 0; + return OperatorResultType::NEED_MORE_INPUT; +} + +void ValidatePostConcurrency(ClientContext &, SetScope, Value &value) { + if (value.GetValue() < 0) { + throw InvalidInputException("quackapi_post_concurrency must be 0 or a positive integer"); + } +} + +void PostScalar(DataChunk &args, ExpressionState &state, Vector &result) { + auto &context = state.GetContext(); + const auto count = args.size(); + result.SetVectorType(VectorType::FLAT_VECTOR); + + vector skip(count, false); + vector requests(count); + const Value default_content_type(LogicalType::VARCHAR); + const Value default_headers(LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + for (idx_t i = 0; i < count; i++) { + const auto url = args.data[0].GetValue(i); + const auto body = args.data[1].GetValue(i); + Value content_type = default_content_type; + if (args.ColumnCount() > 2) { + content_type = args.data[2].GetValue(i); + } + Value headers = default_headers; + if (args.ColumnCount() > 3) { + headers = args.data[3].GetValue(i); + } + skip[i] = !InitializePostRequest(url, body, content_type, headers, requests[i]); + } + + auto responses = QuackapiPostExecutor::Execute(context, std::move(requests), skip); + + for (idx_t i = 0; i < count; i++) { + if (skip[i]) { + result.SetValue(i, Value(QuackapiHttpFetchResultType())); + continue; + } + // Move each response out of the batch before copying it into the output + // vector. Keeping the whole responses vector alive here duplicates every + // body while the result vector is being populated. + auto response = std::move(responses[i]); + result.SetValue(i, QuackapiHttpFetchResultValue(response)); + } +} + +} // namespace + +void RegisterQuackapiPostFunctions(ExtensionLoader &loader) { + auto &config = DBConfig::GetConfig(loader.GetDatabaseInstance()); + config.AddExtensionOption("quackapi_post_concurrency", + "Maximum concurrent outbound quackapi_post requests for this database. " + "0 uses the database thread count (minimum 1).", + LogicalType::BIGINT, Value::BIGINT(0), ValidatePostConcurrency, SetScope::GLOBAL); + const auto header_type = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); + const auto result_type = QuackapiHttpFetchResultType(); + + // POST changes remote state, so all overloads must stay volatile. + ScalarFunctionSet post_set("quackapi_post"); + ScalarFunction post2("quackapi_post", {LogicalType::VARCHAR, LogicalType::VARCHAR}, result_type, PostScalar); + post2.stability = FunctionStability::VOLATILE; + ScalarFunction post2j("quackapi_post", {LogicalType::VARCHAR, LogicalType::JSON()}, result_type, PostScalar); + post2j.stability = FunctionStability::VOLATILE; + ScalarFunction post3("quackapi_post", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, + result_type, PostScalar); + post3.stability = FunctionStability::VOLATILE; + ScalarFunction post4("quackapi_post", + {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, header_type}, result_type, + PostScalar); + post4.stability = FunctionStability::VOLATILE; + post_set.AddFunction(post2); + post_set.AddFunction(post2j); + post_set.AddFunction(post3); + post_set.AddFunction(post4); + loader.RegisterFunction(post_set); + + TableFunction post_table("quackapi_post", {LogicalType::TABLE}, nullptr, BindPostTable, InitPostTableGlobal, + InitPostTableLocal); + post_table.in_out_function = PostTableInOut; + loader.RegisterFunction(post_table); +} + +} // namespace duckdb diff --git a/src/quackapi_post_executor.cpp b/src/quackapi_post_executor.cpp new file mode 100644 index 0000000..8166427 --- /dev/null +++ b/src/quackapi_post_executor.cpp @@ -0,0 +1,358 @@ +#include "quackapi_post_executor.hpp" + +#include "quackapi_limits.hpp" +#include "quackapi_post_provider.hpp" + +#include "duckdb/common/exception.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/main/database.hpp" + +#include +#include +#include +#include +#include + +namespace duckdb { + +namespace { + +constexpr auto POST_CONCURRENCY_SETTING = "quackapi_post_concurrency"; + +struct PostBatch { + vector results; + vector>> cancellation_flags; + idx_t completed = 0; + idx_t accepted = 0; + std::atomic cancelled {false}; + std::atomic interrupted {false}; + std::chrono::steady_clock::time_point deadline; + std::mutex mutex; + std::condition_variable cv; +}; + +struct PostJob { + shared_ptr batch; + idx_t row; + QuackapiPostRequest request; +}; + +bool IsExpired(std::chrono::steady_clock::time_point deadline) { + return deadline != std::chrono::steady_clock::time_point::max() && std::chrono::steady_clock::now() >= deadline; +} + +void CheckCaller(ClientContext &context, std::chrono::steady_clock::time_point deadline) { + if (context.IsInterrupted() || IsExpired(deadline)) { + throw InterruptException(); + } +} + +idx_t EffectiveConcurrencyFor(DatabaseInstance &db) { + Value setting; + auto found = db.TryGetCurrentSetting(POST_CONCURRENCY_SETTING, setting); + if (found) { + const auto configured = setting.GetValue(); + if (configured > 0) { + return NumericCast(configured); + } + } + return MaxValue(1, db.NumberOfThreads()); +} + +idx_t WaitingQueueCapacity(idx_t concurrency) { + return concurrency > NumericLimits::Maximum() / 2 ? NumericLimits::Maximum() : concurrency * 2; +} + +} // namespace + +class QuackapiPostExecutor::Impl { +public: + struct Worker { + std::thread thread; + shared_ptr active_job; + }; + + explicit Impl(DatabaseInstance &db_p) : db(db_p) { + } + + ~Impl() { + { + std::lock_guard guard(lock); + stopping = true; + for (auto &job : queue) { + job->request.cancelled->store(true); + job->batch->cancelled.store(true); + job->batch->interrupted.store(true); + std::lock_guard batch_guard(job->batch->mutex); + job->batch->completed++; + job->batch->cv.notify_all(); + } + queue.clear(); + for (auto &worker : workers) { + if (worker->active_job) { + auto &job = *worker->active_job; + job.request.cancelled->store(true); + job.batch->cancelled.store(true); + job.batch->interrupted.store(true); + job.batch->cv.notify_all(); + } + } + } + queue_cv.notify_all(); + space_cv.notify_all(); + for (auto &worker : workers) { + if (worker->thread.joinable()) { + worker->thread.join(); + } + } + } + + vector Execute(ClientContext &context, vector requests, + const vector &skip) { + const auto deadline = QuackapiCurrentDeadline(context); + auto batch = make_shared_ptr(); + batch->results.resize(requests.size()); + batch->cancellation_flags.resize(requests.size()); + batch->deadline = deadline; + for (idx_t i = 0; i < requests.size(); i++) { + if (!skip[i]) { + requests[i].cancelled = make_shared_ptr>(false); + batch->cancellation_flags[i] = requests[i].cancelled; + } + } + try { + for (idx_t i = 0; i < requests.size(); i++) { + if (skip[i]) { + continue; + } + CheckCaller(context, deadline); + auto job = make_shared_ptr(); + job->batch = batch; + job->row = i; + job->request = std::move(requests[i]); + + std::unique_lock guard(lock); + while (!stopping && queue.size() >= WaitingQueueCapacity(EffectiveConcurrencyFor(db))) { + CheckCaller(context, deadline); + space_cv.wait_for(guard, std::chrono::milliseconds(10)); + } + CheckCaller(context, deadline); + if (stopping || batch->cancelled.load()) { + throw InterruptException(); + } + queue.push_back(std::move(job)); + { + std::lock_guard batch_guard(batch->mutex); + batch->accepted++; + } + const auto concurrency = EffectiveConcurrencyFor(db); + EnsureWorkers(std::min(concurrency, active + queue.size())); + queue_cv.notify_one(); + } + } catch (InterruptException &) { + CancelBatch(batch); + throw; + } catch (...) { + auto error = std::current_exception(); + CancelBatch(batch); + std::rethrow_exception(error); + } + + const auto expected = Accepted(batch); + while (true) { + { + std::lock_guard guard(batch->mutex); + if (batch->completed >= expected) { + break; + } + } + if (context.IsInterrupted() || IsExpired(deadline) || batch->interrupted.load()) { + CancelBatch(batch); + throw InterruptException(); + } + std::unique_lock guard(batch->mutex); + batch->cv.wait_for(guard, std::chrono::milliseconds(10)); + } + if (batch->interrupted.load() || context.IsInterrupted() || IsExpired(deadline)) { + throw InterruptException(); + } + return std::move(batch->results); + } + +private: + static idx_t Accepted(const shared_ptr &batch) { + std::lock_guard guard(batch->mutex); + return batch->accepted; + } + + void EnsureWorkers(idx_t desired) { + while (workers.size() < desired) { + workers.emplace_back(make_uniq()); + auto *worker = workers.back().get(); + try { + worker->thread = std::thread([this, worker]() { WorkerLoop(*worker); }); + } catch (...) { + workers.pop_back(); + throw; + } + } + } + + void CancelQueued(const shared_ptr &batch) { + std::lock_guard guard(lock); + for (auto it = queue.begin(); it != queue.end();) { + if ((*it)->batch == batch) { + (*it)->request.cancelled->store(true); + { + std::lock_guard batch_guard(batch->mutex); + batch->completed++; + } + it = queue.erase(it); + } else { + ++it; + } + } + space_cv.notify_all(); + queue_cv.notify_all(); + batch->cv.notify_all(); + } + + void CancelBatch(const shared_ptr &batch) { + batch->cancelled.store(true); + for (const auto &flag : batch->cancellation_flags) { + if (flag) { + flag->store(true); + } + } + // Queued jobs are removed below. Active jobs retain their shared batch and + // owned request until provider I/O returns, so the SQL caller need not drain them. + // An in-flight POST can still complete remotely; DB teardown joins all workers. + CancelQueued(batch); + } + + void WorkerLoop(Worker &worker) { + unique_ptr state; + try { + state = QuackapiPostProvider::CreateWorkerState(db); + } catch (...) { + // Each task receives a concrete error below if worker state setup failed. + } + for (;;) { + shared_ptr job; + { + std::unique_lock guard(lock); + for (;;) { + if (stopping && queue.empty()) { + break; + } + if (!stopping && !queue.empty()) { + const auto desired = std::min(EffectiveConcurrencyFor(db), active + queue.size()); + try { + EnsureWorkers(desired); + } catch (...) { + // Existing workers can still drain the queue if growth hits a system limit. + } + } + if (!queue.empty() && (stopping || active < EffectiveConcurrencyFor(db))) { + break; + } + queue_cv.wait_for(guard, std::chrono::milliseconds(10)); + } + if (stopping && queue.empty()) { + break; + } + job = std::move(queue.front()); + queue.pop_front(); + active++; + worker.active_job = job; + space_cv.notify_one(); + } + + auto &batch = *job->batch; + bool skip_job; + { + std::lock_guard guard(batch.mutex); + skip_job = batch.cancelled.load() || job->request.cancelled->load() || IsExpired(batch.deadline); + } + QuackapiHttpFetchResult response; + if (!skip_job) { + const auto old_deadline = QuackapiCurrentDeadline(); + QuackapiSetCurrentDeadline(batch.deadline); + try { + if (!state) { + throw IOException("failed to initialize outbound POST worker"); + } + if (job->request.cancelled->load() || IsExpired(batch.deadline)) { + throw InterruptException(); + } + response = QuackapiPostProvider::Post(db, *state, job->request); + } catch (InterruptException &) { + batch.cancelled.store(true); + batch.interrupted.store(true); + } catch (std::exception &e) { + response.request_error = e.what(); + } catch (...) { + response.request_error = "unknown error during outbound HTTP request"; + } + QuackapiSetCurrentDeadline(old_deadline); + } + { + std::lock_guard guard(batch.mutex); + if (!skip_job && !batch.cancelled.load()) { + batch.results[job->row] = std::move(response); + } + batch.completed++; + } + { + std::lock_guard guard(lock); + active--; + worker.active_job.reset(); + } + queue_cv.notify_one(); + batch.cv.notify_all(); + if (batch.cancelled.load()) { + CancelBatch(job->batch); + } + } + state.reset(); + } + + DatabaseInstance &db; + std::mutex lock; + std::condition_variable queue_cv; + std::condition_variable space_cv; + std::deque> queue; + vector> workers; + idx_t active = 0; + bool stopping = false; +}; + +QuackapiPostExecutor::QuackapiPostExecutor(DatabaseInstance &db) : impl(make_uniq(db)) { +} + +QuackapiPostExecutor::~QuackapiPostExecutor() = default; + +string QuackapiPostExecutor::ObjectType() { + return "quackapi_post_executor"; +} + +string QuackapiPostExecutor::GetObjectType() { + return ObjectType(); +} + +optional_idx QuackapiPostExecutor::GetEstimatedCacheMemory() const { + return optional_idx(); +} + +vector +QuackapiPostExecutor::Execute(ClientContext &context, vector requests, const vector &skip) { + auto &db = DatabaseInstance::GetDatabase(context); + auto executor = db.GetObjectCache().GetOrCreateWithTypePrefix("database", db); + return executor->impl->Execute(context, std::move(requests), skip); +} + +idx_t QuackapiPostExecutor::EffectiveConcurrency(ClientContext &context) { + return EffectiveConcurrencyFor(DatabaseInstance::GetDatabase(context)); +} + +} // namespace duckdb diff --git a/src/quackapi_post_provider.cpp b/src/quackapi_post_provider.cpp new file mode 100644 index 0000000..775a18d --- /dev/null +++ b/src/quackapi_post_provider.cpp @@ -0,0 +1,171 @@ +#include "quackapi_post_provider.hpp" + +#include "quackapi_post_executor.hpp" +#include "quackapi_limits.hpp" + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/string_util.hpp" + +namespace duckdb { + +namespace { + +constexpr int64_t OUTBOUND_TIMEOUT_FALLBACK_MS = 600 * 1000; + +//! Identity of a provider client that may be reinitialized by the next request. +struct ClientConfig { + string provider_name; + string origin; + + bool Matches(const HTTPUtil &util, const string &candidate_origin) const { + const auto name = util.GetName(); + const bool reviewed_provider = name == "HTTPFS-Curl" || name == "MultiCurl"; + return reviewed_provider && provider_name == name && origin == candidate_origin; + } + + void Set(const HTTPUtil &util, const string &candidate_origin) { + provider_name = util.GetName(); + origin = candidate_origin; + } +}; + +struct WorkerState : public QuackapiPostWorkerState { + unique_ptr client; + HTTPUtil *client_util = nullptr; + ClientConfig client_config; + bool has_client_config = false; + + void Reset() { + client.reset(); + client_util = nullptr; + has_client_config = false; + } +}; + +void InsertExtraHeaders(HTTPHeaders &headers, const unordered_map &extra_headers) { + for (auto &header : extra_headers) { + headers.Insert(header.first, header.second); + } +} + +QuackapiHttpFetchResult FromResponse(unique_ptr response) { + QuackapiHttpFetchResult result; + if (!response) { + result.request_error = "null HTTP response"; + return result; + } + result.status = response->status; + result.body = std::move(response->body); + result.reason = std::move(response->reason); + result.success = response->Success(); + result.request_error = response->GetRequestError(); + result.headers = std::move(response->headers); + return result; +} + +bool IsCancelled(const QuackapiPostRequest &request) { + if (request.cancelled && request.cancelled->load()) { + return true; + } + return std::chrono::steady_clock::now() >= QuackapiCurrentDeadline(); +} + +void ThrowIfCancelled(const QuackapiPostRequest &request) { + if (IsCancelled(request)) { + throw InterruptException(); + } +} + +string UnsupportedProviderMessage(const string &provider_name) { + return StringUtil::Format("quackapi_post requires an explicitly loaded HTTPUtil provider that supports POST. " + "Load curl_httpfs (recommended) or httpfs, then retry. Active HTTPUtil is '%s'.", + provider_name); +} + +void DiscardClient(WorkerState &state) { + state.Reset(); +} + +} // namespace + +unique_ptr QuackapiPostProvider::CreateWorkerState(DatabaseInstance &) { + return make_uniq(); +} + +QuackapiHttpFetchResult QuackapiPostProvider::Post(DatabaseInstance &db, QuackapiPostWorkerState &state_p, + const QuackapiPostRequest &request) { + ThrowIfCancelled(request); + auto &state = static_cast(state_p); + auto &http_util = HTTPUtil::Get(db); + const auto provider_name = http_util.GetName(); + if (provider_name == "Built-In") { + throw InvalidConfigurationException(UnsupportedProviderMessage(provider_name)); + } + + string path; + string origin; + HTTPUtil::DecomposeURL(request.url, path, origin); + auto params = http_util.InitializeParameters(db, request.url); + params->keep_alive = true; + params->retries = 0; // POST is not safe to replay after an ambiguous transport failure. + const bool reused = state.client && state.client_util == &http_util && state.has_client_config && + state.client_config.Matches(http_util, origin); + + HTTPHeaders headers(db); + if (!request.content_type.empty()) { + headers.Insert("Content-Type", request.content_type); + } else if (StringUtil::StartsWith(StringUtil::Lower(request.url), "http://")) { + // Preserve the previous plain-HTTP empty-content-type default. + headers.Insert("Content-Type", "application/octet-stream"); + } + InsertExtraHeaders(headers, request.headers); + PostRequestInfo info(request.url, headers, *params, const_data_ptr_cast(request.body.data()), request.body.size()); + info.try_request = true; + + // Prepare the provider client with the latest remaining deadline. Parameter + // initialization and client setup happen before this point and can consume + // part of the caller's budget. + ThrowIfCancelled(request); + const auto timeout_ms = QuackapiRemainingTimeoutMillis(OUTBOUND_TIMEOUT_FALLBACK_MS); + params->timeout = static_cast(timeout_ms / 1000); + params->timeout_usec = static_cast((timeout_ms % 1000) * 1000); + if (!reused) { + DiscardClient(state); + state.client = http_util.InitializeClient(*params, origin); + if (!state.client) { + throw InvalidConfigurationException(UnsupportedProviderMessage(provider_name)); + } + state.client_util = &http_util; + state.client_config.Set(http_util, origin); + state.has_client_config = true; + } + + // Client setup itself may use part of the remaining budget. Refresh timeout + // parameters and initialize with the current configuration before sending. + const auto send_timeout_ms = QuackapiRemainingTimeoutMillis(OUTBOUND_TIMEOUT_FALLBACK_MS); + params->timeout = static_cast(send_timeout_ms / 1000); + params->timeout_usec = static_cast((send_timeout_ms % 1000) * 1000); + // The reviewed providers fully apply current HTTPParams on Initialize. This + // includes changed proxy/TLS/auth settings; no short-lived params reference + // is retained. For fresh clients this also replaces the initial timeout. + state.client->Initialize(*params); + + // Never send after setup consumes the remaining deadline. + ThrowIfCancelled(request); + QuackapiHttpFetchResult result; + try { + result = FromResponse(http_util.Request(info, state.client)); + } catch (NotImplementedException &) { + DiscardClient(state); + throw InvalidConfigurationException(UnsupportedProviderMessage(provider_name)); + } + // This reports HTTPClient object reuse only; the provider does not expose + // whether its underlying TCP socket was reused. + result.reused_connection = reused; + if (!result.request_error.empty()) { + DiscardClient(state); + } + return result; +} + +} // namespace duckdb diff --git a/src/quackapi_post_result.cpp b/src/quackapi_post_result.cpp new file mode 100644 index 0000000..addab22 --- /dev/null +++ b/src/quackapi_post_result.cpp @@ -0,0 +1,52 @@ +#include "quackapi_post_result.hpp" + +#include "duckdb/common/types/value.hpp" + +namespace duckdb { + +LogicalType QuackapiHttpFetchResultType() { + child_list_t children; + children.emplace_back("status", LogicalType::INTEGER); + children.emplace_back("reason", LogicalType::VARCHAR); + children.emplace_back("body", LogicalType::VARCHAR); + children.emplace_back("headers", LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + children.emplace_back("error", LogicalType::VARCHAR); + children.emplace_back("reused_connection", LogicalType::BOOLEAN); + return LogicalType::STRUCT(std::move(children)); +} + +Value QuackapiHttpFetchResultValue(const QuackapiHttpFetchResult &result) { + vector header_keys; + vector header_values; + for (auto &entry : result.headers) { + header_keys.emplace_back(entry.first); + header_values.emplace_back(entry.second); + } + child_list_t children; + children.emplace_back("status", Value::INTEGER(static_cast(result.status))); + children.emplace_back("reason", Value(result.reason)); + children.emplace_back("body", Value(result.body)); + children.emplace_back("headers", Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(header_keys), + std::move(header_values))); + children.emplace_back("error", + result.request_error.empty() ? Value(LogicalType::VARCHAR) : Value(result.request_error)); + children.emplace_back("reused_connection", Value::BOOLEAN(result.reused_connection)); + return Value::STRUCT(std::move(children)); +} + +unordered_map QuackapiHttpHeadersFromValue(const Value &value) { + unordered_map out; + if (value.IsNull()) { + return out; + } + for (auto &entry : MapValue::GetChildren(value)) { + auto &kv = StructValue::GetChildren(entry); + if (kv.size() != 2 || kv[0].IsNull()) { + continue; + } + out[kv[0].ToString()] = kv[1].IsNull() ? string() : kv[1].ToString(); + } + return out; +} + +} // namespace duckdb diff --git a/src/quackapi_queue.cpp b/src/quackapi_queue.cpp deleted file mode 100644 index b50b736..0000000 --- a/src/quackapi_queue.cpp +++ /dev/null @@ -1,1035 +0,0 @@ -#include "quackapi_queue.hpp" - -#include "duckdb/common/exception.hpp" -#include "duckdb/common/string_util.hpp" -#include "duckdb/common/types.hpp" -#include "duckdb/common/types/timestamp.hpp" -#include "duckdb/common/types/vector.hpp" -#include "duckdb/common/unordered_map.hpp" -#include "duckdb/function/scalar_function.hpp" -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/main/connection.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/main/extension/extension_loader.hpp" -#include "duckdb/parser/parser_extension.hpp" - -#include "quackapi_state.hpp" - -#include -#include -#include "quackapi_util.hpp" - -namespace duckdb { - -namespace { - -//===--------------------------------------------------------------------===// -// SQL helpers -//===--------------------------------------------------------------------===// - -//! Escape a string for use as a single-quoted SQL literal. -string SqlQuote(const string &s) { - string out = "'"; - for (char c : s) { - if (c == '\'') { - out += "''"; - } else { - out += c; - } - } - out += "'"; - return out; -} - -void CheckQuery(unique_ptr &res, const string &ctx) { - if (res->HasError()) { - throw InvalidInputException("quackapi queue %s: %s", ctx, res->GetError()); - } -} - -void RunSQL(DatabaseInstance &db, const string &sql, const string &ctx) { - Connection con(db); - auto res = con.Query(sql); - CheckQuery(res, ctx); -} - -} // namespace - -void EnsureQuackapiJobsTable(DatabaseInstance &db) { - // One statement per Query — DuckDB does not accept multi-statement strings here. - RunSQL(db, "CREATE SEQUENCE IF NOT EXISTS quackapi_job_seq START 1", "create sequence"); - // payload is VARCHAR holding JSON text — avoids a hard runtime dep on LOAD - // json while still accepting payload::JSON at the function boundary (cast to - // text on insert). Users can CAST(payload AS JSON) when the json extension - // is loaded. - RunSQL(db, - "CREATE TABLE IF NOT EXISTS quackapi_jobs (" - "id BIGINT PRIMARY KEY, " - "queue VARCHAR NOT NULL, " - "payload VARCHAR NOT NULL, " - "status VARCHAR NOT NULL DEFAULT 'pending', " - "attempts INTEGER NOT NULL DEFAULT 0, " - "max_attempts INTEGER NOT NULL DEFAULT 3, " - "visible_at TIMESTAMP NOT NULL, " - "created_at TIMESTAMP NOT NULL, " - "updated_at TIMESTAMP NOT NULL, " - "last_error VARCHAR, " - "delivery_generation BIGINT NOT NULL DEFAULT 0, " - "worker_id VARCHAR" - ")", - "create table"); - // Existing durable queue tables predate delivery fencing. The generation is - // intentionally persisted with the job: a process restart must not make a - // stale worker's completion valid again. - RunSQL(db, - "ALTER TABLE quackapi_jobs " - // DuckDB cannot ADD COLUMN with a NOT NULL constraint. New tables use - // the constraint above; old durable tables are safely backfilled here. - "ADD COLUMN IF NOT EXISTS delivery_generation BIGINT DEFAULT 0", - "migrate delivery generation"); - RunSQL(db, "UPDATE quackapi_jobs SET delivery_generation = 0 WHERE delivery_generation IS NULL", - "backfill delivery generation"); - RunSQL(db, - "CREATE INDEX IF NOT EXISTS idx_quackapi_jobs_ready " - "ON quackapi_jobs (queue, status, visible_at, id)", - "create index"); -} - -namespace { - -//===--------------------------------------------------------------------===// -// Duration parse: 30 | '30' | '30s' | '5m' | '1h' -//===--------------------------------------------------------------------===// - -bool ParseDurationSeconds(const string &raw, int32_t &out_sec, string &err) { - auto s = QuackapiTrim(raw); - if (s.empty()) { - err = "empty duration"; - return false; - } - // Strip surrounding quotes if present. - if (s.size() >= 2 && s.front() == '\'' && s.back() == '\'') { - s = s.substr(1, s.size() - 2); - } - if (s.empty()) { - err = "empty duration"; - return false; - } - idx_t i = 0; - bool neg = false; - if (s[i] == '-') { - neg = true; - i++; - } - if (i >= s.size() || !StringUtil::CharacterIsDigit(s[i])) { - err = "duration must start with a number"; - return false; - } - int64_t num = 0; - while (i < s.size() && StringUtil::CharacterIsDigit(s[i])) { - num = num * 10 + (s[i] - '0'); - if (num > 86400LL * 365) { - err = "duration too large"; - return false; - } - i++; - } - int64_t mult = 1; - if (i < s.size()) { - auto unit = StringUtil::Lower(s.substr(i)); - if (unit == "s" || unit == "sec" || unit == "secs" || unit == "second" || unit == "seconds") { - mult = 1; - } else if (unit == "m" || unit == "min" || unit == "mins" || unit == "minute" || unit == "minutes") { - mult = 60; - } else if (unit == "h" || unit == "hr" || unit == "hrs" || unit == "hour" || unit == "hours") { - mult = 3600; - } else { - err = "unknown duration unit \"" + unit + "\" — use s, m, or h"; - return false; - } - } - int64_t sec = num * mult; - if (neg) { - sec = -sec; - } - if (sec < 0) { - err = "duration must be non-negative"; - return false; - } - if (sec > 86400LL * 7) { - err = "visibility_timeout max is 7 days"; - return false; - } - out_sec = static_cast(sec); - return true; -} - -//===--------------------------------------------------------------------===// -// CREATE / DROP QUEUE parser -//===--------------------------------------------------------------------===// - -struct QueueDdlParseData : public ParserExtensionParseData { - string action; // CREATE / DROP - bool or_replace = false; - QuackapiQueue queue; - - unique_ptr Copy() const override { - auto copy = make_uniq(); - copy->action = action; - copy->or_replace = or_replace; - copy->queue = queue; - return std::move(copy); - } - string ToString() const override { - return action + " QUEUE " + queue.name; - } -}; - -//! Grammar: -//! CREATE [OR REPLACE] QUEUE -//! [WITH ( max_attempts= , visibility_timeout='30s'|30 , backoff_base_seconds= )] -//! DROP QUEUE -ParserExtensionParseResult QueueDdlParse(ParserExtensionInfo *, const string &query) { - auto q = QuackapiTrim(query); - auto upper = StringUtil::Upper(q); - - bool or_replace = false; - idx_t pos; - if (StringUtil::StartsWith(upper, "CREATE QUEUE ")) { - pos = 13; - } else if (StringUtil::StartsWith(upper, "CREATE OR REPLACE QUEUE ")) { - pos = 24; - or_replace = true; - } else if (StringUtil::StartsWith(upper, "DROP QUEUE ")) { - auto name = QuackapiTrim(q.substr(11)); - if (name.empty() || name.find(' ') != string::npos) { - return ParserExtensionParseResult("DROP QUEUE expects a single queue name"); - } - auto data = make_uniq(); - data->action = "DROP"; - data->queue.name = name; - return ParserExtensionParseResult(std::move(data)); - } else { - return ParserExtensionParseResult(); - } - - auto rest = QuackapiTrim(q.substr(pos)); - if (rest.empty()) { - return ParserExtensionParseResult("CREATE QUEUE expects a queue name"); - } - - // — bare identifier up to space or end or WITH - idx_t name_end = 0; - while (name_end < rest.size() && !StringUtil::CharacterIsSpace(rest[name_end]) && rest[name_end] != '(') { - name_end++; - } - auto name = rest.substr(0, name_end); - if (name.empty()) { - return ParserExtensionParseResult("CREATE QUEUE expects a queue name"); - } - // Reject path-like names - for (char c : name) { - if (!(StringUtil::CharacterIsAlpha(c) || StringUtil::CharacterIsDigit(c) || c == '_' || c == '-')) { - return ParserExtensionParseResult("Queue name must be an identifier ([A-Za-z0-9_-]+)"); - } - } - rest = QuackapiTrim(rest.substr(name_end)); - auto rest_upper = StringUtil::Upper(rest); - - QuackapiQueue queue; - queue.name = name; - // defaults already set on QuackapiQueue - - // optional WITH ( ... ) - if (!rest.empty()) { - if (!(StringUtil::StartsWith(rest_upper, "WITH") && - (rest.size() == 4 || StringUtil::CharacterIsSpace(rest[4]) || rest[4] == '('))) { - return ParserExtensionParseResult("Expected WITH (...) after queue name, or end of statement"); - } - rest = QuackapiTrim(rest.substr(4)); - if (rest.empty() || rest[0] != '(') { - return ParserExtensionParseResult("WITH expects a parenthesized option list"); - } - auto close = rest.find(')'); - if (close == string::npos) { - return ParserExtensionParseResult("Unterminated WITH ( ... ) options"); - } - auto opts = QuackapiTrim(rest.substr(1, close - 1)); - rest = QuackapiTrim(rest.substr(close + 1)); - if (!rest.empty()) { - return ParserExtensionParseResult("Unexpected tokens after WITH options"); - } - - // Parse comma-separated key=value pairs - idx_t oi = 0; - while (oi < opts.size()) { - while (oi < opts.size() && (StringUtil::CharacterIsSpace(opts[oi]) || opts[oi] == ',')) { - oi++; - } - if (oi >= opts.size()) { - break; - } - // key - idx_t key_start = oi; - while (oi < opts.size() && (StringUtil::CharacterIsAlpha(opts[oi]) || - StringUtil::CharacterIsDigit(opts[oi]) || opts[oi] == '_')) { - oi++; - } - auto key = StringUtil::Lower(opts.substr(key_start, oi - key_start)); - while (oi < opts.size() && StringUtil::CharacterIsSpace(opts[oi])) { - oi++; - } - if (oi >= opts.size() || opts[oi] != '=') { - return ParserExtensionParseResult("WITH options: expected key=value (got key \"" + key + "\")"); - } - oi++; // = - while (oi < opts.size() && StringUtil::CharacterIsSpace(opts[oi])) { - oi++; - } - if (oi >= opts.size()) { - return ParserExtensionParseResult("WITH options: missing value for \"" + key + "\""); - } - // value: quoted string or bare token up to comma - string val; - if (opts[oi] == '\'') { - idx_t j = oi + 1; - string raw; - while (j < opts.size()) { - if (opts[j] == '\'') { - if (j + 1 < opts.size() && opts[j + 1] == '\'') { - raw += '\''; - j += 2; - continue; - } - break; - } - raw += opts[j]; - j++; - } - if (j >= opts.size() || opts[j] != '\'') { - return ParserExtensionParseResult("Unterminated quoted value for \"" + key + "\""); - } - val = raw; - oi = j + 1; - } else { - idx_t j = oi; - while (j < opts.size() && opts[j] != ',' && !StringUtil::CharacterIsSpace(opts[j])) { - j++; - } - val = opts.substr(oi, j - oi); - oi = j; - } - - if (key == "max_attempts") { - int v = atoi(val.c_str()); - if (v < 1 || v > 1000) { - return ParserExtensionParseResult("max_attempts must be between 1 and 1000"); - } - queue.max_attempts = v; - } else if (key == "visibility_timeout" || key == "visibility_timeout_sec") { - int32_t sec = 0; - string err; - if (!ParseDurationSeconds(val, sec, err)) { - return ParserExtensionParseResult("visibility_timeout: " + err); - } - if (sec < 1) { - return ParserExtensionParseResult("visibility_timeout must be at least 1 second"); - } - queue.visibility_timeout_sec = sec; - } else if (key == "backoff_base_seconds" || key == "backoff_base") { - int v = atoi(val.c_str()); - if (v < 0 || v > 3600) { - return ParserExtensionParseResult("backoff_base_seconds must be between 0 and 3600"); - } - queue.backoff_base_sec = v; - } else { - return ParserExtensionParseResult( - "Unknown QUEUE option \"" + key + - "\" — expected max_attempts, visibility_timeout, backoff_base_seconds"); - } - } - } - - auto data = make_uniq(); - data->action = "CREATE"; - data->or_replace = or_replace; - data->queue = queue; - return ParserExtensionParseResult(std::move(data)); -} - -struct ApplyQueueBindData : public TableFunctionData { - string action; - bool or_replace = false; - string name; - int32_t max_attempts = 3; - int32_t visibility_timeout_sec = 30; - int32_t backoff_base_sec = 2; - bool finished = false; -}; - -unique_ptr ApplyQueueBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - bind_data->action = input.inputs[0].GetValue(); - bind_data->or_replace = input.inputs[1].GetValue(); - bind_data->name = input.inputs[2].GetValue(); - bind_data->max_attempts = input.inputs[3].GetValue(); - bind_data->visibility_timeout_sec = input.inputs[4].GetValue(); - bind_data->backoff_base_sec = input.inputs[5].GetValue(); - BindStatusColumn(return_types, names); - return std::move(bind_data); -} - -void ApplyQueueExec(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &bind_data = data_p.bind_data->CastNoConst(); - if (bind_data.finished) { - return; - } - auto &state = QuackapiState::Get(*context.db); - string message; - if (bind_data.action == "CREATE") { - // Durable jobs table lives in the user's catalog / .db file. - EnsureQuackapiJobsTable(*context.db); - QuackapiQueue q; - q.name = bind_data.name; - q.max_attempts = bind_data.max_attempts; - q.visibility_timeout_sec = bind_data.visibility_timeout_sec; - q.backoff_base_sec = bind_data.backoff_base_sec; - state.AddQueue(q, bind_data.or_replace); - message = StringUtil::Format("Queue %s: max_attempts=%d visibility_timeout=%ds", q.name, q.max_attempts, - q.visibility_timeout_sec); - } else { - if (state.DropQueue(bind_data.name)) { - message = StringUtil::Format("Dropped queue %s", bind_data.name); - } else { - throw InvalidInputException("Queue \"%s\" does not exist", bind_data.name); - } - } - EmitOneShotStatus(output, bind_data.finished, message); -} - -TableFunction MakeApplyQueueFunction() { - return MakeApplyDdlFunction("quackapi_apply_queue", - {LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::VARCHAR, LogicalType::INTEGER, - LogicalType::INTEGER, LogicalType::INTEGER}, - ApplyQueueExec, ApplyQueueBind); -} - -ParserExtensionPlanResult QueueDdlPlan(ParserExtensionInfo *, ClientContext &, - unique_ptr parse_data) { - auto &data = static_cast(*parse_data); - ParserExtensionPlanResult result; - result.function = MakeApplyQueueFunction(); - result.parameters.push_back(Value(data.action)); - result.parameters.push_back(Value::BOOLEAN(data.or_replace)); - result.parameters.push_back(Value(data.queue.name)); - result.parameters.push_back(Value::INTEGER(data.queue.max_attempts)); - result.parameters.push_back(Value::INTEGER(data.queue.visibility_timeout_sec)); - result.parameters.push_back(Value::INTEGER(data.queue.backoff_base_sec)); - FinishDdlPlan(result); - return result; -} - -//===--------------------------------------------------------------------===// -// Lookup helper -//===--------------------------------------------------------------------===// - -QuackapiQueue RequireQueue(DatabaseInstance &db, const string &name) { - QuackapiQueue q; - if (!QuackapiState::Get(db).GetQueue(name, q)) { - throw InvalidInputException("Queue \"%s\" does not exist — CREATE QUEUE first", name); - } - return q; -} - -//===--------------------------------------------------------------------===// -// quackapi_enqueue(queue, payload [, max_attempts]) → job_id BIGINT -//===--------------------------------------------------------------------===// - -int64_t EnqueueJob(DatabaseInstance &db, const string &queue_name, const string &payload_json, - int32_t max_attempts_override) { - auto q = RequireQueue(db, queue_name); - EnsureQuackapiJobsTable(db); - int32_t max_att = max_attempts_override > 0 ? max_attempts_override : q.max_attempts; - if (max_att < 1) { - throw InvalidInputException("quackapi_enqueue: max_attempts must be >= 1"); - } - // Store payload as JSON text (VARCHAR). Callers may pass JSON or VARCHAR. - string sql = - StringUtil::Format("INSERT INTO quackapi_jobs " - "(id, queue, payload, status, attempts, max_attempts, visible_at, created_at, updated_at) " - "SELECT nextval('quackapi_job_seq'), %s, %s, 'pending', 0, %d, " - "now()::TIMESTAMP, now()::TIMESTAMP, now()::TIMESTAMP " - "RETURNING id", - SqlQuote(queue_name), SqlQuote(payload_json), max_att); - Connection con(db); - auto res = con.Query(sql); - CheckQuery(res, "enqueue"); - if (res->RowCount() == 0) { - throw InternalException("quackapi_enqueue: INSERT RETURNING produced no row"); - } - return res->GetValue(0, 0).GetValue(); -} - -//! enqueue(queue, payload) / enqueue(queue, payload, max_attempts) — one body. -//! Optional 3rd arg via args.ColumnCount() (same pattern as nack). -void EnqueueScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - UnifiedVectorFormat qdata, pdata, mdata; - args.data[0].ToUnifiedFormat(args.size(), qdata); - args.data[1].ToUnifiedFormat(args.size(), pdata); - const bool has_max = args.ColumnCount() >= 3; - if (has_max) { - args.data[2].ToUnifiedFormat(args.size(), mdata); - } - result.SetVectorType(VectorType::FLAT_VECTOR); - auto out = FlatVector::GetData(result); - auto &validity = FlatVector::Validity(result); - for (idx_t i = 0; i < args.size(); i++) { - auto qi = qdata.sel->get_index(i); - auto pi = pdata.sel->get_index(i); - if (!qdata.validity.RowIsValid(qi) || !pdata.validity.RowIsValid(pi)) { - validity.SetInvalid(i); - continue; - } - auto qn = UnifiedVectorFormat::GetData(qdata)[qi].GetString(); - auto payload = UnifiedVectorFormat::GetData(pdata)[pi].GetString(); - int32_t max_att = 0; - if (has_max) { - auto mi = mdata.sel->get_index(i); - if (mdata.validity.RowIsValid(mi)) { - max_att = UnifiedVectorFormat::GetData(mdata)[mi]; - } - } - out[i] = EnqueueJob(db, qn, payload, max_att); - } -} - -//===--------------------------------------------------------------------===// -// quackapi_dequeue(queue [, n]) table function — claim jobs -//===--------------------------------------------------------------------===// - -struct DequeueBindData : public TableFunctionData { - string queue_name; - int32_t n = 1; -}; - -struct DequeueGlobalState : public GlobalTableFunctionState { - vector> rows; // pre-claimed rows - idx_t offset = 0; -}; - -unique_ptr DequeueBind(ClientContext &, TableFunctionBindInput &input, vector &return_types, - vector &names) { - auto bind_data = make_uniq(); - bind_data->queue_name = input.inputs[0].GetValue(); - if (input.inputs.size() > 1 && !input.inputs[1].IsNull()) { - bind_data->n = input.inputs[1].GetValue(); - } - if (bind_data->n < 1) { - throw InvalidInputException("quackapi_dequeue: n must be >= 1"); - } - if (bind_data->n > 1000) { - throw InvalidInputException("quackapi_dequeue: n max is 1000"); - } - return_types = {LogicalType::BIGINT, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::VARCHAR, LogicalType::INTEGER, LogicalType::INTEGER, - LogicalType::TIMESTAMP, LogicalType::VARCHAR, LogicalType::BIGINT}; - names = {"id", "queue", "payload", - "status", "attempts", "max_attempts", - "visible_at", "last_error", "delivery_generation"}; - return std::move(bind_data); -} - -//! Recover every expired delivery in a queue. This deliberately does not use -//! a queue-wide "live lease" gate: one active job must never delay recovery of -//! another job whose lease already expired. -void RecoverExpiredJobs(Connection &con, const string &queue_name) { - // A legacy database can contain an exhausted pending row from the old expiry - // path. Dead-letter it before claiming so an upgrade cannot create attempt - // max_attempts + 1. - string exhausted_pending_sql = - StringUtil::Format("UPDATE quackapi_jobs SET status = 'dead', visible_at = now()::TIMESTAMP, " - "updated_at = now()::TIMESTAMP, worker_id = NULL, " - "last_error = COALESCE(last_error, 'attempt limit reached before delivery') " - "WHERE queue = %s AND status = 'pending' AND attempts >= max_attempts", - SqlQuote(queue_name)); - auto exhausted_pending = con.Query(exhausted_pending_sql); - CheckQuery(exhausted_pending, "dequeue exhausted-pending sweep"); - - // A timed-out final delivery is terminal. Other timed-out deliveries become - // pending atomically with the state transition and can be claimed below. - string exhausted_running_sql = - StringUtil::Format("UPDATE quackapi_jobs SET status = 'dead', visible_at = now()::TIMESTAMP, " - "updated_at = now()::TIMESTAMP, worker_id = NULL, " - "last_error = 'visibility lease expired after maximum attempts' " - "WHERE queue = %s AND status = 'running' " - "AND visible_at <= now()::TIMESTAMP AND attempts >= max_attempts", - SqlQuote(queue_name)); - auto exhausted_running = con.Query(exhausted_running_sql); - CheckQuery(exhausted_running, "dequeue exhausted-running sweep"); - - string retry_sql = StringUtil::Format("UPDATE quackapi_jobs SET status = 'pending', visible_at = now()::TIMESTAMP, " - "updated_at = now()::TIMESTAMP, worker_id = NULL " - "WHERE queue = %s AND status = 'running' " - "AND visible_at <= now()::TIMESTAMP AND attempts < max_attempts", - SqlQuote(queue_name)); - auto retry = con.Query(retry_sql); - CheckQuery(retry, "dequeue expiry sweep"); -} - -unique_ptr ClaimPendingJob(Connection &con, const string &queue_name, - int32_t visibility_timeout_sec) { - string sql = - StringUtil::Format("UPDATE quackapi_jobs SET " - "status = 'running', " - "attempts = attempts + 1, " - "delivery_generation = delivery_generation + 1, " - "visible_at = now()::TIMESTAMP + to_seconds(%d), " - "updated_at = now()::TIMESTAMP, " - "worker_id = NULL " - "WHERE id = (" - " SELECT id FROM quackapi_jobs " - " WHERE queue = %s " - " AND status = 'pending' " - " AND attempts < max_attempts " - " AND visible_at <= now()::TIMESTAMP " - " ORDER BY id LIMIT 1" - ") " - "AND status = 'pending' " - "RETURNING id, queue, payload, status, attempts, max_attempts, visible_at, last_error, " - "delivery_generation", - visibility_timeout_sec, SqlQuote(queue_name)); - unique_ptr res; - for (int attempt = 0; attempt < 8; attempt++) { - res = con.Query(sql); - if (!res->HasError()) { - return res; - } - const string &err = res->GetError(); - // Concurrent claim on the same tuple — retry; do not surface 422 Conflict. - if (StringUtil::Contains(err, "Conflict")) { - continue; - } - CheckQuery(res, "dequeue"); - } - CheckQuery(res, "dequeue"); - return res; -} - -unique_ptr DequeueInit(ClientContext &context, TableFunctionInitInput &input) { - auto &bind = input.bind_data->Cast(); - auto state = make_uniq(); - auto q = RequireQueue(*context.db, bind.queue_name); - EnsureQuackapiJobsTable(*context.db); - - // The gate lives on QuackapiState (one per DatabaseInstance) so static and - // loadable extension copies serialize claims, expiry recovery, and delivery - // transitions through the same mutex. - auto &qa = QuackapiState::Get(*context.db); - std::lock_guard claim_lock(qa.DequeueClaimMutex()); - - Connection con(*context.db); - for (int32_t i = 0; i < bind.n; i++) { - RecoverExpiredJobs(con, bind.queue_name); - auto res = ClaimPendingJob(con, bind.queue_name, q.visibility_timeout_sec); - if (res->RowCount() == 0) { - break; - } - vector row; - for (idx_t c = 0; c < 9; c++) { - row.push_back(res->GetValue(c, 0)); - } - state->rows.push_back(std::move(row)); - } - return std::move(state); -} - -void DequeueExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.rows.size() && row < STANDARD_VECTOR_SIZE) { - auto &r = state.rows[state.offset]; - for (idx_t c = 0; c < 9; c++) { - output.SetValue(c, row, r[c]); - } - row++; - state.offset++; - } - output.SetCardinality(row); -} - -//===--------------------------------------------------------------------===// -// quackapi_ack(queue, job_id, delivery_generation) → BOOLEAN -//===--------------------------------------------------------------------===// - -bool AckJob(DatabaseInstance &db, const string &queue_name, int64_t job_id, int64_t delivery_generation) { - RequireQueue(db, queue_name); - auto &qa = QuackapiState::Get(db); - std::lock_guard lock(qa.DequeueClaimMutex()); - EnsureQuackapiJobsTable(db); - string sql = StringUtil::Format( - "UPDATE quackapi_jobs SET status = 'done', updated_at = now()::TIMESTAMP, worker_id = NULL " - "WHERE id = %lld AND queue = %s AND status = 'running' " - "AND delivery_generation = %lld AND visible_at > now()::TIMESTAMP " - "RETURNING id", - static_cast(job_id), SqlQuote(queue_name), static_cast(delivery_generation)); - Connection con(db); - auto res = con.Query(sql); - CheckQuery(res, "ack"); - return res->RowCount() > 0; -} - -void AckScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - UnifiedVectorFormat qdata, idata, gdata; - args.data[0].ToUnifiedFormat(args.size(), qdata); - args.data[1].ToUnifiedFormat(args.size(), idata); - args.data[2].ToUnifiedFormat(args.size(), gdata); - result.SetVectorType(VectorType::FLAT_VECTOR); - auto out = FlatVector::GetData(result); - auto &validity = FlatVector::Validity(result); - for (idx_t i = 0; i < args.size(); i++) { - auto qi = qdata.sel->get_index(i); - auto ii = idata.sel->get_index(i); - auto gi = gdata.sel->get_index(i); - if (!qdata.validity.RowIsValid(qi) || !idata.validity.RowIsValid(ii) || !gdata.validity.RowIsValid(gi)) { - validity.SetInvalid(i); - continue; - } - auto qn = UnifiedVectorFormat::GetData(qdata)[qi].GetString(); - auto jid = UnifiedVectorFormat::GetData(idata)[ii]; - auto generation = UnifiedVectorFormat::GetData(gdata)[gi]; - out[i] = AckJob(db, qn, jid, generation); - } -} - -//===--------------------------------------------------------------------===// -// quackapi_nack(queue, job_id, delivery_generation [, requeue [, error]]) → VARCHAR status -//===--------------------------------------------------------------------===// - -string NackJob(DatabaseInstance &db, const string &queue_name, int64_t job_id, int64_t delivery_generation, - bool requeue, const string &error) { - auto q = RequireQueue(db, queue_name); - auto &qa = QuackapiState::Get(db); - std::lock_guard lock(qa.DequeueClaimMutex()); - EnsureQuackapiJobsTable(db); - - // The shared delivery lock keeps this read and conditional transition one - // queue operation. The generation + unexpired-lease predicate is repeated in - // the UPDATE so the ownership proof remains explicit in the SQL itself. - string read_sql = StringUtil::Format("SELECT attempts, max_attempts FROM quackapi_jobs " - "WHERE id = %lld AND queue = %s AND status = 'running' " - "AND delivery_generation = %lld AND visible_at > now()::TIMESTAMP", - static_cast(job_id), SqlQuote(queue_name), - static_cast(delivery_generation)); - Connection con(db); - auto read = con.Query(read_sql); - CheckQuery(read, "nack read"); - if (read->RowCount() == 0) { - return "stale"; - } - int32_t attempts = read->GetValue(0, 0).GetValue(); - int32_t max_att = read->GetValue(1, 0).GetValue(); - - bool to_dead = !requeue || attempts >= max_att; - int32_t backoff = 0; - if (!to_dead) { - // exponential: base^min(attempts, 6). backoff_base_sec=0 means immediate retry. - if (q.backoff_base_sec > 0) { - int exp = attempts; - if (exp > 6) { - exp = 6; - } - if (exp < 0) { - exp = 0; - } - double secs = std::pow(static_cast(q.backoff_base_sec), exp); - if (secs > 3600.0) { - secs = 3600.0; - } - backoff = static_cast(secs); - } - } - - string sql; - if (to_dead) { - sql = StringUtil::Format("UPDATE quackapi_jobs SET status = 'dead', " - "visible_at = now()::TIMESTAMP, " - "last_error = %s, updated_at = now()::TIMESTAMP, worker_id = NULL " - "WHERE id = %lld AND queue = %s AND status = 'running' " - "AND delivery_generation = %lld AND visible_at > now()::TIMESTAMP " - "RETURNING status", - SqlQuote(error.empty() ? "nack" : error), static_cast(job_id), - SqlQuote(queue_name), static_cast(delivery_generation)); - } else { - sql = StringUtil::Format("UPDATE quackapi_jobs SET status = 'pending', " - "visible_at = now()::TIMESTAMP + to_seconds(%d), " - "last_error = %s, updated_at = now()::TIMESTAMP, worker_id = NULL " - "WHERE id = %lld AND queue = %s AND status = 'running' " - "AND delivery_generation = %lld AND visible_at > now()::TIMESTAMP " - "RETURNING status", - backoff, SqlQuote(error.empty() ? "nack" : error), static_cast(job_id), - SqlQuote(queue_name), static_cast(delivery_generation)); - } - auto res = con.Query(sql); - CheckQuery(res, "nack"); - if (res->RowCount() == 0) { - return "stale"; - } - return res->GetValue(0, 0).GetValue(); -} - -//! nack(queue, job_id, delivery_generation [, requeue [, error]]) — one body, optional args by arity. -void NackScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - UnifiedVectorFormat qdata, idata, gdata, rdata, edata; - args.data[0].ToUnifiedFormat(args.size(), qdata); - args.data[1].ToUnifiedFormat(args.size(), idata); - args.data[2].ToUnifiedFormat(args.size(), gdata); - const bool has_requeue = args.ColumnCount() >= 4; - const bool has_error = args.ColumnCount() >= 5; - if (has_requeue) { - args.data[3].ToUnifiedFormat(args.size(), rdata); - } - if (has_error) { - args.data[4].ToUnifiedFormat(args.size(), edata); - } - result.SetVectorType(VectorType::FLAT_VECTOR); - auto out = FlatVector::GetData(result); - auto &validity = FlatVector::Validity(result); - for (idx_t i = 0; i < args.size(); i++) { - auto qi = qdata.sel->get_index(i); - auto ii = idata.sel->get_index(i); - auto gi = gdata.sel->get_index(i); - if (!qdata.validity.RowIsValid(qi) || !idata.validity.RowIsValid(ii) || !gdata.validity.RowIsValid(gi)) { - validity.SetInvalid(i); - continue; - } - auto qn = UnifiedVectorFormat::GetData(qdata)[qi].GetString(); - auto jid = UnifiedVectorFormat::GetData(idata)[ii]; - auto generation = UnifiedVectorFormat::GetData(gdata)[gi]; - bool requeue = true; - if (has_requeue) { - auto ri = rdata.sel->get_index(i); - if (rdata.validity.RowIsValid(ri)) { - requeue = UnifiedVectorFormat::GetData(rdata)[ri]; - } - } - string err = "nack"; - if (has_error) { - auto ei = edata.sel->get_index(i); - if (edata.validity.RowIsValid(ei)) { - err = UnifiedVectorFormat::GetData(edata)[ei].GetString(); - } - } - auto st = NackJob(db, qn, jid, generation, requeue, err); - out[i] = StringVector::AddString(result, st); - } -} - -//===--------------------------------------------------------------------===// -// quackapi_renew(queue, job_id, delivery_generation) → BOOLEAN -//===--------------------------------------------------------------------===// - -bool RenewJob(DatabaseInstance &db, const string &queue_name, int64_t job_id, int64_t delivery_generation) { - auto q = RequireQueue(db, queue_name); - auto &qa = QuackapiState::Get(db); - std::lock_guard lock(qa.DequeueClaimMutex()); - EnsureQuackapiJobsTable(db); - string sql = StringUtil::Format("UPDATE quackapi_jobs SET visible_at = now()::TIMESTAMP + to_seconds(%d), " - "updated_at = now()::TIMESTAMP " - "WHERE id = %lld AND queue = %s AND status = 'running' " - "AND delivery_generation = %lld AND visible_at > now()::TIMESTAMP " - "RETURNING id", - q.visibility_timeout_sec, static_cast(job_id), SqlQuote(queue_name), - static_cast(delivery_generation)); - Connection con(db); - auto res = con.Query(sql); - CheckQuery(res, "renew"); - return res->RowCount() > 0; -} - -void RenewScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - UnifiedVectorFormat qdata, idata, gdata; - args.data[0].ToUnifiedFormat(args.size(), qdata); - args.data[1].ToUnifiedFormat(args.size(), idata); - args.data[2].ToUnifiedFormat(args.size(), gdata); - result.SetVectorType(VectorType::FLAT_VECTOR); - auto out = FlatVector::GetData(result); - auto &validity = FlatVector::Validity(result); - for (idx_t i = 0; i < args.size(); i++) { - auto qi = qdata.sel->get_index(i); - auto ii = idata.sel->get_index(i); - auto gi = gdata.sel->get_index(i); - if (!qdata.validity.RowIsValid(qi) || !idata.validity.RowIsValid(ii) || !gdata.validity.RowIsValid(gi)) { - validity.SetInvalid(i); - continue; - } - auto qn = UnifiedVectorFormat::GetData(qdata)[qi].GetString(); - auto jid = UnifiedVectorFormat::GetData(idata)[ii]; - auto generation = UnifiedVectorFormat::GetData(gdata)[gi]; - out[i] = RenewJob(db, qn, jid, generation); - } -} - -//===--------------------------------------------------------------------===// -// quackapi_queues() — name, depth, in_flight, dead (+ options) -//===--------------------------------------------------------------------===// - -struct QueuesBindData : public TableFunctionData {}; - -struct QueuesGlobalState : public GlobalTableFunctionState { - vector queues; - // Parallel stats vectors indexed same as queues - vector depth; - vector in_flight; - vector dead; - idx_t offset = 0; -}; - -unique_ptr QueuesBind(ClientContext &, TableFunctionBindInput &, vector &return_types, - vector &names) { - return_types = {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT, LogicalType::BIGINT, - LogicalType::INTEGER, LogicalType::INTEGER, LogicalType::INTEGER}; - names = {"name", "depth", "in_flight", "dead", "max_attempts", "visibility_timeout_sec", "backoff_base_sec"}; - return make_uniq(); -} - -unique_ptr QueuesInit(ClientContext &context, TableFunctionInitInput &) { - auto state = make_uniq(); - state->queues = QuackapiState::Get(*context.db).SnapshotQueues(); - state->depth.assign(state->queues.size(), 0); - state->in_flight.assign(state->queues.size(), 0); - state->dead.assign(state->queues.size(), 0); - - // Jobs table may not exist yet if no CREATE QUEUE ran (empty registry). - if (state->queues.empty()) { - return std::move(state); - } - try { - EnsureQuackapiJobsTable(*context.db); - } catch (...) { - return std::move(state); - } - Connection con(*context.db); - // depth = claimable (pending OR running with expired lease) - // in_flight = running with active lease - // dead = dead - auto res = - con.Query("SELECT queue, " - "count(*) FILTER (WHERE status = 'pending' OR " - " (status = 'running' AND visible_at <= now()::TIMESTAMP))::BIGINT AS depth, " - "count(*) FILTER (WHERE status = 'running' AND visible_at > now()::TIMESTAMP)::BIGINT AS in_flight, " - "count(*) FILTER (WHERE status = 'dead')::BIGINT AS dead " - "FROM quackapi_jobs GROUP BY queue"); - if (res->HasError()) { - return std::move(state); - } - unordered_map idx; - for (idx_t i = 0; i < state->queues.size(); i++) { - idx[state->queues[i].name] = i; - } - for (idx_t r = 0; r < res->RowCount(); r++) { - auto qn = res->GetValue(0, r).GetValue(); - auto it = idx.find(qn); - if (it == idx.end()) { - continue; // orphaned jobs from dropped queue — ignore for this view - } - state->depth[it->second] = res->GetValue(1, r).GetValue(); - state->in_flight[it->second] = res->GetValue(2, r).GetValue(); - state->dead[it->second] = res->GetValue(3, r).GetValue(); - } - return std::move(state); -} - -void QueuesExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.queues.size() && row < STANDARD_VECTOR_SIZE) { - auto &q = state.queues[state.offset]; - output.SetValue(0, row, Value(q.name)); - output.SetValue(1, row, Value::BIGINT(state.depth[state.offset])); - output.SetValue(2, row, Value::BIGINT(state.in_flight[state.offset])); - output.SetValue(3, row, Value::BIGINT(state.dead[state.offset])); - output.SetValue(4, row, Value::INTEGER(q.max_attempts)); - output.SetValue(5, row, Value::INTEGER(q.visibility_timeout_sec)); - output.SetValue(6, row, Value::INTEGER(q.backoff_base_sec)); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -} // namespace - -//===--------------------------------------------------------------------===// -// Public registration -//===--------------------------------------------------------------------===// - -QueueDdlParserExtension::QueueDdlParserExtension() { - parse_function = QueueDdlParse; - plan_function = QueueDdlPlan; -} - -TableFunction GetApplyQueueFunction() { - return MakeApplyQueueFunction(); -} - -void RegisterQuackapiQueueFunctions(ExtensionLoader &loader) { - // enqueue(queue, payload) / enqueue(queue, payload, max_attempts) - ScalarFunctionSet enqueue_set("quackapi_enqueue"); - enqueue_set.AddFunction(ScalarFunction("quackapi_enqueue", {LogicalType::VARCHAR, LogicalType::VARCHAR}, - LogicalType::BIGINT, EnqueueScalar)); - enqueue_set.AddFunction(ScalarFunction("quackapi_enqueue", - {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::INTEGER}, - LogicalType::BIGINT, EnqueueScalar)); - // JSON payload overload (JSON is a logical type alias of VARCHAR storage) - enqueue_set.AddFunction(ScalarFunction("quackapi_enqueue", {LogicalType::VARCHAR, LogicalType::JSON()}, - LogicalType::BIGINT, EnqueueScalar)); - enqueue_set.AddFunction(ScalarFunction("quackapi_enqueue", - {LogicalType::VARCHAR, LogicalType::JSON(), LogicalType::INTEGER}, - LogicalType::BIGINT, EnqueueScalar)); - loader.RegisterFunction(enqueue_set); - - // dequeue(queue) / dequeue(queue, n) - TableFunctionSet dequeue_set("quackapi_dequeue"); - TableFunction dequeue1("quackapi_dequeue", {LogicalType::VARCHAR}, DequeueExec, DequeueBind, DequeueInit); - TableFunction dequeue2("quackapi_dequeue", {LogicalType::VARCHAR, LogicalType::INTEGER}, DequeueExec, DequeueBind, - DequeueInit); - dequeue_set.AddFunction(dequeue1); - dequeue_set.AddFunction(dequeue2); - loader.RegisterFunction(dequeue_set); - - // Delivery ownership is mandatory. The retired two-argument form could let - // an expired worker complete a newer delivery of the same job. - loader.RegisterFunction(ScalarFunction("quackapi_ack", - {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT}, - LogicalType::BOOLEAN, AckScalar)); - - // nack(queue, job_id, delivery_generation [, requeue [, error]]) - ScalarFunctionSet nack_set("quackapi_nack"); - nack_set.AddFunction(ScalarFunction("quackapi_nack", - {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT}, - LogicalType::VARCHAR, NackScalar)); - nack_set.AddFunction(ScalarFunction( - "quackapi_nack", {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT, LogicalType::BOOLEAN}, - LogicalType::VARCHAR, NackScalar)); - nack_set.AddFunction(ScalarFunction( - "quackapi_nack", - {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT, LogicalType::BOOLEAN, LogicalType::VARCHAR}, - LogicalType::VARCHAR, NackScalar)); - loader.RegisterFunction(nack_set); - - // renew(queue, job_id, delivery_generation) → bool - loader.RegisterFunction(ScalarFunction("quackapi_renew", - {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT}, - LogicalType::BOOLEAN, RenewScalar)); - - // queues() inspection - loader.RegisterFunction(TableFunction("quackapi_queues", {}, QueuesExec, QueuesBind, QueuesInit)); -} - -} // namespace duckdb diff --git a/src/quackapi_registry.cpp b/src/quackapi_registry.cpp index 821b2a3..bea4519 100644 --- a/src/quackapi_registry.cpp +++ b/src/quackapi_registry.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include "duckdb/common/exception.hpp" #include "duckdb/common/helper.hpp" @@ -12,7 +11,6 @@ #include "duckdb/main/database.hpp" #include "quackapi_auth.hpp" -#include "quackapi_http_fetch.hpp" #include "quackapi_server.hpp" namespace duckdb { @@ -34,16 +32,10 @@ void QuackapiState::PublishRoutes() { routes_live = make_shared_ptr>(routes); } -void QuackapiState::PublishStreams() { - // Caller holds streams_mutex. - streams_live = make_shared_ptr>(streams); -} - void QuackapiState::AddRoute(const QuackapiRoute &route, bool or_replace) { std::lock_guard lock(routes_mutex); - // Method+pattern must be unique across route names (GraphQL mounts already - // enforce path uniqueness). Otherwise OR REPLACE onto another route's path - // leaves two handlers and last-match / duplicate OpenAPI keys. + // Method+pattern must be unique across route names. Otherwise OR REPLACE + // onto another route's path leaves duplicate handlers and OpenAPI keys. auto collides = [&](const QuackapiRoute &candidate, const string &ignore_name) -> const QuackapiRoute * { for (auto &other : routes) { if (other.name == ignore_name) { @@ -106,66 +98,6 @@ shared_ptr> QuackapiState::LiveRoutes() { return routes_live; } -void QuackapiState::AddStream(const QuackapiStream &stream, bool or_replace) { - std::lock_guard lock(streams_mutex); - auto collides = [&](const QuackapiStream &candidate, const string &ignore_name) -> const QuackapiStream * { - for (auto &other : streams) { - if (other.name == ignore_name) { - continue; - } - if (other.method == candidate.method && other.pattern == candidate.pattern) { - return &other; - } - } - return nullptr; - }; - for (auto it = streams.begin(); it != streams.end(); ++it) { - if (it->name == stream.name) { - if (!or_replace) { - throw InvalidInputException("Stream \"%s\" already exists — use CREATE OR REPLACE STREAM", stream.name); - } - if (auto *hit = collides(stream, stream.name)) { - throw InvalidInputException("Stream path \"%s %s\" already used by stream \"%s\"", stream.method, - stream.pattern, hit->name); - } - *it = stream; - PublishStreams(); - return; - } - } - if (auto *hit = collides(stream, string())) { - throw InvalidInputException("Stream path \"%s %s\" already used by stream \"%s\"", stream.method, - stream.pattern, hit->name); - } - streams.push_back(stream); - PublishStreams(); -} - -bool QuackapiState::DropStream(const string &name) { - std::lock_guard lock(streams_mutex); - for (auto it = streams.begin(); it != streams.end(); ++it) { - if (it->name == name) { - streams.erase(it); - PublishStreams(); - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotStreams() { - auto live = LiveStreams(); - return *live; -} - -shared_ptr> QuackapiState::LiveStreams() { - std::lock_guard lock(streams_mutex); - if (!streams_live) { - PublishStreams(); - } - return streams_live; -} - void QuackapiState::AddGroup(const QuackapiGroup &group, bool or_replace) { std::lock_guard lock(groups_mutex); for (auto it = groups.begin(); it != groups.end(); ++it) { @@ -334,47 +266,6 @@ vector QuackapiState::SnapshotApiKeys(const string &auth_na return result; } -void QuackapiState::AddQueue(const QuackapiQueue &queue, bool or_replace) { - std::lock_guard lock(queues_mutex); - for (auto it = queues.begin(); it != queues.end(); ++it) { - if (it->name == queue.name) { - if (!or_replace) { - throw InvalidInputException("Queue \"%s\" already exists — use CREATE OR REPLACE QUEUE", queue.name); - } - *it = queue; - return; - } - } - queues.push_back(queue); -} - -bool QuackapiState::DropQueue(const string &name) { - std::lock_guard lock(queues_mutex); - for (auto it = queues.begin(); it != queues.end(); ++it) { - if (it->name == name) { - queues.erase(it); - return true; - } - } - return false; -} - -bool QuackapiState::GetQueue(const string &name, QuackapiQueue &out) { - std::lock_guard lock(queues_mutex); - for (auto &q : queues) { - if (q.name == name) { - out = q; - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotQueues() { - std::lock_guard lock(queues_mutex); - return queues; -} - void QuackapiState::StartServer(DatabaseInstance &db, const string &host, int port, const QuackapiServeOptions &opts) { // Mirrors QuackStorageExtensionInfo::CreateServer (duckdb-quack // src/quack_storage.cpp): lock map → reject duplicate key → construct @@ -406,9 +297,6 @@ bool QuackapiState::StopServer(int port) { if (!to_destroy) { return false; } - // Close pooled outbound keep-alives first so Server worker join is not - // blocked by idle clients still holding the listen-side connection open. - QuackapiHttpFetch::ResetPool(); to_destroy->StopAccepting(); // Brief delay so a self-stop from inside a route can finish its response // before ~Server joins the thread pool. @@ -431,8 +319,7 @@ void QuackapiState::StopAllServers() { } servers.clear(); } - // Same as StopServer: drain free-list before joining httplib workers. - QuackapiHttpFetch::ResetPool(); + // Stop accepting before joining listener and request-handler threads. for (auto &srv : to_destroy) { if (srv) { srv->StopAccepting(); @@ -470,455 +357,14 @@ bool QuackapiState::GetServerHost(int port, string &host_out) { return false; } -vector> QuackapiState::ListServers() { +vector> QuackapiState::ListServers() { std::lock_guard lock(servers_mutex); - vector> result; + vector> result; result.reserve(servers.size()); for (auto &kv : servers) { - const auto &opts = kv.second->Options(); - string client = opts.http_client_active.empty() ? string("httplib") : opts.http_client_active; - string reason = opts.http_client_reason; - result.emplace_back(kv.second->Host(), kv.second->Port(), std::move(client), std::move(reason)); + result.emplace_back(kv.second->Host(), kv.second->Port()); } return result; } -vector QuackapiState::SnapshotRequests(int port) { - std::lock_guard lock(servers_mutex); - if (port == 0) { - if (servers.empty()) { - return {}; - } - if (servers.size() > 1) { - throw InvalidInputException("quackapi_requests(): port is required when multiple servers are running"); - } - return servers.begin()->second->SnapshotRequests(); - } - for (auto &kv : servers) { - if (kv.second->Port() == port) { - return kv.second->SnapshotRequests(); - } - } - return {}; -} - -vector QuackapiState::SnapshotTelemetryStatus(int port) { - std::lock_guard lock(servers_mutex); - if (port == 0) { - if (servers.empty()) { - return {}; - } - if (servers.size() > 1) { - throw InvalidInputException( - "quackapi_telemetry_status(): port is required when multiple servers are running"); - } - return servers.begin()->second->SnapshotTelemetryStatus(); - } - for (auto &kv : servers) { - if (kv.second->Port() == port) { - return kv.second->SnapshotTelemetryStatus(); - } - } - return {}; -} - -bool QuackapiState::FlushTelemetry(int port) { - std::lock_guard lock(servers_mutex); - if (port == 0) { - if (servers.empty()) { - return false; - } - if (servers.size() > 1) { - throw InvalidInputException( - "quackapi_telemetry_flush(): port is required when multiple servers are running"); - } - servers.begin()->second->FlushTelemetryPending(); - return true; - } - for (auto &kv : servers) { - if (kv.second->Port() == port) { - kv.second->FlushTelemetryPending(); - return true; - } - } - return false; -} - -//===--------------------------------------------------------------------===// -// Row access + masking policies -//===--------------------------------------------------------------------===// - -void QuackapiState::AddRowAccessPolicy(const QuackapiRowAccessPolicy &policy, bool or_replace) { - std::lock_guard lock(policies_mutex); - for (auto it = row_access_policies.begin(); it != row_access_policies.end(); ++it) { - if (it->name == policy.name) { - if (!or_replace) { - throw InvalidInputException( - "Row access policy \"%s\" already exists — use CREATE OR REPLACE ROW ACCESS POLICY", policy.name); - } - *it = policy; - return; - } - } - // Name must not collide with a masking policy. - for (auto &m : masking_policies) { - if (m.name == policy.name) { - throw InvalidInputException("Policy name \"%s\" is already used by a masking policy", policy.name); - } - } - row_access_policies.push_back(policy); -} - -bool QuackapiState::DropRowAccessPolicy(const string &name) { - std::lock_guard lock(policies_mutex); - bool found = false; - for (auto it = row_access_policies.begin(); it != row_access_policies.end(); ++it) { - if (it->name == name) { - row_access_policies.erase(it); - found = true; - break; - } - } - if (found) { - row_access_bindings.erase( - std::remove_if(row_access_bindings.begin(), row_access_bindings.end(), - [&](const QuackapiRowAccessBinding &b) { return b.policy_name == name; }), - row_access_bindings.end()); - } - return found; -} - -bool QuackapiState::GetRowAccessPolicy(const string &name, QuackapiRowAccessPolicy &out) { - std::lock_guard lock(policies_mutex); - for (auto &p : row_access_policies) { - if (p.name == name) { - out = p; - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotRowAccessPolicies() { - std::lock_guard lock(policies_mutex); - return row_access_policies; -} - -void QuackapiState::AddMaskingPolicy(const QuackapiMaskingPolicy &policy, bool or_replace) { - std::lock_guard lock(policies_mutex); - for (auto it = masking_policies.begin(); it != masking_policies.end(); ++it) { - if (it->name == policy.name) { - if (!or_replace) { - throw InvalidInputException( - "Masking policy \"%s\" already exists — use CREATE OR REPLACE MASKING POLICY", policy.name); - } - *it = policy; - return; - } - } - for (auto &r : row_access_policies) { - if (r.name == policy.name) { - throw InvalidInputException("Policy name \"%s\" is already used by a row access policy", policy.name); - } - } - masking_policies.push_back(policy); -} - -bool QuackapiState::DropMaskingPolicy(const string &name) { - std::lock_guard lock(policies_mutex); - bool found = false; - for (auto it = masking_policies.begin(); it != masking_policies.end(); ++it) { - if (it->name == name) { - masking_policies.erase(it); - found = true; - break; - } - } - if (found) { - masking_bindings.erase(std::remove_if(masking_bindings.begin(), masking_bindings.end(), - [&](const QuackapiMaskingBinding &b) { return b.policy_name == name; }), - masking_bindings.end()); - } - return found; -} - -bool QuackapiState::GetMaskingPolicy(const string &name, QuackapiMaskingPolicy &out) { - std::lock_guard lock(policies_mutex); - for (auto &p : masking_policies) { - if (p.name == name) { - out = p; - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotMaskingPolicies() { - std::lock_guard lock(policies_mutex); - return masking_policies; -} - -void QuackapiState::BindRowAccessPolicy(const QuackapiRowAccessBinding &binding, bool or_replace) { - std::lock_guard lock(policies_mutex); - const QuackapiRowAccessPolicy *pol = nullptr; - for (auto &p : row_access_policies) { - if (p.name == binding.policy_name) { - pol = &p; - break; - } - } - if (!pol) { - throw InvalidInputException("Row access policy \"%s\" does not exist", binding.policy_name); - } - if (binding.columns.size() != pol->arg_columns.size()) { - throw InvalidInputException("Row access policy \"%s\" expects %llu column(s), got %llu", binding.policy_name, - (unsigned long long)pol->arg_columns.size(), - (unsigned long long)binding.columns.size()); - } - // One RAP per table (replace same policy name, or or_replace any). - for (auto it = row_access_bindings.begin(); it != row_access_bindings.end(); ++it) { - if (StringUtil::Lower(it->table_name) == StringUtil::Lower(binding.table_name)) { - if (it->policy_name == binding.policy_name || or_replace) { - *it = binding; - return; - } - throw InvalidInputException( - "Table \"%s\" already has row access policy \"%s\" — DROP it first or use OR REPLACE", - binding.table_name, it->policy_name); - } - } - row_access_bindings.push_back(binding); -} - -bool QuackapiState::UnbindRowAccessPolicy(const string &table_name, const string &policy_name) { - std::lock_guard lock(policies_mutex); - for (auto it = row_access_bindings.begin(); it != row_access_bindings.end(); ++it) { - if (StringUtil::Lower(it->table_name) == StringUtil::Lower(table_name) && - (policy_name.empty() || it->policy_name == policy_name)) { - row_access_bindings.erase(it); - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotRowAccessBindings() { - std::lock_guard lock(policies_mutex); - return row_access_bindings; -} - -void QuackapiState::BindMaskingPolicy(const QuackapiMaskingBinding &binding, bool or_replace) { - std::lock_guard lock(policies_mutex); - bool found = false; - for (auto &p : masking_policies) { - if (p.name == binding.policy_name) { - found = true; - break; - } - } - if (!found) { - throw InvalidInputException("Masking policy \"%s\" does not exist", binding.policy_name); - } - for (auto it = masking_bindings.begin(); it != masking_bindings.end(); ++it) { - if (StringUtil::Lower(it->table_name) == StringUtil::Lower(binding.table_name) && - StringUtil::Lower(it->column_name) == StringUtil::Lower(binding.column_name)) { - if (!or_replace && it->policy_name != binding.policy_name) { - throw InvalidInputException("Column \"%s\".\"%s\" already has masking policy \"%s\"", - binding.table_name, binding.column_name, it->policy_name); - } - *it = binding; - return; - } - } - masking_bindings.push_back(binding); -} - -bool QuackapiState::UnbindMaskingPolicy(const string &table_name, const string &column_name) { - std::lock_guard lock(policies_mutex); - for (auto it = masking_bindings.begin(); it != masking_bindings.end(); ++it) { - if (StringUtil::Lower(it->table_name) == StringUtil::Lower(table_name) && - StringUtil::Lower(it->column_name) == StringUtil::Lower(column_name)) { - masking_bindings.erase(it); - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotMaskingBindings() { - std::lock_guard lock(policies_mutex); - return masking_bindings; -} - -void QuackapiState::AddGraphqlTable(const string &table_name, bool or_replace) { - if (table_name.empty()) { - throw InvalidInputException("CREATE GRAPHQL FOR TABLE expects a non-empty table name"); - } - std::lock_guard lock(graphql_mutex); - for (auto &t : graphql_tables) { - if (t == table_name) { - if (!or_replace) { - throw InvalidInputException( - "GraphQL table \"%s\" already registered — use CREATE OR REPLACE GRAPHQL FOR TABLE", table_name); - } - return; - } - } - graphql_tables.push_back(table_name); -} - -bool QuackapiState::DropGraphqlTable(const string &table_name) { - std::lock_guard lock(graphql_mutex); - for (auto it = graphql_tables.begin(); it != graphql_tables.end(); ++it) { - if (*it == table_name) { - graphql_tables.erase(it); - return true; - } - } - return false; -} - -void QuackapiState::ClearGraphqlTables() { - std::lock_guard lock(graphql_mutex); - graphql_tables.clear(); -} - -bool QuackapiState::GraphqlAllowlistActive() { - std::lock_guard lock(graphql_mutex); - return !graphql_tables.empty(); -} - -bool QuackapiState::IsGraphqlTableAllowed(const string &table_name) { - std::lock_guard lock(graphql_mutex); - if (graphql_tables.empty()) { - return true; // open mode - } - for (auto &t : graphql_tables) { - if (t == table_name) { - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotGraphqlTables() { - std::lock_guard lock(graphql_mutex); - auto out = graphql_tables; - std::sort(out.begin(), out.end()); - return out; -} - -namespace { - -string NormalizeGraphqlMountPath(const string &path) { - string p = path; - while (p.size() > 1 && p.back() == '/') { - p.pop_back(); - } - return p; -} - -bool IsReservedGraphqlPath(const string &path) { - return path == "/graphql" || path == "/graphql/schema"; -} - -} // namespace - -void QuackapiState::AddGraphqlRoute(const QuackapiGraphqlRoute &route_in, bool or_replace) { - if (route_in.name.empty()) { - throw InvalidInputException("CREATE GRAPHQL ROUTE expects a non-empty name"); - } - if (route_in.path.empty() || route_in.path[0] != '/') { - throw InvalidInputException("CREATE GRAPHQL ROUTE path must be absolute (start with '/')"); - } - if (route_in.tables.empty()) { - throw InvalidInputException("CREATE GRAPHQL ROUTE requires FROM

[, …]"); - } - QuackapiGraphqlRoute route = route_in; - route.path = NormalizeGraphqlMountPath(route.path); - route.method = StringUtil::Upper(route.method); - if (route.method != "POST") { - throw InvalidInputException("CREATE GRAPHQL ROUTE only supports POST in v0 (got %s)", route.method); - } - if (IsReservedGraphqlPath(route.path)) { - throw InvalidInputException("CREATE GRAPHQL ROUTE path \"%s\" is reserved for the built-in GraphQL endpoint", - route.path); - } - // Schema sibling must not collide with another mount or reserved paths. - string schema_path = route.path + "/schema"; - if (IsReservedGraphqlPath(schema_path)) { - throw InvalidInputException("CREATE GRAPHQL ROUTE path \"%s\" collides with built-in schema URL", route.path); - } - - std::lock_guard lock(graphql_mutex); - for (auto &existing : graphql_routes) { - if (existing.name == route.name) { - continue; // handled below by name match - } - if (existing.path == route.path) { - throw InvalidInputException("GraphQL route path \"%s\" already used by route \"%s\"", route.path, - existing.name); - } - if (existing.path + "/schema" == route.path || existing.path == schema_path) { - throw InvalidInputException("GraphQL route path \"%s\" collides with schema URL of route \"%s\"", - route.path, existing.name); - } - } - for (auto it = graphql_routes.begin(); it != graphql_routes.end(); ++it) { - if (it->name == route.name) { - if (!or_replace) { - throw InvalidInputException("GraphQL route \"%s\" already exists — use CREATE OR REPLACE GRAPHQL ROUTE", - route.name); - } - *it = route; - return; - } - } - graphql_routes.push_back(route); -} - -bool QuackapiState::DropGraphqlRoute(const string &name) { - std::lock_guard lock(graphql_mutex); - for (auto it = graphql_routes.begin(); it != graphql_routes.end(); ++it) { - if (it->name == name) { - graphql_routes.erase(it); - return true; - } - } - return false; -} - -bool QuackapiState::GetGraphqlRouteByPath(const string &path, const string &method, QuackapiGraphqlRoute &out) { - string norm = NormalizeGraphqlMountPath(path); - string meth = StringUtil::Upper(method); - std::lock_guard lock(graphql_mutex); - for (auto &r : graphql_routes) { - if (r.path == norm && r.method == meth) { - out = r; - return true; - } - } - return false; -} - -bool QuackapiState::GetGraphqlRouteBySchemaPath(const string &path, QuackapiGraphqlRoute &out) { - string norm = NormalizeGraphqlMountPath(path); - std::lock_guard lock(graphql_mutex); - for (auto &r : graphql_routes) { - if (norm == r.path + "/schema") { - out = r; - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotGraphqlRoutes() { - std::lock_guard lock(graphql_mutex); - auto out = graphql_routes; - std::sort(out.begin(), out.end(), - [](const QuackapiGraphqlRoute &a, const QuackapiGraphqlRoute &b) { return a.name < b.name; }); - return out; -} - } // namespace duckdb diff --git a/src/quackapi_request_binding.cpp b/src/quackapi_request_binding.cpp new file mode 100644 index 0000000..239c333 --- /dev/null +++ b/src/quackapi_request_binding.cpp @@ -0,0 +1,114 @@ +#include "quackapi_request_binding.hpp" + +#include + +#include "duckdb/common/string_util.hpp" +#include "yyjson.hpp" + +namespace duckdb { +bool IsJsonMediaType(const string &media) { + return media == "application/json" || StringUtil::EndsWith(media, "+json"); +} + +bool IsFormUrlEncodedMediaType(const string &media) { + return media == "application/x-www-form-urlencoded"; +} + +bool IsMultipartMediaType(const string &media) { + return StringUtil::StartsWith(media, "multipart/form-data"); +} + +bool IsBodyMethod(const string &method) { + return method == "POST" || method == "PUT" || method == "PATCH"; +} + +//! application/x-www-form-urlencoded → key/value map (last wins). +void ParseFormUrlEncoded(const string &body, case_insensitive_map_t &out) { + idx_t i = 0; + while (i < body.size()) { + idx_t amp = body.find('&', i); + if (amp == string::npos) { + amp = body.size(); + } + string pair = body.substr(i, amp - i); + if (!pair.empty()) { + auto eq = pair.find('='); + string key, val; + if (eq == string::npos) { + key = duckdb_httplib::decode_query_component(pair, true); + } else { + key = duckdb_httplib::decode_query_component(pair.substr(0, eq), true); + val = duckdb_httplib::decode_query_component(pair.substr(eq + 1), true); + } + if (!key.empty()) { + out[key] = val; + } + } + i = amp + 1; + } +} +JsonBodyParseResult ParseJsonBodyFields(const string &raw_body, case_insensitive_map_t &fields) { + fields.clear(); + if (raw_body.empty()) { + return JsonBodyParseResult::INVALID_JSON; + } + auto document = + duckdb_yyjson::yyjson_read(raw_body.data(), raw_body.size(), duckdb_yyjson::YYJSON_READ_BIGNUM_AS_RAW); + if (!document) { + return JsonBodyParseResult::INVALID_JSON; + } + unique_ptr doc(document, + duckdb_yyjson::yyjson_doc_free); + auto root = duckdb_yyjson::yyjson_doc_get_root(doc.get()); + if (!root || !duckdb_yyjson::yyjson_is_obj(root)) { + return JsonBodyParseResult::NOT_OBJECT; + } + auto iter = duckdb_yyjson::yyjson_obj_iter_with(root); + duckdb_yyjson::yyjson_val *key = nullptr; + while ((key = duckdb_yyjson::yyjson_obj_iter_next(&iter))) { + auto value = duckdb_yyjson::yyjson_obj_iter_get_val(key); + string name(duckdb_yyjson::yyjson_get_str(key), duckdb_yyjson::yyjson_get_len(key)); + if (duckdb_yyjson::yyjson_is_null(value)) { + fields[name] = {string(), true}; + } else if (duckdb_yyjson::yyjson_is_str(value)) { + fields[name] = {string(duckdb_yyjson::yyjson_get_str(value), duckdb_yyjson::yyjson_get_len(value)), false}; + } else { + size_t length = 0; + auto serialized = duckdb_yyjson::yyjson_val_write(value, duckdb_yyjson::YYJSON_WRITE_NOFLAG, &length); + if (!serialized) { + fields.clear(); + return JsonBodyParseResult::INVALID_JSON; + } + fields[name] = {string(serialized, length), false}; + std::free(serialized); + } + } + return JsonBodyParseResult::SUCCESS; +} +//! Parse application/x-www-form-urlencoded query into httplib Params. +void ParseQueryStringIntoParams(const string &query, duckdb_httplib::Params ¶ms) { + if (query.empty()) { + return; + } + idx_t start = 0; + while (start < query.size()) { + idx_t amp = query.find('&', start); + if (amp == string::npos) { + amp = query.size(); + } + string pair = query.substr(start, amp - start); + start = amp + 1; + if (pair.empty()) { + continue; + } + auto eq = pair.find('='); + if (eq == string::npos) { + params.emplace(duckdb_httplib::decode_query_component(pair, true), string()); + } else { + params.emplace(duckdb_httplib::decode_query_component(pair.substr(0, eq), true), + duckdb_httplib::decode_query_component(pair.substr(eq + 1), true)); + } + } +} + +} // namespace duckdb diff --git a/src/quackapi_response_encoding.cpp b/src/quackapi_response_encoding.cpp new file mode 100644 index 0000000..85b0df7 --- /dev/null +++ b/src/quackapi_response_encoding.cpp @@ -0,0 +1,460 @@ +#include "quackapi_response_encoding.hpp" + +#include +#include + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/csv_writer.hpp" +#include "duckdb/common/serializer/write_stream.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/function/function_binder.hpp" +#include "duckdb/main/connection.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/planner/expression/bound_reference_expression.hpp" +#include "quackapi_state.hpp" +#include "quackapi_util.hpp" + +namespace duckdb { +namespace { + +const string &EmptyString() { + static const string empty; + return empty; +} + +class BoundedResponseStream : public WriteStream { +public: + explicit BoundedResponseStream(QuackapiResponseWriter &writer) : writer(writer) { + } + + void WriteData(const_data_ptr_t buffer, idx_t write_size) override { + writer.Append(reinterpret_cast(buffer), write_size); + } + +private: + QuackapiResponseWriter &writer; +}; + +bool IsNestedCsvCell(const LogicalType &type) { + auto id = type.id(); + return id == LogicalTypeId::LIST || id == LogicalTypeId::MAP || id == LogicalTypeId::STRUCT; +} + +string TrimMediaToken(string value) { + StringUtil::Trim(value); + return value; +} + +} // namespace + +QuackapiResponseWriter::QuackapiResponseWriter(idx_t max_bytes) + : max_bytes(max_bytes), bytes_written(0), exceeded_limit(false) { +} + +bool QuackapiResponseWriter::Append(const char *append_data, idx_t length) { + if (exceeded_limit) { + return false; + } + if (length > max_bytes - bytes_written) { + exceeded_limit = true; + return false; + } + if (length) { + data.append(append_data, length); + } + bytes_written += length; + return true; +} + +bool QuackapiResponseWriter::Append(const string &append_data) { + return Append(append_data.data(), append_data.size()); +} + +idx_t QuackapiResponseWriter::BytesWritten() const { + return bytes_written; +} + +bool QuackapiResponseWriter::ExceededLimit() const { + return exceeded_limit; +} + +const string &QuackapiResponseWriter::Data() const { + return exceeded_limit ? EmptyString() : data; +} + +string QuackapiResponseWriter::TakeData() { + return std::move(data); +} + +class QuackapiJsonChunkEncoder::State { +public: + State(Connection &con, const vector &names, const vector &types, + const vector &data_cols, JsonRowMode mode, QuackapiResponseWriter &writer) + : mode(mode), writer(writer), row_count(0), encoded_rows(0), finished(false) { + if (names.size() != types.size()) { + throw InvalidInputException("JSON encoder: result name/type count mismatch"); + } + // json_object supplies DuckDB's own recursive serializer for every column + // type (including MAP, STRUCT, LIST, and JSON), rather than Value::ToString. + // The JSON extension is a declared runtime dependency; don't LOAD it while + // a route result is live. + vector> args; + for (auto col : data_cols) { + if (col >= names.size()) { + throw InvalidInputException("JSON encoder: data column index is out of bounds"); + } + args.push_back(make_uniq(Value(names[col]))); + args.push_back(make_uniq(types[col], UnsafeNumericCast(col))); + } + con.context->RunFunctionInTransaction([&]() { + FunctionBinder binder(*con.context); + ErrorData error; + expression = binder.BindScalarFunction(DEFAULT_SCHEMA, "json_object", std::move(args), error); + if (!expression) { + throw InvalidInputException("JSON encoder: could not bind json_object: %s", error.Message()); + } + executor = make_uniq(*con.context, *expression); + }); + if (mode == JsonRowMode::ARRAY) { + writer.Append("["); + } + } + + void AppendChunk(DataChunk &chunk) { + if (finished) { + throw InternalException("JSON encoder: cannot append after Finish"); + } + row_count += chunk.size(); + if (writer.ExceededLimit() || chunk.size() == 0 || (mode == JsonRowMode::OBJECT && encoded_rows > 0)) { + return; + } + Vector rows(LogicalType::JSON(), chunk.size()); + executor->ExecuteExpression(chunk, rows); + UnifiedVectorFormat format; + rows.ToUnifiedFormat(chunk.size(), format); + auto data = format.GetData(); + for (idx_t i = 0; i < chunk.size(); i++) { + if (mode == JsonRowMode::ARRAY && encoded_rows > 0 && !writer.Append(",")) { + return; + } + auto index = format.sel->get_index(i); + if (!format.validity.RowIsValid(index)) { + throw InternalException("JSON encoder: json_object returned NULL"); + } + auto row = data[index]; + if (!writer.Append(row.GetData(), row.GetSize())) { + return; + } + if (mode == JsonRowMode::NDJSON && !writer.Append("\n")) { + return; + } + encoded_rows++; + if (mode == JsonRowMode::OBJECT) { + return; + } + } + } + + void Finish() { + if (finished) { + return; + } + finished = true; + if (mode == JsonRowMode::ARRAY) { + writer.Append("]"); + } else if (mode == JsonRowMode::OBJECT && encoded_rows == 0) { + writer.Append("null"); + } + } + + JsonRowMode mode; + QuackapiResponseWriter &writer; + idx_t row_count; + idx_t encoded_rows; + bool finished; + unique_ptr expression; + unique_ptr executor; +}; + +QuackapiJsonChunkEncoder::QuackapiJsonChunkEncoder(Connection &con, const vector &names, + const vector &types, const vector &data_cols, + JsonRowMode mode, QuackapiResponseWriter &writer) + : state(make_uniq(con, names, types, data_cols, mode, writer)) { +} + +QuackapiJsonChunkEncoder::~QuackapiJsonChunkEncoder() = default; + +QuackapiJsonChunkEncoder::QuackapiJsonChunkEncoder(QuackapiJsonChunkEncoder &&) noexcept = default; + +QuackapiJsonChunkEncoder &QuackapiJsonChunkEncoder::operator=(QuackapiJsonChunkEncoder &&) noexcept = default; + +void QuackapiJsonChunkEncoder::AppendChunk(DataChunk &chunk) { + state->AppendChunk(chunk); +} + +idx_t QuackapiJsonChunkEncoder::RowCount() const { + return state->row_count; +} + +void QuackapiJsonChunkEncoder::Finish() { + state->Finish(); +} + +class QuackapiCsvChunkEncoder::State { +public: + State(Connection &con, const vector &names, const vector &types, + const vector &data_cols, QuackapiResponseWriter &response_writer) + : response_writer(response_writer), stream(response_writer), + writer(stream, SelectNames(names, data_cols), false), writer_state(*con.context, 8192), executor(nullptr), + finished(false) { + if (names.size() != types.size()) { + throw InvalidInputException("CSV encoder: result name/type count mismatch"); + } + for (auto col : data_cols) { + if (col >= names.size()) { + throw InvalidInputException("CSV encoder: data column index is out of bounds"); + } + } + con.context->RunFunctionInTransaction([&]() { + for (auto col : data_cols) { + unique_ptr cell = + make_uniq(types[col], UnsafeNumericCast(col)); + if (IsNestedCsvCell(types[col])) { + vector> args; + args.push_back(std::move(cell)); + FunctionBinder binder(*con.context); + ErrorData error; + cell = binder.BindScalarFunction(DEFAULT_SCHEMA, "to_json", std::move(args), error); + if (!cell) { + throw InvalidInputException("CSV encoder: could not bind to_json: %s", error.Message()); + } + } + casts.push_back( + BoundCastExpression::AddCastToType(*con.context, std::move(cell), LogicalType::VARCHAR)); + output_types.push_back(LogicalType::VARCHAR); + } + if (!casts.empty()) { + executor = make_uniq(*con.context, casts); + } + }); + writer.options.null_str = {""}; + writer.options.dialect_options.header = CSVOption(true, true); + writer.writer_options.newline_writing_mode = CSVNewLineMode::WRITE_AFTER; + writer.Initialize(); + writer.WriteRawString(writer.writer_options.newline); + } + + static vector SelectNames(const vector &names, const vector &data_cols) { + vector result; + result.reserve(data_cols.size()); + for (auto col : data_cols) { + if (col >= names.size()) { + throw InvalidInputException("CSV encoder: data column index is out of bounds"); + } + result.push_back(names[col]); + } + return result; + } + + void AppendChunk(DataChunk &chunk) { + if (finished) { + throw InternalException("CSV encoder: cannot append after Finish"); + } + if (chunk.size() == 0) { + return; + } + if (!executor) { + for (idx_t i = 0; i < chunk.size(); i++) { + response_writer.Append("\n"); + } + return; + } + DataChunk output; + output.Initialize(executor->GetContext(), output_types); + executor->Execute(chunk, output); + writer.WriteChunk(output, writer_state); + writer.Flush(writer_state); + } + + void Finish() { + if (finished) { + return; + } + finished = true; + writer.Flush(writer_state); + } + + QuackapiResponseWriter &response_writer; + BoundedResponseStream stream; + CSVWriter writer; + CSVWriterState writer_state; + vector> casts; + vector output_types; + unique_ptr executor; + bool finished; +}; + +QuackapiCsvChunkEncoder::QuackapiCsvChunkEncoder(Connection &con, const vector &names, + const vector &types, const vector &data_cols, + QuackapiResponseWriter &writer) + : state(make_uniq(con, names, types, data_cols, writer)) { +} + +QuackapiCsvChunkEncoder::~QuackapiCsvChunkEncoder() = default; + +QuackapiCsvChunkEncoder::QuackapiCsvChunkEncoder(QuackapiCsvChunkEncoder &&) noexcept = default; + +QuackapiCsvChunkEncoder &QuackapiCsvChunkEncoder::operator=(QuackapiCsvChunkEncoder &&) noexcept = default; + +void QuackapiCsvChunkEncoder::AppendChunk(DataChunk &chunk) { + state->AppendChunk(chunk); +} + +void QuackapiCsvChunkEncoder::Finish() { + state->Finish(); +} + +//! True if column name is a special response control column (stripped from body). +bool IsSpecialResponseColumn(const string &name) { + auto lower = StringUtil::Lower(name); + return lower == "location" || lower == "set_cookie" || lower == "set-cookie"; +} + +void SetJson(duckdb_httplib::Response &res, int status, const string &body) { + res.status = status; + res.set_content(body, "application/json"); +} + +//! Response mode inferred from the single output column's name — the database +//! names the payload. `html` -> text/html, `text` -> text/plain; anything else +//! serializes as row data (JSON / NDJSON / CSV via FORMAT + Accept). + +ResponseMode ResponseModeFor(const vector &names) { + if (names.size() != 1) { + return ResponseMode::JSON; + } + auto lower = StringUtil::Lower(names[0]); + if (lower == "html") { + return ResponseMode::HTML; + } + if (lower == "text") { + return ResponseMode::TEXT; + } + return ResponseMode::JSON; +} + +//! Wire format for multi-row data responses (not html/text column modes). + +BodyFormat ParseBodyFormatToken(const string &token) { + auto t = StringUtil::Lower(token); + if (t == "ndjson") { + return BodyFormat::NDJSON; + } + if (t == "csv") { + return BodyFormat::CSV; + } + return BodyFormat::JSON; +} + +//! Map Accept media-type (no parameters) to BodyFormat. Returns false if unknown. +bool MediaTypeToBodyFormat(const string &media, BodyFormat &out) { + auto m = StringUtil::Lower(media); + // strip parameters already; still trim + m = TrimMediaToken(m); + if (m == "application/x-ndjson" || m == "application/jsonl" || m == "application/ndjson") { + out = BodyFormat::NDJSON; + return true; + } + if (m == "text/csv" || m == "application/csv") { + out = BodyFormat::CSV; + return true; + } + if (m == "application/json" || m == "text/json" || m == "*/*") { + out = BodyFormat::JSON; + return true; + } + return false; +} + +//! Parse one Accept token's q-value (default 1.0). +double ParseAcceptQ(const string &token) { + auto parts = StringUtil::Split(token, ';'); + if (parts.size() < 2) { + return 1.0; + } + for (idx_t i = 1; i < parts.size(); i++) { + auto p = TrimMediaToken(parts[i]); + auto pl = StringUtil::Lower(p); + if (StringUtil::StartsWith(pl, "q=")) { + return atof(p.c_str() + 2); + } + } + return 1.0; +} + +//! When route FORMAT is default/json, honor Accept for ndjson/csv/json. +//! Explicit route FORMAT (ndjson|csv) always wins. +BodyFormat ResolveBodyFormat(const QuackapiRoute &route, const duckdb_httplib::Request &req) { + auto route_fmt = StringUtil::Lower(route.response_format.empty() ? "json" : route.response_format); + if (route_fmt == "ndjson" || route_fmt == "csv") { + return ParseBodyFormatToken(route_fmt); + } + // Default / explicit json: negotiate Accept when present. + auto it = req.headers.find("Accept"); + if (it == req.headers.end() || it->second.empty()) { + return BodyFormat::JSON; + } + // Pick the highest-q recognized type among json / ndjson / csv. + BodyFormat best = BodyFormat::JSON; + double best_q = -1.0; + bool saw_recognized = false; + auto tokens = StringUtil::Split(it->second, ','); + for (auto &tok : tokens) { + auto t = TrimMediaToken(tok); + if (t.empty()) { + continue; + } + double q = ParseAcceptQ(t); + if (q <= 0.0) { + continue; + } + auto semi = t.find(';'); + string media = semi == string::npos ? t : TrimMediaToken(t.substr(0, semi)); + BodyFormat f; + if (!MediaTypeToBodyFormat(media, f)) { + continue; + } + // Prefer a concrete non-json format when q ties with JSON. + bool better = !saw_recognized || q > best_q; + if (!better && q == best_q) { + // Prefer non-json when client listed it at same q after json — first highest wins + // unless current best is from */* and this is concrete. + if (best == BodyFormat::JSON && f != BodyFormat::JSON) { + better = true; + } + } + if (better) { + best = f; + best_q = q; + saw_recognized = true; + } + } + return saw_recognized ? best : BodyFormat::JSON; +} + +string EmptyResultBody(const QuackapiRoute &route) { + if (!route.empty_body.empty()) { + return route.empty_body; + } + return "{\"detail\":\"Not Found\"}"; +} + +void SetInternalError(duckdb_httplib::Response &res, const string &server_side_detail) { + // Return DuckDB's diagnostic so clients can diagnose route execution failures. + fprintf(stderr, "quackapi: internal error: %s\n", server_side_detail.c_str()); + SetJson(res, 500, "{\"detail\":\"" + QuackapiJsonEscape(server_side_detail) + "\"}"); +} + +} // namespace duckdb diff --git a/src/quackapi_server.cpp b/src/quackapi_server.cpp index 386c26c..39ffd26 100644 --- a/src/quackapi_server.cpp +++ b/src/quackapi_server.cpp @@ -1,18 +1,19 @@ #include "quackapi_server.hpp" -#include +#include "quackapi_logging.hpp" +#include "quackapi_request_binding.hpp" +#include "quackapi_response_encoding.hpp" + #include #include #include #include #include -#include #include #include #include "duckdb/common/case_insensitive_map.hpp" #include "duckdb/common/exception.hpp" -#include "duckdb/common/limits.hpp" #include "duckdb/common/file_open_flags.hpp" #include "duckdb/common/file_system.hpp" #include "duckdb/common/helper.hpp" @@ -23,20 +24,17 @@ #include "duckdb/common/types/uuid.hpp" #include "duckdb/common/types/value.hpp" #include "duckdb/common/types/timestamp.hpp" -#include "duckdb/main/appender.hpp" #include "duckdb/main/client_context.hpp" #include "duckdb/main/connection.hpp" #include "duckdb/main/database.hpp" +#include "duckdb/main/extension_helper.hpp" #include "duckdb/main/prepared_statement.hpp" #include "duckdb/main/query_result.hpp" #include "duckdb/parser/keyword_helper.hpp" #include "duckdb/parser/parser.hpp" #include "quackapi_auth.hpp" -#include "quackapi_graphql.hpp" #include "quackapi_openapi.hpp" -#include "quackapi_pg.hpp" -#include "quackapi_policy.hpp" #include "quackapi_limits.hpp" #include "quackapi_middleware.hpp" #include "quackapi_state.hpp" @@ -98,9 +96,9 @@ struct QuackapiHttplibServer : duckdb_httplib::Server { //! SO_* is restored to serve defaults in QuackapiHttplibServer after //! process_request returns (keep-alive safe). Do not restore in a HandleRequest //! destructor — that runs before httplib's write_response. -//! Records the applied write deadline on qa_state only after this path runs -//! (early return leaves the serve-level seed from HandleRequest unchanged). -void ApplyRouteIoTimeout(time_t timeout_sec, QuackapiState &qa_state) { +//! Route timeout introspection was removed; the socket deadlines remain local +//! to the active request. +void ApplyRouteIoTimeout(time_t timeout_sec) { if (timeout_sec <= 0) { return; } @@ -113,7 +111,6 @@ void ApplyRouteIoTimeout(time_t timeout_sec, QuackapiState &qa_state) { duckdb_httplib::detail::set_socket_opt_time(quackapi_tls_sock, SOL_SOCKET, SO_RCVTIMEO, timeout_sec, 0); duckdb_httplib::detail::set_socket_opt_time(quackapi_tls_sock, SOL_SOCKET, SO_SNDTIMEO, timeout_sec, 0); } - qa_state.SetLastEffectiveWriteTimeoutSec(static_cast(timeout_sec)); } //! Content-Encoding choice after Accept-Encoding negotiation. @@ -283,66 +280,6 @@ bool CompressGzip(const string &input, string &output) { } } -//! Render a DuckDB Value as JSON. The database typed the column — the JSON -//! representation follows the type, not string-formatting heuristics. -string ValueToJson(const Value &value) { - if (value.IsNull()) { - return "null"; - } - auto &type = value.type(); - switch (type.id()) { - case LogicalTypeId::BOOLEAN: - return value.GetValue() ? "true" : "false"; - case LogicalTypeId::TINYINT: - case LogicalTypeId::SMALLINT: - case LogicalTypeId::INTEGER: - case LogicalTypeId::BIGINT: - case LogicalTypeId::HUGEINT: - case LogicalTypeId::UTINYINT: - case LogicalTypeId::USMALLINT: - case LogicalTypeId::UINTEGER: - case LogicalTypeId::UBIGINT: - case LogicalTypeId::DECIMAL: - return value.ToString(); - case LogicalTypeId::FLOAT: - case LogicalTypeId::DOUBLE: { - auto d = value.GetValue(); - if (std::isnan(d) || std::isinf(d)) { - // JSON has no NaN/Infinity — mirror FastAPI/ujson and emit null - return "null"; - } - return value.ToString(); - } - case LogicalTypeId::LIST: { - auto &children = ListValue::GetChildren(value); - string result = "["; - for (idx_t i = 0; i < children.size(); i++) { - if (i > 0) { - result += ","; - } - result += ValueToJson(children[i]); - } - result += "]"; - return result; - } - case LogicalTypeId::STRUCT: { - auto &children = StructValue::GetChildren(value); - auto &child_types = StructType::GetChildTypes(type); - string result = "{"; - for (idx_t i = 0; i < children.size(); i++) { - if (i > 0) { - result += ","; - } - result += "\"" + QuackapiJsonEscape(child_types[i].first) + "\":" + ValueToJson(children[i]); - } - result += "}"; - return result; - } - default: - return "\"" + QuackapiJsonEscape(value.ToString()) + "\""; - } -} - //! FastAPI-shaped validation error body (loc = [kind, name]). string ValidationErrorJson(const string &loc_kind, const string ¶m_name, const string &msg, const string &type) { return "{\"detail\":[{\"loc\":[\"" + QuackapiJsonEscape(loc_kind) + "\",\"" + QuackapiJsonEscape(param_name) + @@ -378,7 +315,7 @@ string PairsToJsonObject(const PairRange &pairs) { //! The json_schema extension includes an RFC 6901 pointer in failures (for //! example, "At /items/0/qty: ..."). Surface it as FastAPI's typed loc array //! instead of losing the field path behind a synthetic _schema member. -string ValidationErrorJsonSchema(const string &msg, const string &type, const string &raw_body) { +string ValidationErrorJsonSchema(Connection &con, const string &msg, const string &type, const string &raw_body) { const string markers[] = {"At /", "at /"}; for (auto &marker : markers) { auto start_marker = msg.find(marker); @@ -401,23 +338,12 @@ string ValidationErrorJsonSchema(const string &msg, const string &type, const st } if (end > start) { return QuackapiValidationErrorsJson( - {{QuackapiValidationBodyPointerLoc(msg.substr(start, end - start), raw_body), msg, type}}); + {{QuackapiValidationBodyPointerLoc(con, msg.substr(start, end - start), raw_body), msg, type}}); } } return ValidationErrorJson("body", "_schema", msg, type); } -//! Preserve whether a JSON member was an actual JSON null. A string value of -//! "null" remains a string and must not silently become SQL NULL. -struct JsonBodyField { - string value; - bool explicit_null; - JsonBodyField() : explicit_null(false) { - } - JsonBodyField(string value_p, bool explicit_null_p) : value(std::move(value_p)), explicit_null(explicit_null_p) { - } -}; - //! Media type from Content-Type (strip parameters; lowercased). string ContentTypeMedia(const case_insensitive_map_t &headers) { auto it = headers.find("Content-Type"); @@ -433,275 +359,6 @@ string ContentTypeMedia(const case_insensitive_map_t &headers) { return StringUtil::Lower(ct); } -bool IsJsonMediaType(const string &media) { - return media == "application/json" || StringUtil::EndsWith(media, "+json"); -} - -bool IsFormUrlEncodedMediaType(const string &media) { - return media == "application/x-www-form-urlencoded"; -} - -bool IsMultipartMediaType(const string &media) { - return StringUtil::StartsWith(media, "multipart/form-data"); -} - -bool IsBodyMethod(const string &method) { - return method == "POST" || method == "PUT" || method == "PATCH"; -} - -//! application/x-www-form-urlencoded → key/value map (last wins). -void ParseFormUrlEncoded(const string &body, case_insensitive_map_t &out) { - idx_t i = 0; - while (i < body.size()) { - idx_t amp = body.find('&', i); - if (amp == string::npos) { - amp = body.size(); - } - string pair = body.substr(i, amp - i); - if (!pair.empty()) { - auto eq = pair.find('='); - string key, val; - if (eq == string::npos) { - key = duckdb_httplib::decode_query_component(pair, true); - } else { - key = duckdb_httplib::decode_query_component(pair.substr(0, eq), true); - val = duckdb_httplib::decode_query_component(pair.substr(eq + 1), true); - } - if (!key.empty()) { - out[key] = val; - } - } - i = amp + 1; - } -} - -//! Fast path: flat JSON object of string/number/bool/null scalars (no DuckDB round-trip). -//! Returns true on success. false → caller may fall back to SQL extract or error. -bool TryExtractFlatJsonObject(const string &raw_body, case_insensitive_map_t &fields) { - const char *s = raw_body.c_str(); - const char *end = s + raw_body.size(); - auto skip_ws = [&]() { - while (s < end && (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')) { - s++; - } - }; - auto is_digit = [](char c) { - return c >= '0' && c <= '9'; - }; - // This path deliberately accepts only unescaped JSON strings. Escapes require - // full JSON decoding (in particular, UTF-16 surrogate handling), so leave them - // to DuckDB instead of attempting a partial implementation here. - auto parse_plain_string = [&](string &out) { - if (s >= end || *s != '"') { - return false; - } - s++; - while (s < end && *s != '"') { - const auto ch = static_cast(*s); - if (ch < 0x20 || *s == '\\') { - return false; - } - out.push_back(*s++); - } - if (s >= end) { - return false; - } - s++; - return true; - }; - auto parse_number = [&]() { - const char *start = s; - if (s < end && *s == '-') { - s++; - } - if (s >= end) { - return false; - } - if (*s == '0') { - s++; - if (s < end && is_digit(*s)) { - return false; - } - } else if (*s >= '1' && *s <= '9') { - do { - s++; - } while (s < end && is_digit(*s)); - } else { - return false; - } - if (s < end && *s == '.') { - s++; - if (s >= end || !is_digit(*s)) { - return false; - } - do { - s++; - } while (s < end && is_digit(*s)); - } - if (s < end && (*s == 'e' || *s == 'E')) { - s++; - if (s < end && (*s == '+' || *s == '-')) { - s++; - } - if (s >= end || !is_digit(*s)) { - return false; - } - do { - s++; - } while (s < end && is_digit(*s)); - } - return s > start; - }; - skip_ws(); - if (s >= end || *s != '{') { - return false; - } - s++; - fields.clear(); - skip_ws(); - if (s < end && *s == '}') { - s++; - skip_ws(); - return s == end; - } - while (s < end) { - skip_ws(); - string key; - if (!parse_plain_string(key)) { - return false; - } - skip_ws(); - if (s >= end || *s != ':') { - return false; - } - s++; - skip_ws(); - if (s >= end) { - return false; - } - string val; - bool explicit_null = false; - if (*s == '"') { - if (!parse_plain_string(val)) { - return false; - } - } else if (*s == 'n' && s + 4 <= end && string(s, 4) == "null") { - explicit_null = true; - s += 4; - } else if (*s == 't' && s + 4 <= end && string(s, 4) == "true") { - val = "true"; - s += 4; - } else if (*s == 'f' && s + 5 <= end && string(s, 5) == "false") { - val = "false"; - s += 5; - } else if (*s == '-' || (*s >= '0' && *s <= '9')) { - const char *start = s; - if (!parse_number()) { - return false; - } - val.assign(start, s - start); - } else { - // nested object/array — not flat; fall back - return false; - } - fields[key] = {val, explicit_null}; - skip_ws(); - if (s < end && *s == ',') { - s++; - continue; - } - if (s < end && *s == '}') { - s++; - skip_ws(); - return s == end; - } - return false; - } - return false; -} - -//! Extract top-level JSON object fields as string values for binding. -//! On invalid JSON sets err_json and returns false. Arrays/non-objects → model_attributes_type. -bool ExtractJsonBodyFields(Connection &con, const string &raw_body, case_insensitive_map_t &fields, - string &err_json) { - fields.clear(); - if (raw_body.empty()) { - err_json = ValidationErrorJsonBody("JSON decode error", "json_invalid"); - return false; - } - // Hot path: flat object without spinning DuckDB (POST /write etc.). - if (TryExtractFlatJsonObject(raw_body, fields)) { - return true; - } - // json_each lives in the JSON extension and is not available when automatic - // extension loading is disabled. Loading a bundled extension is idempotent. - auto json_load = con.Query("LOAD json"); - if (json_load->HasError()) { - err_json = ValidationErrorJsonBody("JSON extension unavailable", "value_error"); - return false; - } - // Validate JSON parse first (TRY_CAST → NULL on failure). - auto check = con.Query("SELECT TRY_CAST(? AS JSON) IS NOT NULL", Value(raw_body)); - if (check->HasError()) { - err_json = ValidationErrorJsonBody("JSON decode error", "json_invalid"); - return false; - } - auto check_chunk = check->Fetch(); - if (!check_chunk || check_chunk->size() == 0 || !check_chunk->GetValue(0, 0).GetValue()) { - err_json = ValidationErrorJsonBody("JSON decode error", "json_invalid"); - return false; - } - // Must be an object to extract named fields (FastAPI body model). - auto type_res = con.Query("SELECT json_type(?::JSON)", Value(raw_body)); - if (type_res->HasError()) { - err_json = ValidationErrorJsonBody("JSON decode error", "json_invalid"); - return false; - } - auto type_chunk = type_res->Fetch(); - string jtype = type_chunk && type_chunk->size() > 0 && !type_chunk->GetValue(0, 0).IsNull() - ? type_chunk->GetValue(0, 0).GetValue() - : string(); - if (jtype != "OBJECT") { - err_json = ValidationErrorJsonBody("Input should be a valid dictionary or object to extract fields from", - "model_attributes_type"); - return false; - } - // json_each supplies the literal key, so keys containing dots or other JSONPath - // metacharacters cannot be reinterpreted as a path expression. - auto fields_res = con.Query("SELECT key, " - " CASE type " - " WHEN 'VARCHAR' THEN json_extract_string(value, '$') " - " WHEN 'NULL' THEN NULL " - " ELSE CAST(value AS VARCHAR) " - " END AS val " - "FROM json_each(?::JSON)", - Value(raw_body)); - if (fields_res->HasError()) { - err_json = ValidationErrorJsonBody("JSON decode error", "json_invalid"); - return false; - } - while (true) { - auto chunk = fields_res->Fetch(); - if (!chunk || chunk->size() == 0) { - break; - } - for (idx_t row = 0; row < chunk->size(); row++) { - auto key_v = chunk->GetValue(0, row); - if (key_v.IsNull()) { - continue; - } - string key = key_v.GetValue(); - auto val_v = chunk->GetValue(1, row); - if (val_v.IsNull()) { - fields[key] = {string(), true}; - continue; - } - fields[key] = {val_v.GetValue(), false}; - } - } - return true; -} - //! Validate body against BODY SCHEMA using community json_schema extension. //! json_schema_validate returns true on pass and THROWS on fail — wrap with try(). bool ValidateBodySchema(Connection &con, const string &schema, const string &raw_body, string &err_json) { @@ -741,7 +398,7 @@ bool ValidateBodySchema(Connection &con, const string &schema, const string &raw msg = msg.substr(0, 200); } } - err_json = ValidationErrorJsonSchema(msg, "value_error", raw_body); + err_json = ValidationErrorJsonSchema(con, msg, "value_error", raw_body); return false; } @@ -751,16 +408,6 @@ bool ValidateBodySchema(Connection &con, const string &schema, const string &raw bool TransformTypedBody(Connection &con, const string &body_type, const string &raw_body, Value &out, string &err_json) { auto result = con.Query("SELECT try(json_transform_strict(?::JSON, ?::JSON))", Value(raw_body), Value(body_type)); - // json_transform is supplied by DuckDB's bundled json extension. Test and - // embedded deployments may turn automatic extension loading off, so load it - // explicitly only after DuckDB identifies this precise missing dependency. - if (result->HasError() && result->GetError().find("json extension") != string::npos) { - auto load = con.Query("LOAD json"); - if (!load->HasError()) { - result = - con.Query("SELECT try(json_transform_strict(?::JSON, ?::JSON))", Value(raw_body), Value(body_type)); - } - } if (!result->HasError()) { auto chunk = result->Fetch(); if (chunk && chunk->size() > 0 && !chunk->GetValue(0, 0).IsNull()) { @@ -971,28 +618,6 @@ vector ProtectedSqlText(const string &sql) { return protected_text; } -//! Parse request SQL without executing it. The /sql route delegates the -//! submitted text to a dynamic SQL table function; some remote query paths -//! report parser failures as an empty result instead of an error result. -//! Extracting each statement preserves valid multi-statement input without -//! binding against the route server's catalog instead of the remote database. -bool ValidateRequestSql(Connection &con, const string &sql, string &error) { - try { - auto statements = con.ExtractStatements(sql); - if (statements.empty()) { - error = "No statement to parse!"; - return false; - } - return true; - } catch (std::exception &ex) { - error = ex.what(); - return false; - } catch (...) { - error = "unknown SQL validation error"; - return false; - } -} - //! Resolve the native DuckDB type represented by a BODY TYPE declaration. //! This lets route SQL use $body.items directly: the server inserts the cast //! from the one declaration before DuckDB prepares the handler. @@ -1062,41 +687,6 @@ struct RouteMatch { vector> path_params; }; -struct StreamMatch { - bool matched = false; - QuackapiStream stream; - vector> path_params; -}; - -//! Format one SSE event from a result row. Includes `id:` when a column named -//! `id` (case-insensitive) is present and non-null. -string FormatSseEvent(const vector &names, const vector &cols) { - string event; - idx_t id_col = names.size(); - for (idx_t c = 0; c < names.size(); c++) { - if (StringUtil::Lower(names[c]) == "id") { - id_col = c; - break; - } - } - if (id_col < cols.size() && !cols[id_col].IsNull()) { - event += "id: "; - event += cols[id_col].ToString(); - event += "\n"; - } - event += "data: {"; - bool first = true; - for (idx_t c = 0; c < names.size() && c < cols.size(); c++) { - if (!first) { - event += ","; - } - first = false; - event += "\"" + QuackapiJsonEscape(names[c]) + "\":" + ValueToJson(cols[c]); - } - event += "}\n\n"; - return event; -} - vector SplitPath(const string &path) { vector segments; string current; @@ -1247,439 +837,6 @@ bool FindHeaderValue(const case_insensitive_map_t &headers, const string return false; } -//! True if column name is a special response control column (stripped from body). -bool IsSpecialResponseColumn(const string &name) { - auto lower = StringUtil::Lower(name); - return lower == "location" || lower == "set_cookie" || lower == "set-cookie"; -} - -void SetJson(duckdb_httplib::Response &res, int status, const string &body) { - res.status = status; - res.set_content(body, "application/json"); -} - -//! Response mode inferred from the single output column's name — the database -//! names the payload. `html` -> text/html, `text` -> text/plain; anything else -//! serializes as row data (json / ndjson / csv / parquet / arrow via FORMAT + Accept). -enum class ResponseMode { JSON, HTML, TEXT }; - -ResponseMode ResponseModeFor(const vector &names) { - if (names.size() != 1) { - return ResponseMode::JSON; - } - auto lower = StringUtil::Lower(names[0]); - if (lower == "html") { - return ResponseMode::HTML; - } - if (lower == "text") { - return ResponseMode::TEXT; - } - return ResponseMode::JSON; -} - -//! Wire format for multi-row data responses (not html/text column modes). -enum class BodyFormat { JSON, NDJSON, CSV, PARQUET, ARROW }; - -BodyFormat ParseBodyFormatToken(const string &token) { - auto t = StringUtil::Lower(token); - if (t == "ndjson") { - return BodyFormat::NDJSON; - } - if (t == "csv") { - return BodyFormat::CSV; - } - if (t == "parquet") { - return BodyFormat::PARQUET; - } - if (t == "arrow" || t == "arrows") { - return BodyFormat::ARROW; - } - return BodyFormat::JSON; -} - -//! Map Accept media-type (no parameters) to BodyFormat. Returns false if unknown. -bool MediaTypeToBodyFormat(const string &media, BodyFormat &out) { - auto m = StringUtil::Lower(media); - // strip parameters already; still trim - m = QuackapiTrim(m); - if (m == "application/x-ndjson" || m == "application/jsonl" || m == "application/ndjson") { - out = BodyFormat::NDJSON; - return true; - } - if (m == "text/csv" || m == "application/csv") { - out = BodyFormat::CSV; - return true; - } - if (m == "application/vnd.apache.parquet" || m == "application/parquet") { - out = BodyFormat::PARQUET; - return true; - } - // Arrow IPC stream (what nanoarrow FORMAT ARROWS emits) or file. - if (m == "application/vnd.apache.arrow.stream" || m == "application/vnd.apache.arrow.file" || - m == "application/x-apache-arrow-stream" || m == "application/arrow") { - out = BodyFormat::ARROW; - return true; - } - if (m == "application/json" || m == "text/json" || m == "*/*") { - out = BodyFormat::JSON; - return true; - } - return false; -} - -//! Parse one Accept token's q-value (default 1.0). -double ParseAcceptQ(const string &token) { - auto parts = StringUtil::Split(token, ';'); - if (parts.size() < 2) { - return 1.0; - } - for (idx_t i = 1; i < parts.size(); i++) { - auto p = QuackapiTrim(parts[i]); - auto pl = StringUtil::Lower(p); - if (StringUtil::StartsWith(pl, "q=")) { - return atof(p.c_str() + 2); - } - } - return 1.0; -} - -//! When route FORMAT is default/json, honor Accept for ndjson/csv/parquet/arrow/json. -//! Explicit route FORMAT (ndjson|csv|parquet|arrow) always wins. -BodyFormat ResolveBodyFormat(const QuackapiRoute &route, const duckdb_httplib::Request &req) { - auto route_fmt = StringUtil::Lower(route.response_format.empty() ? "json" : route.response_format); - if (route_fmt == "ndjson" || route_fmt == "csv" || route_fmt == "parquet" || route_fmt == "arrow" || - route_fmt == "arrows") { - return ParseBodyFormatToken(route_fmt); - } - // Default / explicit json: negotiate Accept when present. - auto it = req.headers.find("Accept"); - if (it == req.headers.end() || it->second.empty()) { - return BodyFormat::JSON; - } - // Pick the highest-q recognized type among json / ndjson / csv / parquet / arrow. - BodyFormat best = BodyFormat::JSON; - double best_q = -1.0; - bool saw_recognized = false; - auto tokens = StringUtil::Split(it->second, ','); - for (auto &tok : tokens) { - auto t = QuackapiTrim(tok); - if (t.empty()) { - continue; - } - double q = ParseAcceptQ(t); - if (q <= 0.0) { - continue; - } - auto semi = t.find(';'); - string media = semi == string::npos ? t : QuackapiTrim(t.substr(0, semi)); - BodyFormat f; - if (!MediaTypeToBodyFormat(media, f)) { - continue; - } - // Prefer more specific non-wildcard when q ties: ndjson/csv/parquet/arrow over */* json. - bool better = !saw_recognized || q > best_q; - if (!better && q == best_q) { - // Prefer non-json when client listed it at same q after json — first highest wins - // unless current best is from */* and this is concrete. - if (best == BodyFormat::JSON && f != BodyFormat::JSON) { - better = true; - } - } - if (better) { - best = f; - best_q = q; - saw_recognized = true; - } - } - return saw_recognized ? best : BodyFormat::JSON; -} - -//! RFC 4180-style CSV field escape (quote when needed). -string CsvEscapeField(const string &input) { - bool need_quote = false; - for (unsigned char c : input) { - if (c == '"' || c == ',' || c == '\n' || c == '\r') { - need_quote = true; - break; - } - } - if (!need_quote) { - return input; - } - string out; - out.reserve(input.size() + 2); - out += '"'; - for (unsigned char c : input) { - if (c == '"') { - out += "\"\""; - } else { - out += static_cast(c); - } - } - out += '"'; - return out; -} - -//! Cell text for CSV: null → empty; scalars ToString; nested → JSON text. -string ValueToCsvCell(const Value &value) { - if (value.IsNull()) { - return ""; - } - auto id = value.type().id(); - if (id == LogicalTypeId::LIST || id == LogicalTypeId::STRUCT || id == LogicalTypeId::MAP) { - return ValueToJson(value); - } - return value.ToString(); -} - -string SerializeRowsJsonArray(const vector &names, const vector &data_cols, - const vector> &rows) { - string body = "["; - bool first_row = true; - for (auto &cols : rows) { - if (!first_row) { - body += ","; - } - first_row = false; - body += "{"; - bool first_col = true; - for (auto col : data_cols) { - if (!first_col) { - body += ","; - } - first_col = false; - body += "\"" + QuackapiJsonEscape(names[col]) + "\":" + ValueToJson(cols[col]); - } - body += "}"; - } - body += "]"; - return body; -} - -//! Single-row JSON object (ENVELOPE object). Caller guarantees rows.size() == 1. -string SerializeRowJsonObject(const vector &names, const vector &data_cols, const vector &cols) { - string body = "{"; - bool first_col = true; - for (auto col : data_cols) { - if (!first_col) { - body += ","; - } - first_col = false; - body += "\"" + QuackapiJsonEscape(names[col]) + "\":" + ValueToJson(cols[col]); - } - body += "}"; - return body; -} - -string EmptyResultBody(const QuackapiRoute &route) { - if (!route.empty_body.empty()) { - return route.empty_body; - } - return "{\"detail\":\"Not Found\"}"; -} - -string SerializeRowsNdjson(const vector &names, const vector &data_cols, - const vector> &rows) { - string body; - for (auto &cols : rows) { - body += "{"; - bool first_col = true; - for (auto col : data_cols) { - if (!first_col) { - body += ","; - } - first_col = false; - body += "\"" + QuackapiJsonEscape(names[col]) + "\":" + ValueToJson(cols[col]); - } - body += "}\n"; - } - return body; -} - -string SerializeRowsCsv(const vector &names, const vector &data_cols, - const vector> &rows) { - string body; - // header - for (idx_t i = 0; i < data_cols.size(); i++) { - if (i > 0) { - body += ","; - } - body += CsvEscapeField(names[data_cols[i]]); - } - body += "\n"; - for (auto &cols : rows) { - for (idx_t i = 0; i < data_cols.size(); i++) { - if (i > 0) { - body += ","; - } - auto col = data_cols[i]; - string cell; - if (col < cols.size()) { - cell = ValueToCsvCell(cols[col]); - } - body += CsvEscapeField(cell); - } - body += "\n"; - } - return body; -} - -//! Shared: TEMP table + Appender fill for binary serdes (parquet / arrow). -void FillTempTableForSerdes(Connection &con, const string &table, const vector &names, - const vector &types, const vector &data_cols, - const vector> &rows, const char *label) { - string create_sql = "CREATE TEMP TABLE " + table + " ("; - for (idx_t i = 0; i < data_cols.size(); i++) { - if (i > 0) { - create_sql += ", "; - } - auto c = data_cols[i]; - create_sql += KeywordHelper::WriteOptionallyQuoted(names[c]); - create_sql += " "; - create_sql += types[c].ToString(); - } - create_sql += ")"; - auto create_res = con.Query(create_sql); - if (create_res->HasError()) { - throw InvalidInputException("%s serialize CREATE: %s", label, create_res->GetError()); - } - - if (!rows.empty()) { - Appender appender(con, table); - for (auto &cols : rows) { - appender.BeginRow(); - for (auto c : data_cols) { - if (c < cols.size()) { - appender.Append(cols[c]); - } else { - appender.Append(Value()); - } - } - appender.EndRow(); - } - appender.Close(); - } -} - -//! OS temp dir + basename for parquet/arrow scratch files. Never hardcode /tmp -//! (Windows CI has no /tmp). Paths use forward slashes for SQL COPY literals. -string SerdesTempFilePath(FileSystem &fs, const string &filename) { - string dir; -#if defined(_WIN32) || defined(WIN32) - const char *t = std::getenv("TEMP"); - if (!t || !t[0]) { - t = std::getenv("TMP"); - } - dir = (t && t[0]) ? string(t) : string("."); -#else - const char *t = std::getenv("TMPDIR"); - dir = (t && t[0]) ? string(t) : string("/tmp"); -#endif - string path = fs.JoinPath(dir, filename); - return StringUtil::Replace(path, "\\", "/"); -} - -string SqlQuotePath(const string &path) { - return StringUtil::Replace(path, "'", "''"); -} - -//! Read path bytes then delete the file. On open failure, best-effort remove. -string ReadAndRemoveFileBytes(Connection &con, const string &path) { - auto &fs = FileSystem::GetFileSystem(*con.context); - unique_ptr handle; - try { - handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ); - } catch (...) { - fs.TryRemoveFile(path); - throw; - } - idx_t file_size = handle->GetFileSize(); - string body; - if (file_size > 0) { - // mutable buffer — data_ptr_cast rejects const char* from string::data() - vector buf(file_size); - handle->Read(buf.data(), file_size); - body.assign(reinterpret_cast(buf.data()), file_size); - } - handle->Close(); - fs.TryRemoveFile(path); - return body; -} - -//! Serialize result rows as a Parquet file body (magic "PAR1"). Uses a unique -//! TEMP table + COPY TO (FORMAT PARQUET) + read bytes, then cleans up. -string SerializeRowsParquet(Connection &con, const vector &names, const vector &types, - const vector &data_cols, const vector> &rows) { - if (data_cols.empty()) { - throw InvalidInputException("parquet serialize: no data columns"); - } - auto &fs = FileSystem::GetFileSystem(*con.context); - string uid = StringUtil::Replace(UUID::ToString(UUIDv7::GenerateRandomUUID()), "-", ""); - string table = "qa_parquet_" + uid; - string path = SerdesTempFilePath(fs, table + ".parquet"); - - FillTempTableForSerdes(con, table, names, types, data_cols, rows, "parquet"); - - auto copy_res = con.Query("COPY " + table + " TO '" + SqlQuotePath(path) + "' (FORMAT PARQUET)"); - if (copy_res->HasError()) { - con.Query("DROP TABLE IF EXISTS " + table); - fs.TryRemoveFile(path); - throw InvalidInputException("parquet serialize COPY: %s", copy_res->GetError()); - } - con.Query("DROP TABLE IF EXISTS " + table); - return ReadAndRemoveFileBytes(con, path); -} - -//! Ensure community nanoarrow is available (LOAD, else INSTALL FROM community). -void EnsureNanoarrowLoaded(Connection &con) { - auto load = con.Query("LOAD nanoarrow"); - if (!load->HasError()) { - return; - } - auto inst = con.Query("INSTALL nanoarrow FROM community"); - if (inst->HasError()) { - throw InvalidInputException( - "arrow serialize: nanoarrow not available (%s). INSTALL nanoarrow FROM community and retry.", - inst->GetError()); - } - load = con.Query("LOAD nanoarrow"); - if (load->HasError()) { - throw InvalidInputException("arrow serialize: could not LOAD nanoarrow: %s", load->GetError()); - } -} - -//! Serialize result rows as Arrow IPC stream bytes (nanoarrow FORMAT ARROWS). -//! Magic is 0xFFFFFFFF stream continuation (not ARROW1 file). Content-Type: -//! application/vnd.apache.arrow.stream. Same TEMP+COPY+read pattern as parquet. -string SerializeRowsArrow(Connection &con, const vector &names, const vector &types, - const vector &data_cols, const vector> &rows) { - if (data_cols.empty()) { - throw InvalidInputException("arrow serialize: no data columns"); - } - EnsureNanoarrowLoaded(con); - auto &fs = FileSystem::GetFileSystem(*con.context); - string uid = StringUtil::Replace(UUID::ToString(UUIDv7::GenerateRandomUUID()), "-", ""); - string table = "qa_arrow_" + uid; - string path = SerdesTempFilePath(fs, table + ".arrows"); - - FillTempTableForSerdes(con, table, names, types, data_cols, rows, "arrow"); - - auto copy_res = con.Query("COPY " + table + " TO '" + SqlQuotePath(path) + "' (FORMAT ARROWS)"); - if (copy_res->HasError()) { - con.Query("DROP TABLE IF EXISTS " + table); - fs.TryRemoveFile(path); - throw InvalidInputException("arrow serialize COPY: %s", copy_res->GetError()); - } - con.Query("DROP TABLE IF EXISTS " + table); - return ReadAndRemoveFileBytes(con, path); -} - -void SetInternalError(duckdb_httplib::Response &res, const string &server_side_detail) { - // Return DuckDB's diagnostic so clients can diagnose route execution failures. - fprintf(stderr, "quackapi: internal error: %s\n", server_side_detail.c_str()); - SetJson(res, 500, "{\"detail\":\"" + QuackapiJsonEscape(server_side_detail) + "\"}"); -} - //! Collect request headers into a case-insensitive map (first value wins). case_insensitive_map_t CollectHeaders(const duckdb_httplib::Request &req) { case_insensitive_map_t headers; @@ -1731,465 +888,9 @@ bool MethodListContains(const vector &list, const string &method) { } // namespace -enum class QuackapiAccessLogColumn : uint8_t { - REQUEST_ID, - RECEIVED_AT, - METHOD, - PATH, - ROUTE_NAME, - ROUTE_PATH, - STATUS, - DURATION_MS, - SQL_PREPARE_MS, - SQL_EXECUTE_MS, - ROWS_OUT, - BYTES_IN, - BYTES_OUT, - CLIENT_IP, - USER_AGENT, - HTTP_VERSION, - ERROR_TYPE, - ERROR_MESSAGE, - TRACE_ID, - SPAN_ID, - PARENT_SPAN_ID, - SAMPLED, - HEADERS, - QUERY, -}; - -static constexpr const char *QUACKAPI_ACCESS_LOG_COLUMNS[] = { - "request_id", "received_at", "method", "path", "route_name", "route_path", - "status", "duration_ms", "sql_prepare_ms", "sql_execute_ms", "rows_out", "bytes_in", - "bytes_out", "client_ip", "user_agent", "http_version", "error_type", "error_message", - "trace_id", "span_id", "parent_span_id", "sampled", "headers", "query"}; - -static std::mutex quackapi_access_log_flush_mutex; - -Value QuackapiRequestHeadersValue(const QuackapiRequestRecord &entry) { - if (!entry.headers_logged) { - return Value(); - } - vector keys; - vector values; - keys.reserve(entry.headers.size()); - values.reserve(entry.headers.size()); - for (auto &header : entry.headers) { - keys.emplace_back(header.first); - values.emplace_back(header.second); - } - return Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(keys), std::move(values)); -} - -Value QuackapiRequestQueryValue(const QuackapiRequestRecord &entry) { - return entry.query_logged ? Value(entry.query) : Value(); -} - -static Value AccessLogValue(const QuackapiRequestRecord &entry, QuackapiAccessLogColumn column) { - switch (column) { - case QuackapiAccessLogColumn::REQUEST_ID: - return Value(entry.request_id); - case QuackapiAccessLogColumn::RECEIVED_AT: - return Value::TIMESTAMP(timestamp_t(entry.received_at_micros)); - case QuackapiAccessLogColumn::METHOD: - return Value(entry.method); - case QuackapiAccessLogColumn::PATH: - return Value(entry.path); - case QuackapiAccessLogColumn::ROUTE_NAME: - return entry.route_name.empty() ? Value() : Value(entry.route_name); - case QuackapiAccessLogColumn::ROUTE_PATH: - return entry.route_path.empty() ? Value() : Value(entry.route_path); - case QuackapiAccessLogColumn::STATUS: - return Value::INTEGER(entry.status); - case QuackapiAccessLogColumn::DURATION_MS: - return Value::DOUBLE(entry.duration_ms); - case QuackapiAccessLogColumn::SQL_PREPARE_MS: - return entry.sql_prepare_ms < 0 ? Value() : Value::DOUBLE(entry.sql_prepare_ms); - case QuackapiAccessLogColumn::SQL_EXECUTE_MS: - return entry.sql_execute_ms < 0 ? Value() : Value::DOUBLE(entry.sql_execute_ms); - case QuackapiAccessLogColumn::ROWS_OUT: - return entry.rows_out < 0 ? Value() : Value::BIGINT(entry.rows_out); - case QuackapiAccessLogColumn::BYTES_IN: - return Value::BIGINT(entry.bytes_in); - case QuackapiAccessLogColumn::BYTES_OUT: - return Value::BIGINT(entry.bytes_out); - case QuackapiAccessLogColumn::CLIENT_IP: - return Value(entry.client_ip); - case QuackapiAccessLogColumn::USER_AGENT: - return Value(entry.user_agent); - case QuackapiAccessLogColumn::HTTP_VERSION: - return entry.http_version.empty() ? Value() : Value(entry.http_version); - case QuackapiAccessLogColumn::ERROR_TYPE: - return entry.error_type.empty() ? Value() : Value(entry.error_type); - case QuackapiAccessLogColumn::ERROR_MESSAGE: - return entry.error_message.empty() ? Value() : Value(entry.error_message); - case QuackapiAccessLogColumn::TRACE_ID: - return entry.trace_id.empty() ? Value() : Value(entry.trace_id); - case QuackapiAccessLogColumn::SPAN_ID: - return entry.span_id.empty() ? Value() : Value(entry.span_id); - case QuackapiAccessLogColumn::PARENT_SPAN_ID: - return entry.parent_span_id.empty() ? Value() : Value(entry.parent_span_id); - case QuackapiAccessLogColumn::SAMPLED: - return Value::BOOLEAN(entry.sampled); - case QuackapiAccessLogColumn::HEADERS: - return QuackapiRequestHeadersValue(entry); - case QuackapiAccessLogColumn::QUERY: - return QuackapiRequestQueryValue(entry); - } - throw InternalException("unknown quackapi access-log column"); -} - -static vector SplitAccessLogTable(const string &table) { - vector parts = StringUtil::Split(table, '.'); - for (auto &part : parts) { - StringUtil::Trim(part); - } - if (parts.empty() || parts.size() > 3) { - throw InvalidInputException("access_log table must be a table, schema.table, or database.schema.table"); - } - for (auto &part : parts) { - if (part.empty()) { - throw InvalidInputException("access_log table name contains an empty identifier"); - } - } - return parts; -} - -static unique_ptr AccessLogTableInfo(Connection &con, const vector &parts) { - if (parts.size() == 1) { - return con.TableInfo(parts[0]); - } - if (parts.size() == 2) { - return con.TableInfo(parts[0], parts[1]); - } - return con.TableInfo(parts[0], parts[1], parts[2]); -} - -static unique_ptr AccessLogAppender(Connection &con, const vector &parts) { - if (parts.size() == 1) { - return make_uniq(con, parts[0]); - } - if (parts.size() == 2) { - return make_uniq(con, parts[0], parts[1]); - } - return make_uniq(con, parts[0], parts[1], parts[2]); -} - -static string JsonEscape(const string &value) { - return QuackapiJsonEscape(value); -} - -static void EmitAccessLogStderr(const QuackapiRequestRecord &entry) { - string path_esc; - path_esc.reserve(entry.path.size() + 8); - for (unsigned char c : entry.path) { - if (c == '"' || c == '\\') { - path_esc += '\\'; - path_esc += static_cast(c); - } else if (c < 0x20) { - char buf[8]; - snprintf(buf, sizeof(buf), "\\u%04x", c); - path_esc += buf; - } else { - path_esc += static_cast(c); - } - } - fprintf(stderr, - "{\"type\":\"access\",\"method\":\"%s\",\"path\":\"%s\",\"status\":%d," - "\"latency_ms\":%.3f,\"request_id\":\"%s\",\"bytes\":%llu,\"client_ip\":\"%s\"," - "\"user_agent\":\"%s\",\"route_path\":\"%s\",\"trace_id\":\"%s\"}\n", - JsonEscape(entry.method).c_str(), path_esc.c_str(), entry.status, entry.duration_ms, - JsonEscape(entry.request_id).c_str(), (unsigned long long)entry.bytes_out, - JsonEscape(entry.client_ip).c_str(), JsonEscape(entry.user_agent).c_str(), - JsonEscape(entry.route_path).c_str(), JsonEscape(entry.trace_id).c_str()); -} - static constexpr const char *QUACKAPI_STATIC_LOG_MARKER = "X-Quackapi-Static-Access-Log"; static constexpr const char *QUACKAPI_STATIC_LOG_STARTED = "X-Quackapi-Static-Started"; static constexpr const char *QUACKAPI_STATIC_LOG_RECEIVED = "X-Quackapi-Static-Received"; -static constexpr idx_t QUACKAPI_ACCESS_LOG_QUEUE_CAPACITY = 10000; - -static int64_t TelemetryExportAgeMs(const std::chrono::steady_clock::time_point &last_export, bool has_export) { - if (!has_export) { - return -1; - } - auto age = std::chrono::duration_cast(std::chrono::steady_clock::now() - last_export); - return age.count() < 0 ? 0 : age.count(); -} - -class QuackapiStderrTelemetrySink final : public QuackapiTelemetrySink { -public: - void Enqueue(const QuackapiRequestRecord &entry) override { - // Keep the existing line writer synchronous so stderr retains its current ordering. - EmitAccessLogStderr(entry); - // Atomics, not a mutex: this runs on every request and must not serialize workers. - exported_total.fetch_add(1, std::memory_order_relaxed); - last_export_ticks.store(std::chrono::steady_clock::now().time_since_epoch().count(), std::memory_order_relaxed); - } - - void FlushPending() override { - } - - void Flush() override { - } - - QuackapiTelemetryStatus Status() const override { - QuackapiTelemetryStatus result; - result.sink = "stderr"; - result.exported_total = exported_total.load(std::memory_order_relaxed); - auto ticks = last_export_ticks.load(std::memory_order_relaxed); - result.last_export_age_ms = TelemetryExportAgeMs( - std::chrono::steady_clock::time_point(std::chrono::steady_clock::duration(ticks)), ticks != NO_EXPORT); - return result; - } - -private: - static constexpr int64_t NO_EXPORT = NumericLimits::Minimum(); - std::atomic exported_total {0}; - std::atomic last_export_ticks {NO_EXPORT}; -}; - -class QuackapiTableTelemetrySink final : public QuackapiTelemetrySink { -public: - QuackapiTableTelemetrySink(DatabaseInstance &db, string table_p) - : db_ptr(db.shared_from_this()), table(std::move(table_p)) { - worker = std::thread(&QuackapiTableTelemetrySink::Run, this); - } - - ~QuackapiTableTelemetrySink() override { - Flush(); - } - - void Enqueue(const QuackapiRequestRecord &entry) override { - bool fallback; - bool overflow = false; - bool warn_overflow = false; - idx_t overflow_count_snapshot = 0; - { - std::lock_guard lock(mutex); - fallback = failed && std::chrono::steady_clock::now() < retry_at; - if (!fallback) { - if (pending.size() >= QUACKAPI_ACCESS_LOG_QUEUE_CAPACITY) { - overflow = true; - overflow_count_snapshot = ++overflow_count; - if (!overflow_warning_emitted) { - overflow_warning_emitted = true; - warn_overflow = true; - } - } else { - pending.push_back(entry); - if (pending.size() >= 100) { - condition.notify_one(); - } - } - } - } - if (overflow) { - if (warn_overflow) { - fprintf(stderr, - "quackapi: access_log table \"%s\" queue full; falling back to stderr " - "(overflow_count=%llu)\n", - table.c_str(), (unsigned long long)overflow_count_snapshot); - } - EmitAccessLogStderr(entry); - } else if (fallback) { - EmitAccessLogStderr(entry); - } - } - - void FlushPending() override { - std::unique_lock lock(mutex); - if (!worker.joinable() || stopping || (pending.empty() && !flushing)) { - return; - } - flush_requested = true; - condition.notify_one(); - condition.wait(lock, [&] { return !flush_requested && !flushing; }); - } - - void Flush() override { - Shutdown(); - } - - QuackapiTelemetryStatus Status() const override { - std::lock_guard lock(mutex); - QuackapiTelemetryStatus result; - result.sink = "table"; - result.target = table; - result.queued = pending.size(); - result.exported_total = exported_total; - result.dropped_total = overflow_count; - result.last_error = last_error; - result.last_export_age_ms = TelemetryExportAgeMs(last_export, has_export); - return result; - } - -private: - void Shutdown() { - if (!worker.joinable()) { - return; - } - { - std::lock_guard lock(mutex); - stopping = true; - } - condition.notify_one(); - worker.join(); - } - - void FlushBatch(const vector &batch) { - if (table_parts.empty()) { - table_parts = SplitAccessLogTable(table); - } - std::lock_guard write_lock(quackapi_access_log_flush_mutex); - auto db = db_ptr.lock(); - if (!db) { - throw InvalidInputException("database is shutting down"); - } - Connection con(*db); - auto info = AccessLogTableInfo(con, table_parts); - if (!info) { - throw CatalogException("access_log table \"%s\" does not exist", table); - } - auto appender = AccessLogAppender(con, table_parts); - appender->ClearColumns(); - - vector selected; - for (idx_t i = 0; i < sizeof(QUACKAPI_ACCESS_LOG_COLUMNS) / sizeof(QUACKAPI_ACCESS_LOG_COLUMNS[0]); i++) { - for (auto &column : info->columns) { - if (!column.Generated() && StringUtil::CIEquals(column.Name(), QUACKAPI_ACCESS_LOG_COLUMNS[i])) { - appender->AddColumn(column.Name()); - selected.push_back(static_cast(i)); - break; - } - } - } - if (selected.empty()) { - throw InvalidInputException("access_log table has no recognized request columns"); - } - for (auto &entry : batch) { - appender->BeginRow(); - for (auto column : selected) { - appender->Append(AccessLogValue(entry, column)); - } - appender->EndRow(); - } - appender->Flush(); - } - - void MarkExported(idx_t count) { - std::lock_guard lock(mutex); - exported_total += count; - last_export = std::chrono::steady_clock::now(); - has_export = true; - } - - void MarkFailed(const string &reason) { - std::lock_guard lock(mutex); - last_error = reason; - if (!failed) { - failed = true; - fprintf(stderr, - "quackapi: access_log table \"%s\" failed; falling back to stderr " - "(overflow_count=%llu): %s\n", - table.c_str(), (unsigned long long)overflow_count, reason.c_str()); - } - retry_at = std::chrono::steady_clock::now() + std::chrono::seconds(retry_delay_seconds); - retry_delay_seconds = std::min(retry_delay_seconds * 2, 60); - } - - void MarkRecovered() { - std::lock_guard lock(mutex); - if (!failed) { - return; - } - failed = false; - retry_delay_seconds = 1; - fprintf(stderr, "quackapi: access_log table \"%s\" recovered; resuming table logging\n", table.c_str()); - } - - void Run() { - try { - table_parts = SplitAccessLogTable(table); - } catch (std::exception &ex) { - MarkFailed(ex.what()); - } - for (;;) { - vector batch; - { - std::unique_lock lock(mutex); - condition.wait_for(lock, std::chrono::seconds(1), - [&] { return stopping || flush_requested || pending.size() >= 100; }); - batch.assign(std::make_move_iterator(pending.begin()), std::make_move_iterator(pending.end())); - pending.clear(); - if (batch.empty() && flush_requested) { - flush_requested = false; - condition.notify_all(); - } - if (batch.empty() && stopping) { - break; - } - if (!batch.empty()) { - flushing = true; - } - } - if (batch.empty()) { - continue; - } - bool use_stderr; - { - std::lock_guard lock(mutex); - use_stderr = failed && std::chrono::steady_clock::now() < retry_at; - } - if (!use_stderr) { - try { - FlushBatch(batch); - MarkExported(batch.size()); - MarkRecovered(); - } catch (std::exception &ex) { - MarkFailed(ex.what()); - use_stderr = true; - } catch (...) { - MarkFailed("unknown append error"); - use_stderr = true; - } - } - if (use_stderr) { - for (auto &entry : batch) { - EmitAccessLogStderr(entry); - } - } - { - std::lock_guard lock(mutex); - flushing = false; - if (flush_requested && pending.empty()) { - flush_requested = false; - } - condition.notify_all(); - } - } - } - - weak_ptr db_ptr; - string table; - vector table_parts; - mutable std::mutex mutex; - std::condition_variable condition; - vector pending; - std::thread worker; - bool stopping = false; - bool flush_requested = false; - bool flushing = false; - bool failed = false; - std::chrono::steady_clock::time_point retry_at; - int64_t retry_delay_seconds = 1; - idx_t overflow_count = 0; - idx_t exported_total = 0; - string last_error; - std::chrono::steady_clock::time_point last_export; - bool has_export = false; - bool overflow_warning_emitted = false; -}; string SanitizeClientRequestId(const string &s); static void MakeTraceContext(const string &traceparent, string &trace_id, string &span_id, string &parent_span_id, @@ -2207,12 +908,10 @@ QuackapiHttpServer::QuackapiHttpServer(DatabaseInstance &db, const string &host_ : db_ptr(db.shared_from_this()), host(host_p), port(port_p), cors_origins(opts.cors_origins), options(opts), started_at(std::chrono::steady_clock::now()), compression(opts.compression), compression_min_bytes(opts.compression_min_bytes) { - if (options.access_log) { - if (options.access_log_table.empty()) { - telemetry_sinks.push_back(make_uniq()); - } else { - telemetry_sinks.push_back(make_uniq(db, options.access_log_table)); - } + // JSON is a runtime dependency for typed request bodies and validation. Load + // an available copy once the database is initialized; never install it here. + if (!db.ExtensionIsLoaded("json") && !ExtensionHelper::TryAutoLoadAvailableExtension(db, "json")) { + throw IOException("QuackAPI requires the DuckDB JSON extension; load json before serving requests"); } // In-process only (quackapi_request): no TCP server object, no bind. if (!bind_and_listen) { @@ -2221,9 +920,6 @@ QuackapiHttpServer::QuackapiHttpServer(DatabaseInstance &db, const string &host_ } server = make_uniq(); - if (options.request_ring > 0) { - request_ring = make_uniq(static_cast(options.request_ring)); - } // Static files (FastAPI StaticFiles equivalent). httplib checks file // requests before route handlers, so API routes always win over files. @@ -2318,50 +1014,23 @@ QuackapiHttpServer::QuackapiHttpServer(DatabaseInstance &db, const string &host_ throw IOException("quackapi: failed to instantiate HTTP server for %s:%d", host, port); } is_running.store(true); - - // Bind synchronously so EADDRINUSE etc. propagate to quackapi_serve() - if (!server->bind_to_port(host, port)) { - throw IOException("quackapi: failed to bind to %s:%d (address in use, permission denied, or invalid host)", - host, port); - } - listen_threads.emplace_back(ListenThread, this); -} - -void QuackapiHttpServer::Dispatch(const duckdb_httplib::Request &req, duckdb_httplib::Response &res) { - HandleRequest(req, res); -} - -//! Parse application/x-www-form-urlencoded query into httplib Params. -void ParseQueryStringIntoParams(const string &query, duckdb_httplib::Params ¶ms) { - if (query.empty()) { - return; - } - idx_t start = 0; - while (start < query.size()) { - idx_t amp = query.find('&', start); - if (amp == string::npos) { - amp = query.size(); - } - string pair = query.substr(start, amp - start); - start = amp + 1; - if (pair.empty()) { - continue; - } - auto eq = pair.find('='); - if (eq == string::npos) { - params.emplace(duckdb_httplib::decode_query_component(pair, true), string()); - } else { - params.emplace(duckdb_httplib::decode_query_component(pair.substr(0, eq), true), - duckdb_httplib::decode_query_component(pair.substr(eq + 1), true)); - } + + // Bind synchronously so EADDRINUSE etc. propagate to quackapi_serve() + if (!server->bind_to_port(host, port)) { + throw IOException("quackapi: failed to bind to %s:%d (address in use, permission denied, or invalid host)", + host, port); } + listen_threads.emplace_back(ListenThread, this); +} + +void QuackapiHttpServer::Dispatch(const duckdb_httplib::Request &req, duckdb_httplib::Response &res) { + HandleRequest(req, res); } void QuackapiInProcessRequest(DatabaseInstance &db, const string &method, const string &path_in, const string &body, int &status_out, string &body_out, string &content_type_out, const unordered_map *req_headers, - unordered_map *headers_out, const string &pg_dsn, - const QuackapiServeOptions *request_options) { + unordered_map *headers_out, const QuackapiServeOptions *request_options) { // Quiet defaults for SQL tests unless request_options explicitly enables logging. QuackapiServeOptions opts; if (request_options) { @@ -2371,7 +1040,6 @@ void QuackapiInProcessRequest(DatabaseInstance &db, const string &method, const } opts.compression = QuackapiCompressionMode::OFF; opts.health_routes = true; - opts.pg_dsn = pg_dsn; // No TCP — Dispatch only. QuackapiHttpServer server(db, "127.0.0.1", 0, opts, /*bind_and_listen=*/false); @@ -2657,12 +1325,10 @@ static QuackapiRequestRecord MakeRequestRecord(const duckdb_httplib::Request &re } void QuackapiHttpServer::EmitAccessLog(const QuackapiRequestRecord &entry) { - if (request_ring) { - request_ring->Push(entry); - } if (!options.access_log) { return; } + LogLevel level; bool emit = false; switch (options.log_level) { case QuackapiLogLevel::DEBUG_LEVEL: @@ -2681,53 +1347,19 @@ void QuackapiHttpServer::EmitAccessLog(const QuackapiRequestRecord &entry) { if (!emit) { return; } - for (auto &sink : telemetry_sinks) { - sink->Enqueue(entry); - } -} - -std::vector QuackapiHttpServer::SnapshotRequests() const { - return request_ring ? request_ring->Snapshot() : std::vector(); -} - -std::vector QuackapiHttpServer::SnapshotTelemetryStatus() const { - std::vector result; - result.reserve(telemetry_sinks.size()); - for (auto &sink : telemetry_sinks) { - result.push_back(sink->Status()); - } - return result; -} - -void QuackapiHttpServer::FlushTelemetryPending() { - for (auto &sink : telemetry_sinks) { - sink->FlushPending(); + if (entry.status >= 500) { + level = LogLevel::LOG_ERROR; + } else if (entry.status >= 400 || entry.duration_ms >= static_cast(options.slow_request_ms)) { + level = LogLevel::LOG_WARNING; + } else if (options.log_level == QuackapiLogLevel::DEBUG_LEVEL) { + level = LogLevel::LOG_DEBUG; + } else { + level = LogLevel::LOG_INFO; } -} - -idx_t QuackapiHttpServer::AccessLogOverflowCount() const { - idx_t result = 0; - for (auto &status : SnapshotTelemetryStatus()) { - result += status.dropped_total; + auto db = db_ptr.lock(); + if (db) { + EmitQuackapiAccessLog(*db, entry, level); } - return result; -} - -static string TelemetryHealthJson(const std::vector &statuses) { - string result = "["; - for (idx_t i = 0; i < statuses.size(); i++) { - if (i > 0) { - result += ","; - } - const auto &status = statuses[i]; - result += StringUtil::Format("{\"sink\":\"%s\",\"queued\":%llu,\"exported_total\":%llu," - "\"dropped_total\":%llu,\"last_error\":\"%s\"}", - QuackapiJsonEscape(status.sink), (unsigned long long)status.queued, - (unsigned long long)status.exported_total, - (unsigned long long)status.dropped_total, QuackapiJsonEscape(status.last_error)); - } - result += "]"; - return result; } void QuackapiHttpServer::ApplyCorsHeaders(const duckdb_httplib::Request &req, duckdb_httplib::Response &res) { @@ -3010,25 +1642,14 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd } auto uptime_sec = std::chrono::duration_cast(std::chrono::steady_clock::now() - started_at).count(); - auto telemetry_status = SnapshotTelemetryStatus(); - auto access_log_overflow_count = AccessLogOverflowCount(); if (ready) { - // Surface active outbound HTTP client + reason so operators / - // readiness probes can confirm batteries applied. auto fallback - // is never silent: reason=curl_httpfs_unavailable when httplib. - const string http_client = - options.http_client_active.empty() ? string("httplib") : options.http_client_active; SetJson( res, 200, StringUtil::Format( "{\"status\":\"ok\",\"version\":\"%s\",\"uptime_sec\":%lld," - "\"request_id_source\":\"%s\",\"http_client\":\"%s\"," - "\"http_client_reason\":\"%s\",\"access_log_overflow_count\":%llu," - "\"telemetry\":%s}", + "\"request_id_source\":\"%s\"}", QuackapiJsonEscape(version), (long long)uptime_sec, - QuackapiJsonEscape(options.request_id_source.empty() ? "uuidv7" : options.request_id_source), - QuackapiJsonEscape(http_client), QuackapiJsonEscape(options.http_client_reason), - (unsigned long long)access_log_overflow_count, TelemetryHealthJson(telemetry_status))); + QuackapiJsonEscape(options.request_id_source.empty() ? "uuidv7" : options.request_id_source))); } else { SetJson(res, 503, "{\"status\":\"not_ready\",\"detail\":\"database handle check failed\"}"); } @@ -3037,156 +1658,6 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd } } - // Named GraphQL mounts (CREATE GRAPHQL ROUTE) — before built-in /graphql. - // POST + GET /schema; optional REQUIRE auth; per-route tables. - { - auto &gql_state = QuackapiState::Get(*db); - QuackapiGraphqlRoute gql_route; - if (req.method == "POST" && gql_state.GetGraphqlRouteByPath(req.path, req.method, gql_route)) { - route_name = gql_route.name; - route_path = gql_route.path; - QuackapiAuthResult auth_result; - if (!gql_route.require_auth.empty()) { - QuackapiRoute auth_probe; - auth_probe.require_auth = gql_route.require_auth; - auto headers = CollectHeaders(req); - auth_result = CheckAuth(*db, auth_probe, headers); - if (!auth_result.ok) { - if (!auth_result.www_authenticate.empty()) { - res.set_header("WWW-Authenticate", auth_result.www_authenticate); - } - SetJson(res, auth_result.status, auth_result.body); - finish(); - return; - } - } - string gql_query; - string gql_err; - if (!GraphqlExtractQuery(*db, req.body, gql_query, gql_err)) { - SetJson(res, 400, gql_err); - finish(); - return; - } - try { - GraphqlExecOptions opts; - opts.allowed_tables = &gql_route.tables; - opts.authenticated = !gql_route.require_auth.empty() && auth_result.ok; - opts.claims = &auth_result.claims; - opts.query_timeout_ms = options.query_timeout_ms; - opts.max_response_bytes = options.max_response_bytes; - opts.limit = gql_route.limit == 0 ? QUACKAPI_GRAPHQL_DEFAULT_LIMIT : gql_route.limit; - SetJson(res, 200, ExecuteGraphqlQuery(*db, gql_query, opts)); - } catch (std::exception &ex) { - SetJson(res, 200, string("{\"errors\":[{\"message\":\"") + QuackapiJsonEscape(ex.what()) + "\"}]}"); - } catch (...) { - SetJson(res, 200, "{\"errors\":[{\"message\":\"graphql execution failed\"}]}"); - } - finish(); - return; - } - if ((req.method == "GET" || req.method == "HEAD") && - gql_state.GetGraphqlRouteBySchemaPath(req.path, gql_route)) { - route_name = gql_route.name + ".schema"; - route_path = gql_route.path + "/schema"; - QuackapiAuthResult auth_result; - if (!gql_route.require_auth.empty()) { - QuackapiRoute auth_probe; - auth_probe.require_auth = gql_route.require_auth; - auto headers = CollectHeaders(req); - auth_result = CheckAuth(*db, auth_probe, headers); - if (!auth_result.ok) { - if (!auth_result.www_authenticate.empty()) { - res.set_header("WWW-Authenticate", auth_result.www_authenticate); - } - SetJson(res, auth_result.status, auth_result.body); - finish(); - return; - } - } - try { - GraphqlExecOptions opts; - opts.allowed_tables = &gql_route.tables; - opts.mode = "route"; - opts.authenticated = !gql_route.require_auth.empty() && auth_result.ok; - opts.claims = &auth_result.claims; - opts.query_timeout_ms = options.query_timeout_ms; - opts.max_response_bytes = options.max_response_bytes; - opts.route_name = gql_route.name; - SetJson(res, 200, BuildGraphqlSchema(*db, opts)); - } catch (std::exception &ex) { - SetJson(res, 200, string("{\"errors\":[{\"message\":\"") + QuackapiJsonEscape(ex.what()) + "\"}]}"); - } catch (...) { - SetJson(res, 200, "{\"errors\":[{\"message\":\"graphql schema failed\"}]}"); - } - finish(); - return; - } - } - - // Built-in thin GraphQL v0 (catalog-only table selection; not full GraphQL). - // POST /graphql — JSON body {"query":"query { table { col } }"} - // GET /graphql/schema — main-schema tables → column names - // Public in v0 (no auth). Not listed in quackapi_routes(). - if (req.path == "/graphql" || req.path == "/graphql/") { - route_name = "graphql"; - route_path = "/graphql"; - if (req.method == "POST") { - string gql_query; - string gql_err; - if (!GraphqlExtractQuery(*db, req.body, gql_query, gql_err)) { - SetJson(res, 400, gql_err); - finish(); - return; - } - try { - GraphqlExecOptions opts; - opts.query_timeout_ms = options.query_timeout_ms; - opts.max_response_bytes = options.max_response_bytes; - SetJson(res, 200, ExecuteGraphqlQuery(*db, gql_query, opts)); - } catch (std::exception &ex) { - SetJson(res, 200, string("{\"errors\":[{\"message\":\"") + QuackapiJsonEscape(ex.what()) + "\"}]}"); - } catch (...) { - SetJson(res, 200, "{\"errors\":[{\"message\":\"graphql execution failed\"}]}"); - } - finish(); - return; - } - if (req.method == "OPTIONS") { - if (cors_origins.empty()) { - res.set_header("Allow", "POST"); - SetJson(res, 405, "{\"detail\":\"Method Not Allowed\"}"); - } else { - res.status = 204; - res.set_header("Allow", "POST, OPTIONS"); - res.body.clear(); - } - finish(); - return; - } - // GET /graphql without schema path → 405 with Allow: POST - res.set_header("Allow", "POST"); - SetJson(res, 405, "{\"detail\":\"Method Not Allowed\"}"); - finish(); - return; - } - if ((req.method == "GET" || req.method == "HEAD") && - (req.path == "/graphql/schema" || req.path == "/graphql/schema/")) { - route_name = "graphql.schema"; - route_path = "/graphql/schema"; - try { - GraphqlExecOptions opts; - opts.query_timeout_ms = options.query_timeout_ms; - opts.max_response_bytes = options.max_response_bytes; - SetJson(res, 200, BuildGraphqlSchema(*db, opts)); - } catch (std::exception &ex) { - SetJson(res, 200, string("{\"errors\":[{\"message\":\"") + QuackapiJsonEscape(ex.what()) + "\"}]}"); - } catch (...) { - SetJson(res, 200, "{\"errors\":[{\"message\":\"graphql schema failed\"}]}"); - } - finish(); - return; - } - // Built-in docs routes (always present while serving; not in quackapi_routes()). // FastAPI parity: GET /openapi.json + GET /docs (+ optional /redoc). if (req.method == "GET" || req.method == "HEAD") { @@ -3242,16 +1713,12 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd // Find a route: method + pattern. Collect methods for Allow on 405. // HEAD automatically matches GET when no explicit HEAD route exists. - // Streams (CREATE STREAM) are matched the same way; ordinary routes win - // when both register the same path+method. RouteMatch match; - StreamMatch stream_match; bool path_matched_other_method = false; vector methods_for_path; auto &qa_state = QuackapiState::Get(*db); // Live shared views — no per-request deep copy of the full registry. auto routes = qa_state.LiveRoutes(); - auto streams = qa_state.LiveStreams(); for (auto &route : *routes) { vector> captures; if (MatchPattern(route.pattern, req.path, captures)) { @@ -3269,27 +1736,8 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd } } } - for (auto &stream : *streams) { - vector> captures; - if (MatchPattern(stream.pattern, req.path, captures)) { - if (!MethodListContains(methods_for_path, stream.method)) { - methods_for_path.push_back(stream.method); - } - if (stream.method == req.method) { - // Only take the stream if no ordinary route already matched. - if (!match.matched && !stream_match.matched) { - stream_match.matched = true; - stream_match.stream = stream; - stream_match.path_params = std::move(captures); - } - } else if (!match.matched && !stream_match.matched) { - path_matched_other_method = true; - } - } - } - // Auto-HEAD: if HEAD and no explicit HEAD route, reuse the GET handler - // (routes first, then streams). - if (!match.matched && !stream_match.matched && req.method == "HEAD") { + // Auto-HEAD: if HEAD and no explicit HEAD route, reuse the GET handler. + if (!match.matched && req.method == "HEAD") { for (auto &route : *routes) { vector> captures; if (MatchPattern(route.pattern, req.path, captures) && route.method == "GET") { @@ -3302,20 +1750,6 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd break; } } - if (!match.matched) { - for (auto &stream : *streams) { - vector> captures; - if (MatchPattern(stream.pattern, req.path, captures) && stream.method == "GET") { - stream_match.matched = true; - stream_match.stream = stream; - stream_match.path_params = std::move(captures); - if (!MethodListContains(methods_for_path, "HEAD")) { - methods_for_path.push_back("HEAD"); - } - break; - } - } - } } // OPTIONS: FastAPI without CORSMiddleware returns 405 for unregistered @@ -3349,9 +1783,9 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd return; } - if (!match.matched && !stream_match.matched) { + if (!match.matched) { // Starlette redirect_slashes: if the alternate trailing-slash form would - // match a registered route or stream, 307 to that path (preserve query). + // match a registered route, 307 to that path (preserve query). // Built-in docs paths already accept both forms above. if (!path_matched_other_method && methods_for_path.empty() && req.path != "/") { string alt_path; @@ -3370,15 +1804,6 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd alt_matches = true; break; } - if (!alt_matches) { - for (auto &stream : *streams) { - vector> captures; - if (MatchPattern(stream.pattern, alt_path, captures)) { - alt_matches = true; - break; - } - } - } if (alt_matches) { string location = alt_path; string qs = BuildQueryString(req); @@ -3410,236 +1835,11 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd return; } - // ---- CREATE STREAM (SSE) path — no auth schemes on streams in v1 ---- - if (!match.matched && stream_match.matched) { - route_name = stream_match.stream.name; - route_path = stream_match.stream.pattern; - try { - bool policy_denied = false; - string stream_policy_error; - const auto stream_sql = RewriteHandlerWithPolicies(*db, stream_match.stream.handler_sql, false, - policy_denied, stream_policy_error); - if (policy_denied || !stream_policy_error.empty()) { - SetJson(res, 403, "{\"detail\":\"Policy denies unauthenticated access\"}"); - finish(); - return; - } - auto headers = CollectHeaders(req); - // HEAD: headers only — do not install a long-lived provider. - if (req.method == "HEAD") { - res.status = 200; - res.set_header("Content-Type", "text/event-stream"); - res.set_header("Cache-Control", "no-cache"); - res.set_header("X-Accel-Buffering", "no"); - res.body.clear(); - finish(); - return; - } - - // Bind path + query (+ Last-Event-ID → $last_id) before installing provider. - auto con = make_shared_ptr(*db); - QuackapiQueryDeadline initial_deadline(*con, options.query_timeout_ms); - auto prepared = con->Prepare(stream_sql); - if (prepared->HasError()) { - SetInternalError(res, prepared->GetError()); - finish(); - return; - } - - case_insensitive_map_t provided; - for (auto &kv : req.params) { - provided[kv.first] = kv.second; - } - for (auto &kv : stream_match.path_params) { - provided[kv.first] = kv.second; - } - provided["request_id"] = request_id; - provided["trace_id"] = trace_id; - provided["span_id"] = span_id; - // Last-Event-ID header → last_id (query ?last_id= wins if both set). - if (provided.find("last_id") == provided.end()) { - string last_event_id; - if (FindHeaderValue(headers, "Last-Event-ID", last_event_id) && !last_event_id.empty()) { - provided["last_id"] = last_event_id; - } - } - - auto expected_types = prepared->GetExpectedParameterTypes(); - case_insensitive_map_t named_values; - for (auto &entry : prepared->named_param_map) { - auto ¶m_name = entry.first; - auto type_it = expected_types.find(param_name); - LogicalType expected = LogicalType::UNKNOWN; - if (type_it != expected_types.end()) { - expected = type_it->second; - } - auto it = provided.find(param_name); - if (it == provided.end()) { - // Optional last_id: missing → SQL NULL so WHERE id > $last_id works with COALESCE. - if (param_name == "last_id") { - named_values[param_name] = BoundParameterData(Value()); - continue; - } - string loc = IsPathParam(stream_match.stream.pattern, param_name) ? "path" : "query"; - SetJson(res, 422, ValidationErrorJson(loc, param_name, "Field required", "missing")); - finish(); - return; - } - string loc = IsPathParam(stream_match.stream.pattern, param_name) ? "path" : "query"; - BoundParameterData bound; - string err_json; - if (!BindParamValue(it->second, expected, loc, param_name, bound, err_json)) { - SetJson(res, 422, err_json); - finish(); - return; - } - named_values[param_name] = bound; - } - - // Prove the first execute works before committing to chunked transfer. - auto first_result = prepared->Execute(named_values, true); - if (first_result->HasError()) { - SetInternalError(res, first_result->GetError()); - finish(); - return; - } - - struct SseProviderState { - shared_ptr con; - string handler_sql; - case_insensitive_map_t named_values; - int64_t interval_ms = 0; - unique_ptr prepared; - unique_ptr result; - bool need_execute = false; - bool closed = false; - int64_t query_timeout_ms = 30000; - idx_t max_bytes = 0; - idx_t sent_bytes = 0; - }; - auto state = make_shared_ptr(); - state->con = con; - state->handler_sql = stream_sql; - state->query_timeout_ms = options.query_timeout_ms; - state->max_bytes = options.max_response_bytes; - state->named_values = std::move(named_values); - state->interval_ms = stream_match.stream.interval_ms; - // Carry the first execution into the provider; never execute side effects twice. - state->prepared = std::move(prepared); - state->result = std::move(first_result); - - res.status = 200; - res.set_header("Cache-Control", "no-cache"); - res.set_header("X-Accel-Buffering", "no"); - // CORS before provider install — headers already on res when provider runs. - ApplyCorsHeaders(req, res); - - res.set_chunked_content_provider( - "text/event-stream", - [state](size_t /*offset*/, duckdb_httplib::DataSink &sink) -> bool { - if (state->closed) { - sink.done(); - return true; - } - try { - QuackapiQueryDeadline deadline(*state->con, state->query_timeout_ms); - if (state->need_execute) { - state->result = state->prepared->Execute(state->named_values, true); - if (state->result->HasError()) { - fprintf(stderr, "quackapi stream execute error: %s\n", - state->result->GetError().c_str()); - state->closed = true; - return false; - } - state->need_execute = false; - } - - auto chunk = state->result->Fetch(); - if (!chunk || chunk->size() == 0) { - state->result.reset(); - if (state->interval_ms <= 0) { - state->closed = true; - sink.done(); - return true; - } - // Polling interval: sleep then re-run SELECT (compose cron-style). - // Split sleep so disconnect can surface via write fail next loop. - auto remaining = state->interval_ms; - while (remaining > 0) { - auto step = remaining > 100 ? 100 : remaining; - std::this_thread::sleep_for(std::chrono::milliseconds(step)); - remaining -= step; - if (!sink.is_writable()) { - state->closed = true; - return false; - } - } - state->need_execute = true; - return true; - } - - auto &names = state->result->names; - string buf; - for (idx_t row = 0; row < chunk->size(); row++) { - vector cols(chunk->ColumnCount()); - for (idx_t col = 0; col < chunk->ColumnCount(); col++) { - cols[col] = chunk->GetValue(col, row); - } - auto event = FormatSseEvent(names, cols); - if (state->sent_bytes + buf.size() + event.size() > state->max_bytes) { - state->closed = true; - return false; - } - buf += event; - } - state->sent_bytes += buf.size(); - if (!buf.empty() && !sink.write(buf.data(), buf.size())) { - state->closed = true; - return false; - } - return true; - } catch (std::exception &ex) { - fprintf(stderr, "quackapi stream provider exception: %s\n", ex.what()); - state->closed = true; - return false; - } catch (...) { - fprintf(stderr, "quackapi stream provider: unknown exception\n"); - state->closed = true; - return false; - } - }, - [state](bool /*success*/) { - try { - if (state->con) { - state->con->Interrupt(); - } - } catch (...) { - } - state->result.reset(); - state->prepared.reset(); - state->closed = true; - }); - // Do not call finish() again — CORS already applied; provider owns body. - return; - } catch (std::exception &ex) { - SetInternalError(res, ex.what()); - finish(); - return; - } catch (...) { - SetInternalError(res, "unknown exception"); - finish(); - return; - } - } - // Per-route IO timeout (CREATE ROUTE … TIMEOUT / WITH (timeout_sec := N)). // Extends httplib SocketStream select deadlines + SO_RCVTIMEO/SO_SNDTIMEO for // this connection through handler execute and response write. - // Seed introspection with serve write_timeout_sec; ApplyRouteIoTimeout - // overwrites only when it actually applies a route override. - qa_state.SetLastEffectiveWriteTimeoutSec(options.write_timeout_sec); if (match.route.timeout_sec > 0) { - ApplyRouteIoTimeout(static_cast(match.route.timeout_sec), qa_state); + ApplyRouteIoTimeout(static_cast(match.route.timeout_sec)); } // ---- AUTH ENFORCEMENT (before prepare/execute) ---- @@ -3674,19 +1874,8 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd } // auth_result.claims is ready for $claims_ binding below. - // ---- ROW ACCESS + MASKING (claims-keyed policies on tables) ---- - // When the handler references a table with RAP/masking bindings, wrap - // table refs as secure subqueries. Unauthenticated requests fail closed. bool authenticated = !match.route.require_auth.empty() && auth_result.ok; - bool deny_unauth = false; - string policy_error; - string handler_sql = - RewriteHandlerWithPolicies(*db, match.route.handler_sql, authenticated, deny_unauth, policy_error); - if (deny_unauth || !policy_error.empty()) { - SetJson(res, 403, "{\"detail\":\"Policy denies unauthenticated access\"}"); - finish(); - return; - } + string handler_sql = match.route.handler_sql; try { // Destroy the connection before leaving the request. Retaining it in TLS @@ -3762,8 +1951,8 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd for (auto &kv : match.path_params) { provided[kv.first] = {"path", kv.second}; } - // Server-stamped request id — bindable as $request_id in handler SQL - // (and libpq path). Wins over any client query/path of the same name. + // Server-stamped request id — bindable as $request_id in handler SQL. + // Wins over any client query/path of the same name. provided["request_id"] = {"server", request_id}; provided["trace_id"] = {"server", trace_id}; provided["span_id"] = {"server", span_id}; @@ -3802,9 +1991,8 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd } // ---- REQUEST BODY BINDING (POST/PUT/PATCH) ---- - // Body fills `provided` BEFORE prepare + before optional libpq path. - // INSERT…RETURNING cannot prepare via DuckDB ATTACH; libpq handles it. - // Named params inferred from handler SQL (prepare may never run). + // Body fills `provided` before handler preparation. + // Named params are inferred from handler SQL. bool body_method = IsBodyMethod(req.method); bool has_body_named = false; bool needs_json_field_extraction = false; @@ -3875,10 +2063,18 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd // A native BODY TYPE bound only through $body can be an object or // array. It does not need the flat-field extractor; the native // transform below validates the complete payload directly. - if (needs_json_field_extraction && !ExtractJsonBodyFields(con, req.body, body_fields, err_json)) { - SetJson(res, 422, err_json); - finish(); - return; + if (needs_json_field_extraction) { + auto parse_result = ParseJsonBodyFields(req.body, body_fields); + if (parse_result != JsonBodyParseResult::SUCCESS) { + err_json = parse_result == JsonBodyParseResult::INVALID_JSON + ? ValidationErrorJsonBody("JSON decode error", "json_invalid") + : ValidationErrorJsonBody( + "Input should be a valid dictionary or object to extract fields from", + "model_attributes_type"); + SetJson(res, 422, err_json); + finish(); + return; + } } if (!match.route.body_schema.empty()) { if (!ValidateBodySchema(con, match.route.body_schema, req.body, err_json)) { @@ -3950,33 +2146,7 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd } } - // Native Postgres path (optional): same model as FastAPI+psycopg — - // thread-local libpq, fresh PQexecParams per request. BEFORE DuckDB - // prepare so INSERT…RETURNING and other ATTACH-hostile SQL still work. - if (!options.pg_dsn.empty()) { - string pg_body, pg_err; - auto pg_result = - QuackapiTryPgNative(options.pg_dsn, handler_sql, provided, options.max_response_bytes, pg_body, pg_err); - if (pg_result == QuackapiPgNativeResult::SUCCESS) { - SetJson(res, match.route.status, pg_body); - finish(); - return; - } - if (pg_result == QuackapiPgNativeResult::FAILED) { - if (pg_err == "PostgreSQL deadline exceeded") { - SetJson(res, 504, "{\"detail\":\"Query execution deadline exceeded\"}"); - } else if (pg_err == "PostgreSQL response exceeds configured byte limit") { - SetJson(res, 507, "{\"detail\":\"Response exceeds configured byte limit\"}"); - } else { - SetJson(res, 502, "{\"detail\":\"Upstream database request failed\"}"); - } - finish(); - return; - } - // Fall through only when native execution is proven inapplicable. - } - - // DuckDB prepare (after libpq short-circuit). Fresh prepare every request: + // DuckDB prepare. Fresh prepare every request: // no SQL-keyed body/plan cache — mutable tables and volatile functions // (now/uuid) must re-execute. if (has_body_named && !match.route.body_type.empty()) { @@ -4230,21 +2400,6 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd return; } - // The SQL route delegates the body value to a dynamic SQL function. Validate - // it locally first so remote parser/binder failures cannot become 200 [] - // when the delegate reports an empty result instead of an error. - if (match.route.pattern == "/sql") { - auto sql_it = provided.find("sql"); - if (sql_it != provided.end()) { - string sql_error; - if (!ValidateRequestSql(con, sql_it->second.second, sql_error)) { - SetJson(res, 422, ValidationErrorJson("body", "sql", sql_error, "value_error")); - finish(); - return; - } - } - } - auto execute_started = std::chrono::steady_clock::now(); auto result = prepared->Execute(named_values, false); sql_execute_ms = @@ -4344,138 +2499,170 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd } } - // Materialize rows once so we can apply headers then serialize body. - vector> rows; - idx_t response_value_bytes = 0; + // HTML/TEXT mode uses the single remaining data column name. + vector data_names; + for (auto c : data_cols) { + data_names.push_back(names[c]); + } + auto mode = ResponseModeFor(data_names); + const bool object_envelope = + StringUtil::Lower(match.route.response_envelope.empty() ? "array" : match.route.response_envelope) == + "object"; + + idx_t max_response_bytes = static_cast(options.max_response_bytes); + QuackapiResponseWriter response_writer(max_response_bytes); + unique_ptr json_encoder; + unique_ptr csv_encoder; + if (!data_cols.empty() && mode == ResponseMode::JSON) { + if (body_format == BodyFormat::CSV) { + csv_encoder = make_uniq(con, names, result->types, data_cols, response_writer); + } else { + JsonRowMode row_mode = JsonRowMode::ARRAY; + if (body_format == BodyFormat::NDJSON) { + row_mode = JsonRowMode::NDJSON; + } else if (object_envelope) { + row_mode = JsonRowMode::OBJECT; + } + json_encoder = make_uniq(con, names, result->types, data_cols, row_mode, + response_writer); + } + } + + string location_value; + vector set_cookie_values; + idx_t rows_fetched = 0; + bool limit_exceeded = false; while (true) { auto chunk = result->Fetch(); if (!chunk || chunk->size() == 0) { break; } + rows_fetched += chunk->size(); + rows_out = static_cast(rows_fetched); + + // Keep response-control columns outside the body while reading them + // from every chunk, matching the old first-Location/all-Cookies rules. for (idx_t row = 0; row < chunk->size(); row++) { - vector cols; - cols.resize(chunk->ColumnCount()); - for (idx_t col = 0; col < chunk->ColumnCount(); col++) { - cols[col] = chunk->GetValue(col, row); - response_value_bytes += cols[col].ToString().size(); - if (response_value_bytes > static_cast(options.max_response_bytes)) { - SetJson(res, 507, "{\"detail\":\"Response exceeds configured byte limit\"}"); - finish(); - return; + if (location_col < chunk->ColumnCount()) { + auto value = chunk->GetValue(location_col, row); + if (location_value.empty() && !value.IsNull()) { + location_value = value.ToString(); + } + } + if (set_cookie_col < chunk->ColumnCount()) { + auto value = chunk->GetValue(set_cookie_col, row); + if (!value.IsNull()) { + set_cookie_values.push_back(value.ToString()); } } - rows.push_back(std::move(cols)); } - } - rows_out = static_cast(rows.size()); - // Apply Location / Set-Cookie from first (or each) row. - string location_value; - vector set_cookie_values; - for (auto &cols : rows) { - if (location_col < cols.size() && !cols[location_col].IsNull() && location_value.empty()) { - location_value = cols[location_col].ToString(); + if (mode == ResponseMode::HTML || mode == ResponseMode::TEXT) { + auto col = data_cols[0]; + auto &vector = chunk->data[col]; + UnifiedVectorFormat format; + vector.ToUnifiedFormat(chunk->size(), format); + const bool raw_string_bytes = result->types[col].id() == LogicalTypeId::BLOB || + result->types[col].id() == LogicalTypeId::VARCHAR || + result->types[col].IsJSONType(); + const string_t *values = nullptr; + if (raw_string_bytes) { + values = format.GetData(); + } + for (idx_t row = 0; row < chunk->size(); row++) { + auto index = format.sel->get_index(row); + if (!format.validity.RowIsValid(index)) { + continue; + } + if (raw_string_bytes) { + auto &value = values[index]; + response_writer.Append(value.GetData(), value.GetSize()); + } else { + auto value = chunk->GetValue(col, row); + auto text = value.ToString(); + response_writer.Append(text); + } + if (response_writer.ExceededLimit()) { + limit_exceeded = true; + break; + } + } + } else if (csv_encoder) { + csv_encoder->AppendChunk(*chunk); + } else if (json_encoder) { + json_encoder->AppendChunk(*chunk); } - if (set_cookie_col < cols.size() && !cols[set_cookie_col].IsNull()) { - set_cookie_values.push_back(cols[set_cookie_col].ToString()); + if (response_writer.ExceededLimit()) { + limit_exceeded = true; + break; } } - if (!location_value.empty()) { - res.set_header("Location", location_value); - } - for (auto &cv : set_cookie_values) { - // httplib Headers is multimap — set_header appends. - res.set_header("Set-Cookie", cv); - } - // HTML/TEXT mode uses the single remaining data column name. - vector data_names; - for (auto c : data_cols) { - data_names.push_back(names[c]); + if (limit_exceeded) { + SetJson(res, 507, "{\"detail\":\"Response exceeds configured byte limit\"}"); + finish(); + return; } - auto mode = ResponseModeFor(data_names); // 0-row EMPTY STATUS: override success STATUS (any FORMAT / html/text). - if (rows.empty() && match.route.empty_status != 0) { - SetJson(res, match.route.empty_status, EmptyResultBody(match.route)); + if (rows_fetched == 0 && match.route.empty_status != 0) { + auto empty_body = EmptyResultBody(match.route); + if (empty_body.size() > max_response_bytes) { + SetJson(res, 507, "{\"detail\":\"Response exceeds configured byte limit\"}"); + finish(); + return; + } + SetJson(res, match.route.empty_status, empty_body); finish(); return; } + if (!location_value.empty()) { + res.set_header("Location", location_value); + } + for (auto &cookie : set_cookie_values) { + // httplib Headers is multimap — set_header appends. + res.set_header("Set-Cookie", cookie); + } + // No data columns: empty body (redirect / cookie-only responses). if (data_cols.empty()) { res.status = match.route.status; res.body.clear(); - // Drop Content-Type when there is no body. finish(); return; } - // HTML/TEXT mode: a single data column named `html`/`text` serves its raw - // string value (e.g. SELECT tera_render(...) AS html). Multiple rows are - // concatenated in order, so a query returning fragments streams a page. - // Column-name modes win over FORMAT / Accept. - if (mode != ResponseMode::JSON) { - string body; - idx_t col = data_cols[0]; - for (auto &cols : rows) { - if (col < cols.size() && !cols[col].IsNull()) { - body += cols[col].ToString(); - } - } - res.status = match.route.status; - res.set_content(body, - mode == ResponseMode::HTML ? "text/html; charset=utf-8" : "text/plain; charset=utf-8"); - // Keep body so Content-Length is correct; httplib omits the body for HEAD. + if (json_encoder) { + json_encoder->Finish(); + } + if (csv_encoder) { + csv_encoder->Finish(); + } + if (response_writer.ExceededLimit()) { + SetJson(res, 507, "{\"detail\":\"Response exceeds configured byte limit\"}"); finish(); return; } - const bool object_envelope = - StringUtil::Lower(match.route.response_envelope.empty() ? "array" : match.route.response_envelope) == - "object"; - - // Serialize rows: json array | json object | ndjson | csv | parquet | arrow. - string body; - const char *content_type = "application/json"; - if (body_format == BodyFormat::NDJSON) { - body = SerializeRowsNdjson(names, data_cols, rows); - content_type = "application/x-ndjson"; + if (mode == ResponseMode::HTML || mode == ResponseMode::TEXT) { + res.status = match.route.status; + res.set_content(response_writer.TakeData(), + mode == ResponseMode::HTML ? "text/html; charset=utf-8" : "text/plain; charset=utf-8"); } else if (body_format == BodyFormat::CSV) { - body = SerializeRowsCsv(names, data_cols, rows); - content_type = "text/csv; charset=utf-8"; - } else if (body_format == BodyFormat::PARQUET) { - body = SerializeRowsParquet(con, names, result->types, data_cols, rows); - content_type = "application/vnd.apache.parquet"; - } else if (body_format == BodyFormat::ARROW) { - body = SerializeRowsArrow(con, names, result->types, data_cols, rows); - // nanoarrow FORMAT ARROWS produces IPC *stream* (0xFFFFFFFF magic). - content_type = "application/vnd.apache.arrow.stream"; - } else if (object_envelope) { - if (rows.size() > 1) { - SetJson(res, 500, - "{\"detail\":\"ENVELOPE object requires exactly one row, got " + std::to_string(rows.size()) + - "\"}"); - finish(); - return; - } - if (rows.empty()) { - // No EMPTY STATUS → JSON null (array default remains []). - body = "null"; - } else { - body = SerializeRowJsonObject(names, data_cols, rows[0]); - } - SetJson(res, match.route.status, body); - finish(); - return; + res.status = match.route.status; + res.set_content(response_writer.TakeData(), "text/csv; charset=utf-8"); + } else if (body_format == BodyFormat::NDJSON) { + res.status = match.route.status; + res.set_content(response_writer.TakeData(), "application/x-ndjson"); + } else if (object_envelope && rows_fetched > 1) { + SetJson(res, 500, + "{\"detail\":\"ENVELOPE object requires exactly one row, got " + std::to_string(rows_fetched) + + "\"}"); } else { - body = SerializeRowsJsonArray(names, data_cols, rows); - SetJson(res, match.route.status, body); - finish(); - return; + res.status = match.route.status; + res.set_content(response_writer.TakeData(), "application/json"); } - res.status = match.route.status; - res.set_content(body, content_type); // Keep body so Content-Length is correct; httplib omits the body for HEAD. } catch (std::exception &ex) { SetInternalError(res, ex.what()); @@ -4515,9 +2702,6 @@ void QuackapiHttpServer::Close() { QuackapiHttpServer::~QuackapiHttpServer() { try { Close(); - for (auto &sink : telemetry_sinks) { - sink->Flush(); - } } catch (std::exception &) { } } diff --git a/src/quackapi_stream.cpp b/src/quackapi_stream.cpp deleted file mode 100644 index 65412fa..0000000 --- a/src/quackapi_stream.cpp +++ /dev/null @@ -1,425 +0,0 @@ -#include "quackapi_stream.hpp" - -#include "duckdb/common/exception.hpp" -#include "duckdb/common/string_util.hpp" -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/main/connection.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/main/extension/extension_loader.hpp" -#include "duckdb/main/prepared_statement.hpp" -#include "duckdb/parser/parser_extension.hpp" - -#include "quackapi_state.hpp" -#include "quackapi_util.hpp" - -namespace duckdb { - -namespace { - -//! Parse duration to milliseconds: 30 | '30' | '30s' | '500ms' | '1m' | '1h' -bool ParseIntervalMs(const string &raw, int64_t &out_ms, string &err) { - auto s = QuackapiTrim(raw); - if (s.empty()) { - err = "empty interval"; - return false; - } - if (s.size() >= 2 && s.front() == '\'' && s.back() == '\'') { - s = s.substr(1, s.size() - 2); - } - if (s.empty()) { - err = "empty interval"; - return false; - } - idx_t i = 0; - if (s[i] == '-') { - err = "interval must be non-negative"; - return false; - } - if (i >= s.size() || !StringUtil::CharacterIsDigit(s[i])) { - err = "interval must start with a number"; - return false; - } - int64_t num = 0; - while (i < s.size() && StringUtil::CharacterIsDigit(s[i])) { - num = num * 10 + (s[i] - '0'); - if (num > 86400LL * 1000) { - err = "interval too large"; - return false; - } - i++; - } - int64_t mult_ms = 1000; // bare number = seconds (same as queue-style durations) - if (i < s.size()) { - auto unit = StringUtil::Lower(s.substr(i)); - if (unit == "ms" || unit == "msec" || unit == "millis" || unit == "millisecond" || unit == "milliseconds") { - mult_ms = 1; - } else if (unit == "s" || unit == "sec" || unit == "secs" || unit == "second" || unit == "seconds") { - mult_ms = 1000; - } else if (unit == "m" || unit == "min" || unit == "mins" || unit == "minute" || unit == "minutes") { - mult_ms = 60 * 1000; - } else if (unit == "h" || unit == "hr" || unit == "hrs" || unit == "hour" || unit == "hours") { - mult_ms = 3600 * 1000; - } else { - err = "unknown interval unit \"" + unit + "\" — use ms, s, m, or h"; - return false; - } - } - int64_t ms = num * mult_ms; - if (ms > 86400LL * 1000) { - err = "interval max is 24 hours"; - return false; - } - out_ms = ms; - return true; -} - -//===--------------------------------------------------------------------===// -// CREATE / DROP STREAM parser -//===--------------------------------------------------------------------===// - -struct StreamDdlParseData : public ParserExtensionParseData { - string action; // CREATE / DROP - bool or_replace = false; - QuackapiStream stream; - - unique_ptr Copy() const override { - auto copy = make_uniq(); - copy->action = action; - copy->or_replace = or_replace; - copy->stream = stream; - return std::move(copy); - } - string ToString() const override { - return action + " STREAM " + stream.name; - } -}; - -//! Grammar: -//! CREATE [OR REPLACE] STREAM GET '' -//! [WITH (interval='1s'|1000)] -//! AS "); - } - auto name = rest.substr(0, first_space); - for (char c : name) { - if (!(StringUtil::CharacterIsAlpha(c) || StringUtil::CharacterIsDigit(c) || c == '_' || c == '-')) { - return ParserExtensionParseResult("Stream name must be an identifier ([A-Za-z0-9_-]+)"); - } - } - rest = QuackapiTrim(rest.substr(first_space)); - auto second_space = rest.find(' '); - if (second_space == string::npos) { - return ParserExtensionParseResult("Expected GET '' after stream name"); - } - auto method = StringUtil::Upper(rest.substr(0, second_space)); - if (method == "WS" || method == "WEBSOCKET" || method == "WSS") { - return ParserExtensionParseResult( - "CREATE STREAM WebSocket is not supported: bundled cpp-httplib has no WebSocket/Upgrade API. " - "Use CREATE STREAM GET '' for Server-Sent Events (text/event-stream). " - "Bidirectional duplex belongs on the quack protocol, not HTTP Upgrade."); - } - if (method != "GET") { - return ParserExtensionParseResult("CREATE STREAM only supports GET (SSE). Unknown method \"" + method + - "\" — WebSocket is deferred (no WS on httplib transport)"); - } - rest = QuackapiTrim(rest.substr(second_space)); - - // '' - if (rest.empty() || rest[0] != '\'') { - return ParserExtensionParseResult("Expected quoted '' after GET"); - } - auto path_end = rest.find('\'', 1); - if (path_end == string::npos) { - return ParserExtensionParseResult("Unterminated stream path"); - } - auto pattern = rest.substr(1, path_end - 1); - if (pattern.empty() || pattern[0] != '/') { - return ParserExtensionParseResult("Stream path must start with '/'"); - } - rest = QuackapiTrim(rest.substr(path_end + 1)); - auto rest_upper = StringUtil::Upper(rest); - - QuackapiStream stream; - stream.name = name; - stream.method = "GET"; - stream.pattern = pattern; - stream.interval_ms = 0; - stream.transport = QuackapiStreamTransport::SSE; - - // optional WITH ( interval=... ) - if (StringUtil::StartsWith(rest_upper, "WITH") && - (rest.size() == 4 || StringUtil::CharacterIsSpace(rest[4]) || rest[4] == '(')) { - rest = QuackapiTrim(rest.substr(4)); - if (rest.empty() || rest[0] != '(') { - return ParserExtensionParseResult("WITH expects a parenthesized option list"); - } - auto close = rest.find(')'); - if (close == string::npos) { - return ParserExtensionParseResult("Unterminated WITH ( ... ) options"); - } - auto opts = QuackapiTrim(rest.substr(1, close - 1)); - rest = QuackapiTrim(rest.substr(close + 1)); - rest_upper = StringUtil::Upper(rest); - - idx_t oi = 0; - while (oi < opts.size()) { - while (oi < opts.size() && (StringUtil::CharacterIsSpace(opts[oi]) || opts[oi] == ',')) { - oi++; - } - if (oi >= opts.size()) { - break; - } - idx_t key_start = oi; - while (oi < opts.size() && (StringUtil::CharacterIsAlpha(opts[oi]) || - StringUtil::CharacterIsDigit(opts[oi]) || opts[oi] == '_')) { - oi++; - } - auto key = StringUtil::Lower(opts.substr(key_start, oi - key_start)); - while (oi < opts.size() && StringUtil::CharacterIsSpace(opts[oi])) { - oi++; - } - if (oi >= opts.size() || opts[oi] != '=') { - return ParserExtensionParseResult("WITH options: expected key=value (got key \"" + key + "\")"); - } - oi++; - while (oi < opts.size() && StringUtil::CharacterIsSpace(opts[oi])) { - oi++; - } - if (oi >= opts.size()) { - return ParserExtensionParseResult("WITH options: missing value for \"" + key + "\""); - } - string val; - if (opts[oi] == '\'') { - idx_t j = oi + 1; - string raw; - while (j < opts.size()) { - if (opts[j] == '\'') { - if (j + 1 < opts.size() && opts[j + 1] == '\'') { - raw += '\''; - j += 2; - continue; - } - break; - } - raw += opts[j]; - j++; - } - if (j >= opts.size() || opts[j] != '\'') { - return ParserExtensionParseResult("Unterminated quoted value for \"" + key + "\""); - } - val = raw; - oi = j + 1; - } else { - idx_t j = oi; - while (j < opts.size() && opts[j] != ',' && !StringUtil::CharacterIsSpace(opts[j])) { - j++; - } - val = opts.substr(oi, j - oi); - oi = j; - } - - if (key == "interval") { - int64_t ms = 0; - string err; - if (!ParseIntervalMs(val, ms, err)) { - return ParserExtensionParseResult("interval: " + err); - } - stream.interval_ms = ms; - } else { - return ParserExtensionParseResult("Unknown STREAM option \"" + key + "\" — expected interval"); - } - } - } - - // AS in CREATE STREAM"); - } - auto handler = QuackapiTrim(rest.substr(2)); - if (handler.empty()) { - return ParserExtensionParseResult("Empty handler after AS"); - } - stream.handler_sql = handler; - - auto data = make_uniq(); - data->action = "CREATE"; - data->or_replace = or_replace; - data->stream = stream; - return ParserExtensionParseResult(std::move(data)); -} - -struct ApplyStreamBindData : public TableFunctionData { - string action; - bool or_replace = false; - QuackapiStream stream; - bool finished = false; -}; - -unique_ptr ApplyStreamBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - bind_data->action = input.inputs[0].GetValue(); - bind_data->or_replace = input.inputs[1].GetValue(); - bind_data->stream.name = input.inputs[2].GetValue(); - bind_data->stream.method = input.inputs[3].GetValue(); - bind_data->stream.pattern = input.inputs[4].GetValue(); - bind_data->stream.handler_sql = input.inputs[5].GetValue(); - bind_data->stream.interval_ms = input.inputs[6].GetValue(); - bind_data->stream.transport = QuackapiStreamTransport::SSE; - BindStatusColumn(return_types, names); - return std::move(bind_data); -} - -void ApplyStreamExec(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &bind_data = data_p.bind_data->CastNoConst(); - if (bind_data.finished) { - return; - } - auto &state = QuackapiState::Get(*context.db); - string message; - if (bind_data.action == "CREATE") { - { - Connection con(*context.db); - auto prepared = con.Prepare(bind_data.stream.handler_sql); - if (prepared->HasError()) { - throw InvalidInputException("Invalid handler SQL for stream \"%s\": %s", bind_data.stream.name, - prepared->GetError()); - } - } - state.AddStream(bind_data.stream, bind_data.or_replace); - if (bind_data.stream.interval_ms > 0) { - message = StringUtil::Format("Stream %s: GET %s SSE interval=%lldms", bind_data.stream.name, - bind_data.stream.pattern, (long long)bind_data.stream.interval_ms); - } else { - message = StringUtil::Format("Stream %s: GET %s SSE", bind_data.stream.name, bind_data.stream.pattern); - } - } else { - if (state.DropStream(bind_data.stream.name)) { - message = StringUtil::Format("Dropped stream %s", bind_data.stream.name); - } else { - throw InvalidInputException("Stream \"%s\" does not exist", bind_data.stream.name); - } - } - EmitOneShotStatus(output, bind_data.finished, message); -} - -TableFunction MakeApplyStreamFunction() { - return MakeApplyDdlFunction("quackapi_apply_stream", - {LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::BIGINT}, - ApplyStreamExec, ApplyStreamBind); -} - -ParserExtensionPlanResult StreamDdlPlan(ParserExtensionInfo *, ClientContext &, - unique_ptr parse_data) { - auto &data = static_cast(*parse_data); - ParserExtensionPlanResult result; - result.function = MakeApplyStreamFunction(); - result.parameters.push_back(Value(data.action)); - result.parameters.push_back(Value::BOOLEAN(data.or_replace)); - result.parameters.push_back(Value(data.stream.name)); - result.parameters.push_back(Value(data.stream.method)); - result.parameters.push_back(Value(data.stream.pattern)); - result.parameters.push_back(Value(data.stream.handler_sql)); - result.parameters.push_back(Value::BIGINT(data.stream.interval_ms)); - FinishDdlPlan(result); - return result; -} - -//===--------------------------------------------------------------------===// -// quackapi_streams() -//===--------------------------------------------------------------------===// - -struct StreamsBindData : public TableFunctionData {}; - -struct StreamsGlobalState : public GlobalTableFunctionState { - vector streams; - idx_t offset = 0; -}; - -unique_ptr StreamsBind(ClientContext &, TableFunctionBindInput &, vector &return_types, - vector &names) { - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("name"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("method"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("pattern"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("transport"); - return_types.emplace_back(LogicalType::BIGINT); - names.emplace_back("interval_ms"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("handler"); - return make_uniq(); -} - -unique_ptr StreamsInit(ClientContext &context, TableFunctionInitInput &) { - auto state = make_uniq(); - state->streams = QuackapiState::Get(*context.db).SnapshotStreams(); - return std::move(state); -} - -void StreamsExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.streams.size() && row < STANDARD_VECTOR_SIZE) { - auto &s = state.streams[state.offset]; - output.SetValue(0, row, Value(s.name)); - output.SetValue(1, row, Value(s.method)); - output.SetValue(2, row, Value(s.pattern)); - output.SetValue(3, row, Value("sse")); - output.SetValue(4, row, Value::BIGINT(s.interval_ms)); - output.SetValue(5, row, Value(s.handler_sql)); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -} // namespace - -StreamDdlParserExtension::StreamDdlParserExtension() { - parse_function = StreamDdlParse; - plan_function = StreamDdlPlan; -} - -TableFunction GetApplyStreamFunction() { - return MakeApplyStreamFunction(); -} - -void RegisterQuackapiStreamFunctions(ExtensionLoader &loader) { - loader.RegisterFunction(TableFunction("quackapi_streams", {}, StreamsExec, StreamsBind, StreamsInit)); -} - -} // namespace duckdb diff --git a/src/quackapi_table_api.cpp b/src/quackapi_table_api.cpp deleted file mode 100644 index 63af3e7..0000000 --- a/src/quackapi_table_api.cpp +++ /dev/null @@ -1,285 +0,0 @@ -#include "duckdb/common/exception.hpp" -#include "duckdb/common/string_util.hpp" -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/main/connection.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/parser/parser_extension.hpp" - -#include "quackapi_state.hpp" -#include "quackapi_table_api.hpp" -#include "quackapi_util.hpp" - -namespace duckdb { - -namespace { - -//! A quoted '' token starting at rest[0]. Returns the inner value and -//! advances `rest` past the closing quote. Throws a parse-style error via the -//! bool return (false) on a missing/unterminated quote. -bool ParseQuoted(string &rest, string &out) { - rest = QuackapiTrim(rest); - if (rest.empty() || rest[0] != '\'') { - return false; - } - auto close = rest.find('\'', 1); - if (close == string::npos) { - return false; - } - out = rest.substr(1, close - 1); - rest = QuackapiTrim(rest.substr(close + 1)); - return true; -} - -//! Parse a SQL identifier: bare token OR double-quoted with "" escapes. -//! Bare tokens must not contain '"' (those need proper "quoting") so the -//! statement splitter / shell never sees an unbalanced double-quote. -bool ParseIdent(string &rest, string &out) { - rest = QuackapiTrim(rest); - if (rest.empty()) { - return false; - } - if (rest[0] == '"') { - string result; - idx_t i = 1; - while (i < rest.size()) { - if (rest[i] == '"') { - if (i + 1 < rest.size() && rest[i + 1] == '"') { - result += '"'; - i += 2; - continue; - } - out = result; - rest = QuackapiTrim(rest.substr(i + 1)); - return !out.empty(); - } - result += rest[i]; - i++; - } - return false; // unterminated - } - // Bare identifier: up to whitespace. Reject embedded double-quotes so callers - // use standard "quoted""idents" (balanced for the shell / statement splitter). - auto space = rest.find(' '); - string tok = space == string::npos ? rest : rest.substr(0, space); - if (tok.empty() || tok.find('"') != string::npos) { - return false; - } - out = tok; - rest = space == string::npos ? string() : QuackapiTrim(rest.substr(space)); - return true; -} - -//! Path/query param names must be safe bare identifiers ($name). -bool IsSafeParamName(const string &name) { - if (name.empty()) { - return false; - } - for (char c : name) { - if (!(StringUtil::CharacterIsAlpha(c) || StringUtil::CharacterIsDigit(c) || c == '_')) { - return false; - } - } - // leading digit is fine for path captures; DuckDB named params accept it - return true; -} - -//! Parsed CREATE API FOR TABLE, carried from parse to plan. -struct TableApiParseData : public ParserExtensionParseData { - bool or_replace = false; - string table; - string base_path; // empty => default '/
' - string key = "id"; - - unique_ptr Copy() const override { - auto copy = make_uniq(); - copy->or_replace = or_replace; - copy->table = table; - copy->base_path = base_path; - copy->key = key; - return std::move(copy); - } - string ToString() const override { - return "CREATE API FOR TABLE " + table; - } -}; - -//! Grammar: -//! CREATE [OR REPLACE] API FOR TABLE
[AT ''] [KEY ''] -ParserExtensionParseResult TableApiParse(ParserExtensionInfo *, const string &query) { - auto q = QuackapiTrim(query); - auto upper = StringUtil::Upper(q); - - bool or_replace = false; - idx_t pos; - if (StringUtil::StartsWith(upper, "CREATE API FOR TABLE ")) { - pos = 21; - } else if (StringUtil::StartsWith(upper, "CREATE OR REPLACE API FOR TABLE ")) { - pos = 32; - or_replace = true; - } else { - // Not ours — let the core parser (or another extension) handle it. - return ParserExtensionParseResult(); - } - - auto rest = QuackapiTrim(q.substr(pos)); - if (rest.empty()) { - return ParserExtensionParseResult("CREATE API FOR TABLE expects a table name"); - } - - //
: bare ident OR "quoted""ident" (SQL standard). Quoted form is - // required when the name embeds double-quotes so the statement splitter - // sees balanced quotes. - auto data = make_uniq(); - data->or_replace = or_replace; - if (!ParseIdent(rest, data->table)) { - return ParserExtensionParseResult( - "CREATE API FOR TABLE: invalid table name — use a bare identifier or \"quoted\"\"ident\""); - } - if (data->table.find('\'') != string::npos) { - return ParserExtensionParseResult("CREATE API FOR TABLE: invalid table name"); - } - - // Optional clauses in any order: AT '', KEY '' - while (!rest.empty()) { - auto rest_upper = StringUtil::Upper(rest); - if (StringUtil::StartsWith(rest_upper, "AT ")) { - rest = rest.substr(3); - if (!ParseQuoted(rest, data->base_path) || data->base_path.empty() || data->base_path[0] != '/') { - return ParserExtensionParseResult("CREATE API FOR TABLE: AT expects a quoted '/path'"); - } - } else if (StringUtil::StartsWith(rest_upper, "KEY ")) { - rest = rest.substr(4); - if (!ParseQuoted(rest, data->key) || data->key.empty()) { - return ParserExtensionParseResult("CREATE API FOR TABLE: KEY expects a quoted 'column'"); - } - } else { - return ParserExtensionParseResult("CREATE API FOR TABLE: unexpected clause \"" + rest + "\""); - } - } - - // KEY is also the path-param / $name — must be a safe bare identifier. - if (!IsSafeParamName(data->key)) { - return ParserExtensionParseResult( - "CREATE API FOR TABLE: KEY must be a bare identifier (letters, digits, underscore)"); - } - - return ParserExtensionParseResult(std::move(data)); -} - -struct ApplyApiBindData : public TableFunctionData { - bool or_replace = false; - string table; - string base_path; - string key; - bool finished = false; -}; - -unique_ptr ApplyApiBind(ClientContext &, TableFunctionBindInput &input, vector &return_types, - vector &names) { - auto bind_data = make_uniq(); - bind_data->or_replace = input.inputs[0].GetValue(); - bind_data->table = input.inputs[1].GetValue(); - bind_data->base_path = input.inputs[2].GetValue(); - bind_data->key = input.inputs[3].GetValue(); - BindStatusColumn(return_types, names); - return std::move(bind_data); -} - -//! Backtick-safe identifier quoting for embedding a table/column into generated -//! handler SQL (double-quote, doubling any embedded quote). -string QuoteIdent(const string &ident) { - string out = "\""; - for (char c : ident) { - if (c == '"') { - out += "\"\""; - } else { - out += c; - } - } - out += "\""; - return out; -} - -void ApplyApiExec(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &bind_data = data_p.bind_data->CastNoConst(); - if (bind_data.finished) { - return; - } - auto &state = QuackapiState::Get(*context.db); - - auto base = bind_data.base_path.empty() ? ("/" + bind_data.table) : bind_data.base_path; - // Normalize: no trailing slash (except the root itself). - while (base.size() > 1 && base.back() == '/') { - base.pop_back(); - } - auto quoted_table = QuoteIdent(bind_data.table); - auto quoted_key = QuoteIdent(bind_data.key); - - // GET -> list - QuackapiRoute list_route; - list_route.name = bind_data.table + "_list"; - list_route.method = "GET"; - list_route.pattern = base; - list_route.handler_sql = "SELECT * FROM " + quoted_table; - list_route.status = 200; - - // GET /: -> by key - QuackapiRoute get_route; - get_route.name = bind_data.table + "_get"; - get_route.method = "GET"; - get_route.pattern = base + "/:" + bind_data.key; - get_route.handler_sql = "SELECT * FROM " + quoted_table + " WHERE " + quoted_key + " = $" + bind_data.key; - get_route.status = 200; - - // Validate handlers before mutating the registry (CREATE ROUTE parity) so a - // missing table cannot leave list/get routes that 500 on first request. - { - Connection con(*context.db); - auto prepared_list = con.Prepare(list_route.handler_sql); - if (prepared_list->HasError()) { - throw InvalidInputException("Invalid handler SQL for API table \"%s\": %s", bind_data.table, - prepared_list->GetError()); - } - auto prepared_get = con.Prepare(get_route.handler_sql); - if (prepared_get->HasError()) { - throw InvalidInputException("Invalid handler SQL for API table \"%s\": %s", bind_data.table, - prepared_get->GetError()); - } - } - - state.AddRoute(list_route, bind_data.or_replace); - state.AddRoute(get_route, bind_data.or_replace); - - EmitOneShotStatus(output, bind_data.finished, - StringUtil::Format("API for %s: GET %s, GET %s/:%s", bind_data.table, base, base, bind_data.key)); -} - -TableFunction MakeApplyApiFunction() { - return MakeApplyDdlFunction( - "quackapi_apply_table_api", - {LogicalType::BOOLEAN, LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, ApplyApiExec, - ApplyApiBind); -} - -ParserExtensionPlanResult TableApiPlan(ParserExtensionInfo *, ClientContext &, - unique_ptr parse_data) { - auto &data = static_cast(*parse_data); - ParserExtensionPlanResult result; - result.function = MakeApplyApiFunction(); - result.parameters.push_back(Value::BOOLEAN(data.or_replace)); - result.parameters.push_back(Value(data.table)); - result.parameters.push_back(Value(data.base_path)); - result.parameters.push_back(Value(data.key)); - FinishDdlPlan(result); - return result; -} - -} // namespace - -TableApiDdlParserExtension::TableApiDdlParserExtension() { - parse_function = TableApiParse; - plan_function = TableApiPlan; -} - -} // namespace duckdb diff --git a/src/quackapi_validation.cpp b/src/quackapi_validation.cpp index 0bc1b3c..3bbb0cb 100644 --- a/src/quackapi_validation.cpp +++ b/src/quackapi_validation.cpp @@ -1,5 +1,8 @@ #include "quackapi_validation.hpp" +#include "duckdb/common/unordered_map.hpp" +#include "duckdb/main/connection.hpp" + #include "quackapi_util.hpp" namespace duckdb { @@ -10,269 +13,74 @@ enum class JsonNodeKind : uint8_t { NIL, BOOL, NUMBER, STRING, ARRAY, OBJECT }; struct JsonNode { JsonNodeKind kind = JsonNodeKind::NIL; + idx_t id = 0; + string key; string scalar; - vector children; - vector> members; }; -//! Small, dependency-free JSON reader used only for declared structures and -//! error locations. DuckDB still owns request parsing and type validation. -class JsonReader { -public: - explicit JsonReader(const string &input_p) : input(input_p) { - } - - bool ParseDocument(JsonNode &result) { - SkipWhitespace(); - if (!ParseValue(result, 0)) { - return false; - } - SkipWhitespace(); - return position == input.size(); - } - -private: - static constexpr idx_t MAX_NESTING_DEPTH = 128; - const string &input; - idx_t position = 0; - - void SkipWhitespace() { - while (position < input.size() && (input[position] == ' ' || input[position] == '\n' || - input[position] == '\r' || input[position] == '\t')) { - position++; - } - } +struct JsonTree { + vector nodes; + unordered_map> children_by_parent; +}; - bool Consume(char expected) { - SkipWhitespace(); - if (position >= input.size() || input[position] != expected) { - return false; - } - position++; - return true; +JsonNodeKind JsonTypeToKind(const string &type) { + if (type == "OBJECT") { + return JsonNodeKind::OBJECT; } - - static int HexValue(char c) { - if (c >= '0' && c <= '9') { - return c - '0'; - } - if (c >= 'a' && c <= 'f') { - return 10 + c - 'a'; - } - if (c >= 'A' && c <= 'F') { - return 10 + c - 'A'; - } - return -1; + if (type == "ARRAY") { + return JsonNodeKind::ARRAY; } - - static void AppendUtf8(string &target, uint32_t codepoint) { - if (codepoint <= 0x7f) { - target += char(codepoint); - } else if (codepoint <= 0x7ff) { - target += char(0xc0 | (codepoint >> 6)); - target += char(0x80 | (codepoint & 0x3f)); - } else if (codepoint <= 0xffff) { - target += char(0xe0 | (codepoint >> 12)); - target += char(0x80 | ((codepoint >> 6) & 0x3f)); - target += char(0x80 | (codepoint & 0x3f)); - } else { - target += char(0xf0 | (codepoint >> 18)); - target += char(0x80 | ((codepoint >> 12) & 0x3f)); - target += char(0x80 | ((codepoint >> 6) & 0x3f)); - target += char(0x80 | (codepoint & 0x3f)); - } + if (type == "VARCHAR") { + return JsonNodeKind::STRING; } - - bool ParseUnicodeEscape(uint32_t &codepoint) { - if (position + 4 > input.size()) { - return false; - } - codepoint = 0; - for (idx_t i = 0; i < 4; i++) { - auto hex = HexValue(input[position++]); - if (hex < 0) { - return false; - } - codepoint = (codepoint << 4) | uint32_t(hex); - } - return true; + if (type == "BOOLEAN") { + return JsonNodeKind::BOOL; } - - bool ParseString(string &result) { - if (!Consume('"')) { - return false; - } - result.clear(); - while (position < input.size()) { - auto c = input[position++]; - if (c == '"') { - return true; - } - if (uint8_t(c) < 0x20) { - return false; - } - if (c != '\\') { - result += c; - continue; - } - if (position >= input.size()) { - return false; - } - auto escaped = input[position++]; - switch (escaped) { - case '"': - case '\\': - case '/': - result += escaped; - break; - case 'b': - result += '\b'; - break; - case 'f': - result += '\f'; - break; - case 'n': - result += '\n'; - break; - case 'r': - result += '\r'; - break; - case 't': - result += '\t'; - break; - case 'u': { - uint32_t codepoint; - if (!ParseUnicodeEscape(codepoint)) { - return false; - } - if (codepoint >= 0xd800 && codepoint <= 0xdbff && position + 6 <= input.size() && - input[position] == '\\' && input[position + 1] == 'u') { - position += 2; - uint32_t low; - if (!ParseUnicodeEscape(low) || low < 0xdc00 || low > 0xdfff) { - return false; - } - codepoint = 0x10000 + ((codepoint - 0xd800) << 10) + (low - 0xdc00); - } - AppendUtf8(result, codepoint); - break; - } - default: - return false; - } - } - return false; + if (type == "TINYINT" || type == "SMALLINT" || type == "INTEGER" || type == "BIGINT" || type == "HUGEINT" || + type == "UTINYINT" || type == "USMALLINT" || type == "UINTEGER" || type == "UBIGINT" || type == "UHUGEINT" || + type == "DECIMAL" || type == "FLOAT" || type == "DOUBLE") { + return JsonNodeKind::NUMBER; } + return JsonNodeKind::NIL; +} - bool ParseValue(JsonNode &result, idx_t depth) { - SkipWhitespace(); - if (position >= input.size()) { - return false; - } - auto c = input[position]; - if (c == '"') { - result.kind = JsonNodeKind::STRING; - return ParseString(result.scalar); - } - if (c == '{') { - return depth < MAX_NESTING_DEPTH && ParseObject(result, depth); - } - if (c == '[') { - return depth < MAX_NESTING_DEPTH && ParseArray(result, depth); - } - if (input.compare(position, 4, "true") == 0 || input.compare(position, 5, "false") == 0) { - result.kind = JsonNodeKind::BOOL; - position += input.compare(position, 4, "true") == 0 ? 4 : 5; - return true; - } - if (input.compare(position, 4, "null") == 0) { - result.kind = JsonNodeKind::NIL; - position += 4; - return true; - } - if (c == '-' || (c >= '0' && c <= '9')) { - auto start = position++; - while (position < input.size()) { - auto number_char = input[position]; - if (!((number_char >= '0' && number_char <= '9') || number_char == '.' || number_char == 'e' || - number_char == 'E' || number_char == '+' || number_char == '-')) { - break; - } - position++; - } - result.kind = JsonNodeKind::NUMBER; - result.scalar = input.substr(start, position - start); - return true; - } +//! Materialize one DuckDB-native JSON traversal. json_tree owns parsing, +//! escaping, Unicode handling, and node classification; this code only stores +//! its rows so callers can walk the resulting tree without reparsing input. +bool ReadJsonTree(Connection &con, const string &json, JsonTree &tree) { + const string sql = "SELECT id, parent, key, type, CASE WHEN type = 'VARCHAR' THEN json_extract_string(atom, '$') " + "ELSE atom::VARCHAR END " + "FROM json_tree(?::JSON) ORDER BY id"; + auto result = con.Query(sql, Value(json)); + if (result->HasError()) { return false; } - - bool ParseArray(JsonNode &result, idx_t depth) { - if (!Consume('[')) { - return false; - } - result.kind = JsonNodeKind::ARRAY; - result.children.clear(); - SkipWhitespace(); - if (position < input.size() && input[position] == ']') { - position++; - return true; - } - while (true) { - JsonNode child; - if (!ParseValue(child, depth + 1)) { - return false; - } - result.children.push_back(std::move(child)); - SkipWhitespace(); - if (position >= input.size()) { - return false; - } - if (input[position] == ']') { - position++; - return true; - } - if (input[position++] != ',') { - return false; - } - } - } - - bool ParseObject(JsonNode &result, idx_t depth) { - if (!Consume('{')) { - return false; - } - result.kind = JsonNodeKind::OBJECT; - result.members.clear(); - SkipWhitespace(); - if (position < input.size() && input[position] == '}') { - position++; - return true; - } - while (true) { - string key; - if (!ParseString(key) || !Consume(':')) { - return false; - } - JsonNode value; - if (!ParseValue(value, depth + 1)) { - return false; + auto chunk = result->Fetch(); + while (chunk && chunk->size() > 0) { + for (idx_t row = 0; row < chunk->size(); row++) { + JsonNode node; + node.id = chunk->GetValue(0, row).GetValue(); + auto parent = chunk->GetValue(1, row); + auto key = chunk->GetValue(2, row); + if (!key.IsNull()) { + node.key = key.GetValue(); } - result.members.emplace_back(std::move(key), std::move(value)); - SkipWhitespace(); - if (position >= input.size()) { - return false; + auto type = chunk->GetValue(3, row).GetValue(); + node.kind = JsonTypeToKind(type); + auto scalar = chunk->GetValue(4, row); + if (!scalar.IsNull()) { + node.scalar = scalar.GetValue(); } - if (input[position] == '}') { - position++; - return true; - } - if (input[position++] != ',') { - return false; + const auto index = tree.nodes.size(); + if (!parent.IsNull()) { + tree.children_by_parent[parent.GetValue()].push_back(index); } + tree.nodes.push_back(std::move(node)); } + chunk = result->Fetch(); } -}; + return !tree.nodes.empty(); +} bool IsCanonicalArrayIndex(const string &segment) { if (segment.empty() || (segment.size() > 1 && segment[0] == '0')) { @@ -307,25 +115,18 @@ string DecodeJsonPointerSegment(const string &segment) { return result; } -const JsonNode *FindMember(const JsonNode &node, const string &name) { - for (auto &member : node.members) { - if (member.first == name) { - return &member.second; - } +const JsonNode *FindChild(const JsonTree &tree, const JsonNode &parent, const string &key) { + auto children = tree.children_by_parent.find(parent.id); + if (children == tree.children_by_parent.end()) { + return nullptr; } - return nullptr; -} - -idx_t ParseIndex(const string &segment) { - idx_t index = 0; - for (auto c : segment) { - auto digit = idx_t(c - '0'); - if (index > (idx_t(-1) - digit) / 10) { - return idx_t(-1); + for (auto child_index : children->second) { + const auto &child = tree.nodes[child_index]; + if (child.key == key) { + return &child; } - index = index * 10 + digit; } - return index; + return nullptr; } string TransformScalarToOpenApi(const string &declared_type) { @@ -366,21 +167,32 @@ string TransformScalarToOpenApi(const string &declared_type) { return "{\"type\":\"string\"}"; } -string TransformNodeToOpenApi(const JsonNode &node) { +string TransformNodeToOpenApi(const JsonTree &tree, idx_t node_index) { + const auto &node = tree.nodes[node_index]; switch (node.kind) { case JsonNodeKind::STRING: return TransformScalarToOpenApi(node.scalar); - case JsonNodeKind::ARRAY: + case JsonNodeKind::ARRAY: { + auto children = tree.children_by_parent.find(node.id); return "{\"type\":\"array\",\"items\":" + - (node.children.empty() ? string("{}") : TransformNodeToOpenApi(node.children[0])) + "}"; + (children == tree.children_by_parent.end() || children->second.empty() + ? string("{}") + : TransformNodeToOpenApi(tree, children->second[0])) + + "}"; + } case JsonNodeKind::OBJECT: { string result = "{\"type\":\"object\",\"properties\":{"; - for (idx_t i = 0; i < node.members.size(); i++) { - if (i > 0) { - result += ","; + auto children = tree.children_by_parent.find(node.id); + if (children != tree.children_by_parent.end()) { + bool first = true; + for (auto child_index : children->second) { + const auto &child = tree.nodes[child_index]; + if (!first) { + result += ","; + } + first = false; + result += "\"" + QuackapiJsonEscape(child.key) + "\":" + TransformNodeToOpenApi(tree, child_index); } - result += "\"" + QuackapiJsonEscape(node.members[i].first) + - "\":" + TransformNodeToOpenApi(node.members[i].second); } return result + "}}"; } @@ -391,13 +203,13 @@ string TransformNodeToOpenApi(const JsonNode &node) { } // namespace -string QuackapiValidationBodyPointerLoc(const string &pointer, const string &json_body) { +string QuackapiValidationBodyPointerLoc(Connection &con, const string &pointer, const string &json_body) { string result = "[\"body\""; if (pointer.empty()) { return result + "]"; } - JsonNode root; - const JsonNode *current = JsonReader(json_body).ParseDocument(root) ? &root : nullptr; + JsonTree tree; + const JsonNode *current = ReadJsonTree(con, json_body, tree) ? &tree.nodes[0] : nullptr; idx_t start = pointer.size() > 0 && pointer[0] == '/' ? 1 : 0; while (true) { auto end = pointer.find('/', start); @@ -406,11 +218,10 @@ string QuackapiValidationBodyPointerLoc(const string &pointer, const string &jso result += ","; if (current && current->kind == JsonNodeKind::ARRAY && IsCanonicalArrayIndex(segment)) { result += segment; - auto index = ParseIndex(segment); - current = index < current->children.size() ? ¤t->children[index] : nullptr; + current = FindChild(tree, *current, segment); } else { result += "\"" + QuackapiJsonEscape(segment) + "\""; - current = current && current->kind == JsonNodeKind::OBJECT ? FindMember(*current, segment) : nullptr; + current = current && current->kind == JsonNodeKind::OBJECT ? FindChild(tree, *current, segment) : nullptr; } if (end == string::npos) { break; @@ -420,12 +231,12 @@ string QuackapiValidationBodyPointerLoc(const string &pointer, const string &jso return result + "]"; } -string QuackapiDuckdbTransformToOpenApi(const string &body_type) { - JsonNode structure; - if (!JsonReader(body_type).ParseDocument(structure)) { +string QuackapiDuckdbTransformToOpenApi(Connection &con, const string &body_type) { + JsonTree tree; + if (!ReadJsonTree(con, body_type, tree)) { return "{}"; } - return TransformNodeToOpenApi(structure); + return TransformNodeToOpenApi(tree, 0); } string QuackapiValidationErrorsJson(const vector &issues) { diff --git a/src/sql/quack_from_express_models.sql b/src/sql/quack_from_express_models.sql deleted file mode 100644 index 3e92497..0000000 --- a/src/sql/quack_from_express_models.sql +++ /dev/null @@ -1,124 +0,0 @@ --- quack_from_express_models — TS interfaces/classes/zod via sitting_duck --- Placeholder: __REPO__ (app root; C++ expands bare dirs to ts+js globs) --- Returns: model_name, field_name, field_type, is_required, is_optional, has_default, --- default_expr, file, field_line -WITH express_ast AS ( - SELECT * FROM read_ast('__REPO__/**/*.ts', 'typescript', peek := 400, ignore_errors := true) - UNION ALL BY NAME - SELECT * FROM read_ast('__REPO__/**/*.js', 'javascript', peek := 400, ignore_errors := true) -), -interface_fields AS ( - SELECT cls.name AS model_name, ps.name AS field_name, - COALESCE( - (SELECT nullif(trim(trim(ta.peek), ': '), '') FROM express_ast ta - WHERE ta.parent_id = ps.node_id AND ta.file_path = ps.file_path AND ta.type = 'type_annotation' LIMIT 1), - (SELECT pi.signature_type FROM express_ast pi - WHERE pi.parent_id = ps.node_id AND pi.file_path = ps.file_path AND pi.type = 'property_identifier' LIMIT 1), - 'unknown') AS field_type, - EXISTS (SELECT 1 FROM express_ast q WHERE q.parent_id = ps.node_id AND q.file_path = ps.file_path AND q.type = '?') AS is_optional, - false AS has_default, CAST(NULL AS VARCHAR) AS default_expr, - ps.file_path AS file, ps.start_line AS field_line - FROM express_ast ps - JOIN express_ast cls ON cls.node_id = ps.scope.class AND cls.file_path = ps.file_path - AND cls.type IN ('interface_declaration','class_declaration','type_alias_declaration') - WHERE ps.type = 'property_signature' AND ps.name IS NOT NULL AND cls.name IS NOT NULL -), -class_fields AS ( - SELECT cls.name AS model_name, f.name AS field_name, - COALESCE( - (SELECT nullif(trim(trim(ta.peek), ': '), '') FROM express_ast ta - WHERE ta.parent_id = f.node_id AND ta.file_path = f.file_path AND ta.type = 'type_annotation' LIMIT 1), - f.signature_type, 'unknown') AS field_type, - (EXISTS (SELECT 1 FROM express_ast q WHERE q.parent_id = f.node_id AND q.file_path = f.file_path AND q.type = '?') - OR coalesce(f.annotations,'') ILIKE '%IsOptional%' - OR coalesce(f.peek,'') LIKE '%?:%') AS is_optional, - (EXISTS (SELECT 1 FROM express_ast eq WHERE eq.parent_id = f.node_id AND eq.file_path = f.file_path - AND eq.type IN ('=','initializer','assignment_expression')) - OR regexp_matches(coalesce(f.peek,''), '=\s*\S')) AS has_default, - CASE WHEN position('=' IN coalesce(f.peek,'')) > 0 THEN trim(regexp_extract(f.peek, '=\s*(.+)$', 1)) END AS default_expr, - f.file_path AS file, f.start_line AS field_line - FROM express_ast f - JOIN express_ast cls ON cls.node_id = f.scope.class AND cls.file_path = f.file_path AND cls.type = 'class_declaration' - WHERE f.type IN ('public_field_definition','field_definition','property_definition') - AND f.name IS NOT NULL AND cls.name IS NOT NULL -), -ctor_fields AS ( - SELECT cls.name AS model_name, p.name AS field_name, - COALESCE( - (SELECT nullif(trim(trim(ta.peek), ': '), '') FROM express_ast ta - WHERE ta.parent_id = p.node_id AND ta.file_path = p.file_path AND ta.type = 'type_annotation' LIMIT 1), - 'unknown') AS field_type, - (p.type = 'optional_parameter') AS is_optional, false AS has_default, CAST(NULL AS VARCHAR) AS default_expr, - p.file_path AS file, p.start_line AS field_line - FROM express_ast p - JOIN express_ast cls ON cls.node_id = p.scope.class AND cls.file_path = p.file_path AND cls.type = 'class_declaration' - WHERE p.type IN ('required_parameter','optional_parameter') AND p.name IS NOT NULL - AND (list_contains(coalesce(p.modifiers, []), 'public') - OR list_contains(coalesce(p.modifiers, []), 'private') - OR list_contains(coalesce(p.modifiers, []), 'protected') - OR list_contains(coalesce(p.modifiers, []), 'readonly') - OR p.peek ILIKE 'public %' OR p.peek ILIKE 'private %' - OR p.peek ILIKE 'protected %' OR p.peek ILIKE 'readonly %') -), -zod_objects AS ( - SELECT c.node_id, c.file_path, - COALESCE( - (SELECT v.name FROM express_ast v WHERE v.file_path = c.file_path AND v.type = 'variable_declarator' - AND v.name IS NOT NULL AND c.start_line BETWEEN v.start_line AND v.end_line AND v.depth < c.depth - ORDER BY v.depth DESC LIMIT 1), 'ZodObject') AS model_name - FROM express_ast c - WHERE c.type = 'call_expression' AND c.name = 'object' - AND EXISTS ( - SELECT 1 FROM express_ast m WHERE m.parent_id = c.node_id AND m.file_path = c.file_path - AND m.type = 'member_expression' - AND (m.peek LIKE 'z.object%' OR m.peek LIKE 'zod.object%' - OR EXISTS (SELECT 1 FROM express_ast id WHERE id.parent_id = m.node_id AND id.file_path = m.file_path - AND id.type = 'identifier' AND lower(id.name) IN ('z','zod')))) -), -zod_fields AS ( - SELECT zo.model_name, COALESCE(key.name, key.peek) AS field_name, - COALESCE((SELECT left(val.peek, 80) FROM express_ast val - WHERE val.parent_id = pair.node_id AND val.file_path = pair.file_path - AND val.node_id <> key.node_id AND val.type NOT IN (':',',','}') - ORDER BY val.sibling_index DESC LIMIT 1), 'zod') AS field_type, - COALESCE((SELECT left(val.peek, 80) FROM express_ast val - WHERE val.parent_id = pair.node_id AND val.file_path = pair.file_path - AND val.node_id <> key.node_id AND val.type NOT IN (':',',','}') - ORDER BY val.sibling_index DESC LIMIT 1), '') ILIKE '%optional%' AS is_optional, - COALESCE((SELECT left(val.peek, 80) FROM express_ast val - WHERE val.parent_id = pair.node_id AND val.file_path = pair.file_path - AND val.node_id <> key.node_id AND val.type NOT IN (':',',','}') - ORDER BY val.sibling_index DESC LIMIT 1), '') ILIKE '%default%' AS has_default, - CAST(NULL AS VARCHAR) AS default_expr, - zo.file_path AS file, pair.start_line AS field_line - FROM zod_objects zo - JOIN express_ast a ON a.parent_id = zo.node_id AND a.file_path = zo.file_path AND a.type = 'arguments' - JOIN express_ast obj ON obj.parent_id = a.node_id AND obj.file_path = a.file_path AND obj.type IN ('object','object_expression') - JOIN express_ast pair ON pair.parent_id = obj.node_id AND pair.file_path = obj.file_path AND pair.type = 'pair' - JOIN express_ast key ON key.parent_id = pair.node_id AND key.file_path = pair.file_path - AND key.type IN ('property_identifier','identifier','string') - QUALIFY row_number() OVER (PARTITION BY pair.node_id, pair.file_path ORDER BY key.sibling_index) = 1 -), -models AS ( - -- Defaults imply not required (Pydantic/FastAPI parity). - SELECT model_name, field_name, field_type, - (NOT is_optional AND NOT has_default) AS is_required, is_optional, - has_default, default_expr, file, field_line FROM interface_fields - UNION ALL BY NAME - SELECT model_name, field_name, field_type, - (NOT is_optional AND NOT has_default) AS is_required, is_optional, - has_default, default_expr, file, field_line FROM class_fields - UNION ALL BY NAME - SELECT model_name, field_name, field_type, - (NOT is_optional AND NOT has_default) AS is_required, is_optional, - has_default, default_expr, file, field_line FROM ctor_fields - UNION ALL BY NAME - SELECT model_name, field_name, field_type, - (NOT coalesce(is_optional,false) AND NOT coalesce(has_default,false)) AS is_required, - coalesce(is_optional,false) AS is_optional, coalesce(has_default,false) AS has_default, - default_expr, file, field_line FROM zod_fields -) -SELECT model_name, field_name, field_type, is_required, is_optional, has_default, - default_expr, file, field_line -FROM models -ORDER BY model_name, field_line, field_name diff --git a/src/sql/quack_from_express_routes.sql b/src/sql/quack_from_express_routes.sql deleted file mode 100644 index ae27dca..0000000 --- a/src/sql/quack_from_express_routes.sql +++ /dev/null @@ -1,128 +0,0 @@ --- quack_from_express routes — Express HTTP routes via sitting_duck --- Placeholder: __REPO__ (app root; C++ expands bare dirs to ts+js globs) --- Returns: method, path, handler_name, file, start_line, evidence -WITH express_ast AS ( - SELECT * FROM read_ast('__REPO__/**/*.ts', 'typescript', peek := 400, ignore_errors := true) - UNION ALL BY NAME - SELECT * FROM read_ast('__REPO__/**/*.js', 'javascript', peek := 400, ignore_errors := true) -), -route_calls AS ( - SELECT c.*, - (SELECT trim(s.name, '''"') - FROM express_ast a JOIN express_ast s ON s.parent_id = a.node_id AND s.file_path = a.file_path - WHERE a.parent_id = c.node_id AND a.file_path = c.file_path AND a.type = 'arguments' AND s.type = 'string' - ORDER BY s.sibling_index LIMIT 1) AS path_raw, - (SELECT obj.name FROM express_ast m JOIN express_ast obj - ON obj.parent_id = m.node_id AND obj.file_path = m.file_path AND obj.type = 'identifier' - WHERE m.parent_id = c.node_id AND m.file_path = c.file_path AND m.type = 'member_expression' - ORDER BY obj.sibling_index LIMIT 1) AS router_obj - FROM express_ast c - WHERE c.type = 'call_expression' - AND lower(c.name) IN ('get','post','put','delete','patch','all','options','head') - AND EXISTS (SELECT 1 FROM express_ast m - WHERE m.parent_id = c.node_id AND m.file_path = c.file_path AND m.type = 'member_expression') -), -routes_base AS ( - SELECT * FROM route_calls - WHERE path_raw IS NOT NULL AND (path_raw LIKE '/%' OR path_raw IN ('*') OR path_raw LIKE '*%') -), -handlers AS ( - SELECT rb.node_id AS rid, rb.file_path, h.node_id AS hid, h.type AS htype, - h.name AS hname, h.start_line AS hs, h.end_line AS he, h.depth AS hdepth - FROM routes_base rb - JOIN express_ast a ON a.parent_id = rb.node_id AND a.file_path = rb.file_path AND a.type = 'arguments' - JOIN express_ast h ON h.parent_id = a.node_id AND h.file_path = a.file_path - AND h.type IN ('arrow_function','function_expression','function_declaration', - 'identifier','member_expression','call_expression') - QUALIFY row_number() OVER (PARTITION BY rb.node_id, rb.file_path ORDER BY h.sibling_index DESC) = 1 -), -handler_names AS ( - SELECT h.rid, h.file_path, - CASE - WHEN h.htype = 'identifier' THEN nullif(h.hname, '') - WHEN h.htype = 'member_expression' THEN COALESCE( - (SELECT p.name FROM express_ast p WHERE p.parent_id = h.hid AND p.file_path = h.file_path - AND p.type = 'property_identifier' ORDER BY p.sibling_index DESC LIMIT 1), - nullif(h.hname, ''), 'member_handler') - WHEN h.htype = 'call_expression' THEN COALESCE(nullif(h.hname, ''), 'bound_handler') - ELSE COALESCE(nullif(h.hname, ''), - (SELECT c.name FROM express_ast c - WHERE c.file_path = h.file_path AND c.type = 'call_expression' - AND c.semantic_type = 'COMPUTATION_CALL' - AND c.start_line BETWEEN h.hs AND h.he AND c.end_line <= h.he - AND c.depth > h.hdepth AND nullif(c.name,'') IS NOT NULL - AND lower(c.name) NOT IN ( - 'json','status','send','sendstatus','next','jsonp','redirect','render','end', - 'set','cookie','clearcookie','get','post','put','delete','patch','all','use') - ORDER BY c.start_line, c.node_id LIMIT 1), - 'anonymous') - END AS handler_name - FROM handlers h -), -string_mounts AS ( - SELECT c.file_path, trim(s.name, '''"') AS prefix - FROM express_ast c - JOIN express_ast m ON m.parent_id = c.node_id AND m.file_path = c.file_path AND m.type = 'member_expression' - JOIN express_ast a ON a.parent_id = c.node_id AND a.file_path = c.file_path AND a.type = 'arguments' - JOIN express_ast s ON s.parent_id = a.node_id AND s.file_path = a.file_path AND s.type = 'string' - WHERE c.type = 'call_expression' AND lower(c.name) = 'use' AND trim(s.name, '''"') LIKE '/%' - QUALIFY row_number() OVER (PARTITION BY c.node_id, c.file_path ORDER BY s.sibling_index) = 1 -), -bare_mounts AS ( - SELECT c.file_path, id.name AS mounted_ident - FROM express_ast c - JOIN express_ast m ON m.parent_id = c.node_id AND m.file_path = c.file_path AND m.type = 'member_expression' - JOIN express_ast a ON a.parent_id = c.node_id AND a.file_path = c.file_path AND a.type = 'arguments' - JOIN express_ast id ON id.parent_id = a.node_id AND id.file_path = a.file_path AND id.type = 'identifier' - WHERE c.type = 'call_expression' AND lower(c.name) = 'use' - AND NOT EXISTS (SELECT 1 FROM express_ast s WHERE s.parent_id = a.node_id AND s.file_path = a.file_path - AND s.type = 'string' AND s.sibling_index < id.sibling_index) -), -imports AS ( - SELECT imp.file_path AS from_file, - COALESCE( - (SELECT id.name FROM express_ast clause - JOIN express_ast id ON id.parent_id = clause.node_id AND id.file_path = clause.file_path - WHERE clause.parent_id = imp.node_id AND clause.file_path = imp.file_path - AND clause.type = 'import_clause' AND id.type = 'identifier' AND id.name IS NOT NULL - ORDER BY id.sibling_index LIMIT 1), - (SELECT sp.name FROM express_ast n - JOIN express_ast sp ON sp.parent_id = n.node_id AND sp.file_path = n.file_path - WHERE n.parent_id IN ( - SELECT c.node_id FROM express_ast c - WHERE c.parent_id = imp.node_id AND c.file_path = imp.file_path AND c.type = 'import_clause') - AND n.file_path = imp.file_path AND n.type = 'named_imports' - AND sp.type = 'import_specifier' AND sp.name IS NOT NULL - ORDER BY sp.sibling_index LIMIT 1) - ) AS local_name, - COALESCE( - trim(imp.name, '''"'), - trim((SELECT s.name FROM express_ast s WHERE s.parent_id = imp.node_id AND s.file_path = imp.file_path - AND s.type = 'string' ORDER BY s.sibling_index DESC LIMIT 1), '''"') - ) AS source_mod - FROM express_ast imp - WHERE imp.type = 'import_statement' AND imp.semantic_type = 'EXTERNAL_IMPORT' -), -file_prefixes AS ( - SELECT DISTINCT sm.prefix, replace(replace(im.source_mod, './', ''), '../', '') AS source_key - FROM string_mounts sm - JOIN bare_mounts bm ON bm.file_path = sm.file_path - JOIN imports im ON im.from_file = bm.file_path AND im.local_name = bm.mounted_ident -) -SELECT upper(rb.name) AS method, - CASE - WHEN fp.prefix IS NOT NULL AND rb.path_raw = '/' THEN fp.prefix - WHEN fp.prefix IS NOT NULL AND starts_with(rb.path_raw, fp.prefix) THEN rb.path_raw - WHEN fp.prefix IS NOT NULL THEN rtrim(fp.prefix, '/') || '/' || ltrim(rb.path_raw, '/') - ELSE rb.path_raw - END AS path, - COALESCE(hn.handler_name, 'anonymous') AS handler_name, - rb.file_path AS file, rb.start_line, - left(rb.peek, 200) AS evidence -FROM routes_base rb -LEFT JOIN handler_names hn ON hn.rid = rb.node_id AND hn.file_path = rb.file_path -LEFT JOIN file_prefixes fp - ON rb.file_path LIKE '%' || fp.source_key || '%' - OR rb.file_path LIKE '%' || fp.source_key || '.ts' - OR rb.file_path LIKE '%' || fp.source_key || '.js' -ORDER BY file, start_line, method, path diff --git a/src/sql/quack_from_fastapi_models.sql b/src/sql/quack_from_fastapi_models.sql deleted file mode 100644 index 02831b3..0000000 --- a/src/sql/quack_from_fastapi_models.sql +++ /dev/null @@ -1,101 +0,0 @@ --- quack_from_fastapi_models — Pydantic BaseModel/SQLModel fields via sitting_duck --- Placeholder: __REPO__ (directory or glob; C++ expands bare dirs to __REPO__/**/*.py) --- Returns: model_name, field_name, field_type, is_required, is_optional, has_default, --- default_expr, file, field_line -WITH source_ast AS ( - SELECT * FROM read_ast( - '__REPO__', - 'python', - ignore_errors := true, - peek := 'full' - ) -), -class_defs AS ( - SELECT node_id, name AS class_name, file_path, start_line, end_line - FROM source_ast - WHERE type = 'class_definition' AND semantic_type = 'DEFINITION_CLASS' - AND name IS NOT NULL AND name <> '' -), -class_bases AS ( - SELECT c.node_id, c.class_name, c.file_path, c.start_line, c.end_line, - list(DISTINCT CASE - WHEN b.name IS NULL THEN NULL - WHEN position('.' IN b.name) > 0 THEN list_last(string_split(b.name, '.')) - ELSE b.name - END) FILTER (WHERE b.name IS NOT NULL AND b.name <> '') AS bases - FROM class_defs c - JOIN source_ast al - ON al.file_path = c.file_path AND al.parent_id = c.node_id AND al.type = 'argument_list' - JOIN source_ast b - ON b.file_path = c.file_path AND b.parent_id = al.node_id - AND b.type IN ('identifier', 'attribute') - GROUP BY ALL -), -pydantic_models AS ( - WITH RECURSIVE reach AS ( - SELECT class_name, file_path, start_line, end_line, node_id, bases, 0 AS depth - FROM class_bases - WHERE list_contains(bases, 'BaseModel') OR list_contains(bases, 'SQLModel') - UNION ALL - SELECT c.class_name, c.file_path, c.start_line, c.end_line, c.node_id, c.bases, r.depth + 1 - FROM class_bases c - JOIN reach r ON list_contains(c.bases, r.class_name) - WHERE r.depth < 16 - ) - SELECT class_name, file_path, start_line, end_line, node_id, bases, min(depth) AS inheritance_depth - FROM reach GROUP BY ALL -), -field_nodes AS ( - SELECT m.class_name AS model_name, a.name AS field_name, - a.file_path AS file, a.start_line AS field_line, a.node_id AS field_id - FROM pydantic_models m - JOIN source_ast a - ON a.file_path = m.file_path AND a.scope.class = m.node_id - AND a.type = 'assignment' AND a.semantic_type = 'DEFINITION_VARIABLE' - AND a.scope.function IS NULL - WHERE a.name IS NOT NULL AND a.name <> '' - AND a.name NOT IN ('model_config', 'Config', 'Meta') - AND NOT starts_with(a.name, '_') -), -field_types AS ( - SELECT f.field_id, f.file, t.peek AS field_type - FROM field_nodes f - JOIN source_ast t - ON t.file_path = f.file AND t.parent_id = f.field_id AND t.type = 'type' -), -field_defaults AS ( - SELECT f.field_id, f.file, true AS has_default, rhs.peek AS default_expr - FROM field_nodes f - JOIN source_ast eq - ON eq.file_path = f.file AND eq.parent_id = f.field_id AND eq.type = '=' - LEFT JOIN source_ast rhs - ON rhs.file_path = f.file AND rhs.parent_id = f.field_id - AND rhs.sibling_index = eq.sibling_index + 1 -) -SELECT - f.model_name, - f.field_name, - coalesce(nullif(trim(ft.field_type), ''), 'Any') AS field_type, - ( - NOT coalesce(fd.has_default, false) - AND NOT ( - coalesce(ft.field_type, '') ILIKE '%Optional[%' - OR coalesce(ft.field_type, '') ILIKE '%| None%' - OR coalesce(ft.field_type, '') ILIKE '%None |%' - ) - ) AS is_required, - ( - coalesce(ft.field_type, '') ILIKE '%Optional[%' - OR coalesce(ft.field_type, '') ILIKE '%| None%' - OR coalesce(ft.field_type, '') ILIKE '%None |%' - OR coalesce(fd.default_expr, '') = 'None' - ) AS is_optional, - coalesce(fd.has_default, false) AS has_default, - fd.default_expr, - f.file, - f.field_line -FROM field_nodes f -LEFT JOIN field_types ft ON ft.field_id = f.field_id AND ft.file = f.file -LEFT JOIN field_defaults fd ON fd.field_id = f.field_id AND fd.file = f.file -WHERE ft.field_type IS NOT NULL OR fd.has_default -ORDER BY model_name, field_line, field_name diff --git a/src/sql/quack_from_fastapi_routes.sql b/src/sql/quack_from_fastapi_routes.sql deleted file mode 100644 index 4092789..0000000 --- a/src/sql/quack_from_fastapi_routes.sql +++ /dev/null @@ -1,65 +0,0 @@ --- quack_from_fastapi routes — sitting_duck AST extraction --- Placeholder: __REPO__ (directory or glob; C++ expands bare dirs to __REPO__/**/*.py) --- Returns: method, path, handler_name, file, start_line, evidence -WITH source_ast AS ( - SELECT * FROM read_ast( - '__REPO__', - 'python', - ignore_errors := true, - peek := 'full' - ) -), -route_calls AS ( - SELECT - d.file_path, d.node_id AS decorated_id, dec.start_line, - c.node_id AS call_id, lower(c.name) AS method_raw, c.peek AS evidence - FROM source_ast d - JOIN source_ast dec - ON dec.file_path = d.file_path AND dec.parent_id = d.node_id AND dec.type = 'decorator' - JOIN source_ast c - ON c.file_path = dec.file_path AND c.parent_id = dec.node_id - AND c.type = 'call' AND c.semantic_type = 'COMPUTATION_CALL' - WHERE d.type = 'decorated_definition' - AND lower(c.name) IN ( - 'get','post','put','delete','patch','options','head','trace','api_route','websocket' - ) -), -path_args AS ( - SELECT - rc.*, - CASE - WHEN s_raw IS NULL THEN NULL - WHEN starts_with(s_raw, 'r''') OR starts_with(s_raw, 'r"') - OR starts_with(s_raw, 'R''') OR starts_with(s_raw, 'R"') - THEN substring(s_raw, 3, length(s_raw) - 3) - WHEN length(s_raw) >= 2 AND ( - (s_raw[1] = '''' AND s_raw[length(s_raw)] = '''') - OR (s_raw[1] = '"' AND s_raw[length(s_raw)] = '"') - ) THEN substring(s_raw, 2, length(s_raw) - 2) - ELSE s_raw - END AS path - FROM ( - SELECT rc.*, coalesce(s.name, s.peek) AS s_raw - FROM route_calls rc - JOIN source_ast al - ON al.file_path = rc.file_path AND al.parent_id = rc.call_id AND al.type = 'argument_list' - JOIN source_ast s - ON s.file_path = rc.file_path AND s.parent_id = al.node_id AND s.type = 'string' - QUALIFY row_number() OVER (PARTITION BY rc.file_path, rc.call_id ORDER BY s.sibling_index) = 1 - ) rc -) -SELECT - CASE WHEN method_raw = 'api_route' THEN 'ANY' - WHEN method_raw = 'websocket' THEN 'WEBSOCKET' - ELSE upper(method_raw) END AS method, - p.path, - fn.name AS handler_name, - p.file_path AS file, - p.start_line, - p.evidence -FROM path_args p -JOIN source_ast fn - ON fn.file_path = p.file_path AND fn.parent_id = p.decorated_id - AND fn.type = 'function_definition' AND fn.semantic_type = 'DEFINITION_FUNCTION' -WHERE p.path IS NOT NULL AND p.path <> '' AND fn.name IS NOT NULL -ORDER BY file, start_line, method, path diff --git a/src/sql/quack_from_gin_models.sql b/src/sql/quack_from_gin_models.sql deleted file mode 100644 index cbf75ab..0000000 --- a/src/sql/quack_from_gin_models.sql +++ /dev/null @@ -1,58 +0,0 @@ --- quack_from_gin_models — Go struct fields with tags via sitting_duck --- Placeholder: __REPO__ (app root; C++ expands bare dirs to __REPO__/**/*.go) --- Returns: model_name, field_name, field_type, is_required, is_optional, has_default, --- default_expr, file, field_line -WITH -ast AS ( - SELECT * FROM read_ast('__REPO__', 'go', ignore_errors := true, peek := 300) -), -type_specs AS ( - SELECT ts.file_path, ts.name AS model_name, st.node_id AS struct_id - FROM ast ts - JOIN ast st ON st.file_path=ts.file_path AND st.parent_id=ts.node_id - AND st.type='struct_type' - WHERE ts.type='type_spec' AND ts.name IS NOT NULL -), -nested_structs AS ( - SELECT fd.file_path, fd.name AS nest_field, st.node_id AS struct_id, - fd.scope.class AS outer_struct_id - FROM ast fd - JOIN ast st ON st.file_path=fd.file_path AND st.parent_id=fd.node_id - AND st.type='struct_type' - WHERE fd.type='field_declaration' AND fd.name IS NOT NULL -), -struct_names AS ( - SELECT file_path, struct_id, model_name FROM type_specs - UNION ALL - SELECT ns.file_path, ns.struct_id, - COALESCE(ts.model_name,'_anon')||'.'||ns.nest_field - FROM nested_structs ns - LEFT JOIN type_specs ts ON ts.file_path=ns.file_path AND ts.struct_id=ns.outer_struct_id -), -fields AS ( - SELECT sn.model_name, f.name AS field_name, f.file_path, - f.start_line AS field_line, - (SELECT COALESCE(t.peek,t.name) FROM ast t - WHERE t.file_path=f.file_path AND t.parent_id=f.node_id - AND t.type IN ('type_identifier','pointer_type','slice_type','array_type', - 'map_type','qualified_type','struct_type','interface_type', - 'channel_type','function_type') - ORDER BY t.sibling_index LIMIT 1) AS field_type, - (SELECT trim(both '`' FROM t.peek) FROM ast t - WHERE t.file_path=f.file_path AND t.parent_id=f.node_id - AND t.type='raw_string_literal' LIMIT 1) AS tag - FROM ast f - JOIN struct_names sn ON sn.file_path=f.file_path AND sn.struct_id=f.scope.class - WHERE f.type='field_declaration' AND f.name IS NOT NULL AND f.scope.class IS NOT NULL -) -SELECT model_name, field_name, field_type, - COALESCE(regexp_matches(tag, 'binding:"[^"]*\brequired\b'), false) AS is_required, - (starts_with(COALESCE(field_type,''), '*') - OR COALESCE(regexp_matches(tag, '\bomitempty\b'), false)) AS is_optional, - false AS has_default, CAST(NULL AS VARCHAR) AS default_expr, - file_path AS file, field_line -FROM fields -WHERE tag IS NOT NULL - AND (tag LIKE '%json:%' OR tag LIKE '%binding:%' OR tag LIKE '%form:%' - OR tag LIKE '%uri:%' OR tag LIKE '%header:%' OR tag LIKE '%gorm:%') -ORDER BY model_name, field_line, field_name diff --git a/src/sql/quack_from_gin_routes.sql b/src/sql/quack_from_gin_routes.sql deleted file mode 100644 index ea5423c..0000000 --- a/src/sql/quack_from_gin_routes.sql +++ /dev/null @@ -1,170 +0,0 @@ --- quack_from_gin routes — Gin HTTP routes via sitting_duck --- Placeholder: __REPO__ (app root; C++ expands bare dirs to __REPO__/**/*.go) --- Returns: method, path, handler_name, file, start_line, evidence -WITH -ast AS ( - SELECT * FROM read_ast('__REPO__', 'go', ignore_errors := true, peek := 300) -), -route_calls AS ( - SELECT c.file_path, c.node_id AS route_id, c.name AS method_raw, c.start_line, - c.peek AS evidence, c.scope.function AS func_scope_id - FROM ast c - WHERE c.semantic_type = 'COMPUTATION_CALL' - AND c.name IN ('GET','POST','PUT','DELETE','PATCH','HEAD','OPTIONS','Any','Handle') -), -receivers AS ( - SELECT r.file_path, r.route_id, id.name AS receiver - FROM route_calls r - JOIN ast sel ON sel.file_path = r.file_path AND sel.parent_id = r.route_id - AND sel.type = 'selector_expression' - JOIN ast id ON id.file_path = sel.file_path AND id.parent_id = sel.node_id - AND id.type = 'identifier' -), -args AS ( - SELECT r.file_path, r.route_id, a.sibling_index, a.type AS arg_type, - a.name AS arg_name, a.peek AS arg_peek - FROM route_calls r - JOIN ast al ON al.file_path = r.file_path AND al.parent_id = r.route_id - AND al.type = 'argument_list' - JOIN ast a ON a.file_path = al.file_path AND a.parent_id = al.node_id - AND a.type IN ('interpreted_string_literal','identifier', - 'selector_expression','func_literal') -), -route_core AS ( - SELECT r.*, rv.receiver, - CASE WHEN r.method_raw = 'Handle' THEN ( - SELECT trim(a.arg_peek,'"') FROM args a - WHERE a.file_path=r.file_path AND a.route_id=r.route_id - AND a.arg_type='interpreted_string_literal' - ORDER BY a.sibling_index OFFSET 1 LIMIT 1) - ELSE ( - SELECT trim(a.arg_peek,'"') FROM args a - WHERE a.file_path=r.file_path AND a.route_id=r.route_id - AND a.arg_type='interpreted_string_literal' - ORDER BY a.sibling_index LIMIT 1) - END AS rel_path, - CASE WHEN r.method_raw='Handle' THEN ( - SELECT upper(trim(a.arg_peek,'"')) FROM args a - WHERE a.file_path=r.file_path AND a.route_id=r.route_id - AND a.arg_type='interpreted_string_literal' - ORDER BY a.sibling_index LIMIT 1) - WHEN r.method_raw='Any' THEN 'ANY' ELSE r.method_raw END AS method, - COALESCE( - (SELECT COALESCE(a.arg_name, regexp_extract(a.arg_peek,'([A-Za-z_][A-Za-z0-9_]*)$',1)) - FROM args a WHERE a.file_path=r.file_path AND a.route_id=r.route_id - AND a.arg_type IN ('identifier','selector_expression') - ORDER BY a.sibling_index DESC LIMIT 1), - (SELECT '' FROM args a - WHERE a.file_path=r.file_path AND a.route_id=r.route_id - AND a.arg_type='func_literal' LIMIT 1) - ) AS handler_name - FROM route_calls r - LEFT JOIN receivers rv ON rv.file_path=r.file_path AND rv.route_id=r.route_id -), -gin_routes AS ( - SELECT rc.*, f.name AS enclosing_fn - FROM route_core rc - LEFT JOIN ast f ON f.file_path=rc.file_path AND f.node_id=rc.func_scope_id - AND f.semantic_type='DEFINITION_FUNCTION' - WHERE rc.rel_path IS NOT NULL - AND (rc.rel_path='' OR rc.rel_path='/' OR rc.rel_path LIKE '/%') -), -gin_groups AS ( - SELECT g.file_path, g.node_id AS group_id, - (SELECT id.name FROM ast sel - JOIN ast id ON id.file_path=sel.file_path AND id.parent_id=sel.node_id - AND id.type='identifier' - WHERE sel.file_path=g.file_path AND sel.parent_id=g.node_id - AND sel.type='selector_expression' LIMIT 1) AS receiver, - (SELECT trim(s.peek,'"') FROM ast al - JOIN ast s ON s.file_path=al.file_path AND s.parent_id=al.node_id - AND s.type='interpreted_string_literal' - WHERE al.file_path=g.file_path AND al.parent_id=g.node_id - AND al.type='argument_list' ORDER BY s.sibling_index LIMIT 1) AS path_seg - FROM ast g - WHERE g.semantic_type='COMPUTATION_CALL' AND g.name='Group' -), -gin_groups_f AS ( - SELECT * FROM gin_groups - WHERE path_seg IS NOT NULL AND (path_seg='' OR path_seg='/' OR path_seg LIKE '/%') -), -group_bindings AS ( - SELECT DISTINCT gg.file_path, gg.group_id, gg.receiver, gg.path_seg, - left_id.name AS bound_var - FROM gin_groups_f gg - JOIN ast svd ON svd.file_path=gg.file_path AND svd.type='short_var_declaration' - AND gg.group_id IN ( - SELECT d.node_id FROM ast d - WHERE d.file_path=svd.file_path - AND d.start_line>=svd.start_line AND d.end_line<=svd.end_line - AND d.type='call_expression' AND d.name='Group') - JOIN ast left_list ON left_list.file_path=svd.file_path - AND left_list.parent_id=svd.node_id AND left_list.type='expression_list' - AND left_list.sibling_index=0 - JOIN ast left_id ON left_id.file_path=left_list.file_path - AND left_id.parent_id=left_list.node_id AND left_id.type='identifier' -), -group_prefix_resolved AS ( - WITH RECURSIVE chain AS ( - SELECT b.file_path, b.bound_var, b.path_seg AS full_prefix, b.receiver, 0 AS depth - FROM group_bindings b - WHERE NOT EXISTS (SELECT 1 FROM group_bindings p - WHERE p.file_path=b.file_path AND p.bound_var=b.receiver) - UNION ALL - SELECT c.file_path, b.bound_var, - CASE WHEN c.full_prefix='' THEN b.path_seg - WHEN b.path_seg IN ('','/') THEN rtrim(c.full_prefix,'/')||b.path_seg - ELSE rtrim(c.full_prefix,'/')|| - CASE WHEN starts_with(b.path_seg,'/') THEN b.path_seg ELSE '/'||b.path_seg END - END, b.receiver, c.depth+1 - FROM chain c - JOIN group_bindings b ON b.file_path=c.file_path AND b.receiver=c.bound_var - WHERE c.depth < 8 - ) - SELECT file_path, bound_var, full_prefix FROM chain -), -register_prefix_best AS ( - SELECT register_fn, full_prefix FROM ( - SELECT c.name AS register_fn, - CASE WHEN gp.full_prefix IS NULL OR gp.full_prefix='' THEN gg.path_seg - WHEN gg.path_seg IN ('','/') THEN rtrim(gp.full_prefix,'/')||gg.path_seg - ELSE rtrim(gp.full_prefix,'/')|| - CASE WHEN starts_with(gg.path_seg,'/') THEN gg.path_seg ELSE '/'||gg.path_seg END - END AS full_prefix, - row_number() OVER (PARTITION BY c.name ORDER BY - (c.file_path NOT LIKE '%_test.go') DESC, - length(COALESCE(gp.full_prefix,'')||gg.path_seg) DESC) AS rn - FROM ast c - JOIN ast al ON al.file_path=c.file_path AND al.parent_id=c.node_id - AND al.type='argument_list' - JOIN gin_groups_f gg ON gg.file_path=c.file_path AND gg.group_id IN ( - SELECT a.node_id FROM ast a WHERE a.file_path=al.file_path - AND a.parent_id=al.node_id AND a.type='call_expression' AND a.name='Group') - LEFT JOIN group_prefix_resolved gp - ON gp.file_path=gg.file_path AND gp.bound_var=gg.receiver - WHERE c.semantic_type='COMPUTATION_CALL' - AND c.name NOT IN ('Group','GET','POST','PUT','DELETE','PATCH', - 'HEAD','OPTIONS','Any','Handle','Use') - ) x WHERE rn=1 -) -SELECT gr.method, - CASE WHEN pref.prefix IS NULL OR pref.prefix='' THEN - CASE WHEN gr.rel_path='' THEN '/' ELSE gr.rel_path END - WHEN gr.rel_path IS NULL OR gr.rel_path='' THEN pref.prefix - WHEN gr.rel_path='/' THEN rtrim(pref.prefix,'/')||'/' - ELSE rtrim(pref.prefix,'/')|| - CASE WHEN starts_with(gr.rel_path,'/') THEN gr.rel_path ELSE '/'||gr.rel_path END - END AS path, - gr.handler_name, gr.file_path AS file, gr.start_line, gr.evidence -FROM gin_routes gr -LEFT JOIN LATERAL ( - SELECT gpr.full_prefix AS prefix FROM group_prefix_resolved gpr - WHERE gpr.file_path=gr.file_path AND gpr.bound_var=gr.receiver - UNION ALL - SELECT rpb.full_prefix FROM register_prefix_best rpb - WHERE rpb.register_fn=gr.enclosing_fn - AND NOT EXISTS (SELECT 1 FROM group_prefix_resolved g2 - WHERE g2.file_path=gr.file_path AND g2.bound_var=gr.receiver) - LIMIT 1 -) pref ON TRUE -ORDER BY file, start_line, method, path diff --git a/src/sql/quack_from_rails_models.sql b/src/sql/quack_from_rails_models.sql deleted file mode 100644 index 1a81847..0000000 --- a/src/sql/quack_from_rails_models.sql +++ /dev/null @@ -1,97 +0,0 @@ --- quack_from_rails_models — validates + strong params via sitting_duck --- Placeholder: __REPO__ (app root; C++ maps bare dirs to models+controllers globs) --- Returns: model_name, field_name, field_type, is_required, is_optional, has_default, --- default_expr, file, field_line -WITH -ast AS ( - SELECT * FROM read_ast( - ['__REPO__/app/models/**/*.rb', - '__REPO__/app/controllers/**/*.rb'], - 'ruby', peek := 400, ignore_errors := true) -), -classes AS ( - SELECT file_path, node_id, name FROM ast - WHERE semantic_type = 'DEFINITION_CLASS' AND name IS NOT NULL AND name <> '' -), -val_calls AS ( - SELECT * FROM ast WHERE semantic_type = 'COMPUTATION_CALL' AND name = 'validates' -), -val_fields AS ( - SELECT v.file_path, v.node_id AS cid, v.start_line, v.peek, v.scope.class AS cls, - trim(BOTH ':' FROM a.name) AS field_name - FROM val_calls v - JOIN ast al ON al.parent_id = v.node_id AND al.file_path = v.file_path AND al.type = 'argument_list' - JOIN ast a ON a.parent_id = al.node_id AND a.file_path = v.file_path AND a.type = 'simple_symbol' -), -val_opts AS ( - SELECT v.file_path, v.node_id AS cid, hk.name AS vkey - FROM val_calls v - JOIN ast al ON al.parent_id = v.node_id AND al.file_path = v.file_path AND al.type = 'argument_list' - JOIN ast p ON p.parent_id = al.node_id AND p.file_path = v.file_path AND p.type = 'pair' - JOIN ast hk ON hk.parent_id = p.node_id AND hk.file_path = v.file_path AND hk.type = 'hash_key_symbol' -), -from_val AS ( - SELECT c.name AS model_name, f.field_name, - array_to_string(list_sort(list(DISTINCT o.vkey)), ',') AS field_type, - list_contains(list(DISTINCT o.vkey), 'presence') AS is_required, - false AS has_default, CAST(NULL AS VARCHAR) AS default_expr, - f.file_path AS file, f.start_line AS field_line - FROM val_fields f - JOIN classes c ON c.node_id = f.cls AND c.file_path = f.file_path - LEFT JOIN val_opts o ON o.cid = f.cid AND o.file_path = f.file_path - GROUP BY ALL -), -permits AS ( - SELECT p.* FROM ast p - WHERE p.semantic_type = 'COMPUTATION_CALL' AND p.name = 'permit' - AND EXISTS (SELECT 1 FROM ast r WHERE r.parent_id = p.node_id AND r.file_path = p.file_path - AND r.semantic_type = 'COMPUTATION_CALL' AND r.name = 'require') -), -req_model AS ( - SELECT p.node_id AS pid, p.file_path, p.start_line, p.peek, p.scope.class AS cls, - trim(BOTH ':' FROM s.name) AS model_key - FROM permits p - JOIN ast r ON r.parent_id = p.node_id AND r.file_path = p.file_path - AND r.semantic_type = 'COMPUTATION_CALL' AND r.name = 'require' - JOIN ast ra ON ra.parent_id = r.node_id AND ra.file_path = p.file_path AND ra.type = 'argument_list' - JOIN ast s ON s.parent_id = ra.node_id AND s.file_path = p.file_path AND s.type = 'simple_symbol' -), -pfields AS ( - SELECT p.node_id AS pid, p.file_path, trim(BOTH ':' FROM s.name) AS field_name, 'strong_param' AS field_type - FROM permits p - JOIN ast pa ON pa.parent_id = p.node_id AND pa.file_path = p.file_path AND pa.type = 'argument_list' - JOIN ast s ON s.parent_id = pa.node_id AND s.file_path = p.file_path AND s.type = 'simple_symbol' - UNION ALL - SELECT p.node_id, p.file_path, hk.name, 'array' - FROM permits p - JOIN ast pa ON pa.parent_id = p.node_id AND pa.file_path = p.file_path AND pa.type = 'argument_list' - JOIN ast pr ON pr.parent_id = pa.node_id AND pr.file_path = p.file_path AND pr.type = 'pair' - JOIN ast hk ON hk.parent_id = pr.node_id AND hk.file_path = p.file_path AND hk.type = 'hash_key_symbol' -), -from_sp AS ( - -- permit is a whitelist only; presence/required comes from validates (from_val). - SELECT upper(substr(rm.model_key,1,1)) || substr(rm.model_key,2) AS model_name, - pf.field_name, pf.field_type, false AS is_required, - false AS has_default, CAST(NULL AS VARCHAR) AS default_expr, - rm.file_path AS file, rm.start_line AS field_line - FROM req_model rm - JOIN pfields pf ON pf.pid = rm.pid AND pf.file_path = rm.file_path -), -models AS ( - SELECT model_name, field_name, field_type, is_required, - NOT is_required AS is_optional, has_default, default_expr, file, field_line - FROM from_val - UNION ALL - -- Keep permit-only fields; drop duplicates already covered by validates. - SELECT sp.model_name, sp.field_name, sp.field_type, sp.is_required, - NOT sp.is_required AS is_optional, sp.has_default, sp.default_expr, sp.file, sp.field_line - FROM from_sp sp - WHERE NOT EXISTS ( - SELECT 1 FROM from_val v - WHERE lower(v.model_name) = lower(sp.model_name) AND v.field_name = sp.field_name - ) -) -SELECT model_name, field_name, field_type, is_required, is_optional, has_default, - default_expr, file, field_line -FROM models -ORDER BY model_name, field_name diff --git a/src/sql/quack_from_rails_routes.sql b/src/sql/quack_from_rails_routes.sql deleted file mode 100644 index 169cff2..0000000 --- a/src/sql/quack_from_rails_routes.sql +++ /dev/null @@ -1,141 +0,0 @@ --- quack_from_rails routes — Rails routes.rb via sitting_duck --- Placeholder: __REPO__ (app root; C++ maps bare dirs to __REPO__/config/routes.rb) --- Returns: method, path, handler_name, file, start_line, evidence -WITH -ast AS ( - SELECT * FROM read_ast('__REPO__', 'ruby', peek := 400, ignore_errors := true) -), -dsl AS ( - SELECT node_id, name, start_line, depth, descendant_count, file_path, peek - FROM ast - WHERE semantic_type = 'COMPUTATION_CALL' - AND name IN ('resources','resource','scope','namespace','get','post','put','patch', - 'delete','root','match','devise_for','member','collection') -), -args AS ( - SELECT d.node_id AS call_id, d.file_path, a.node_id AS arg_id, a.type, a.name, a.peek, a.sibling_index - FROM dsl d - JOIN ast al ON al.parent_id = d.node_id AND al.file_path = d.file_path AND al.type = 'argument_list' - JOIN ast a ON a.parent_id = al.node_id AND a.file_path = d.file_path - WHERE a.type NOT IN (',','(',')',':') -), -res AS ( - SELECT call_id, COALESCE( - (SELECT trim(BOTH ':' FROM name) FROM args x WHERE x.call_id = a.call_id AND x.type = 'simple_symbol' - ORDER BY sibling_index LIMIT 1), - (SELECT trim(BOTH '''"' FROM peek) FROM args x WHERE x.call_id = a.call_id AND x.type IN ('string','string_content') - ORDER BY sibling_index LIMIT 1) - ) AS rname - FROM (SELECT DISTINCT call_id FROM args) a -), -pairs AS ( - SELECT a.call_id, a.file_path, hk.name AS k, p.node_id AS pair_id - FROM args a - JOIN ast p ON p.node_id = a.arg_id AND p.file_path = a.file_path AND p.type = 'pair' - JOIN ast hk ON hk.parent_id = p.node_id AND hk.file_path = a.file_path AND hk.type = 'hash_key_symbol' -), -opt AS ( - SELECT d.node_id, - COALESCE((SELECT trim(BOTH ':' FROM s.name) FROM pairs op - JOIN ast s ON s.parent_id = op.pair_id AND s.file_path = op.file_path AND s.type = 'simple_symbol' - WHERE op.call_id = d.node_id AND op.k = 'param' LIMIT 1), 'id') AS param, - (SELECT trim(BOTH ':' FROM s.name) FROM pairs op - JOIN ast s ON s.parent_id = op.pair_id AND s.file_path = op.file_path AND s.type = 'simple_symbol' - WHERE op.call_id = d.node_id AND op.k = 'on' LIMIT 1) AS on_type, - (SELECT trim(BOTH '''"' FROM COALESCE(s.name, t.peek)) FROM pairs op - LEFT JOIN ast s ON s.parent_id = op.pair_id AND s.file_path = op.file_path AND s.type = 'simple_symbol' - LEFT JOIN ast t ON t.parent_id = op.pair_id AND t.file_path = op.file_path AND t.type = 'string' - WHERE op.call_id = d.node_id AND op.k = 'to' LIMIT 1) AS to_h, - (SELECT list(trim(BOTH ':' FROM s.name) ORDER BY s.sibling_index) FROM pairs op - JOIN ast arr ON arr.parent_id = op.pair_id AND arr.file_path = op.file_path AND arr.type = 'array' - JOIN ast s ON s.parent_id = arr.node_id AND s.file_path = op.file_path AND s.type = 'simple_symbol' - WHERE op.call_id = d.node_id AND op.k = 'only') AS only_a, - (SELECT list(trim(BOTH ':' FROM s.name) ORDER BY s.sibling_index) FROM pairs op - JOIN ast arr ON arr.parent_id = op.pair_id AND arr.file_path = op.file_path AND arr.type = 'array' - JOIN ast s ON s.parent_id = arr.node_id AND s.file_path = op.file_path AND s.type = 'simple_symbol' - WHERE op.call_id = d.node_id AND op.k = 'except') AS except_a - FROM dsl d -), -reg AS ( - SELECT d.*, r.rname, o.param, o.on_type, o.to_h, o.only_a, o.except_a, - (SELECT p.node_id FROM dsl p - WHERE d.file_path = p.file_path AND d.node_id > p.node_id - AND d.node_id <= p.node_id + p.descendant_count - AND p.name IN ('resources','resource','scope','namespace','member','collection') - AND p.node_id <> d.node_id - ORDER BY p.depth DESC LIMIT 1) AS parent_id - FROM dsl d - LEFT JOIN res r ON r.call_id = d.node_id - LEFT JOIN opt o ON o.node_id = d.node_id -), -path_built AS ( - WITH RECURSIVE pb AS ( - SELECT *, CAST('' AS VARCHAR) AS pref FROM reg WHERE parent_id IS NULL - UNION ALL - SELECT c.*, CAST(p.pref || CASE - WHEN p.name IN ('scope','namespace') AND p.rname IS NOT NULL THEN '/' || p.rname - WHEN p.name = 'resources' AND p.rname IS NOT NULL THEN '/' || p.rname || '/:' || p.param - WHEN p.name = 'resource' AND p.rname IS NOT NULL THEN '/' || p.rname - ELSE '' END AS VARCHAR) - FROM reg c JOIN pb p ON c.parent_id = p.node_id AND c.file_path = p.file_path - ) SELECT * FROM pb -), -rest AS ( - SELECT * FROM (VALUES - ('index','GET','collection',false),('create','POST','collection',false), - ('new','GET','collection',true),('show','GET','member',false), - ('edit','GET','member',true),('update','PUT','member',false), - ('update','PATCH','member',false),('destroy','DELETE','member',false) - ) t(action, method, nest, sfx) -), -expanded AS ( - SELECT ra.method, - CASE - WHEN pb.name = 'resources' AND ra.nest = 'collection' AND NOT ra.sfx - THEN pb.pref || '/' || pb.rname - WHEN pb.name = 'resources' AND ra.nest = 'collection' AND ra.sfx - THEN pb.pref || '/' || pb.rname || '/' || ra.action - WHEN pb.name = 'resources' AND ra.nest = 'member' AND NOT ra.sfx - THEN pb.pref || '/' || pb.rname || '/:' || pb.param - WHEN pb.name = 'resources' AND ra.nest = 'member' AND ra.sfx - THEN pb.pref || '/' || pb.rname || '/:' || pb.param || '/' || ra.action - WHEN pb.name = 'resource' AND ra.action = 'index' THEN NULL - WHEN pb.name = 'resource' AND ra.sfx THEN pb.pref || '/' || pb.rname || '/' || ra.action - WHEN pb.name = 'resource' THEN pb.pref || '/' || pb.rname - END AS path, - CASE WHEN pb.name = 'resources' THEN pb.rname || '#' || ra.action - ELSE pb.rname || 's#' || ra.action END AS handler_name, - pb.file_path AS file, pb.start_line, pb.peek AS evidence - FROM path_built pb CROSS JOIN rest ra - WHERE pb.name IN ('resources','resource') AND pb.rname IS NOT NULL - AND (pb.only_a IS NULL OR list_contains(pb.only_a, ra.action)) - AND (pb.except_a IS NULL OR NOT list_contains(pb.except_a, ra.action)) - AND NOT (pb.name = 'resource' AND ra.action = 'index') - - UNION ALL - SELECT upper(pb.name), - CASE - WHEN pb.on_type = 'collection' - THEN regexp_replace(pb.pref, '/:[^/]+$', '') || '/' || pb.rname - WHEN pb.on_type = 'member' THEN pb.pref || '/' || pb.rname - WHEN pb.rname LIKE '%/%' OR pb.rname LIKE '%:%' - THEN CASE WHEN pb.rname LIKE '/%' THEN pb.rname ELSE pb.pref || '/' || pb.rname END - ELSE pb.pref || '/' || pb.rname - END, - COALESCE(pb.to_h, - (SELECT CASE WHEN p.name = 'resource' THEN p.rname || 's' ELSE p.rname END - FROM path_built p WHERE p.node_id = pb.parent_id) || '#' || pb.rname), - pb.file_path, pb.start_line, pb.peek - FROM path_built pb - WHERE pb.name IN ('get','post','put','patch','delete','match') AND pb.rname IS NOT NULL - - UNION ALL - SELECT t.m, pb.pref || t.p, 'sessions#' || t.a, pb.file_path, pb.start_line, pb.peek - FROM path_built pb - CROSS JOIN (VALUES ('GET','/users/login','new'),('POST','/users/login','create'), - ('DELETE','/users/sign_out','destroy')) t(m,p,a) - WHERE pb.name = 'devise_for' -) -SELECT method, path, handler_name, file, start_line, evidence -FROM expanded WHERE path IS NOT NULL -ORDER BY file, start_line, method, path diff --git a/test/conformance/run.sh b/test/conformance/run.sh index 5110837..a3d95ca 100755 --- a/test/conformance/run.sh +++ b/test/conformance/run.sh @@ -6,20 +6,28 @@ set -euo pipefail CONF="$(cd "$(dirname "$0")" && pwd)" REPO="$(cd "$CONF/../.." && pwd)" DUCK="${DUCKDB:-$REPO/build/release/duckdb}" +EXT="${QUACKAPI_EXT:-$REPO/build/release/extension/quackapi/quackapi.duckdb_extension}" +EXPECTED_EXT_VERSION="${QUACKAPI_EXPECTED_VERSION:-$(git -C "$REPO" rev-parse --short HEAD)}" PORT="${PORT:-18770}" BASE="http://127.0.0.1:${PORT}" FIFO="${TMPDIR:-/tmp}/quackapi_conformance_$$.fifo" LOG="${TMPDIR:-/tmp}/quackapi_conformance_$$.log" PIDFILE="${TMPDIR:-/tmp}/quackapi_conformance_$$.pid" +DUCKDB_CLEAN_HOME="$(mktemp -d "${TMPDIR:-/tmp}/quackapi_conformance_home_XXXXXX")" RESULTS_DIR="${RESULTS_DIR:-$CONF/results}" DPID="" +duckdb_cli() { + HOME="$DUCKDB_CLEAN_HOME" "$DUCK" "$@" +} + cleanup() { if [[ -n "$DPID" ]] && kill -0 "$DPID" 2>/dev/null; then kill "$DPID" 2>/dev/null || true wait "$DPID" 2>/dev/null || true fi rm -f "$FIFO" "$PIDFILE" + rm -rf "$DUCKDB_CLEAN_HOME" } trap cleanup EXIT @@ -29,20 +37,47 @@ if [[ ! -x "$DUCK" ]]; then exit 2 fi +if [[ ! -f "$EXT" ]]; then + echo "quackapi extension artifact not found: $EXT" >&2 + exit 2 +fi +if [[ "$EXT" == *"'"* ]]; then + echo "quackapi artifact path contains a SQL quote: $EXT" >&2 + exit 2 +fi +core_version="$(duckdb_cli -unsigned -noheader -list -c 'SELECT library_version, source_id FROM pragma_version();')" +if [[ "$core_version" != 'v1.5.6|069cc9f9' && "$core_version" != 'v1.5.6|069cc9f9b5' ]]; then + echo "wrong DuckDB core: $core_version (expected v1.5.6 and source ID 069cc9f9 or 069cc9f9b5)" >&2 + exit 2 +fi +initially_loaded="$(duckdb_cli -unsigned -noheader -list -c "SELECT count(extension_name) FROM duckdb_extensions() WHERE loaded AND extension_name IN ('quackapi', 'curl_httpfs', 'httpfs');")" +if [[ "$initially_loaded" != '0' ]]; then + echo "test core unexpectedly preloaded QuackAPI or an HTTP provider before explicit LOAD" >&2 + exit 2 +fi +actual_ext_version="$(duckdb_cli -unsigned -noheader -list -c "LOAD '$EXT'; SELECT extension_version FROM duckdb_extensions() WHERE extension_name = 'quackapi' AND loaded;")" +if [[ "$actual_ext_version" != "$EXPECTED_EXT_VERSION" ]]; then + echo "wrong quackapi artifact: $actual_ext_version (expected $EXPECTED_EXT_VERSION from $EXT)" >&2 + exit 2 +fi + if lsof -nP -iTCP:"$PORT" -sTCP:LISTEN -t >/dev/null 2>&1; then echo "Port $PORT is already occupied; choose PORT for an isolated run" >&2 exit 2 fi mkfifo "$FIFO" -"$DUCK" -init /dev/null -unsigned <"$FIFO" >"$LOG" 2>&1 & +duckdb_cli -unsigned <"$FIFO" >"$LOG" 2>&1 & echo $! >"$PIDFILE" DPID=$! exec 3>"$FIFO" { - echo "LOAD quackapi;" + echo "SET threads = 4;" + echo "SET memory_limit = '256MB';" + echo "SET preserve_insertion_order = false;" + printf "LOAD '%s';\n" "$EXT" cat "$CONF/routes.sql" echo "SELECT * FROM quackapi_serve(${PORT}, health_routes := false, access_log := false);" echo "SELECT * FROM quackapi_servers();" diff --git a/test/cpp/CMakeLists.txt b/test/cpp/CMakeLists.txt new file mode 100644 index 0000000..ce29e20 --- /dev/null +++ b/test/cpp/CMakeLists.txt @@ -0,0 +1,123 @@ +cmake_minimum_required(VERSION 3.16) +project(quackapi_http_tests LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +enable_testing() + +get_filename_component(QUACKAPI_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../.." + ABSOLUTE) +set(QUACKAPI_BUILD_ROOT + "${QUACKAPI_ROOT}/build/release" + CACHE PATH "Existing DuckDB release build") +set(QUACKAPI_EXTENSION + "${QUACKAPI_BUILD_ROOT}/extension/quackapi/quackapi.duckdb_extension" + CACHE FILEPATH "Exact QuackAPI loadable artifact") +set(JSON_EXTENSION + "${QUACKAPI_BUILD_ROOT}/extension/json/json.duckdb_extension" + CACHE FILEPATH "Exact JSON loadable artifact QuackAPI requires") +set(POST_PROVIDER_EXTENSION + "" + CACHE FILEPATH "Exact HTTP provider loadable artifact") +set(POST_PROVIDER_NAME + "" + CACHE STRING "Expected HTTP provider extension name") +set(QUACKAPI_VERSION + "" + CACHE STRING "Expected QuackAPI extension_version") +set(POST_PROVIDER_VERSION + "" + CACHE STRING "Expected HTTP provider extension_version") + +if(NOT EXISTS "${QUACKAPI_EXTENSION}" + OR NOT EXISTS "${JSON_EXTENSION}" + OR NOT EXISTS "${POST_PROVIDER_EXTENSION}" + OR QUACKAPI_VERSION STREQUAL "" + OR POST_PROVIDER_NAME STREQUAL "" + OR POST_PROVIDER_VERSION STREQUAL "") + message( + FATAL_ERROR + "Pass the two existing artifact paths and their expected extension versions" + ) +endif() + +find_library( + DUCKDB_LIBRARY duckdb + PATHS "${QUACKAPI_BUILD_ROOT}/src" + NO_DEFAULT_PATH REQUIRED) +add_executable(quackapi_post_http_test quackapi_post_http_test.cpp) +find_package(OpenSSL REQUIRED) +target_include_directories( + quackapi_post_http_test SYSTEM PRIVATE "${QUACKAPI_ROOT}/duckdb/src/include" + "${QUACKAPI_ROOT}/third_party/httplib") +target_compile_definitions(quackapi_post_http_test + PRIVATE CPPHTTPLIB_LISTEN_BACKLOG=SOMAXCONN) +target_link_libraries(quackapi_post_http_test PRIVATE "${DUCKDB_LIBRARY}") +add_executable(quackapi_post_tls_test quackapi_post_tls_test.cpp) +target_include_directories( + quackapi_post_tls_test SYSTEM PRIVATE "${QUACKAPI_ROOT}/duckdb/src/include" + "${QUACKAPI_ROOT}/third_party/httplib") +target_compile_definitions(quackapi_post_tls_test + PRIVATE CPPHTTPLIB_OPENSSL_SUPPORT) +target_link_libraries(quackapi_post_tls_test + PRIVATE "${DUCKDB_LIBRARY}" OpenSSL::SSL OpenSSL::Crypto) +if(NOT WIN32) + find_package(Threads REQUIRED) + target_link_libraries(quackapi_post_http_test PRIVATE Threads::Threads) + target_link_libraries(quackapi_post_tls_test PRIVATE Threads::Threads) +endif() + +add_test( + NAME post_http_contract + COMMAND + quackapi_post_http_test "${QUACKAPI_EXTENSION}" "${QUACKAPI_VERSION}" + "${POST_PROVIDER_EXTENSION}" "${POST_PROVIDER_NAME}" + "${POST_PROVIDER_VERSION}") +set_tests_properties( + post_http_contract + PROPERTIES TIMEOUT 30 ENVIRONMENT + "QUACKAPI_TEST_JSON_EXTENSION=${JSON_EXTENSION}") +add_test( + NAME post_tls_contract + COMMAND + quackapi_post_tls_test "${QUACKAPI_EXTENSION}" "${QUACKAPI_VERSION}" + "${POST_PROVIDER_EXTENSION}" "${POST_PROVIDER_NAME}" + "${POST_PROVIDER_VERSION}" + "${CMAKE_CURRENT_SOURCE_DIR}/fixtures/loopback.crt" + "${CMAKE_CURRENT_SOURCE_DIR}/fixtures/loopback.key") +set_tests_properties( + post_tls_contract PROPERTIES TIMEOUT 20 ENVIRONMENT + "QUACKAPI_TEST_JSON_EXTENSION=${JSON_EXTENSION}") + +option(ENABLE_PROVIDER_REPLACEMENT "Build the two-provider lifetime contract" + OFF) +if(ENABLE_PROVIDER_REPLACEMENT) + if(NOT EXISTS "${CURL_HTTPFS_EXTENSION}" + OR NOT EXISTS "${HTTPFS_EXTENSION}" + OR CURL_HTTPFS_VERSION STREQUAL "" + OR HTTPFS_VERSION STREQUAL "") + message( + FATAL_ERROR + "Provider replacement requires exact CURL_HTTPFS_EXTENSION/VERSION and HTTPFS_EXTENSION/VERSION" + ) + endif() + add_executable(quackapi_provider_replacement_test + quackapi_provider_replacement_test.cpp) + target_include_directories( + quackapi_provider_replacement_test SYSTEM + PRIVATE "${QUACKAPI_ROOT}/duckdb/src/include" + "${QUACKAPI_ROOT}/third_party/httplib") + target_link_libraries(quackapi_provider_replacement_test + PRIVATE "${DUCKDB_LIBRARY}") + if(NOT WIN32) + target_link_libraries(quackapi_provider_replacement_test + PRIVATE Threads::Threads) + endif() + add_test( + NAME provider_replacement_contract + COMMAND + quackapi_provider_replacement_test "${QUACKAPI_EXTENSION}" + "${QUACKAPI_VERSION}" "${CURL_HTTPFS_EXTENSION}" "${CURL_HTTPFS_VERSION}" + "${HTTPFS_EXTENSION}" "${HTTPFS_VERSION}") + set_tests_properties(provider_replacement_contract PROPERTIES TIMEOUT 20) +endif() diff --git a/test/cpp/fixtures/loopback.crt b/test/cpp/fixtures/loopback.crt new file mode 100644 index 0000000..542bff1 --- /dev/null +++ b/test/cpp/fixtures/loopback.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDJTCCAg2gAwIBAgIUGAGIJXxOLFfW9fjpJduSTwMFkcowDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI2MTAxNDIwMDQxNVoXDTM2MTAx +MTIwMDQxNVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEA0N8DdUux0FhX7SvSy8lY9MC6QoNkb4E/BCPpiUQ8E9j0 +8E03uXP3niVHe2RAxsavhRRzDqQyMlfdVB9Sq7mtELAFANq7GFjPQbpDqk7LSwTK +pxdhGt9aAbzJWT7iqRY+GX5lcj4mEbkIncKVqjk9ru04nvy1A+/9wuoPPL5oOmbu +YAjcEMNWKEuYb7DWjqpv5ujCjDJSvYq5NWWHpyFOqzLbJ+h+EBV1BH6bj9/GXNp0 +U9XTziVgf0Sz5hxvtmvt7UbHskDoZzfzx1q5gxaFPVDECkUSyvK6J6BsQ5645W+H +3fEKA283MoOW2cbbyxBurjWLJUuBTCgrJGgLh8jMZwIDAQABo28wbTAdBgNVHQ4E +FgQUeu+iYqI8L1JPtuP3ZfMgWcQzDngwHwYDVR0jBBgwFoAUeu+iYqI8L1JPtuP3 +ZfMgWcQzDngwDwYDVR0TAQH/BAUwAwEB/zAaBgNVHREEEzARgglsb2NhbGhvc3SH +BH8AAAEwDQYJKoZIhvcNAQELBQADggEBAKUp8q+XkR/zzRORINQ76C/etVesotfe ++JuUliP9uarNtLHVxL83CA5XQglxUxdGCjkJTbl9DESzlEgUAafsxVl22uNSczvK +EeHA8FGIAXNzlAOApjGWFFJ4Zv361j5cUsXn7d5dGvhUyf/gkqaawY5N8vAeH19I +I1/2zqNUgR5O3j3gznIOWlxsuqWzk0HrQtLa/PWvyp8ZTBHGZyJBjynyDNE3ZXTw +huksdwt6YEK7zJIwDbZlcn5AKQJbpsca3YeWGz6cyUUWNeWLAcTItEWDcdQdsc3V +sNES/YkzyLKPXW2ie12qHJPQWQnmRIsCrCa1hiB0fAs3h1sPKI7nMWg= +-----END CERTIFICATE----- diff --git a/test/cpp/fixtures/loopback.key b/test/cpp/fixtures/loopback.key new file mode 100644 index 0000000..ef908a7 --- /dev/null +++ b/test/cpp/fixtures/loopback.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDQ3wN1S7HQWFft +K9LLyVj0wLpCg2RvgT8EI+mJRDwT2PTwTTe5c/eeJUd7ZEDGxq+FFHMOpDIyV91U +H1Krua0QsAUA2rsYWM9BukOqTstLBMqnF2Ea31oBvMlZPuKpFj4ZfmVyPiYRuQid +wpWqOT2u7Tie/LUD7/3C6g88vmg6Zu5gCNwQw1YoS5hvsNaOqm/m6MKMMlK9irk1 +ZYenIU6rMtsn6H4QFXUEfpuP38Zc2nRT1dPOJWB/RLPmHG+2a+3tRseyQOhnN/PH +WrmDFoU9UMQKRRLK8ronoGxDnrjlb4fd8QoDbzcyg5bZxtvLEG6uNYslS4FMKCsk +aAuHyMxnAgMBAAECggEAPaLp5tWxAb3oKH1rpaZjoUo0TMdwPnVh7qtYUaD4p5S4 +5yaqQIhHtHvkxbVWZjIgfoGD2Hi8tV13W0kpALL4rrl6qH9jgbJHaup36Ex7MkaW +NMux5JevfF7E/hiQE3yUU7qYXieX3VgAgml29/YPTU0Ejeod0q0h8cBXQcIgiMbp +pnpZQxPCH4gkt0JMHPnS2iGzlL9GmWbgv1gDsWpspUrPuEayfLWxLuO01iKcZ59R +73VBfyyNhY2XxdFS1edpiR17VfYJFuualXXXEctWfBFwiOYpYWgxGdYhAInCZax5 +Xe1Am22enuYXP78rNhg5Epcu+gCCHxboi6kdB495iQKBgQDriWRmQHXZs+F8OQJD +SUlqgbcg1WcVL3VI2/NLfc3Lyxsn8IcZyxtguCVj44Wo9stulZ4MM1sUrDN1CRJi +eN8XX8GPFD9fecpBLDG+V1gQJoUyQSTAHa3q4RctNhA+Vihi4M/2nN6Hq6AValtu +pAtRk1kCwbL3kTA+j9pgl3jcCQKBgQDjBIyEg7gF03yh4IjKoAw4MYdObuKL50T3 +A8I2osaMRVga9GvfPLv/mdIHrxzW6hHjNCAV9oKD3oQX68BHbY7biUcUeVIntS5n +durx3rfa3uwSa3voOrT0ayq6gB7loRp7Vgj1DVSbsNdP7i+1Ho5BSonOZ4+CNdk9 +FnelK/Bg7wKBgF7P3o9ekVICG9lSZXvKa3gpIRtha+5WPUMH7wbLbGUFrDfGlcun +O60SU1LDSpF6TueD6KwGnDpVGx2YekMTXtJc5GCBCQWY0/aqIJf7NuLBbsqM+Isx +J0ouM5MqeR1utzEeR/iAqv838efM+WMvO7MpuiHm8PDBDAeTBpftQWRpAoGBANa7 +KMb3aItqPctIfVtcA/LrNlcEH5nBycP0dDIFydRIi0wDwj1IT3M35iiKhLbsdLr6 +RRdBMrbVNREDV764bJZPxUIid4lDoX8QRq89wZ5vmdDahjtlvGAtNh9urMr7lfbv +rE1OrKcPoLnkTI//NQOhI7Ru1JmPocpchBNh+l9VAoGAIgd7+bnO4s8VKFcS0h7L +0nlyHLqq4HGiUsrmoq7W8Sy5OvLlmxSBctnaH32Kjt+z00mzePOCRAE4BPVNSGUu +RXlJxrfBWHh+AgcH8L3PrurqPGUgUA/Dxec+QTv5MRj96D+YDEgzbwxI1VmjZ3MY +AeS2Pdt0nxySKT07Y6TE7FA= +-----END PRIVATE KEY----- diff --git a/test/cpp/quackapi_core_identity.hpp b/test/cpp/quackapi_core_identity.hpp new file mode 100644 index 0000000..63993c6 --- /dev/null +++ b/test/cpp/quackapi_core_identity.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +inline bool MatchesPinnedDuckDBIdentity(const std::string &library_version, const std::string &source_id) { + static const std::string pinned_commit = "069cc9f9b5be802405797faecc284961b07c70ef"; + return library_version == "v1.5.6" && (source_id.size() == 8 || source_id.size() == 10) && + std::equal(source_id.begin(), source_id.end(), pinned_commit.begin()); +} diff --git a/test/cpp/quackapi_post_http_test.cpp b/test/cpp/quackapi_post_http_test.cpp new file mode 100644 index 0000000..40264b7 --- /dev/null +++ b/test/cpp/quackapi_post_http_test.cpp @@ -0,0 +1,680 @@ +// Real HTTP regression checks for row-local POST, side effects, and the +// database-wide concurrency limit. Arguments are explicit artifact paths and +// their expected extension versions; this must never load an installed copy. +#include "duckdb.hpp" +#include "duckdb/common/http_util.hpp" +#include "httplib.hpp" +#include "quackapi_core_identity.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace duckdb; + +static void Check(bool condition, const std::string &message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +static void VerifyCoreIdentityGuard() { + Check(MatchesPinnedDuckDBIdentity("v1.5.6", "069cc9f9"), "core identity guard rejected the 8-character pinned ID"); + Check(MatchesPinnedDuckDBIdentity("v1.5.6", "069cc9f9b5"), + "core identity guard rejected the 10-character pinned ID"); + Check(!MatchesPinnedDuckDBIdentity("v1.5.5", "069cc9f9"), "core identity guard accepted the wrong version"); + Check(!MatchesPinnedDuckDBIdentity("v1.5.6", "069cc9f8"), "core identity guard accepted the wrong source ID"); +} + +static std::unique_ptr Run(Connection &connection, const std::string &sql) { + auto result = connection.Query(sql); + Check(!result->HasError(), sql + ": " + result->GetError()); + return result; +} + +static std::string SqlQuote(const std::string &value) { + Check(value.find('\'') == std::string::npos, "artifact path contains a SQL quote"); + return "'" + value + "'"; +} + +// The test core has no built-in extensions, so load the JSON artifact QuackAPI +// requires explicitly instead of relying on autoload from the user directory. +static std::string LoadQuackapiSql(const std::string &quackapi_path) { + const char *json_path = std::getenv("QUACKAPI_TEST_JSON_EXTENSION"); + Check(json_path && *json_path, "QUACKAPI_TEST_JSON_EXTENSION is not set"); + return "LOAD " + SqlQuote(json_path) + "; LOAD " + SqlQuote(quackapi_path); +} + +static void CheckExtension(Connection &connection, const std::string &name, const std::string &expected_version) { + auto result = Run(connection, "SELECT extension_version, loaded FROM duckdb_extensions() WHERE extension_name = " + + SqlQuote(name)); + Check(result->RowCount() == 1, name + " extension metadata missing"); + Check(result->GetValue(0, 0).ToString() == expected_version, name + " artifact version mismatch"); + Check(result->GetValue(1, 0).GetValue(), name + " extension is not loaded"); +} + +static void CheckExtensionsInitiallyUnloaded(Connection &connection) { + auto loaded = Run(connection, "SELECT count(extension_name) FROM duckdb_extensions() " + "WHERE loaded AND extension_name IN ('quackapi', 'curl_httpfs', 'httpfs')"); + Check(loaded->GetValue(0, 0).GetValue() == 0, + "test core unexpectedly preloaded QuackAPI or an HTTP provider"); +} + +struct ListenerGuard { + duckdb_httplib::Server &server; + std::thread &listener; + ~ListenerGuard() { + server.stop(); + if (listener.joinable()) { + listener.join(); + } + } +}; + +int main(int argc, char **argv) { + try { + Check(argc == 6, "usage: quackapi_post_http_test QUACKAPI_PATH QUACKAPI_VERSION PROVIDER_PATH PROVIDER_NAME " + "PROVIDER_VERSION"); + VerifyCoreIdentityGuard(); + Check(std::string(argv[4]) == "curl_httpfs" || std::string(argv[4]) == "httpfs", + "expected curl_httpfs or httpfs provider"); + std::atomic received {0}; + std::atomic ambiguous_received {0}; + std::atomic redirect_received {0}; + std::atomic large_received {0}; + std::atomic active {0}; + std::atomic peak {0}; + std::mutex peer_mutex; + std::set reuse_peer_ports; + std::string large_json_body = "[{\"payload\":["; + for (int i = 0; i < 10000; i++) { + if (i > 0) { + large_json_body += ','; + } + large_json_body += '0'; + } + large_json_body += "]}]"; + duckdb_httplib::Server server; + server.set_keep_alive_max_count(20); + server.new_task_queue = []() { + return new duckdb_httplib::ThreadPool(48); + }; + server.Post("/echo", [&](const duckdb_httplib::Request &request, duckdb_httplib::Response &response) { + received.fetch_add(1); + if (request.body == "reuse") { + std::lock_guard guard(peer_mutex); + reuse_peer_ports.insert(request.remote_port); + } + response.set_header("X-Echo-Content-Type", request.get_header_value("Content-Type")); + response.set_content(request.body, "text/plain"); + }); + server.Post("/headers", [&](const duckdb_httplib::Request &request, duckdb_httplib::Response &response) { + received.fetch_add(1); + const auto content_types = request.headers.equal_range("Content-Type"); + response.set_content(request.get_header_value("Content-Type") + "|" + request.get_header_value("X-Trace") + + "|" + std::to_string(std::distance(content_types.first, content_types.second)), + "text/plain"); + }); + server.Post("/drop", [&](const duckdb_httplib::Request &, duckdb_httplib::Response &response) { + ambiguous_received.fetch_add(1); + response.set_content_provider(10, "text/plain", + [](size_t, size_t, duckdb_httplib::DataSink &) { return false; }); + }); + server.Post("/redirect", [&](const duckdb_httplib::Request &, duckdb_httplib::Response &response) { + redirect_received.fetch_add(1); + response.set_redirect("/echo", 307); + }); + server.Post("/large-json", [&](const duckdb_httplib::Request &, duckdb_httplib::Response &response) { + large_received.fetch_add(1); + response.set_content(large_json_body, "application/json"); + }); + server.Post("/slow", [&](const duckdb_httplib::Request &request, duckdb_httplib::Response &response) { + received.fetch_add(1); + const auto now = active.fetch_add(1) + 1; + int seen = peak.load(); + while (now > seen && !peak.compare_exchange_weak(seen, now)) { + } + std::this_thread::sleep_for(std::chrono::milliseconds(request.body == "wide" ? 900 : 180)); + active.fetch_sub(1); + response.set_content(request.body, "text/plain"); + }); + const int port = server.bind_to_any_port("127.0.0.1"); + Check(port > 0, "could not bind loopback HTTP fixture"); + std::thread listener([&]() { server.listen_after_bind(); }); + ListenerGuard guard {server, listener}; + server.wait_until_ready(); + duckdb_httplib::Client probe("127.0.0.1", port); + auto probe_response = probe.Post("/echo", "probe", "text/plain"); + Check(probe_response && probe_response->status == 200 && probe_response->body == "probe", + "loopback fixture did not serve its own POST probe"); + received.store(0); + + DBConfig config; + config.SetOptionByName("allow_unsigned_extensions", Value::BOOLEAN(true)); + DuckDB database(nullptr, &config); + Connection connection(database); + auto version = Run(connection, "PRAGMA version"); + const auto actual_library_version = version->GetValue(0, 0).ToString(); + const auto actual_source_id = version->GetValue(1, 0).ToString(); + Check(MatchesPinnedDuckDBIdentity(actual_library_version, actual_source_id), + "test core mismatch: actual library_version='" + actual_library_version + "', source_id='" + + actual_source_id + + "'; expected v1.5.6 and an 8- or 10-character prefix of pinned core " + "069cc9f9b5be802405797faecc284961b07c70ef"); + CheckExtensionsInitiallyUnloaded(connection); + Run(connection, "LOAD " + SqlQuote(argv[3])); + CheckExtension(connection, argv[4], argv[5]); + if (std::string(argv[4]) == "httpfs") { + Run(connection, "SET httpfs_client_implementation = 'curl'"); + } + const auto expected_provider = std::string(argv[4]) == "httpfs" ? "HTTPFS-Curl" : "MultiCurl"; + Check(HTTPUtil::Get(*connection.context->db).GetName() == expected_provider, + "loaded artifact did not select the expected active HTTPUtil"); + const std::string core_settings = "SELECT current_setting('threads')::VARCHAR || '|' || " + "current_setting('memory_limit')::VARCHAR || '|' || " + "current_setting('preserve_insertion_order')::VARCHAR || '|' || " + "current_setting('autoinstall_known_extensions')::VARCHAR || '|' || " + "current_setting('autoload_known_extensions')::VARCHAR"; + const std::string installed_extensions = + "SELECT string_agg(extension_name, ',' ORDER BY extension_name) " + "FROM duckdb_extensions() WHERE installed AND extension_name != 'quackapi'"; + const auto settings_before = Run(connection, core_settings)->GetValue(0, 0).ToString(); + const auto extensions_before = Run(connection, installed_extensions)->GetValue(0, 0).ToString(); + Run(connection, LoadQuackapiSql(argv[1])); + CheckExtension(connection, "quackapi", argv[2]); + Check(Run(connection, core_settings)->GetValue(0, 0).ToString() == settings_before, + "loading QuackAPI changed unrelated DuckDB settings"); + Check(Run(connection, installed_extensions)->GetValue(0, 0).ToString() == extensions_before, + "loading QuackAPI installed another extension"); + const std::string echo_url = "http://127.0.0.1:" + std::to_string(port) + "/echo"; + const std::string headers_url = "http://127.0.0.1:" + std::to_string(port) + "/headers"; + const std::string slow_url = "http://127.0.0.1:" + std::to_string(port) + "/slow"; + auto headers = Run(connection, "SELECT receipt.status, receipt.body FROM " + "(SELECT quackapi_post(" + + SqlQuote(headers_url) + + ", '{}', 'application/vnd.quackapi-test+json', " + "MAP {'X-Trace': 'trace-42'}) AS receipt)"); + Check(headers->GetValue(0, 0).GetValue() == 200 && + headers->GetValue(1, 0).ToString() == "application/vnd.quackapi-test+json|trace-42|1", + "provider changed explicit Content-Type or custom headers"); + auto default_headers = Run(connection, "SELECT receipt.status, receipt.body FROM " + "(SELECT quackapi_post(" + + SqlQuote(headers_url) + ", '{}') AS receipt)"); + Check(default_headers->GetValue(0, 0).GetValue() == 200 && + default_headers->GetValue(1, 0).ToString() == "application/json||1", + "provider duplicated or changed default Content-Type"); + Check(received.load() == 2, "header probes made an unexpected number of requests"); + + // Distinct input rows must each reach the wire and return their own body. + auto rows = + Run(connection, "SELECT i, receipt.status, receipt.body, receipt.error FROM " + "(SELECT i, quackapi_post(" + + SqlQuote(echo_url) + + ", printf('row:%d', i), 'text/plain') AS receipt FROM range(40) t(i)) ORDER BY i"); + Check(rows->RowCount() == 40, "POST collapsed distinct rows"); + for (idx_t i = 0; i < 40; i++) { + Check(rows->GetValue(1, i).GetValue() == 200, + "POST row failed at " + std::to_string(i) + ": " + rows->GetValue(3, i).ToString()); + Check(rows->GetValue(2, i).ToString() == "row:" + std::to_string(i), "POST body crossed rows"); + } + Check(received.load() == 42, "distinct rows made the wrong number of HTTP requests"); + + // A constant volatile call still sends once per row. + auto constant = Run(connection, "SELECT (quackapi_post(" + SqlQuote(echo_url) + + ", 'same', 'text/plain')).status FROM range(16)"); + Check(constant->RowCount() == 16 && received.load() == 58, "constant POST was deduplicated"); + + // One transport failure must not discard successful peers in the same vector. + auto mixed = + Run(connection, "SELECT i, receipt.status, receipt.error FROM " + "(SELECT i, quackapi_post(CASE WHEN i = 1 THEN 'http://127.0.0.1:1/' ELSE " + + SqlQuote(echo_url) + " END, 'mixed') AS receipt FROM range(3) t(i)) ORDER BY i"); + Check(mixed->GetValue(1, 0).GetValue() == 200 && mixed->GetValue(1, 2).GetValue() == 200, + "a failed POST poisoned successful rows"); + Check(mixed->GetValue(1, 1).GetValue() == 0 && !mixed->GetValue(2, 1).IsNull(), + "transport failure lost its row-local error"); + Check(received.load() == 60, "mixed batch retried or lost requests"); + + auto concurrent = [&](int callers_count, const std::string &body) { + std::atomic ready {0}; + std::atomic go {false}; + std::atomic passed {0}; + std::vector callers; + for (int i = 0; i < callers_count; i++) { + callers.emplace_back([&]() { + Connection caller(database); + ready.fetch_add(1); + while (!go.load()) { + std::this_thread::yield(); + } + auto result = caller.Query("SELECT (quackapi_post(" + SqlQuote(slow_url) + ", " + SqlQuote(body) + + ")).status"); + if (!result->HasError() && result->GetValue(0, 0).GetValue() == 200) { + passed.fetch_add(1); + } + }); + } + while (ready.load() != callers_count) { + std::this_thread::yield(); + } + go.store(true); + for (auto &caller : callers) { + caller.join(); + } + return passed.load(); + }; + + Run(connection, "SET quackapi_post_concurrency = 4"); + Check(concurrent(10, "x") == 10, "concurrent POST lost a request"); + Check(peak.load() > 1 && peak.load() <= 4, "POST ignored database-wide concurrency limit"); + Check(received.load() == 70, "concurrent POST retried or lost requests"); + + // A positive bound above 32 must be effective, not silently clamped. + peak.store(0); + Run(connection, "SET quackapi_post_concurrency = 40"); + Check(concurrent(36, "wide") == 36, "wide POST batch lost a request"); + Check(peak.load() > 32 && peak.load() <= 40, "POST concurrency was capped at 32"); + Check(received.load() == 106, "wide POST batch retried or lost requests"); + DBConfig other_config; + other_config.SetOptionByName("allow_unsigned_extensions", Value::BOOLEAN(true)); + DuckDB other_database(nullptr, &other_config); + Connection other_setup(other_database); + Run(other_setup, "LOAD " + SqlQuote(argv[3])); + Run(other_setup, LoadQuackapiSql(argv[1])); + CheckExtension(other_setup, argv[4], argv[5]); + CheckExtension(other_setup, "quackapi", argv[2]); + if (std::string(argv[4]) == "httpfs") { + Run(other_setup, "SET httpfs_client_implementation = 'curl'"); + } + Run(connection, "SET quackapi_post_concurrency = 1"); + Run(other_setup, "SET quackapi_post_concurrency = 1"); + peak.store(0); + std::atomic independent_ready {0}; + std::atomic independent_go {false}; + std::atomic independent_passed {0}; + auto call_database = [&](DuckDB &target) { + Connection caller(target); + independent_ready.fetch_add(1); + while (!independent_go.load()) { + std::this_thread::yield(); + } + auto result = caller.Query("SELECT (quackapi_post(" + SqlQuote(slow_url) + ", 'wide')).status"); + if (!result->HasError() && result->GetValue(0, 0).GetValue() == 200) { + independent_passed.fetch_add(1); + } + }; + std::thread first_database_call([&]() { call_database(database); }); + std::thread second_database_call([&]() { call_database(other_database); }); + while (independent_ready.load() != 2) { + std::this_thread::yield(); + } + independent_go.store(true); + first_database_call.join(); + second_database_call.join(); + Check(independent_passed.load() == 2 && peak.load() == 2, + "independent databases shared one POST concurrency limit"); + Check(received.load() == 108, "independent databases lost or retried a request"); + + // Interrupt a request waiting behind the sole active worker. It must + // leave the queue promptly and never reach the HTTP fixture. + Connection first(database); + Connection queued(database); + std::atomic first_ok {false}; + std::atomic queued_failed {false}; + std::thread first_call([&]() { + auto result = first.Query("SELECT (quackapi_post(" + SqlQuote(slow_url) + ", 'wide')).status"); + first_ok.store(!result->HasError() && result->GetValue(0, 0).GetValue() == 200); + }); + while (active.load() == 0) { + std::this_thread::yield(); + } + const auto queued_at = std::chrono::steady_clock::now(); + std::thread queued_call([&]() { + auto result = queued.Query("SELECT (quackapi_post(" + SqlQuote(slow_url) + ", 'queued')).status"); + queued_failed.store(result->HasError()); + }); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + queued.Interrupt(); + queued_call.join(); + const auto queued_elapsed = std::chrono::steady_clock::now() - queued_at; + first_call.join(); + Check(first_ok.load(), "active POST did not finish after peer cancellation"); + Check(queued_failed.load() && queued_elapsed < std::chrono::milliseconds(600), + "queued POST did not cancel promptly"); + Check(received.load() == 109, "cancelled queued POST reached the server"); + Run(connection, "CREATE ROUTE post_wait GET '/post-wait' AS SELECT " + "(quackapi_post(" + + SqlQuote(slow_url) + ", 'queued')).status AS upstream_status"); + Connection occupied(database); + std::thread occupied_call([&]() { + auto result = occupied.Query("SELECT (quackapi_post(" + SqlQuote(slow_url) + ", 'wide')).status"); + first_ok.store(!result->HasError() && result->GetValue(0, 0).GetValue() == 200); + }); + while (active.load() == 0) { + std::this_thread::yield(); + } + const auto waiting_at = std::chrono::steady_clock::now(); + auto waiting = Run(connection, "SELECT status FROM quackapi_request('GET', '/post-wait', " + "query_timeout_ms := 100)"); + const auto waiting_elapsed = std::chrono::steady_clock::now() - waiting_at; + occupied_call.join(); + Check(first_ok.load() && waiting->GetValue(0, 0).GetValue() == 504 && + waiting_elapsed < std::chrono::milliseconds(600), + "queued POST ignored its route deadline"); + Check(received.load() == 110, "expired queued POST reached the server"); + + // The inbound route deadline must bound its outbound POST, even after + // the provider has started receiving a slow response. + Run(connection, "CREATE ROUTE post_deadline GET '/post-deadline' AS SELECT " + "(quackapi_post(" + + SqlQuote(slow_url) + ", 'wide')).status AS upstream_status"); + const auto deadline_at = std::chrono::steady_clock::now(); + auto timed = Run(connection, "SELECT status, decode(body) FROM quackapi_request('GET', '/post-deadline', " + "query_timeout_ms := 100)"); + const auto deadline_elapsed = std::chrono::steady_clock::now() - deadline_at; + const auto deadline_ms = std::chrono::duration_cast(deadline_elapsed).count(); + Check(timed->GetValue(0, 0).GetValue() == 504 && deadline_elapsed < std::chrono::milliseconds(600), + "in-flight POST outlived its inbound route deadline: status=" + timed->GetValue(0, 0).ToString() + + " elapsed_ms=" + std::to_string(deadline_ms) + " body=" + timed->GetValue(1, 0).ToString()); + while (active.load() != 0) { + std::this_thread::yield(); + } + Connection interrupted(database); + std::atomic active_failed {false}; + std::thread active_call([&]() { + auto result = interrupted.Query("SELECT (quackapi_post(" + SqlQuote(slow_url) + ", 'wide')).status"); + active_failed.store(result->HasError()); + }); + while (active.load() == 0) { + std::this_thread::yield(); + } + const auto active_at = std::chrono::steady_clock::now(); + interrupted.Interrupt(); + active_call.join(); + const auto active_elapsed = std::chrono::steady_clock::now() - active_at; + const auto active_ms = std::chrono::duration_cast(active_elapsed).count(); + Check(active_failed.load() && active_elapsed < std::chrono::milliseconds(600), + "active POST ignored caller interrupt: elapsed_ms=" + std::to_string(active_ms)); + + // Real HTTP access logs must use DuckDB's native logger and redact + // explicitly allowed credential headers and sensitive query values. + Run(connection, "SET enable_logging = true"); + Run(connection, "SET logging_storage = 'memory'"); + Run(connection, "SET logging_level = 'debug'"); + duckdb_httplib::Server reserve; + const int log_port = reserve.bind_to_any_port("127.0.0.1"); + Check(log_port > 0, "could not reserve access-log fixture port"); + reserve.stop(); + Run(connection, "CREATE ROUTE log_secret GET '/log-secret' AS SELECT 1 AS ok"); + Run(connection, "SELECT * FROM quackapi_serve(" + std::to_string(log_port) + + ", host := '127.0.0.1', access_log := true, log_level := 'debug', " + "log_headers := 'Authorization,X-Trace', log_query := true)"); + duckdb_httplib::Client inbound("127.0.0.1", log_port); + auto logged_response = inbound.Get("/log-secret?api_key=secret42&normal=yes", + {{"Authorization", "Bearer secret42"}, {"X-Trace", "trace-42"}}); + Check(logged_response && logged_response->status == 200, "real HTTP access-log request failed"); + auto logs = Run(connection, "SELECT message FROM duckdb_logs() WHERE contains(message, '\"type\":\"access\"') " + "ORDER BY timestamp DESC"); + Check(logs->RowCount() > 0, "real HTTP request did not write a native DuckDB access log"); + const auto access_log = logs->GetValue(0, 0).ToString(); + Check(access_log.find("secret42") == std::string::npos && access_log.find("Bearer") == std::string::npos && + access_log.find("[REDACTED]") != std::string::npos && + access_log.find("trace-42") != std::string::npos && + access_log.find("normal=yes") != std::string::npos, + "native access log did not redact credentials while preserving allowed metadata"); + // The interrupted 'wide' POST above still drains in this database's only + // POST worker (concurrency 1). Let it finish so the loopback's own POST + // does not queue behind it and spend its budget waiting. Note `active` + // is the fixture-side counter: it hits 0 when the fixture finishes + // sleeping, which races the worker actually releasing its slot, so the + // loopback can still briefly queue behind the drain on a loaded runner. + // The 10 s budget absorbs that; the tight 100 ms deadline checks above + // still cover real deadline behavior. + while (active.load() != 0) { + std::this_thread::yield(); + } + Run(connection, "CREATE ROUTE loopback_inner POST '/loopback-inner' AS SELECT 9 AS value"); + Run(connection, "CREATE ROUTE loopback_outer GET '/loopback-outer' AS SELECT " + "(quackapi_post('http://127.0.0.1:" + + std::to_string(log_port) + "/loopback-inner', '{}')).status AS upstream_status"); + auto loopback = Run(connection, "SELECT status, decode(body) FROM quackapi_request('GET', '/loopback-outer', " + "query_timeout_ms := 10000)"); + Check(loopback->GetValue(0, 0).GetValue() == 200 && + loopback->GetValue(1, 0).ToString() == "[{\"upstream_status\":200}]", + "same-database HTTP loopback stalled or returned the wrong row: status=" + + loopback->GetValue(0, 0).ToString() + " body=" + loopback->GetValue(1, 0).ToString()); + Run(connection, "SELECT * FROM quackapi_stop(" + std::to_string(log_port) + ")"); + + // The SQL caller has already been interrupted. Database shutdown must + // finish after the bounded provider response, even though it safely + // drains its DB-owned worker rather than detaching it. + auto closing_database = make_uniq(nullptr, &config); + auto closing_connection = make_uniq(*closing_database); + Run(*closing_connection, "LOAD " + SqlQuote(argv[3])); + Run(*closing_connection, LoadQuackapiSql(argv[1])); + if (std::string(argv[4]) == "httpfs") { + Run(*closing_connection, "SET httpfs_client_implementation = 'curl'"); + } + std::atomic closing_failed {false}; + std::thread closing_call([&]() { + auto result = + closing_connection->Query("SELECT (quackapi_post(" + SqlQuote(slow_url) + ", 'wide')).status"); + closing_failed.store(result->HasError()); + }); + while (active.load() == 0) { + std::this_thread::yield(); + } + closing_connection->Interrupt(); + closing_call.join(); + Check(closing_failed.load(), "shutdown probe did not interrupt its caller"); + closing_connection.reset(); + const auto closing_at = std::chrono::steady_clock::now(); + closing_database.reset(); + const auto closing_elapsed = std::chrono::steady_clock::now() - closing_at; + Check(closing_elapsed < std::chrono::milliseconds(1500), + "database shutdown outlived a bounded interrupted provider request: elapsed_ms=" + + std::to_string(std::chrono::duration_cast(closing_elapsed).count())); + auto ambiguous = Run(connection, "SELECT receipt.status, receipt.error FROM " + "(SELECT quackapi_post(" + + SqlQuote("http://127.0.0.1:" + std::to_string(port) + "/drop") + + ", 'unsafe') AS receipt)"); + Check(ambiguous_received.load() == 1, "ambiguous POST transport failure was automatically replayed"); + Check(!ambiguous->GetValue(1, 0).IsNull(), "ambiguous POST failure lost its row-local error"); + Run(connection, "SET threads = 1"); + Run(connection, "SET quackapi_post_concurrency = 0"); + peak.store(0); + Check(concurrent(3, "x") == 3 && peak.load() == 1, + "automatic POST concurrency did not follow the database's one thread"); + DBConfig reuse_config; + reuse_config.SetOptionByName("allow_unsigned_extensions", Value::BOOLEAN(true)); + DuckDB reuse_database(nullptr, &reuse_config); + Connection reuse_connection(reuse_database); + Run(reuse_connection, "LOAD " + SqlQuote(argv[3])); + Run(reuse_connection, LoadQuackapiSql(argv[1])); + CheckExtension(reuse_connection, argv[4], argv[5]); + CheckExtension(reuse_connection, "quackapi", argv[2]); + if (std::string(argv[4]) == "httpfs") { + Run(reuse_connection, "SET httpfs_client_implementation = 'curl'"); + } + Check(HTTPUtil::Get(*reuse_connection.context->db).GetName() == expected_provider, + "reuse probe selected the wrong active HTTPUtil"); + Run(reuse_connection, "SET threads = 1"); + Run(reuse_connection, "SET quackapi_post_concurrency = 0"); + for (int i = 0; i < 10; i++) { + auto reused = Run(reuse_connection, "SELECT (quackapi_post(" + SqlQuote(echo_url) + ", 'reuse')).status"); + Check(reused->GetValue(0, 0).GetValue() == 200, "sequential keep-alive POST failed"); + } + { + std::lock_guard guard(peer_mutex); + Check(reuse_peer_ports.size() == 1, + "10 sequential POSTs used " + std::to_string(reuse_peer_ports.size()) + " peer ports"); + } + + Check(received.load() == 126, "bounded HTTP fixture lost or retried a request"); + auto redirect = Run(connection, "SELECT receipt.status, receipt.body FROM " + "(SELECT quackapi_post(" + + SqlQuote("http://127.0.0.1:" + std::to_string(port) + "/redirect") + + ", 'redirect') AS receipt)"); + Check(redirect->GetValue(0, 0).GetValue() == 200 && redirect->GetValue(1, 0).ToString() == "redirect" && + redirect_received.load() == 1 && received.load() == 127, + "HTTP 307 POST redirect did not reach its target exactly once"); + + // The table overload must not reintroduce the old 32-request cap. + Run(connection, "SET threads = 4"); + Run(connection, "SET quackapi_post_concurrency = 40"); + peak.store(0); + auto table_wide = + Run(connection, "SELECT i, response.status, response.body FROM quackapi_post((SELECT i, " + + SqlQuote(slow_url) + " AS url, 'wide' AS body FROM range(40) t(i))) ORDER BY i"); + Check(table_wide->RowCount() == 40, "table POST dropped an input row"); + for (idx_t i = 0; i < 40; i++) { + Check(table_wide->GetValue(0, i).GetValue() == static_cast(i) && + table_wide->GetValue(1, i).GetValue() == 200 && + table_wide->GetValue(2, i).ToString() == "wide", + "table POST crossed or lost rows above concurrency 32"); + } + Check(peak.load() > 32 && peak.load() <= 40 && received.load() == 167, + "table POST ignored its configured concurrency above 32"); + + // Separate table callers share the database's limit, while a second + // DuckDB instance has its own limit even with the same HTTP provider. + Run(connection, "SET quackapi_post_concurrency = 2"); + peak.store(0); + std::atomic table_ready {0}; + std::atomic table_go {false}; + std::atomic table_passed {0}; + auto table_caller = [&]() { + Connection caller(database); + table_ready.fetch_add(1); + while (!table_go.load()) { + std::this_thread::yield(); + } + auto result = caller.Query("SELECT count(i) FROM quackapi_post((SELECT i, " + SqlQuote(slow_url) + + " AS url, 'wide' AS body FROM range(3) t(i))) WHERE response.status = 200"); + if (!result->HasError() && result->GetValue(0, 0).GetValue() == 3) { + table_passed.fetch_add(1); + } + }; + std::thread table_first(table_caller); + std::thread table_second(table_caller); + while (table_ready.load() != 2) { + std::this_thread::yield(); + } + table_go.store(true); + table_first.join(); + table_second.join(); + Check(table_passed.load() == 2 && peak.load() > 1 && peak.load() <= 2 && received.load() == 173, + "table POST callers did not share the database-wide limit"); + + Run(connection, "SET quackapi_post_concurrency = 1"); + Run(other_setup, "SET quackapi_post_concurrency = 1"); + peak.store(0); + std::atomic table_independent_ready {0}; + std::atomic table_independent_go {false}; + std::atomic table_independent_passed {0}; + auto table_database_call = [&](DuckDB &target) { + Connection caller(target); + table_independent_ready.fetch_add(1); + while (!table_independent_go.load()) { + std::this_thread::yield(); + } + auto result = caller.Query("SELECT response.status FROM quackapi_post((SELECT " + SqlQuote(slow_url) + + " AS url, 'wide' AS body))"); + if (!result->HasError() && result->GetValue(0, 0).GetValue() == 200) { + table_independent_passed.fetch_add(1); + } + }; + std::thread table_db_first([&]() { table_database_call(database); }); + std::thread table_db_second([&]() { table_database_call(other_database); }); + while (table_independent_ready.load() != 2) { + std::this_thread::yield(); + } + table_independent_go.store(true); + table_db_first.join(); + table_db_second.join(); + Check(table_independent_passed.load() == 2 && peak.load() == 2 && received.load() == 175, + "independent databases shared the table POST concurrency limit"); + + // Interrupt an in-flight table batch before it can admit the next row. + Connection table_interrupted(database); + std::atomic table_cancelled {false}; + std::thread table_call([&]() { + auto result = table_interrupted.Query("SELECT count(i) FROM quackapi_post((SELECT i, " + + SqlQuote(slow_url) + " AS url, 'wide' AS body FROM range(33) t(i)))"); + table_cancelled.store(result->HasError()); + }); + const auto table_started_by = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (active.load() == 0 && std::chrono::steady_clock::now() < table_started_by) { + std::this_thread::yield(); + } + if (active.load() == 0) { + table_interrupted.Interrupt(); + table_call.join(); + Check(false, "table POST never reached the slow HTTP fixture"); + } + const auto table_cancel_at = std::chrono::steady_clock::now(); + table_interrupted.Interrupt(); + table_call.join(); + const auto table_cancel_elapsed = std::chrono::steady_clock::now() - table_cancel_at; + const auto table_drained_by = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (active.load() != 0 && std::chrono::steady_clock::now() < table_drained_by) { + std::this_thread::yield(); + } + Check(active.load() == 0 && table_cancelled.load() && table_cancel_elapsed < std::chrono::milliseconds(600) && + received.load() == 176, + "interrupted table POST did not cancel promptly or admitted another row"); + auto table_headers = Run(connection, "SELECT response.status, response.body FROM quackapi_post((SELECT " + + SqlQuote(headers_url) + + " AS url, '{}'::JSON AS body, " + "'application/vnd.quackapi-test+json' AS content_type, " + "MAP {'X-Trace': 'table-42'} AS headers))"); + Check(table_headers->GetValue(0, 0).GetValue() == 200 && + table_headers->GetValue(1, 0).ToString() == "application/vnd.quackapi-test+json|table-42|1", + "table POST changed typed body, Content-Type or custom headers"); + auto table_default_headers = Run(connection, "SELECT response.status, response.body FROM " + "quackapi_post((SELECT " + + SqlQuote(headers_url) + " AS url, '{}' AS body))"); + Check(table_default_headers->GetValue(0, 0).GetValue() == 200 && + table_default_headers->GetValue(1, 0).ToString() == "application/json||1" && received.load() == 178, + "table POST changed default Content-Type or request count"); + + // A full scalar response vector exhausts this limit. The table operator + // must let downstream JSON parsing consume a bounded batch at a time. + DBConfig memory_config; + memory_config.SetOptionByName("allow_unsigned_extensions", Value::BOOLEAN(true)); + DuckDB memory_database(nullptr, &memory_config); + Connection memory_connection(memory_database); + Run(memory_connection, "LOAD " + SqlQuote(argv[3])); + Run(memory_connection, LoadQuackapiSql(argv[1])); + CheckExtension(memory_connection, argv[4], argv[5]); + CheckExtension(memory_connection, "quackapi", argv[2]); + if (std::string(argv[4]) == "httpfs") { + Run(memory_connection, "SET httpfs_client_implementation = 'curl'"); + } + Run(memory_connection, "LOAD json"); + Run(memory_connection, "SET threads = 4"); + Run(memory_connection, "SET memory_limit = '128MB'"); + Run(memory_connection, "SET quackapi_post_concurrency = 4"); + auto large_rows = + Run(memory_connection, "SELECT count(i), sum(json_array_length(response.body::JSON, '$[0].payload')) " + "FROM quackapi_post((SELECT i, " + + SqlQuote("http://127.0.0.1:" + std::to_string(port) + "/large-json") + + " AS url, '{}' AS body FROM range(256) t(i)))"); + Check(large_rows->GetValue(0, 0).GetValue() == 256 && + large_rows->GetValue(1, 0).GetValue() == 2560000 && large_received.load() == 256, + "table POST did not return all 256 large JSON bodies for downstream parsing under 128 MB"); + std::cout << "POST HTTP integration passed: 178 requests including table batches\n"; + return 0; + } catch (const std::exception &error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/test/cpp/quackapi_post_tls_test.cpp b/test/cpp/quackapi_post_tls_test.cpp new file mode 100644 index 0000000..0378d2a --- /dev/null +++ b/test/cpp/quackapi_post_tls_test.cpp @@ -0,0 +1,159 @@ +// HTTPS counterpart to the plain-HTTP provider contract. The checked-in key +// belongs only to this disposable loopback fixture. +#include "duckdb.hpp" +#include "httplib.hpp" +#include "quackapi_core_identity.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace duckdb; + +static void Check(bool condition, const std::string &message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +static std::unique_ptr Run(Connection &connection, const std::string &sql) { + auto result = connection.Query(sql); + Check(!result->HasError(), sql + ": " + result->GetError()); + return result; +} + +static std::string Quote(const std::string &value) { + Check(value.find('\'') == std::string::npos, "artifact path contains SQL quote"); + return "'" + value + "'"; +} + +// The test core has no built-in extensions, so load the JSON artifact QuackAPI +// requires explicitly instead of relying on autoload from the user directory. +static std::string LoadQuackapiSql(const std::string &quackapi_path) { + const char *json_path = std::getenv("QUACKAPI_TEST_JSON_EXTENSION"); + Check(json_path && *json_path, "QUACKAPI_TEST_JSON_EXTENSION is not set"); + return "LOAD " + Quote(json_path) + "; LOAD " + Quote(quackapi_path); +} + +static void CheckExtensionsInitiallyUnloaded(Connection &connection) { + auto loaded = Run(connection, "SELECT count(extension_name) FROM duckdb_extensions() " + "WHERE loaded AND extension_name IN ('quackapi', 'curl_httpfs', 'httpfs')"); + Check(loaded->GetValue(0, 0).GetValue() == 0, + "test core unexpectedly preloaded QuackAPI or an HTTP provider"); +} + +int main(int argc, char **argv) { + try { + Check(argc == 8, "usage: post_tls_test QUACKAPI_PATH QUACKAPI_VERSION PROVIDER_PATH PROVIDER_NAME " + "PROVIDER_VERSION TEST_CERT TEST_KEY"); + std::atomic received {0}; + std::atomic redirects {0}; + duckdb_httplib_openssl::SSLServer server(argv[6], argv[7]); + Check(server.is_valid(), "test TLS certificate or key is invalid"); + server.Post("/headers", [&](const duckdb_httplib_openssl::Request &request, + duckdb_httplib_openssl::Response &response) { + received.fetch_add(1); + const auto content_types = request.headers.equal_range("Content-Type"); + response.set_content(request.get_header_value("Content-Type") + "|" + request.get_header_value("X-Trace") + + "|" + std::to_string(std::distance(content_types.first, content_types.second)), + "text/plain"); + }); + server.Post("/slow", + [&](const duckdb_httplib_openssl::Request &request, duckdb_httplib_openssl::Response &response) { + received.fetch_add(1); + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); + response.set_content(request.body, "text/plain"); + }); + server.Post("/redirect", + [&](const duckdb_httplib_openssl::Request &, duckdb_httplib_openssl::Response &response) { + redirects.fetch_add(1); + response.set_redirect("/headers", 307); + }); + const int port = server.bind_to_any_port("127.0.0.1"); + Check(port > 0, "could not bind loopback TLS fixture"); + std::thread listener([&]() { server.listen_after_bind(); }); + struct ListenerGuard { + duckdb_httplib_openssl::SSLServer &server; + std::thread &listener; + ~ListenerGuard() { + server.stop(); + if (listener.joinable()) { + listener.join(); + } + } + } guard {server, listener}; + server.wait_until_ready(); + + DBConfig config; + config.SetOptionByName("allow_unsigned_extensions", Value::BOOLEAN(true)); + DuckDB database(nullptr, &config); + Connection connection(database); + auto version = Run(connection, "PRAGMA version"); + const auto actual_library_version = version->GetValue(0, 0).ToString(); + const auto actual_source_id = version->GetValue(1, 0).ToString(); + Check(MatchesPinnedDuckDBIdentity(actual_library_version, actual_source_id), + "test core mismatch: actual library_version='" + actual_library_version + "', source_id='" + + actual_source_id + + "'; expected v1.5.6 and an 8- or 10-character prefix of pinned core " + "069cc9f9b5be802405797faecc284961b07c70ef"); + CheckExtensionsInitiallyUnloaded(connection); + Run(connection, "LOAD " + Quote(argv[3])); + Run(connection, LoadQuackapiSql(argv[1])); + auto loaded = Run(connection, "SELECT extension_name, extension_version FROM duckdb_extensions() " + "WHERE extension_name IN ('quackapi', " + + Quote(argv[4]) + ") AND loaded ORDER BY extension_name"); + Check(loaded->RowCount() == 2, "expected provider and QuackAPI artifacts were not both loaded"); + for (idx_t row = 0; row < 2; row++) { + const auto name = loaded->GetValue(0, row).ToString(); + const auto expected = name == "quackapi" ? argv[2] : argv[5]; + Check(loaded->GetValue(1, row).ToString() == expected, name + " artifact version mismatch"); + } + if (std::string(argv[4]) == "httpfs") { + Run(connection, "SET httpfs_client_implementation = 'curl'"); + } + Run(connection, "SET GLOBAL ca_cert_file = " + Quote(argv[6])); + const std::string url = "https://127.0.0.1:" + std::to_string(port); + auto custom = Run(connection, "SELECT receipt.status, receipt.body, receipt.error FROM " + "(SELECT quackapi_post(" + + Quote(url + "/headers") + + ", '{}', 'application/vnd.quackapi-test+json', " + "MAP {'X-Trace': 'tls-42'}) AS receipt)"); + Check(custom->GetValue(0, 0).GetValue() == 200 && + custom->GetValue(1, 0).ToString() == "application/vnd.quackapi-test+json|tls-42|1", + "HTTPS custom Content-Type/header contract failed: " + custom->GetValue(2, 0).ToString()); + auto standard = Run(connection, "SELECT receipt.status, receipt.body FROM " + "(SELECT quackapi_post(" + + Quote(url + "/headers") + ", '{}') AS receipt)"); + Check(standard->GetValue(0, 0).GetValue() == 200 && + standard->GetValue(1, 0).ToString() == "application/json||1", + "HTTPS default Content-Type contract failed"); + Check(received.load() == 2, "HTTPS POSTs were retried or lost"); + Run(connection, "CREATE ROUTE tls_deadline GET '/tls-deadline' AS SELECT " + "(quackapi_post(" + + Quote(url + "/slow") + ", 'slow')).status AS upstream_status"); + const auto started = std::chrono::steady_clock::now(); + auto timed = + Run(connection, "SELECT status FROM quackapi_request('GET', '/tls-deadline', query_timeout_ms := 750)"); + const auto elapsed = std::chrono::steady_clock::now() - started; + Check(timed->GetValue(0, 0).GetValue() == 504 && elapsed < std::chrono::milliseconds(1400), + "HTTPS POST exceeded its 750ms route deadline"); + Check(received.load() == 3, "HTTPS timed POST was replayed"); + auto redirect = Run(connection, "SELECT receipt.status, receipt.body FROM " + "(SELECT quackapi_post(" + + Quote(url + "/redirect") + ", '{}') AS receipt)"); + Check(redirect->GetValue(0, 0).GetValue() == 200 && + redirect->GetValue(1, 0).ToString() == "application/json||1" && redirects.load() == 1 && + received.load() == 4, + "HTTPS 307 POST redirect did not reach its target exactly once"); + std::cout << "HTTPS provider contract passed: 4 requests\n"; + return 0; + } catch (const std::exception &error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/test/cpp/quackapi_provider_replacement_test.cpp b/test/cpp/quackapi_provider_replacement_test.cpp new file mode 100644 index 0000000..9368006 --- /dev/null +++ b/test/cpp/quackapi_provider_replacement_test.cpp @@ -0,0 +1,164 @@ +// A live POST retains its original HTTPUtil while another provider replaces +// the database's active implementation. Later POSTs must use a fresh client. +#include "duckdb.hpp" +#include "duckdb/common/http_util.hpp" +#include "httplib.hpp" +#include "quackapi_core_identity.hpp" + +#include +#include +#include +#include +#include +#include + +using namespace duckdb; + +static void Check(bool condition, const std::string &message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +static void VerifyCoreIdentityGuard() { + Check(MatchesPinnedDuckDBIdentity("v1.5.6", "069cc9f9"), "core identity guard rejected the 8-character pinned ID"); + Check(MatchesPinnedDuckDBIdentity("v1.5.6", "069cc9f9b5"), + "core identity guard rejected the 10-character pinned ID"); + Check(!MatchesPinnedDuckDBIdentity("v1.5.5", "069cc9f9"), "core identity guard accepted the wrong version"); + Check(!MatchesPinnedDuckDBIdentity("v1.5.6", "069cc9f8"), "core identity guard accepted the wrong source ID"); +} + +static std::unique_ptr Run(Connection &connection, const std::string &sql) { + auto result = connection.Query(sql); + Check(!result->HasError(), sql + ": " + result->GetError()); + return result; +} + +static std::string Quote(const std::string &value) { + Check(value.find('\'') == std::string::npos, "artifact path contains SQL quote"); + return "'" + value + "'"; +} + +static void CheckExtensionsInitiallyUnloaded(Connection &connection) { + auto loaded = Run(connection, "SELECT count(extension_name) FROM duckdb_extensions() " + "WHERE loaded AND extension_name IN ('quackapi', 'curl_httpfs', 'httpfs')"); + Check(loaded->GetValue(0, 0).GetValue() == 0, + "test core unexpectedly preloaded QuackAPI or an HTTP provider"); +} + +int main(int argc, char **argv) { + try { + Check(argc == 7, "usage: provider_replacement QUACKAPI_PATH QUACKAPI_VERSION " + "CURL_HTTPFS_PATH CURL_HTTPFS_VERSION HTTPFS_PATH HTTPFS_VERSION"); + VerifyCoreIdentityGuard(); + std::atomic received {0}; + std::atomic slow_active {false}; + duckdb_httplib::Server server; + server.Post("/echo", [&](const duckdb_httplib::Request &request, duckdb_httplib::Response &response) { + received.fetch_add(1); + response.set_content(request.body, "text/plain"); + }); + server.Post("/slow", [&](const duckdb_httplib::Request &request, duckdb_httplib::Response &response) { + received.fetch_add(1); + slow_active.store(true); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + response.set_content(request.body, "text/plain"); + }); + const int port = server.bind_to_any_port("127.0.0.1"); + Check(port > 0, "could not bind replacement fixture"); + std::thread listener([&]() { server.listen_after_bind(); }); + struct ListenerGuard { + duckdb_httplib::Server &server; + std::thread &listener; + ~ListenerGuard() { + server.stop(); + if (listener.joinable()) { + listener.join(); + } + } + } guard {server, listener}; + server.wait_until_ready(); + + DBConfig config; + config.SetOptionByName("allow_unsigned_extensions", Value::BOOLEAN(true)); + DuckDB database(nullptr, &config); + Connection connection(database); + auto version = Run(connection, "PRAGMA version"); + const auto actual_library_version = version->GetValue(0, 0).ToString(); + const auto actual_source_id = version->GetValue(1, 0).ToString(); + Check(MatchesPinnedDuckDBIdentity(actual_library_version, actual_source_id), + "provider replacement core mismatch: actual library_version='" + actual_library_version + + "', source_id='" + actual_source_id + + "'; expected v1.5.6 and an 8- or 10-character prefix of " + "pinned core 069cc9f9b5be802405797faecc284961b07c70ef"); + CheckExtensionsInitiallyUnloaded(connection); + Run(connection, "LOAD " + Quote(argv[5])); + Run(connection, "SET httpfs_client_implementation = 'curl'"); + Run(connection, "LOAD " + Quote(argv[1])); + auto httpfs_version = Run(connection, "SELECT extension_version FROM duckdb_extensions() " + "WHERE extension_name = 'httpfs' AND loaded"); + Check(httpfs_version->RowCount() == 1 && httpfs_version->GetValue(0, 0).ToString() == argv[6], + "wrong HTTPFS-Curl artifact"); + auto quackapi_version = Run(connection, "SELECT extension_version FROM duckdb_extensions() " + "WHERE extension_name = 'quackapi' AND loaded"); + Check(quackapi_version->RowCount() == 1 && quackapi_version->GetValue(0, 0).ToString() == argv[2], + "wrong QuackAPI artifact"); + Check(HTTPUtil::Get(*connection.context->db).GetName() == "HTTPFS-Curl", "HTTPFS-Curl did not become active"); + Run(connection, "SET quackapi_post_concurrency = 1"); + const std::string root = "http://127.0.0.1:" + std::to_string(port); + auto first = Run(connection, "SELECT receipt.status, receipt.reused_connection FROM " + "(SELECT quackapi_post(" + + Quote(root + "/echo") + ", 'first') AS receipt)"); + Check(first->GetValue(0, 0).GetValue() == 200 && !first->GetValue(1, 0).GetValue(), + "initial HTTPFS-Curl POST did not create its own client"); + auto reused = Run(connection, "SELECT receipt.status, receipt.reused_connection FROM " + "(SELECT quackapi_post(" + + Quote(root + "/echo") + ", 'second') AS receipt)"); + Check(reused->GetValue(0, 0).GetValue() == 200 && reused->GetValue(1, 0).GetValue(), + "HTTPFS-Curl worker did not retain its client"); + + Connection in_flight(database); + std::atomic slow_ok {false}; + std::thread slow_call([&]() { + auto result = in_flight.Query("SELECT (quackapi_post(" + Quote(root + "/slow") + ", 'slow')).status"); + slow_ok.store(!result->HasError() && result->GetValue(0, 0).GetValue() == 200); + }); + struct ThreadJoin { + std::thread &thread; + ~ThreadJoin() { + if (thread.joinable()) { + thread.join(); + } + } + } slow_join {slow_call}; + while (!slow_active.load()) { + std::this_thread::yield(); + } + Connection loader(database); + Run(loader, "LOAD " + Quote(argv[3])); + auto curl_version = Run(loader, "SELECT extension_version FROM duckdb_extensions() " + "WHERE extension_name = 'curl_httpfs' AND loaded"); + Check(curl_version->RowCount() == 1 && curl_version->GetValue(0, 0).ToString() == argv[4], + "wrong curl_httpfs artifact"); + Check(HTTPUtil::Get(*connection.context->db).GetName() == "MultiCurl", + "loading MultiCurl did not replace HTTPFS-Curl"); + slow_call.join(); + Check(slow_ok.load(), "active HTTPFS-Curl POST failed during provider replacement"); + auto after = Run(connection, "SELECT receipt.status, receipt.reused_connection FROM " + "(SELECT quackapi_post(" + + Quote(root + "/echo") + ", 'after') AS receipt)"); + Check(after->GetValue(0, 0).GetValue() == 200 && !after->GetValue(1, 0).GetValue(), + "MultiCurl reused a HTTPFS-Curl client after provider replacement"); + auto after_reuse = Run(connection, "SELECT receipt.status, receipt.reused_connection FROM " + "(SELECT quackapi_post(" + + Quote(root + "/echo") + ", 'after-reuse') AS receipt)"); + Check(after_reuse->GetValue(0, 0).GetValue() == 200 && after_reuse->GetValue(1, 0).GetValue(), + "MultiCurl did not retain its replacement client"); + Check(received.load() == 5, "provider replacement lost or retried a POST"); + std::cout << "provider replacement passed: 5 requests\n"; + return 0; + } catch (const std::exception &error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/test/cpp/quackapi_thread_join_test.cpp b/test/cpp/quackapi_thread_join_test.cpp deleted file mode 100644 index d508257..0000000 --- a/test/cpp/quackapi_thread_join_test.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// Standalone regression: compile with -std=c++17 -pthread -Isrc/include. -// Simulate a failure after one worker starts without exhausting host threads. -#include "quackapi_thread_join.hpp" - -#include -#include -#include -#include -#include - -int main() { - std::atomic completed {0}; - try { - std::vector threads; - duckdb::JoinOutboundThreads> join_threads {threads}; - threads.emplace_back([&completed]() { completed++; }); - throw std::runtime_error("simulated later thread-start failure"); - } catch (const std::runtime_error &) { - assert(completed == 1); - } - { - std::vector threads; - duckdb::JoinOutboundThreads> join_threads {threads}; - threads.emplace_back([&completed]() { completed++; }); - threads.emplace_back([&completed]() { completed++; }); - threads[0].join(); // Already-joined workers must also be safe. - } - assert(completed == 3); -} diff --git a/test/sql/fixtures/dequeue_conflict_repro.md b/test/sql/fixtures/dequeue_conflict_repro.md deleted file mode 100644 index b7011bc..0000000 --- a/test/sql/fixtures/dequeue_conflict_repro.md +++ /dev/null @@ -1,15 +0,0 @@ -# Dequeue claim race (not runnable in single-threaded SQLLogic) - -**Where:** `src/quackapi_queue.cpp` — `DequeueInit` UPDATE…WHERE id=(SELECT…LIMIT 1) without re-checking claim predicates on the outer UPDATE. - -**Repro (5 lines):** - -```text -1. LOAD quackapi; CREATE QUEUE q; SELECT quackapi_enqueue('q','only-one'); -2. CREATE ROUTE dq GET '/dq' AS SELECT id,payload FROM quackapi_dequeue('q',1); -3. SELECT * FROM quackapi_serve(PORT, worker_threads:=8); -4. Parallel: curl PORT/dq ×8 -5. Observe: one 200 with the job; losers → 422 "Conflict on tuple deletion!" (should be empty 200 []) -``` - -**Observable wrong behaviour:** concurrent claimants raise TransactionContext conflicts (surfaced as 422) instead of a clean exclusive claim / empty result for losers. diff --git a/test/sql/quackapi_access_log.test b/test/sql/quackapi_access_log.test deleted file mode 100644 index d45a101..0000000 --- a/test/sql/quackapi_access_log.test +++ /dev/null @@ -1,176 +0,0 @@ -# name: test/sql/quackapi_access_log.test -# description: Table-backed access logging for serve and the in-process TestClient -# group: [sql] - -require quackapi - -statement ok -CREATE TABLE access_log ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - route_name VARCHAR, - status INTEGER, - duration_ms DOUBLE, - bytes_out BIGINT, - client_ip VARCHAR, - user_agent VARCHAR, - operator_note VARCHAR DEFAULT 'kept' -); - -statement ok -CREATE ROUTE access_log_route GET '/access-log' AS SELECT 'ok' AS result; - -query I -SELECT status -FROM quackapi_request('GET', '/access-log', access_log := 'access_log'); ----- -200 - -query I -SELECT count(*) FROM access_log; ----- -1 - -query III -SELECT route_name, status, operator_note -FROM access_log -WHERE path = '/access-log'; ----- -access_log_route 200 kept - -query I -SELECT status -FROM quackapi_request('GET', '/missing', access_log := 'access_log'); ----- -404 - -query II -SELECT route_name IS NULL, status -FROM access_log -WHERE path = '/missing'; ----- -true 404 - -query I -SELECT count(*) FROM access_log; ----- -2 - -# A transiently unavailable table falls back to stderr, then resumes table -# logging after the capped retry backoff when the table is recreated. -statement ok -CREATE TABLE access_log_retry ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - route_name VARCHAR, - status INTEGER, - duration_ms DOUBLE, - bytes_out BIGINT, - client_ip VARCHAR, - user_agent VARCHAR -); - -query I -SELECT * FROM quackapi_serve(29461, host := '127.0.0.1', access_log := 'access_log_retry'); ----- -http://127.0.0.1:29461 - -query I -SELECT (quackapi_fetch('http://127.0.0.1:29461/access-log')).status; ----- -200 - -statement ok -SELECT sleep_ms(1100); - -query I -SELECT count(*) FROM access_log_retry; ----- -1 - -statement ok -DROP TABLE access_log_retry; - -# The request still succeeds while the writer is backing off and the missing -# table means its access row cannot be appended. -query I -SELECT (quackapi_fetch('http://127.0.0.1:29461/access-log')).status; ----- -200 - -statement ok -SELECT sleep_ms(1200); - -statement ok -CREATE TABLE access_log_retry ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - route_name VARCHAR, - status INTEGER, - duration_ms DOUBLE, - bytes_out BIGINT, - client_ip VARCHAR, - user_agent VARCHAR -); - -statement ok -SELECT sleep_ms(1200); - -query I -SELECT (quackapi_fetch('http://127.0.0.1:29461/access-log')).status; ----- -200 - -statement ok -SELECT sleep_ms(1100); - -query I -SELECT count(*) FROM access_log_retry; ----- -1 - -query I -SELECT * FROM quackapi_stop(29461); ----- -Stopped quackapi server on port 29461 - -# The existing readiness surface reports table-writer overflow state. -query I -SELECT * FROM quackapi_serve(29231, host := '127.0.0.1', access_log := 'access_log'); ----- -http://127.0.0.1:29231 - -query I -SELECT contains(body, '"access_log_overflow_count":0') -FROM quackapi_fetch('http://127.0.0.1:29231/healthz'); ----- -true - -query I -SELECT * FROM quackapi_stop(29231); ----- -Stopped quackapi server on port 29231 - -# A missing operator table must not fail the in-process request. -query I -SELECT status -FROM quackapi_request('GET', '/access-log', access_log := 'missing_access_log'); ----- -200 - -query I -SELECT status -FROM quackapi_request('GET', '/access-log', access_log := false); ----- -200 - -query I -SELECT count(*) FROM access_log; ----- -2 diff --git a/test/sql/quackapi_batteries.test b/test/sql/quackapi_batteries.test deleted file mode 100644 index 71f0ac5..0000000 --- a/test/sql/quackapi_batteries.test +++ /dev/null @@ -1,82 +0,0 @@ -# name: test/sql/quackapi_batteries.test -# description: batteries-included overrides — preserve_insertion_order, health_routes off, http_client force -# group: [sql] - -require quackapi - -# log_level extension option defaults to info -query I -SELECT current_setting('quackapi_log_level'); ----- -info - -# http_client extension option defaults to auto (prefer curl_httpfs) -query I -SELECT current_setting('quackapi_http_client'); ----- -auto - -# --------------------------------------------------------------------------- -# Overridable: preserve_insertion_order := true keeps insertion order -# --------------------------------------------------------------------------- -query I -SELECT * FROM quackapi_serve(29202, host := '127.0.0.1', preserve_insertion_order := true); ----- -http://127.0.0.1:29202 - -query I -SELECT current_setting('preserve_insertion_order'); ----- -true - -query I -SELECT * FROM quackapi_stop(29202); ----- -Stopped quackapi server on port 29202 - -# --------------------------------------------------------------------------- -# health_routes := false → not re-registered after explicit drop -# --------------------------------------------------------------------------- -query I -DROP ROUTE __quackapi_health; ----- -Dropped route __quackapi_health - -query I -DROP ROUTE __quackapi_healthz; ----- -Dropped route __quackapi_healthz - -query I -SELECT * FROM quackapi_serve(29203, host := '127.0.0.1', health_routes := false); ----- -http://127.0.0.1:29203 - -query I -SELECT count(*) FROM quackapi_routes() WHERE name LIKE '__quackapi_health%'; ----- -0 - -query I -SELECT * FROM quackapi_stop(29203); ----- -Stopped quackapi server on port 29203 - -# --------------------------------------------------------------------------- -# http_client := 'httplib' forces stock client (always available; no network -# needed for the force path). Active backend + reason on quackapi_servers(). -# --------------------------------------------------------------------------- -query I -SELECT * FROM quackapi_serve(29204, host := '127.0.0.1', http_client := 'httplib'); ----- -http://127.0.0.1:29204 - -query II -SELECT http_client, http_client_reason FROM quackapi_servers() WHERE port = 29204; ----- -httplib operator_forced - -query I -SELECT * FROM quackapi_stop(29204); ----- -Stopped quackapi server on port 29204 diff --git a/test/sql/quackapi_body_schema.test b/test/sql/quackapi_body_schema.test new file mode 100644 index 0000000..6b07a87 --- /dev/null +++ b/test/sql/quackapi_body_schema.test @@ -0,0 +1,91 @@ +# name: test/sql/quackapi_body_schema.test +# description: BODY SCHEMA validation with the optional json_schema extension +# group: [sql] + +require quackapi + +require json + +require json_schema + +# Invalid schema text and an unterminated BODY SCHEMA remain DDL errors. +statement error +CREATE ROUTE invalid_schema POST '/invalid-schema' + BODY SCHEMA '{"type":"object"' + AS SELECT 1; +---- +Invalid BODY SCHEMA JSON + +statement error +CREATE ROUTE bad_schema POST '/x' BODY SCHEMA '{"type":"object" AS SELECT 1; +---- +Unterminated BODY SCHEMA + +# JSON Pointer numeric segments use the actual payload container: object keys +# remain JSON strings, including a leading-zero key, while array indexes are numbers. +query I +CREATE ROUTE numeric_object_key POST '/numeric-object-key' + BODY SCHEMA '{"type":"object","properties":{"01":{"type":"integer","maximum":1}}}' + AS SELECT 1 AS ok; +---- +Route numeric_object_key: POST /numeric-object-key + +query II +SELECT status, contains(decode(body), '"loc":["body","01"]') +FROM quackapi_request('POST', '/numeric-object-key', '{"01":2}'); +---- +422 true + +query I +DROP ROUTE numeric_object_key; +---- +Dropped route numeric_object_key + +# Error-location parsing bounds its own JSON walk. A hostile deeply nested body +# still returns a normal 422 instead of recursing on the request stack. +query I +CREATE ROUTE deeply_nested_body POST '/deeply-nested-body' + BODY SCHEMA '{"type":"array","items":{"type":"integer"}}' + AS SELECT 1 AS ok; +---- +Route deeply_nested_body: POST /deeply-nested-body + +query I +SELECT status +FROM quackapi_request('POST', '/deeply-nested-body', repeat('[', 129) || '0' || repeat(']', 129)); +---- +422 + +query I +DROP ROUTE deeply_nested_body; +---- +Dropped route deeply_nested_body + +# maximum:10 is schema-only; SQL INTEGER binding alone would accept 100. +query I +CREATE ROUTE with_schema POST '/with-schema' + BODY SCHEMA '{"type":"object","required":["n"],"properties":{"n":{"type":"integer","maximum":10}}}' + AS SELECT $n::INTEGER AS n; +---- +Route with_schema: POST /with-schema + +query II +SELECT status, contains(decode(body), '"loc":["body","n"]') +FROM quackapi_request('POST', '/with-schema', '{"n":100}'); +---- +422 true + +query I +SELECT status FROM quackapi_request('POST', '/with-schema', '{"n":"not-int"}'); +---- +422 + +query II +SELECT status, decode(body) FROM quackapi_request('POST', '/with-schema', '{"n":1}'); +---- +200 [{"n":1}] + +query I +DROP ROUTE with_schema; +---- +Dropped route with_schema diff --git a/test/sql/quackapi_bug_state.test b/test/sql/quackapi_bug_state.test index a0a7aa0..733f8b3 100644 --- a/test/sql/quackapi_bug_state.test +++ b/test/sql/quackapi_bug_state.test @@ -16,6 +16,8 @@ require quackapi +require json + # --------------------------------------------------------------------------- # 1) Thread-local static JSON body cache (ThreadRequestCache::static_json_bodies) # Zero-param handlers must not freeze the first serialized body forever. diff --git a/test/sql/quackapi_cache_key.test b/test/sql/quackapi_cache_key.test index a878bc6..6bf6bcb 100644 --- a/test/sql/quackapi_cache_key.test +++ b/test/sql/quackapi_cache_key.test @@ -4,6 +4,8 @@ require quackapi +require json + # Keep quackapi_request on one worker thread so TLS state is shared across calls. statement ok SET threads = 1; diff --git a/test/sql/quackapi_compression.test b/test/sql/quackapi_compression.test index fc0070f..9ee4a97 100644 --- a/test/sql/quackapi_compression.test +++ b/test/sql/quackapi_compression.test @@ -2,6 +2,8 @@ # description: response compression mode and threshold configuration # group: [sql] +require json + require quackapi query I diff --git a/test/sql/quackapi_concurrent_request.test b/test/sql/quackapi_concurrent_request.test new file mode 100644 index 0000000..90ee28b --- /dev/null +++ b/test/sql/quackapi_concurrent_request.test @@ -0,0 +1,35 @@ +# name: test/sql/quackapi_concurrent_request.test +# description: Concurrent in-process route requests keep independent path bindings +# group: [sql] + +require quackapi + +require json + +statement ok +CREATE ROUTE concurrent_echo GET '/concurrent/:id' AS + SELECT $id::INTEGER AS id, + {'seen': [$id::INTEGER, NULL]} AS nested, + repeat('x', 64) AS payload; + +# Each iteration uses a separate DuckDB connection. A route registration, +# request context or path binding shared incorrectly between callers changes +# the result of at least one request. +concurrentloop threadid 0 40 + +query III +SELECT status, + json_extract_string(decode(body)::JSON, '$[0].id') = '${threadid}', + json_extract_string(decode(body)::JSON, '$[0].nested.seen[0]') = '${threadid}' +FROM quackapi_request('GET', '/concurrent/${threadid}'); +---- +200 true true + +endloop + +# The same registry remains usable after all concurrent callers finish. +query II +SELECT status, json_extract_string(decode(body)::JSON, '$[0].id') +FROM quackapi_request('GET', '/concurrent/42'); +---- +200 42 diff --git a/test/sql/quackapi_encoding_contract.test b/test/sql/quackapi_encoding_contract.test new file mode 100644 index 0000000..d7d75b1 --- /dev/null +++ b/test/sql/quackapi_encoding_contract.test @@ -0,0 +1,91 @@ +# name: test/sql/quackapi_encoding_contract.test +# description: Route encoders preserve typed JSON, CSV cells, and encoded byte limits +# group: [sql] + +require quackapi + +require json + +# A JSON column must remain a JSON object, not a quoted JSON string. Nested +# DuckDB values and SQL NULL must keep their distinct JSON representations. +statement ok +CREATE ROUTE typed_json GET '/encoding/typed' AS + SELECT '{"nested":[1,null,{"ok":true}]}'::JSON AS document, + {'numbers': [2, 3], 'label': 'x'} AS object_value, + MAP {'k': [4, 5]} AS mapping, + [6, NULL, 7] AS values, + NULL::VARCHAR AS absent; + +query IIIII +SELECT json_type(payload, '$[0].document'), + json_extract(payload, '$[0].document.nested[2].ok') = 'true'::JSON, + json_extract(payload, '$[0].object_value.numbers') = '[2,3]'::JSON, + json_extract(payload, '$[0].mapping.k') = '[4,5]'::JSON, + json_extract(payload, '$[0].values') = '[6,null,7]'::JSON +FROM (SELECT decode(body)::JSON AS payload FROM quackapi_request('GET', '/encoding/typed')); +---- +OBJECT true true true true + +query I +SELECT json_extract(payload, '$[0].absent') = 'null'::JSON +FROM (SELECT decode(body)::JSON AS payload FROM quackapi_request('GET', '/encoding/typed')); +---- +true + +statement ok +CREATE ROUTE typed_ndjson GET '/encoding/ndjson' FORMAT ndjson AS + SELECT i, {'n': [i, NULL]} AS nested + FROM range(2049) t(i); + +# More than one DuckDB vector must serialize without a missing or extra row. +query III +SELECT len(string_split(trim(decode(body), E'\n'), E'\n')), + json_extract_string(split_part(decode(body), E'\n', 1)::JSON, '$.i'), + json_extract_string(split_part(decode(body), E'\n', 2049)::JSON, '$.i') +FROM quackapi_request('GET', '/encoding/ndjson'); +---- +2049 0 2048 + +statement ok +CREATE ROUTE typed_csv GET '/encoding/csv' FORMAT csv AS + SELECT 'comma,quote"newline' || chr(10) || 'tail' AS cell, + NULL::VARCHAR AS absent, + {'n': [1, 2]} AS nested; + +# DuckDB's CSV writer quotes punctuation/newlines and JSON-encodes nested cells. +query III +SELECT contains(decode(body), '"comma,quote""newline' || chr(10) || 'tail"'), + contains(decode(body), ',,"{""n"":[1,2]}"'), + starts_with(decode(body), 'cell,absent,nested' || chr(10)) +FROM quackapi_request('GET', '/encoding/csv'); +---- +true true true + +# The limit applies to the complete encoded response, including brackets, +# property names, quotes and punctuation, and permits an exact-size response. +statement ok +CREATE ROUTE encoded_limit GET '/encoding/limit' AS SELECT 'x' AS a; + +query II +SELECT status, decode(body) +FROM quackapi_request('GET', '/encoding/limit', max_response_bytes := 11); +---- +200 [{"a":"x"}] + +query I +SELECT status +FROM quackapi_request('GET', '/encoding/limit', max_response_bytes := 10); +---- +507 + +# Text mode carries BLOB bytes unchanged, including NUL, invalid UTF-8, +# quotation marks and a newline. Decoding this body would lose that contract. +statement ok +CREATE ROUTE binary_body GET '/encoding/blob' AS + SELECT from_hex('00ff220a5c') AS text; + +query III +SELECT status, hex(body), content_type +FROM quackapi_request('GET', '/encoding/blob'); +---- +200 00FF220A5C text/plain; charset=utf-8 diff --git a/test/sql/quackapi_format.test b/test/sql/quackapi_format.test index 3ceda07..d6456a6 100644 --- a/test/sql/quackapi_format.test +++ b/test/sql/quackapi_format.test @@ -1,10 +1,10 @@ # name: test/sql/quackapi_format.test -# description: FORMAT json|ndjson|csv|parquet + Accept via quackapi_request +# description: FORMAT json|ndjson|csv + Accept via quackapi_request # group: [sql] require quackapi -require parquet +require json statement ok CREATE OR REPLACE ROUTE items GET '/items' AS @@ -18,19 +18,14 @@ statement ok CREATE OR REPLACE ROUTE items_csv GET '/items_csv' FORMAT csv AS SELECT 1 AS id, 'alice' AS name UNION ALL SELECT 2, 'bob,jr'; -statement ok -CREATE OR REPLACE ROUTE items_parquet GET '/items_parquet' FORMAT parquet AS - SELECT 1 AS id, 'alice' AS name UNION ALL SELECT 2, 'bob'; - query II SELECT name, format FROM quackapi_routes() -WHERE name IN ('items','items_ndjson','items_csv','items_parquet') +WHERE name IN ('items','items_ndjson','items_csv') ORDER BY name; ---- items json items_csv csv items_ndjson ndjson -items_parquet parquet query II SELECT status, content_type FROM quackapi_request('GET', '/items'); @@ -75,15 +70,3 @@ FROM quackapi_request( ); ---- 200 true - -query II -SELECT status, starts_with(content_type, 'application/vnd.apache.parquet') -FROM quackapi_request('GET', '/items_parquet'); ----- -200 true - -# PAR1 magic on parquet body -query I -SELECT decode(body[1:4]) FROM quackapi_request('GET', '/items_parquet'); ----- -PAR1 diff --git a/test/sql/quackapi_from_x_native.test b/test/sql/quackapi_from_x_native.test deleted file mode 100644 index f1b8a5e..0000000 --- a/test/sql/quackapi_from_x_native.test +++ /dev/null @@ -1,162 +0,0 @@ -# name: test/sql/quackapi_from_x_native.test -# description: Native quack_from_* table functions — fixtures + embed drift -# group: [sql] - -require quackapi - -# --------------------------------------------------------------------------- -# FastAPI — routes from committed fastapi_mini fixture (2 routes) -# --------------------------------------------------------------------------- -query III -SELECT method, path, handler_name -FROM quack_from_fastapi('bridges/from_x/fixtures/fastapi_mini') -ORDER BY method, path; ----- -GET /articles/{slug} get_article -POST /login login - -# --------------------------------------------------------------------------- -# FastAPI — models: UserInLogin with 3 fields (2 required, 1 defaulted) -# --------------------------------------------------------------------------- -query IIIII -SELECT model_name, field_name, field_type, is_required, has_default -FROM quack_from_fastapi_models('bridges/from_x/fixtures/fastapi_mini') -ORDER BY field_line, field_name; ----- -UserInLogin email str true false -UserInLogin password str true false -UserInLogin remember_me bool false true - -query I -SELECT default_expr FROM quack_from_fastapi_models('bridges/from_x/fixtures/fastapi_mini') -WHERE field_name = 'remember_me'; ----- -False - -# --------------------------------------------------------------------------- -# Rails — routes from committed rails_mini fixture (2 routes under /api) -# --------------------------------------------------------------------------- -query III -SELECT method, path, handler_name -FROM quack_from_rails('bridges/from_x/fixtures/rails_mini') -ORDER BY method, path; ----- -GET /api/articles/:slug articles#show -POST /api/login sessions#create - -# --------------------------------------------------------------------------- -# Rails — models: User validates (2 presence-required, 1 optional) -# --------------------------------------------------------------------------- -query IIIII -SELECT model_name, field_name, field_type, is_required, has_default -FROM quack_from_rails_models('bridges/from_x/fixtures/rails_mini') -ORDER BY field_name; ----- -User email presence true false -User password presence true false -User remember_me allow_blank,inclusion false false - -# --------------------------------------------------------------------------- -# Express — routes from committed express_mini fixture (2 routes) -# --------------------------------------------------------------------------- -query III -SELECT method, path, handler_name -FROM quack_from_express('bridges/from_x/fixtures/express_mini') -ORDER BY method, path; ----- -GET /articles/:slug getArticle -POST /login login - -# --------------------------------------------------------------------------- -# Express — models: UserInLogin class (2 required, 1 defaulted) -# --------------------------------------------------------------------------- -query IIIII -SELECT model_name, field_name, field_type, is_required, has_default -FROM quack_from_express_models('bridges/from_x/fixtures/express_mini') -ORDER BY field_line, field_name; ----- -UserInLogin email string true false -UserInLogin password string true false -UserInLogin remember_me boolean false true - -query I -SELECT default_expr FROM quack_from_express_models('bridges/from_x/fixtures/express_mini') -WHERE field_name = 'remember_me'; ----- -false - -# --------------------------------------------------------------------------- -# Gin — routes from committed gin_mini fixture (2 routes under /api) -# --------------------------------------------------------------------------- -query III -SELECT method, path, handler_name -FROM quack_from_gin('bridges/from_x/fixtures/gin_mini') -ORDER BY method, path; ----- -GET /api/articles/:slug GetArticle -POST /api/login Login - -# --------------------------------------------------------------------------- -# Gin — models: UserInLogin struct tags (2 required, 1 omitempty optional) -# --------------------------------------------------------------------------- -query IIIII -SELECT model_name, field_name, field_type, is_required, has_default -FROM quack_from_gin_models('bridges/from_x/fixtures/gin_mini') -ORDER BY field_line, field_name; ----- -UserInLogin Email string true false -UserInLogin Password string true false -UserInLogin RememberMe bool false false - -# --------------------------------------------------------------------------- -# Embed drift: on-disk src/sql/*.sql must byte-match embedded constants. -# (unittest cwd is the extension repo root. read_text needs literal paths — -# no lateral column params — so each pair is checked separately.) -# --------------------------------------------------------------------------- -query I -SELECT (SELECT content FROM read_text('src/sql/quack_from_fastapi_routes.sql')) - = quack_from_x_sql('fastapi', 'routes'); ----- -true - -query I -SELECT (SELECT content FROM read_text('src/sql/quack_from_fastapi_models.sql')) - = quack_from_x_sql('fastapi', 'models'); ----- -true - -query I -SELECT (SELECT content FROM read_text('src/sql/quack_from_rails_routes.sql')) - = quack_from_x_sql('rails', 'routes'); ----- -true - -query I -SELECT (SELECT content FROM read_text('src/sql/quack_from_rails_models.sql')) - = quack_from_x_sql('rails', 'models'); ----- -true - -query I -SELECT (SELECT content FROM read_text('src/sql/quack_from_express_routes.sql')) - = quack_from_x_sql('express', 'routes'); ----- -true - -query I -SELECT (SELECT content FROM read_text('src/sql/quack_from_express_models.sql')) - = quack_from_x_sql('express', 'models'); ----- -true - -query I -SELECT (SELECT content FROM read_text('src/sql/quack_from_gin_routes.sql')) - = quack_from_x_sql('gin', 'routes'); ----- -true - -query I -SELECT (SELECT content FROM read_text('src/sql/quack_from_gin_models.sql')) - = quack_from_x_sql('gin', 'models'); ----- -true diff --git a/test/sql/quackapi_graphql.test b/test/sql/quackapi_graphql.test deleted file mode 100644 index 585dbd8..0000000 --- a/test/sql/quackapi_graphql.test +++ /dev/null @@ -1,325 +0,0 @@ -# name: test/sql/quackapi_graphql.test -# description: Thin GraphQL v0 + closed built-in allowlist + named routes -# group: [sql] - -require quackapi - -require json - -statement ok -CREATE TABLE users AS SELECT 1 AS id, 'ada' AS name; - -statement ok -CREATE TABLE posts AS SELECT 10 AS id, 'hello' AS title; - -statement ok -CREATE TABLE secrets AS SELECT 99 AS id, 'hidden' AS payload; - -# --------------------------------------------------------------------------- -# Built-in mode is closed by default: registration is required before /graphql -# --------------------------------------------------------------------------- -query I -SELECT mode FROM quackapi_graphql_tables(); ----- -disabled - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ users { id name } posts { id title } }"}' -); ----- -200 {"errors":[{"message":"table 'users' is not registered for GraphQL — CREATE GRAPHQL FOR TABLE users"}]} - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/graphql/schema'); ----- -200 {"mode":"disabled","tables":{},"note":"GraphQL v0 catalog-only schema. Full __schema introspection not implemented. CREATE GRAPHQL FOR TABLE registers an allowlist for POST /graphql; empty = closed. SET GLOBAL quackapi_graphql_allow_all = true restores legacy open catalog mode. CREATE GRAPHQL ROUTE mounts named paths with per-route tables."} - -# Legacy open-catalog behavior needs an explicit database-wide migration opt-in. -statement ok -SET GLOBAL quackapi_graphql_allow_all = true; - -query I -SELECT mode FROM quackapi_graphql_tables(); ----- -open - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ secrets { id } }"}' -); ----- -200 {"data":{"secrets":[{"id":99}]}} - -statement ok -SET GLOBAL quackapi_graphql_allow_all = false; - -# --------------------------------------------------------------------------- -# Allowlist mode: only registered tables -# --------------------------------------------------------------------------- -query I -CREATE GRAPHQL FOR TABLE users, posts; ----- -GraphQL registered: users, posts (allowlist mode on POST /graphql) - -query TT -SELECT mode, table_name FROM quackapi_graphql_tables() ORDER BY table_name; ----- -allowlist posts -allowlist users - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ users { id name } }"}' -); ----- -200 {"data":{"users":[{"id":1,"name":"ada"}]}} - -# secrets is not on the allowlist -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ secrets { id payload } }"}' -); ----- -200 {"errors":[{"message":"table 'secrets' is not registered for GraphQL — CREATE GRAPHQL FOR TABLE secrets"}]} - -# schema only lists allowlisted tables that exist -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/graphql/schema'); ----- -200 {"mode":"allowlist","tables":{"posts":["id","title"],"users":["id","name"]},"note":"GraphQL v0 catalog-only schema. Full __schema introspection not implemented. CREATE GRAPHQL FOR TABLE registers an allowlist for POST /graphql; empty = closed. SET GLOBAL quackapi_graphql_allow_all = true restores legacy open catalog mode. CREATE GRAPHQL ROUTE mounts named paths with per-route tables."} - -# Duplicate without OR REPLACE fails -statement error -CREATE GRAPHQL FOR TABLE users; ----- -already registered - -# OR REPLACE is idempotent -query I -CREATE OR REPLACE GRAPHQL FOR TABLE users; ----- -GraphQL registered: users (allowlist mode on POST /graphql) - -# Unknown catalog table rejected at CREATE -statement error -CREATE GRAPHQL FOR TABLE no_such_table; ----- -not found in schema main - -# DROP one table; allowlist still active while non-empty -query I -DROP GRAPHQL FOR TABLE posts; ----- -GraphQL unregistered: posts - -query TT -SELECT mode, table_name FROM quackapi_graphql_tables(); ----- -allowlist users - -# DROP last table → closed mode again -query I -DROP GRAPHQL FOR TABLE users; ----- -GraphQL unregistered: users (allowlist empty → catalog closed) - -query I -SELECT mode FROM quackapi_graphql_tables(); ----- -disabled - -# secrets remains unavailable after the final registration is dropped -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ secrets { id } }"}' -); ----- -200 {"errors":[{"message":"table 'secrets' is not registered for GraphQL — CREATE GRAPHQL FOR TABLE secrets"}]} - -# DROP GRAPHQL ALL clears even a multi-table allowlist -query I -CREATE GRAPHQL FOR TABLE users, posts, secrets; ----- -GraphQL registered: users, posts, secrets (allowlist mode on POST /graphql) - -query I -DROP GRAPHQL ALL; ----- -GraphQL allowlist cleared (catalog closed) - -query I -SELECT mode FROM quackapi_graphql_tables(); ----- -disabled - -# DROP missing registration fails when none match -statement error -DROP GRAPHQL FOR TABLE users; ----- -no matching registered tables - -# --------------------------------------------------------------------------- -# CREATE GRAPHQL ROUTE — named mounts, independent of global allowlist -# --------------------------------------------------------------------------- -statement ok -CREATE TABLE multi AS SELECT * FROM (VALUES (1),(2),(3)) t(id); - -query I -CREATE GRAPHQL ROUTE public_api POST '/api/gql' FROM users, posts LIMIT 2; ----- -GraphQL route public_api: POST /api/gql FROM users, posts LIMIT 2 - -query IIIII -SELECT name, method, path, require_auth, "limit" FROM quackapi_graphql_routes(); ----- -public_api POST /api/gql NULL 2 - -# Route serves only FROM tables -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ users { id name } }"}' -); ----- -200 {"data":{"users":[{"id":1,"name":"ada"}]}} - -# secrets is not on the named route and remains closed on /graphql -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ secrets { id } }"}' -); ----- -200 {"errors":[{"message":"table 'secrets' is not on this GraphQL route — CREATE GRAPHQL ROUTE … FROM secrets"}]} - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ secrets { id } }"}' -); ----- -200 {"errors":[{"message":"table 'secrets' is not registered for GraphQL — CREATE GRAPHQL FOR TABLE secrets"}]} - -# Global allowlist does not shrink the route's FROM set -query I -CREATE GRAPHQL FOR TABLE users; ----- -GraphQL registered: users (allowlist mode on POST /graphql) - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ posts { id title } }"}' -); ----- -200 {"data":{"posts":[{"id":10,"title":"hello"}]}} - -query I -DROP GRAPHQL ALL; ----- -GraphQL allowlist cleared (catalog closed) - -# Schema for the route -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/api/gql/schema'); ----- -200 {"mode":"route","route":"public_api","tables":{"posts":["id","title"],"users":["id","name"]},"note":"GraphQL v0 catalog-only schema. Full __schema introspection not implemented. CREATE GRAPHQL FOR TABLE registers an allowlist for POST /graphql; empty = closed. SET GLOBAL quackapi_graphql_allow_all = true restores legacy open catalog mode. CREATE GRAPHQL ROUTE mounts named paths with per-route tables."} - -# LIMIT honored on multi-row table (route with LIMIT 1) -query I -CREATE OR REPLACE GRAPHQL ROUTE public_api POST '/api/gql' FROM multi LIMIT 1; ----- -GraphQL route public_api: POST /api/gql FROM multi LIMIT 1 - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ multi { id } }"}' -); ----- -200 {"data":{"multi":[{"id":1}]}} - -# Reserved path rejected -statement error -CREATE GRAPHQL ROUTE bad POST '/graphql' FROM users; ----- -reserved - -# REQUIRE auth -query I -CREATE AUTH gql_site AS API_KEY; ----- -Auth gql_site: API_KEY - -query I -SELECT * FROM quackapi_add_api_key('gql_site', 'gql-secret-key', 'bob'); ----- -bob - -query I -CREATE OR REPLACE GRAPHQL ROUTE public_api POST '/api/gql' FROM users REQUIRE gql_site; ----- -GraphQL route public_api: POST /api/gql FROM users REQUIRE gql_site LIMIT 100 - -query I -SELECT status FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ users { id } }"}' -); ----- -401 - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ users { id } }"}', - headers := MAP {'X-API-Key': 'gql-secret-key'} -); ----- -200 {"data":{"users":[{"id":1}]}} - -# DROP route → 404 on path; built-in remains closed until explicitly registered -query I -DROP GRAPHQL ROUTE public_api; ----- -Dropped GraphQL route public_api - -query I -SELECT status FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ users { id } }"}' -); ----- -404 - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ users { id } }"}' -); ----- -200 {"errors":[{"message":"table 'users' is not registered for GraphQL — CREATE GRAPHQL FOR TABLE users"}]} - -query I -CREATE GRAPHQL FOR TABLE users; ----- -GraphQL registered: users (allowlist mode on POST /graphql) - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ users { id } }"}' -); ----- -200 {"data":{"users":[{"id":1}]}} - -query I -SELECT count(*) FROM quackapi_graphql_routes(); ----- -0 diff --git a/test/sql/quackapi_http_pool.test b/test/sql/quackapi_http_pool.test deleted file mode 100644 index 8737739..0000000 --- a/test/sql/quackapi_http_pool.test +++ /dev/null @@ -1,136 +0,0 @@ -# name: test/sql/quackapi_http_pool.test -# description: Outbound quackapi_fetch / quackapi_post reuse pooled connections -# group: [sql] - -require quackapi - -require json - -# Serve a tiny echo API and then call it from the SAME process. No external -# network: the pool is exercised against quackapi's own server. -statement ok -CREATE ROUTE ping GET '/ping' AS SELECT 'pong' AS msg; - -statement ok -CREATE ROUTE echo POST '/echo' AS SELECT $body::VARCHAR AS got; - -query I -SELECT * FROM quackapi_serve(29301, host := '127.0.0.1', access_log := false); ----- -http://127.0.0.1:29301 - -# --- GET --------------------------------------------------------------------- -query I -SELECT (quackapi_fetch('http://127.0.0.1:29301/ping')).status; ----- -200 - -query I -SELECT (quackapi_fetch('http://127.0.0.1:29301/ping')).body; ----- -[{"msg":"pong"}] - -# Extra headers travel as a MAP argument. -query I -SELECT (quackapi_fetch('http://127.0.0.1:29301/ping', MAP{'X-Test': 'yes'})).status; ----- -200 - -# --- POST -------------------------------------------------------------------- -# Plain http:// needs NO companion extension — the vendored httplib implements -# POST, unlike DuckDB's built-in HTTPUtil client. -query I -SELECT (quackapi_post('http://127.0.0.1:29301/echo', to_json({a: 1})::VARCHAR)).status; ----- -200 - -# The body arrives intact — $body binds the raw payload we sent. -query I -SELECT (quackapi_post('http://127.0.0.1:29301/echo', to_json({a: 1})::VARCHAR)).body; ----- -[{"got":"{\"a\":1}"}] - -# --- pooling ----------------------------------------------------------------- -# The point of the whole exercise: the second call must NOT dial again. -query I -SELECT (quackapi_fetch('http://127.0.0.1:29301/ping')).reused_connection; ----- -true - -# One origin, one dial, many reuses. -query I -SELECT count(*) FROM quackapi_http_pool() WHERE host = 'http://127.0.0.1:29301'; ----- -1 - -query I -SELECT dialed = 1 AND reused >= 4 AND idle >= 1 -FROM quackapi_http_pool() WHERE host = 'http://127.0.0.1:29301'; ----- -true - -# --- VOLATILE ---------------------------------------------------------------- -# Without FunctionStability::VOLATILE, DuckDB constant-folds a literal-argument -# call and range(N) would issue ONE request while reporting N rows. Assert the -# server actually saw every one of them by watching TOTAL requests (dialed + -# reused) climb. -# -# Total, not `reused` alone: a chunk carrying N independent calls fans them out -# concurrently, so the burst DIALS up to the fan-out width instead of reusing one -# socket N times sequentially. Which counter moves is an artefact of the execution -# model; the invariant this test exists to defend is that N rows issue N requests. -statement ok -CREATE OR REPLACE TEMP TABLE before AS - SELECT dialed + reused AS n FROM quackapi_http_pool() WHERE host = 'http://127.0.0.1:29301'; - -statement ok -SELECT (quackapi_fetch('http://127.0.0.1:29301/ping')).status AS s FROM range(10) t(i); - -query I -SELECT (SELECT dialed + reused FROM quackapi_http_pool() WHERE host = 'http://127.0.0.1:29301') - - (SELECT n FROM before) >= 10; ----- -true - -# --- fan-out: one chunk, many concurrent calls -------------------------------- -# A chunk carrying N independent calls issues them concurrently rather than one -# blocking round trip at a time. Asserted on RESULTS, not elapsed time: a -# "finishes within X ms" check would be measuring the machine under CI load, not -# this code. What must hold is that concurrency loses nothing — every row gets -# its own correct answer, none are dropped, none error, and the total request -# count still matches the row count. -statement ok -CREATE OR REPLACE TEMP TABLE fanout_before AS - SELECT dialed + reused AS n FROM quackapi_http_pool() WHERE host = 'http://127.0.0.1:29301'; - -query III -SELECT count(*) AS n, - count(CASE WHEN (r).status = 200 THEN 1 END) AS ok, - count(CASE WHEN (r).error IS NOT NULL THEN 1 END) AS errs -FROM (SELECT quackapi_fetch('http://127.0.0.1:29301/ping') AS r FROM range(64) t(i)); ----- -64 64 0 - -# Every one of the 64 rows became a real request (no folding, no coalescing). -query I -SELECT (SELECT dialed + reused FROM quackapi_http_pool() WHERE host = 'http://127.0.0.1:29301') - - (SELECT n FROM fanout_before) = 64; ----- -true - -# --- transport errors are values, not exceptions ------------------------------ -query I -SELECT (quackapi_fetch('http://127.0.0.1:29399/nope')).error IS NOT NULL; ----- -true - -# NULL url → NULL struct, no request. -query I -SELECT (quackapi_fetch(NULL::VARCHAR)).status IS NULL; ----- -true - -# stop closes the listener and (via StopServer) drains the free-list so idle -# keep-alive clients cannot pin httplib workers across process teardown. -statement ok -SELECT * FROM quackapi_stop(29301); diff --git a/test/sql/quackapi_lifecycle.test b/test/sql/quackapi_lifecycle.test index d196d94..c342be9 100644 --- a/test/sql/quackapi_lifecycle.test +++ b/test/sql/quackapi_lifecycle.test @@ -4,6 +4,8 @@ require quackapi +require json + # --------------------------------------------------------------------------- # Bug: CREATE OR REPLACE AUTH API_KEY → JWT leaves hashed keys that resurrect # when replaced back to API_KEY (DropAuth clears keys; AddAuth did not). @@ -118,82 +120,6 @@ DROP ROUTE keeper; statement ok DROP ROUTE mover; -# --------------------------------------------------------------------------- -# Bug: CREATE GRAPHQL FOR TABLE a, b half-applied when a later name failed -# (missing table or duplicate) — earlier tables stayed on the allowlist. -# --------------------------------------------------------------------------- -statement ok -CREATE TABLE users AS SELECT 1 AS id; - -statement ok -CREATE TABLE posts AS SELECT 1 AS id; - -statement error -CREATE GRAPHQL FOR TABLE users, no_such_table; ----- -not found - -query I -SELECT count(*) FROM quackapi_graphql_tables() WHERE table_name = 'users'; ----- -0 - -statement ok -CREATE GRAPHQL FOR TABLE users; - -statement error -CREATE GRAPHQL FOR TABLE posts, users; ----- -already registered - -query I -SELECT count(*) FROM quackapi_graphql_tables() WHERE table_name = 'posts'; ----- -0 - -statement ok -DROP GRAPHQL FOR TABLE users; - -# --------------------------------------------------------------------------- -# Bug: CREATE API FOR TABLE skipped handler validation — missing table still -# registered list/get routes that 500'd on first request. -# --------------------------------------------------------------------------- -statement error -CREATE API FOR TABLE no_such_table; ----- -Invalid handler SQL - -query I -SELECT count(*) FROM quackapi_routes() WHERE name IN ('no_such_table_list', 'no_such_table_get'); ----- -0 - -# --------------------------------------------------------------------------- -# Bug: serve(health_routes=true) → stop → serve(health_routes=false) left -# __quackapi_health* in the registry (RegisterQuackapiHealthRoutes no-op). -# --------------------------------------------------------------------------- -statement ok -SELECT * FROM quackapi_serve(29451, host := '127.0.0.1'); - -query I -SELECT count(*) FROM quackapi_routes() WHERE starts_with(name, '__quackapi_health'); ----- -2 - -statement ok -SELECT * FROM quackapi_stop(29451); - -statement ok -SELECT * FROM quackapi_serve(29451, host := '127.0.0.1', health_routes := false); - -query I -SELECT count(*) FROM quackapi_routes() WHERE starts_with(name, '__quackapi_health'); ----- -0 - -statement ok -SELECT * FROM quackapi_stop(29451); - # OpenAPI after DROP ROUTE: path must vanish (live snapshot — regression lock). statement ok CREATE ROUTE oa GET '/oa' AS SELECT 1 AS n; diff --git a/test/sql/quackapi_logging_defaults.test b/test/sql/quackapi_logging_defaults.test deleted file mode 100644 index 1ca1864..0000000 --- a/test/sql/quackapi_logging_defaults.test +++ /dev/null @@ -1,62 +0,0 @@ -# name: test/sql/quackapi_logging_defaults.test -# description: quackapi_serve preserves prior DuckDB logging unless explicitly disabled -# group: [sql] - -require quackapi - -# An operator-enabled logger survives the serve defaults. -statement ok -CALL enable_logging(storage := 'memory'); - -query I -SELECT * FROM quackapi_serve(29501, host := '127.0.0.1'); ----- -http://127.0.0.1:29501 - -query I -SELECT current_setting('enable_logging'); ----- -1 - -query I -SELECT * FROM quackapi_stop(29501); ----- -Stopped quackapi server on port 29501 - -# Without prior logging, the serve default remains off. -statement ok -SET enable_logging = false; - -query I -SELECT * FROM quackapi_serve(29502, host := '127.0.0.1'); ----- -http://127.0.0.1:29502 - -query I -SELECT current_setting('enable_logging'); ----- -0 - -query I -SELECT * FROM quackapi_stop(29502); ----- -Stopped quackapi server on port 29502 - -# An explicit false still overrides an operator-enabled logger. -statement ok -CALL enable_logging(storage := 'memory'); - -query I -SELECT * FROM quackapi_serve(29503, host := '127.0.0.1', enable_logging := false); ----- -http://127.0.0.1:29503 - -query I -SELECT current_setting('enable_logging'); ----- -0 - -query I -SELECT * FROM quackapi_stop(29503); ----- -Stopped quackapi server on port 29503 diff --git a/test/sql/quackapi_memory_limit.test b/test/sql/quackapi_memory_limit.test deleted file mode 100644 index 4ac38b2..0000000 --- a/test/sql/quackapi_memory_limit.test +++ /dev/null @@ -1,54 +0,0 @@ -# name: test/sql/quackapi_memory_limit.test -# description: memory_limit knob — named param and SET quackapi_memory_limit reach DuckDB via serve -# group: [sql] - -require quackapi - -# Default: empty string (non-clobber / safe-default logic) -query I -SELECT current_setting('quackapi_memory_limit'); ----- -(empty) - -# --------------------------------------------------------------------------- -# Named param wins and applies into DuckDB memory_limit -# --------------------------------------------------------------------------- -query I -SELECT * FROM quackapi_serve(29102, host := '127.0.0.1', memory_limit := '4GB'); ----- -http://127.0.0.1:29102 - -query I -SELECT current_setting('memory_limit'); ----- -3.7 GiB - -query I -SELECT * FROM quackapi_stop(29102); ----- -Stopped quackapi server on port 29102 - -# --------------------------------------------------------------------------- -# Extension setting applies when named param omitted -# --------------------------------------------------------------------------- -statement ok -SET quackapi_memory_limit = '512MB'; - -query I -SELECT * FROM quackapi_serve(29103, host := '127.0.0.1'); ----- -http://127.0.0.1:29103 - -query I -SELECT current_setting('memory_limit'); ----- -488.2 MiB - -query I -SELECT * FROM quackapi_stop(29103); ----- -Stopped quackapi server on port 29103 - -# Cleanup setting -statement ok -SET quackapi_memory_limit = ''; diff --git a/test/sql/quackapi_middleware.test b/test/sql/quackapi_middleware.test index 38ca5d4..73a4ccf 100644 --- a/test/sql/quackapi_middleware.test +++ b/test/sql/quackapi_middleware.test @@ -4,6 +4,8 @@ require quackapi +require json + # --------------------------------------------------------------------------- # Definitions are introspectable and OR REPLACE preserves creation order. # --------------------------------------------------------------------------- diff --git a/test/sql/quackapi_native_validation.test b/test/sql/quackapi_native_validation.test index 2a04718..890cf29 100644 --- a/test/sql/quackapi_native_validation.test +++ b/test/sql/quackapi_native_validation.test @@ -86,46 +86,6 @@ DROP ROUTE typed_increment; ---- Dropped route typed_increment -# JSON Pointer numeric segments use the actual payload container: object keys -# remain JSON strings, including a leading-zero key, while array indexes are numbers. -query I -CREATE ROUTE numeric_object_key POST '/numeric-object-key' - BODY SCHEMA '{"type":"object","properties":{"01":{"type":"integer","maximum":1}}}' - AS SELECT 1 AS ok; ----- -Route numeric_object_key: POST /numeric-object-key - -query II -SELECT status, contains(decode(body), '"loc":["body","01"]') -FROM quackapi_request('POST', '/numeric-object-key', '{"01":2}'); ----- -422 true - -query I -DROP ROUTE numeric_object_key; ----- -Dropped route numeric_object_key - -# Error-location parsing bounds its own JSON walk. A hostile deeply nested body -# still returns a normal 422 instead of recursing on the request stack. -query I -CREATE ROUTE deeply_nested_body POST '/deeply-nested-body' - BODY SCHEMA '{"type":"array","items":{"type":"integer"}}' - AS SELECT 1 AS ok; ----- -Route deeply_nested_body: POST /deeply-nested-body - -query I -SELECT status -FROM quackapi_request('POST', '/deeply-nested-body', repeat('[', 129) || '0' || repeat(']', 129)); ----- -422 - -query I -DROP ROUTE deeply_nested_body; ----- -Dropped route deeply_nested_body - # Missing input takes DEFAULT NULL; explicit JSON null takes the separate native # null path; the string "null" is still a string and fails INTEGER conversion. query I diff --git a/test/sql/quackapi_pg_prepare_cache.test b/test/sql/quackapi_pg_prepare_cache.test deleted file mode 100644 index 6637944..0000000 --- a/test/sql/quackapi_pg_prepare_cache.test +++ /dev/null @@ -1,99 +0,0 @@ -# name: test/sql/quackapi_pg_prepare_cache.test -# description: PG PQprepare cache must not key on SQL alone (catalog/definition identity) -# group: [sql] - -# -# Requires a live Postgres reachable at QUACKAPI_TEST_PG_DSN (libpq DSN). -# Throwaway setup used in development: -# postgresql://quackapi_pg_cache:qa_pg_cache_test@/quackapi_pg_cache_test?host=/tmp - -require quackapi - -require-env QUACKAPI_TEST_PG_DSN - -# Keep quackapi_request on one worker thread so TLS PGconn + prepare map are shared. -statement ok -SET threads = 1; - -statement ok -SET quackapi_pg_dsn = '${QUACKAPI_TEST_PG_DSN}'; - -# --------------------------------------------------------------------------- -# PQprepare was keyed only by pg_sql. Missing: catalog type OID identity. -# Prepare SELECT …::qa_dpos, DROP+CREATE the domain (new OID, same name), then -# reuse the cached statement name → sticky "cache lookup failed for type". -# Fresh exec (no SQL-only prepare cache) returns the post-redefinition row. -# --------------------------------------------------------------------------- - -statement ok -CREATE OR REPLACE ROUTE qa_pg_redef POST '/qa_pg_redef' AS -DO $$ -BEGIN - DROP DOMAIN IF EXISTS qa_dpos CASCADE; - CREATE DOMAIN qa_dpos AS integer CHECK (VALUE > 0); -END -$$; - -statement ok -CREATE OR REPLACE ROUTE qa_pg_cast GET '/qa_pg_cast/:id' AS - SELECT ($id::INTEGER)::qa_dpos AS v, $id::INTEGER AS id; - -# Boot domain on the TLS libpq connection. -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/qa_pg_redef'); ----- -200 [] - -# First cast prepares with the live domain OID. -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/qa_pg_cast/5'); ----- -200 [{"v":5,"id":5}] - -# A PostgreSQL execution failure is authoritative once the native path has -# dispatched it. It must not silently replay the route against DuckDB. -statement ok -CREATE OR REPLACE ROUTE qa_pg_no_replay GET '/qa_pg_no_replay' AS - SELECT current_setting('quackapi_missing_setting') AS v; - -query I -SELECT status FROM quackapi_request('GET', '/qa_pg_no_replay'); ----- -502 - -# Native libpq I/O and the backend statement both inherit the request deadline. -# The response must terminate as a timeout rather than falling through to the -# DuckDB path after PostgreSQL has accepted the command. -statement ok -CREATE OR REPLACE ROUTE qa_pg_deadline GET '/qa_pg_deadline' AS - SELECT pg_sleep(0.25) AS delayed; - -query I -SELECT status FROM quackapi_request('GET', '/qa_pg_deadline', query_timeout_ms := 100); ----- -504 - -# Wire text must retain its declared SQL type and SQL rewriting must leave -# literals/comments alone. Explicit parameter casts must survive translation. -statement ok -CREATE OR REPLACE ROUTE qa_pg_types GET '/qa_pg_types/:id' AS -SELECT '0012'::TEXT AS code, true AS enabled, false AS disabled, - 'pg.$phantom postgres_query'::TEXT AS literal, - $id::INTEGER AS id, NULL::TEXT AS missing; -- $comment pg.nope - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/qa_pg_types/7'); ----- -200 [{"code":"0012","enabled":true,"disabled":false,"literal":"pg.$phantom postgres_query","id":7,"missing":null}] - -# Redefine domain (new OID, same SQL text for the cast route). -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/qa_pg_redef'); ----- -200 [] - -# Must return the post-redefinition result — not a sticky dead type OID failure. -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/qa_pg_cast/5'); ----- -200 [{"v":5,"id":5}] diff --git a/test/sql/quackapi_policy.test b/test/sql/quackapi_policy.test deleted file mode 100644 index 580708c..0000000 --- a/test/sql/quackapi_policy.test +++ /dev/null @@ -1,535 +0,0 @@ -# name: test/sql/quackapi_policy.test -# description: ROW ACCESS / MASKING enforcement via quackapi_request (claims-keyed) -# group: [sql] - -require quackapi - -require json - -# JWTs in test/sql/fixtures/jwt_pol_*.txt — SECRET 'policy-test-secret' (HS256). -# Loaded via SET VARIABLE so signature bytes that contain '--' are not SQL comments. -# require json: JWT claims parse uses json_* inside quackapi_verify_auth. - -# --------------------------------------------------------------------------- -# DDL setup — policies, binds, seed, routes -# --------------------------------------------------------------------------- -statement ok -CREATE TABLE policy_orders (id INTEGER, tenant_id VARCHAR, amount INTEGER); - -statement ok -CREATE TABLE policy_users (id INTEGER, email VARCHAR, tenant_id VARCHAR); - -statement ok -INSERT INTO policy_orders VALUES - (1, 'acme', 100), - (2, 'beta', 200), - (3, 'acme', 150); - -statement ok -INSERT INTO policy_users VALUES - (1, 'alice@acme.test', 'acme'), - (2, 'bob@beta.test', 'beta'); - -query I -CREATE AUTH jwt_pol AS JWT ( SECRET 'policy-test-secret' ); ----- -Auth jwt_pol: JWT_HS256 - -query I -CREATE ROW ACCESS POLICY tenant_isolation - AS (tenant_id VARCHAR) RETURNS BOOLEAN - USING (tenant_id = $claims_tenant OR $claims_role = 'admin'); ----- -Row access policy tenant_isolation - -query I -CREATE MASKING POLICY mask_email ON VARCHAR - USING (CASE WHEN $claims_role = 'admin' THEN val ELSE '***' END); ----- -Masking policy mask_email ON VARCHAR - -# Duplicate without OR REPLACE errors -statement error -CREATE ROW ACCESS POLICY tenant_isolation - AS (tenant_id VARCHAR) RETURNS BOOLEAN - USING (true); ----- -already exists - -# Name collision across kinds errors -statement error -CREATE MASKING POLICY tenant_isolation ON VARCHAR USING (val); ----- -already used by a row access policy - -query I -ALTER TABLE policy_orders ADD ROW ACCESS POLICY tenant_isolation ON (tenant_id); ----- -Bound row access policy tenant_isolation on policy_orders - -query I -ALTER TABLE policy_users ADD ROW ACCESS POLICY tenant_isolation ON (tenant_id); ----- -Bound row access policy tenant_isolation on policy_users - -query I -ALTER TABLE policy_users MODIFY COLUMN email SET MASKING POLICY mask_email; ----- -Set masking policy mask_email on policy_users.email - -query I -CREATE ROUTE list_orders GET '/orders' REQUIRE jwt_pol AS - SELECT id, tenant_id, amount FROM policy_orders ORDER BY id; ----- -Route list_orders: GET /orders - -query I -CREATE ROUTE list_users GET '/users' REQUIRE jwt_pol AS - SELECT id, email, tenant_id FROM policy_users ORDER BY id; ----- -Route list_users: GET /users - -# Public route on policied table — CREATE ok; request must fail closed (403) -query I -CREATE ROUTE public_orders GET '/public_orders' AS - SELECT id, tenant_id FROM policy_orders; ----- -Route public_orders: GET /public_orders - -# Parsed policy enforcement covers schema-qualified base tables. Views and -# table macros are deliberately rejected while a bound expansion is unavailable, -# rather than leaking their protected dependencies. -statement ok -CREATE VIEW policy_orders_view AS SELECT * FROM policy_orders; - -statement ok -CREATE MACRO policy_orders_macro() AS TABLE SELECT * FROM policy_orders; - -query I -CREATE ROUTE list_orders_qualified GET '/orders-qualified' REQUIRE jwt_pol AS - SELECT id, tenant_id, amount FROM main.policy_orders ORDER BY id; ----- -Route list_orders_qualified: GET /orders-qualified - -query I -CREATE ROUTE public_orders_view GET '/public_orders_view' AS - SELECT id, tenant_id FROM policy_orders_view; ----- -Route public_orders_view: GET /public_orders_view - -query I -CREATE ROUTE public_orders_macro GET '/public_orders_macro' AS - SELECT id, tenant_id FROM policy_orders_macro(); ----- -Route public_orders_macro: GET /public_orders_macro - -# Nested scalar expressions and CTEs must be walked by the policy rewriter; -# a relation hidden outside the top-level FROM is not a safe bypass. The CTE -# deliberately shadows the bound table name while its body uses a qualified -# alias for the actual catalog table. -query I -CREATE ROUTE public_users_scalar GET '/public_users_scalar' AS - SELECT (SELECT email FROM policy_users ORDER BY id LIMIT 1) AS email; ----- -Route public_users_scalar: GET /public_users_scalar - -query I -CREATE ROUTE list_users_scalar GET '/users-scalar' REQUIRE jwt_pol AS - SELECT (SELECT email FROM policy_users WHERE tenant_id = $claims_tenant ORDER BY id LIMIT 1) AS email; ----- -Route list_users_scalar: GET /users-scalar - -query I -CREATE ROUTE list_users_cte GET '/users-cte' REQUIRE jwt_pol AS - WITH policy_users AS ( - SELECT p.id, p.email, p.tenant_id FROM main.policy_users AS p - ) - SELECT u.email FROM policy_users AS u ORDER BY u.id; ----- -Route list_users_cte: GET /users-cte - -# Table macros remain an indirect relation source for all callers until a -# bound expansion exists; authenticated credentials do not make it safe. -query I -CREATE ROUTE auth_orders_macro GET '/auth_orders_macro' REQUIRE jwt_pol AS - SELECT id, tenant_id FROM policy_orders_macro(); ----- -Route auth_orders_macro: GET /auth_orders_macro - -# Policy enforcement is read-only today. Reject protected-table mutations -# rather than executing an update without a row-level write policy. -query I -CREATE ROUTE mutate_orders POST '/mutate_orders' REQUIRE jwt_pol AS - UPDATE policy_orders SET amount = 0 WHERE id = 1 RETURNING id; ----- -Route mutate_orders: POST /mutate_orders - -query I -CREATE STREAM policy_orders_sse GET '/policy-orders-sse' AS - SELECT id, tenant_id FROM policy_orders; ----- -Stream policy_orders_sse: GET /policy-orders-sse SSE - -query I -CREATE GRAPHQL ROUTE policy_orders_gql POST '/orders-gql' FROM policy_orders REQUIRE jwt_pol; ----- -GraphQL route policy_orders_gql: POST /orders-gql FROM policy_orders REQUIRE jwt_pol LIMIT 100 - -query I -CREATE GRAPHQL ROUTE policy_users_gql POST '/users-gql' FROM policy_users REQUIRE jwt_pol; ----- -GraphQL route policy_users_gql: POST /users-gql FROM policy_users REQUIRE jwt_pol LIMIT 100 - -# This registration must never turn a policy-protected table into a public -# built-in GraphQL endpoint. -query I -CREATE GRAPHQL FOR TABLE policy_orders; ----- -GraphQL registered: policy_orders (allowlist mode on POST /graphql) - -statement ok -SET VARIABLE jwt_acme_user = ( - SELECT rtrim(content, E'\n\r ') FROM read_text('test/sql/fixtures/jwt_pol_acme_user.txt') -); - -statement ok -SET VARIABLE jwt_acme_admin = ( - SELECT rtrim(content, E'\n\r ') FROM read_text('test/sql/fixtures/jwt_pol_acme_admin.txt') -); - -statement ok -SET VARIABLE jwt_beta_user = ( - SELECT rtrim(content, E'\n\r ') FROM read_text('test/sql/fixtures/jwt_pol_beta_user.txt') -); - -# --------------------------------------------------------------------------- -# Row access policy — filters rows by claim (via quackapi_request) -# --------------------------------------------------------------------------- -# acme user → only acme rows -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/orders', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 [{"id":1,"tenant_id":"acme","amount":100},{"id":3,"tenant_id":"acme","amount":150}] - -# beta user → only beta rows -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/orders', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_beta_user')} -); ----- -200 [{"id":2,"tenant_id":"beta","amount":200}] - -# admin → all tenants -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/orders', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_admin')} -); ----- -200 [{"id":1,"tenant_id":"acme","amount":100},{"id":2,"tenant_id":"beta","amount":200},{"id":3,"tenant_id":"acme","amount":150}] - -# A qualified direct table reference is rewritten from the parsed base-table -# identity, not a bare-name text match. -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/orders-qualified', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 [{"id":1,"tenant_id":"acme","amount":100},{"id":3,"tenant_id":"acme","amount":150}] - -# --------------------------------------------------------------------------- -# Masking policy — column masked unless claims permit -# --------------------------------------------------------------------------- -# acme user → email masked; RAP also limits to acme row -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/users', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 [{"id":1,"email":"***","tenant_id":"acme"}] - -# admin → email unmasked; sees all tenants -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/users', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_admin')} -); ----- -200 [{"id":1,"email":"alice@acme.test","tenant_id":"acme"},{"id":2,"email":"bob@beta.test","tenant_id":"beta"}] - -# Scalar subqueries and CTE bodies are parsed and rewritten recursively. The -# first verifies authenticated RAP/masking; the second verifies a public -# scalar source fails closed rather than reading the underlying table. -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/users-scalar', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 [{"email":"***"}] - -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/users-cte', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 [{"email":"***"}] - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/public_users_scalar'); ----- -403 {"detail":"Policy denies unauthenticated access"} - -# Indirect table macros and protected-table writes are denied even after JWT -# verification. Confirm the rejected mutation left the table unchanged. -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/auth_orders_macro', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -403 {"detail":"Policy denies unauthenticated access"} - -query II -SELECT status, decode(body) -FROM quackapi_request( - 'POST', '/mutate_orders', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -403 {"detail":"Policy denies unauthenticated access"} - -query I -SELECT amount FROM policy_orders WHERE id = 1; ----- -100 - -# --------------------------------------------------------------------------- -# Fail closed — unauthenticated request to policied table → 403 (not open) -# --------------------------------------------------------------------------- -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/public_orders'); ----- -403 {"detail":"Policy denies unauthenticated access"} - -# Views and table macros are indirect sources. They fail closed instead of -# bypassing a policy bound to policy_orders. -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/public_orders_view'); ----- -403 {"detail":"Policy denies unauthenticated access"} - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/public_orders_macro'); ----- -403 {"detail":"Policy denies unauthenticated access"} - -# Streams have no auth/claims contract yet, so a protected source must not -# become an unauthenticated SSE bypass. -query I -SELECT status FROM quackapi_request('GET', '/policy-orders-sse'); ----- -403 - -# REQUIRE without credentials still 401 (auth before policy) -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/orders'); ----- -401 {"detail":"Not authenticated"} - -# Named GraphQL applies the same RAP/masking expansion and verified claims as -# REST. Built-in GraphQL remains unauthenticated and therefore cannot read a -# registered protected table. -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/orders-gql', - '{"query":"{ policy_orders { id tenant_id amount } }"}', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 {"data":{"policy_orders":[{"id":1,"tenant_id":"acme","amount":100},{"id":3,"tenant_id":"acme","amount":150}]}} - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/users-gql', - '{"query":"{ policy_users { id email tenant_id } }"}', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 {"data":{"policy_users":[{"id":1,"email":"***","tenant_id":"acme"}]}} - -query I -SELECT status FROM quackapi_request( - 'POST', '/orders-gql', - '{"query":"{ policy_orders { id } }"}' -); ----- -401 - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ policy_orders { id } }"}' -); ----- -200 {"errors":[{"message":"access denied by table policy"}]} - -# Schema discovery uses the same named-route auth gate and does not expose a -# registered protected table through the unauthenticated built-in schema. -query I -SELECT status FROM quackapi_request('GET', '/orders-gql/schema'); ----- -401 - -query I -SELECT status FROM quackapi_request( - 'GET', '/orders-gql/schema', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 - -query II -SELECT status, contains(decode(body), 'policy_orders') -FROM quackapi_request('GET', '/graphql/schema'); ----- -200 false - -# --------------------------------------------------------------------------- -# Bad grammar / missing policy -# --------------------------------------------------------------------------- -statement error -CREATE ROW ACCESS POLICY bad AS (x) USING (true); ----- -RETURNS BOOLEAN - -statement error -CREATE MASKING POLICY bad ON VARCHAR AS (val); ----- -USING - -statement error -ALTER TABLE policy_orders ADD ROW ACCESS POLICY nope ON (tenant_id); ----- -does not exist - -# --------------------------------------------------------------------------- -# Cleanup -# --------------------------------------------------------------------------- -query I -DROP GRAPHQL ROUTE policy_orders_gql; ----- -Dropped GraphQL route policy_orders_gql - -query I -DROP GRAPHQL ROUTE policy_users_gql; ----- -Dropped GraphQL route policy_users_gql - -query I -DROP GRAPHQL FOR TABLE policy_orders; ----- -GraphQL unregistered: policy_orders (allowlist empty → catalog closed) - -query I -DROP STREAM policy_orders_sse; ----- -Dropped stream policy_orders_sse - -query I -DROP ROUTE list_orders; ----- -Dropped route list_orders - -query I -DROP ROUTE list_users; ----- -Dropped route list_users - -query I -DROP ROUTE public_orders; ----- -Dropped route public_orders - -query I -DROP ROUTE list_orders_qualified; ----- -Dropped route list_orders_qualified - -query I -DROP ROUTE public_orders_view; ----- -Dropped route public_orders_view - -query I -DROP ROUTE public_orders_macro; ----- -Dropped route public_orders_macro - -query I -DROP ROUTE public_users_scalar; ----- -Dropped route public_users_scalar - -query I -DROP ROUTE list_users_scalar; ----- -Dropped route list_users_scalar - -query I -DROP ROUTE list_users_cte; ----- -Dropped route list_users_cte - -query I -DROP ROUTE auth_orders_macro; ----- -Dropped route auth_orders_macro - -query I -DROP ROUTE mutate_orders; ----- -Dropped route mutate_orders - -query I -DROP AUTH jwt_pol; ----- -Dropped auth jwt_pol - -query I -DROP MASKING POLICY mask_email; ----- -Dropped masking policy mask_email - -query I -DROP ROW ACCESS POLICY tenant_isolation; ----- -Dropped row access policy tenant_isolation - -statement ok -DROP MACRO policy_orders_macro; - -statement ok -DROP VIEW policy_orders_view; - -statement ok -DROP TABLE policy_orders; - -statement ok -DROP TABLE policy_users; diff --git a/test/sql/quackapi_post_surface.test b/test/sql/quackapi_post_surface.test new file mode 100644 index 0000000..49e5292 --- /dev/null +++ b/test/sql/quackapi_post_surface.test @@ -0,0 +1,63 @@ +# name: test/sql/quackapi_post_surface.test +# description: Frozen outbound POST signatures, provider boundary and concurrency setting +# group: [sql] + +require quackapi + +require json + +# Applications depend on these exact overloads and a side-effecting function. +query I +SELECT len(array_agg(parameter_types ORDER BY len(parameter_types), parameter_types::VARCHAR)) +FROM duckdb_functions() +WHERE function_name = 'quackapi_post' + AND stability = 'VOLATILE' + AND return_type = + 'STRUCT(status INTEGER, reason VARCHAR, body VARCHAR, headers MAP(VARCHAR, VARCHAR), "error" VARCHAR, reused_connection BOOLEAN)'; +---- +4 + +query I +SELECT (quackapi_post(NULL::VARCHAR, '{}')).status IS NULL; +---- +true + +query I +SELECT quackapi_post(NULL::VARCHAR, '{}'::JSON) IS NULL; +---- +true + +# Loading QuackAPI alone must not silently install or select an HTTP provider. +# The row-local transport adapter reports the configuration error in its +# receipt, allowing mixed rows to keep their own outcomes. +query II +SELECT receipt.status, + contains(receipt.error, 'requires an explicitly loaded HTTPUtil provider') +FROM (SELECT quackapi_post('http://127.0.0.1:9/', '{}') AS receipt); +---- +0 true + +statement ok +SET quackapi_post_concurrency = 64; + +query I +SELECT current_setting('quackapi_post_concurrency'); +---- +64 + +statement error +SET quackapi_post_concurrency = -1; +---- +must be 0 or a positive integer + +# Removed entry points must fail at binding/parsing, rather than shadowing the +# retained route and POST surface with stale registrations. +statement error +SELECT quackapi_fetch('http://127.0.0.1:9/'); +---- +quackapi_fetch + +statement error +SELECT * FROM quackapi_http_pool(); +---- +quackapi_http_pool diff --git a/test/sql/quackapi_queue.test b/test/sql/quackapi_queue.test deleted file mode 100644 index 860c5ac..0000000 --- a/test/sql/quackapi_queue.test +++ /dev/null @@ -1,353 +0,0 @@ -# name: test/sql/quackapi_queue.test -# description: Queue delivery semantics via quackapi_request / serve+fetch -# group: [sql] - -require quackapi - -require json - -# --------------------------------------------------------------------------- -# Setup: durable queue + HTTP routes (enqueue / claim / ack / nack / stats) -# --------------------------------------------------------------------------- -query I -CREATE QUEUE jobs WITH (max_attempts=3, visibility_timeout='1s', backoff_base_seconds=0); ----- -Queue jobs: max_attempts=3 visibility_timeout=1s - -statement ok -CREATE ROUTE q_enq POST '/q/enq' AS - SELECT quackapi_enqueue('jobs', $payload::VARCHAR) AS job_id; - -# $t probe defeats zero-param static JSON body cache on this branch -statement ok -CREATE ROUTE q_claim GET '/q/claim' AS - SELECT id, payload, status, attempts, delivery_generation FROM quackapi_dequeue('jobs', 1) - WHERE $t::INT >= 0; - -statement ok -CREATE ROUTE q_ack POST '/q/ack' AS - SELECT quackapi_ack('jobs', $id::BIGINT, $delivery_generation::BIGINT) AS ok; - -statement ok -CREATE ROUTE q_nack POST '/q/nack' AS - SELECT quackapi_nack('jobs', $id::BIGINT, $delivery_generation::BIGINT, - $requeue::BOOLEAN, $error::VARCHAR) AS status; - -statement ok -CREATE ROUTE q_stats GET '/q/stats' AS - SELECT name, depth, in_flight, dead FROM quackapi_queues() - WHERE name = 'jobs' AND $t::INT >= 0; - -# --------------------------------------------------------------------------- -# HTTP enqueue → depth; claim leases; second claim empty (visibility hold) -# --------------------------------------------------------------------------- -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/q/enq', '{"payload":"{\"task\":\"a\"}"}'); ----- -200 [{"job_id":1}] - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/stats?t=1'); ----- -[{"name":"jobs","depth":1,"in_flight":0,"dead":0}] - -query II -SELECT status, - json_extract(decode(body)::JSON, '$[0].id')::BIGINT AS id -FROM quackapi_request('GET', '/q/claim?t=1'); ----- -200 1 - -# Leased — invisible to a second dequeue -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/claim?t=2'); ----- -[] - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/stats?t=2'); ----- -[{"name":"jobs","depth":0,"in_flight":1,"dead":0}] - -# Visibility timeout lapses → redelivery (attempts increment) -statement ok -SELECT sleep_ms(1100); - -query III -SELECT status, - json_extract(decode(body)::JSON, '$[0].id')::BIGINT, - json_extract(decode(body)::JSON, '$[0].attempts')::INTEGER -FROM quackapi_request('GET', '/q/claim?t=3'); ----- -200 1 2 - -# --------------------------------------------------------------------------- -# ack removes permanently — no redelivery after visibility would have lapsed -# --------------------------------------------------------------------------- -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/q/ack', '{"id":1,"delivery_generation":2}'); ----- -200 [{"ok":true}] - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/claim?t=4'); ----- -[] - -statement ok -SELECT sleep_ms(1100); - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/claim?t=5'); ----- -[] - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/stats?t=3'); ----- -[{"name":"jobs","depth":0,"in_flight":0,"dead":0}] - -# --------------------------------------------------------------------------- -# nack returns the message for redelivery -# --------------------------------------------------------------------------- -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/q/enq', '{"payload":"{\"task\":\"b\"}"}'); ----- -200 [{"job_id":2}] - -query I -SELECT json_extract(decode(body)::JSON, '$[0].id')::BIGINT FROM quackapi_request('GET', '/q/claim?t=6'); ----- -2 - -query II -SELECT status, decode(body) -FROM quackapi_request('POST', '/q/nack', '{"id":2,"delivery_generation":1,"requeue":true,"error":"try_again"}'); ----- -200 [{"status":"pending"}] - -query III -SELECT status, - json_extract(decode(body)::JSON, '$[0].id')::BIGINT, - json_extract(decode(body)::JSON, '$[0].attempts')::INTEGER -FROM quackapi_request('GET', '/q/claim?t=7'); ----- -200 2 2 - -# nack without requeue → dead; not redelivered -query II -SELECT status, decode(body) -FROM quackapi_request('POST', '/q/nack', '{"id":2,"delivery_generation":2,"requeue":false,"error":"no_retry"}'); ----- -200 [{"status":"dead"}] - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/claim?t=8'); ----- -[] - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/stats?t=4'); ----- -[{"name":"jobs","depth":0,"in_flight":0,"dead":1}] - -# --------------------------------------------------------------------------- -# Concurrent dequeue: one message → exactly one winner across parallel GETs -# -# Uses its own queue with a visibility timeout far longer than the storm can -# take. The `jobs` queue above is deliberately 1s so the tests before this one -# can exercise lease lapse + redelivery — but redelivery after an expired lease -# is CORRECT queue behavior, so reusing `jobs` here made this test assert -# "the 8-request storm finishes within 1s", a wall-clock property CI cannot -# guarantee. On a loaded runner the lease lapsed mid-storm and a second -# claimant legitimately won job 3 (attempts 1 then 2 -> winner_ids [3,3]). -# A long lease isolates the property actually under test: exclusive claim. -# --------------------------------------------------------------------------- -query I -CREATE QUEUE race WITH (max_attempts=3, visibility_timeout='300s', backoff_base_seconds=0); ----- -Queue race: max_attempts=3 visibility_timeout=300s - -statement ok -CREATE ROUTE q_rclaim GET '/q/rclaim' AS - SELECT id, payload, status, attempts, delivery_generation FROM quackapi_dequeue('race', 1) - WHERE $t::INT >= 0; - -statement ok -CREATE ROUTE q_renq POST '/q/renq' AS - SELECT quackapi_enqueue('race', $payload::VARCHAR) AS job_id; - -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/q/renq', '{"payload":"{\"task\":\"race\"}"}'); ----- -200 [{"job_id":3}] - -query I -SELECT * FROM quackapi_serve(29270, host := '127.0.0.1', access_log := false, worker_threads := 8); ----- -http://127.0.0.1:29270 - -# Exactly one winner with job 3; losers empty []; every attempt 200 with no error. -query III -SELECT list(json_extract(body::JSON, '$[0].id')::BIGINT) - FILTER (WHERE status = 200 AND body != '[]') AS winner_ids, - bool_and(status = 200 AND error IS NULL) AS all_ok, - len(list(idx) FILTER (WHERE body = '[]')) AS empty_n -FROM quackapi_parallel_fetch('http://127.0.0.1:29270/q/rclaim?t=9', 8); ----- -[3] true 7 - -query I -SELECT * FROM quackapi_stop(29270); ----- -Stopped quackapi server on port 29270 - -# --------------------------------------------------------------------------- -# Delivery generations fence stale workers; renew only extends the active lease -# --------------------------------------------------------------------------- -query I -CREATE QUEUE fence WITH (max_attempts=3, visibility_timeout='1s', backoff_base_seconds=0); ----- -Queue fence: max_attempts=3 visibility_timeout=1s - -query I -SELECT quackapi_enqueue('fence', '{"task":"fence"}'); ----- -4 - -query II -SELECT id, delivery_generation FROM quackapi_dequeue('fence'); ----- -4 1 - -statement ok -SELECT sleep_ms(1100); - -query III -SELECT id, attempts, delivery_generation FROM quackapi_dequeue('fence'); ----- -4 2 2 - -# The first worker cannot mutate the redelivered job in any direction. -query I -SELECT quackapi_ack('fence', 4, 1); ----- -false - -query I -SELECT quackapi_nack('fence', 4, 1, true, 'old worker'); ----- -stale - -query I -SELECT quackapi_renew('fence', 4, 1); ----- -false - -query III -SELECT status, attempts, delivery_generation FROM quackapi_jobs WHERE id = 4; ----- -running 2 2 - -# The current receipt can renew the lease and remains exclusively owned beyond -# its original expiry; a duplicate ack has the defined false result. -statement ok -SELECT sleep_ms(300); - -query I -SELECT quackapi_renew('fence', 4, 2); ----- -true - -statement ok -SELECT sleep_ms(800); - -query I -SELECT count(*) FROM quackapi_dequeue('fence'); ----- -0 - -query I -SELECT quackapi_ack('fence', 4, 2); ----- -true - -query I -SELECT quackapi_ack('fence', 4, 2); ----- -false - -# --------------------------------------------------------------------------- -# Lease expiry obeys max_attempts instead of creating an unbounded retry loop -# --------------------------------------------------------------------------- -query I -CREATE QUEUE expiry WITH (max_attempts=2, visibility_timeout='1s', backoff_base_seconds=0); ----- -Queue expiry: max_attempts=2 visibility_timeout=1s - -query I -SELECT quackapi_enqueue('expiry', '{"task":"timeout"}'); ----- -5 - -query III -SELECT id, attempts, delivery_generation FROM quackapi_dequeue('expiry'); ----- -5 1 1 - -statement ok -SELECT sleep_ms(1100); - -query III -SELECT id, attempts, delivery_generation FROM quackapi_dequeue('expiry'); ----- -5 2 2 - -statement ok -SELECT sleep_ms(1100); - -query I -SELECT count(*) FROM quackapi_dequeue('expiry'); ----- -0 - -query II -SELECT status, attempts FROM quackapi_jobs WHERE id = 5; ----- -dead 2 - -# --------------------------------------------------------------------------- -# A live lease cannot starve recovery of another expired job in the queue -# --------------------------------------------------------------------------- -query I -CREATE QUEUE recovery WITH (max_attempts=3, visibility_timeout='300s', backoff_base_seconds=0); ----- -Queue recovery: max_attempts=3 visibility_timeout=300s - -query I -SELECT quackapi_enqueue('recovery', '{"task":"live"}'); ----- -6 - -query I -SELECT quackapi_enqueue('recovery', '{"task":"expired"}'); ----- -7 - -query I -SELECT id FROM quackapi_dequeue('recovery'); ----- -6 - -# Simulate a second process's timed-out delivery while job 6 has a valid local -# lease. This was previously suppressed by the queue-wide live-lease gate. -statement ok -UPDATE quackapi_jobs -SET status = 'running', attempts = 1, delivery_generation = 1, - visible_at = now()::TIMESTAMP - to_seconds(1) -WHERE id = 7; - -query III -SELECT id, attempts, delivery_generation FROM quackapi_dequeue('recovery'); ----- -7 2 2 diff --git a/test/sql/quackapi_rate_limit.test b/test/sql/quackapi_rate_limit.test index 9feeae4..93428d7 100644 --- a/test/sql/quackapi_rate_limit.test +++ b/test/sql/quackapi_rate_limit.test @@ -4,6 +4,8 @@ require quackapi +require json + # --- DDL registration --- statement ok CREATE ROUTE rl1 GET '/rl1' RATE LIMIT 5 PER 10 AS SELECT 1 AS x; diff --git a/test/sql/quackapi_removed_surface.test b/test/sql/quackapi_removed_surface.test new file mode 100644 index 0000000..073371b --- /dev/null +++ b/test/sql/quackapi_removed_surface.test @@ -0,0 +1,89 @@ +# name: test/sql/quackapi_removed_surface.test +# description: Removed SQL registrations and parser statements stay absent while routes remain usable +# group: [sql] + +require quackapi + +require json + +# Verify the live catalog after loading QuackAPI. A stale registration would +# otherwise remain callable even if its implementation source was deleted. +query I +SELECT coalesce(len(list(function_name)), 0) = 0 +FROM duckdb_functions() +WHERE function_name IN ( + 'quackapi_ack', 'quackapi_dequeue', 'quackapi_enqueue', 'quackapi_nack', + 'quackapi_renew', 'quackapi_queues', + 'quackapi_graphql_routes', 'quackapi_graphql_tables', + 'quackapi_policies', 'quackapi_streams', + 'quackapi_fetch', 'quackapi_parallel_fetch', 'quackapi_http_pool', + 'quackapi_http_util_name', 'quackapi_last_write_timeout_sec', + 'quackapi_telemetry_flush', 'quackapi_telemetry_status', + 'quackapi_request_ring'); +---- +true + +query I +SELECT coalesce(len(list(name)), 0) = 0 +FROM duckdb_settings() +WHERE name IN ( + 'quackapi_graphql_allow_all', 'quackapi_http_client', + 'quackapi_memory_limit', 'quackapi_pg_dsn', 'quackapi_request_ring'); +---- +true + +statement error +CREATE GRAPHQL ROUTE removed_graphql POST '/removed-graphql' FROM items; +---- +syntax error at or near "GRAPHQL" + +statement error +CREATE QUEUE removed_queue; +---- +syntax error at or near "QUEUE" + +statement error +CREATE ROW ACCESS POLICY removed_row_policy AS (tenant_id) RETURNS BOOLEAN USING (true); +---- +syntax error at or near "ROW" + +statement error +CREATE MASKING POLICY removed_masking_policy ON VARCHAR USING ('masked'); +---- +syntax error at or near "MASKING" + +statement error +CREATE STREAM removed_stream GET '/removed-stream' AS SELECT 1; +---- +syntax error at or near "STREAM" + +statement error +CREATE API FOR TABLE removed_table; +---- +syntax error at or near "API" + +statement error +CREATE ROUTE removed_arrow GET '/removed-arrow' FORMAT arrow AS SELECT 1; +---- +FORMAT must be json, ndjson, or csv + +statement error +CREATE ROUTE removed_parquet GET '/removed-parquet' FORMAT parquet AS SELECT 1; +---- +FORMAT must be json, ndjson, or csv + +# The parser still accepts explicit routes and the request adapter still runs. +query I +CREATE ROUTE kept_route GET '/kept-route' AS SELECT 7 AS answer; +---- +Route kept_route: GET /kept-route + +query II +SELECT status, decode(body) FROM quackapi_request('GET', '/kept-route'); +---- +200 [{"answer":7}] + +query I +DROP ROUTE kept_route; +---- +Dropped route kept_route diff --git a/test/sql/quackapi_request.test b/test/sql/quackapi_request.test index e1fefe8..dcff317 100644 --- a/test/sql/quackapi_request.test +++ b/test/sql/quackapi_request.test @@ -4,6 +4,8 @@ require quackapi +require json + # --------------------------------------------------------------------------- # CREATE routes (registry only — no quackapi_serve / no port) # --------------------------------------------------------------------------- diff --git a/test/sql/quackapi_request_id.test b/test/sql/quackapi_request_id.test index 64a4817..2aff6c3 100644 --- a/test/sql/quackapi_request_id.test +++ b/test/sql/quackapi_request_id.test @@ -4,6 +4,8 @@ require quackapi +require json + statement ok CREATE OR REPLACE ROUTE echo_rid GET '/echo-rid' AS SELECT $request_id::VARCHAR AS id; diff --git a/test/sql/quackapi_requests_ring.test b/test/sql/quackapi_requests_ring.test deleted file mode 100644 index 23136e0..0000000 --- a/test/sql/quackapi_requests_ring.test +++ /dev/null @@ -1,209 +0,0 @@ -# name: test/sql/quackapi_requests_ring.test -# description: Recent TCP requests retained by quackapi_requests() -# group: [sql] - -require quackapi - -# No listener means the built-in ring surface is an empty relation. -query I -SELECT count(*) FROM quackapi_requests(); ----- -0 - -statement ok -CREATE OR REPLACE ROUTE ring_first GET '/ring/first' AS SELECT 'first' AS result; - -statement ok -CREATE OR REPLACE ROUTE ring_second GET '/ring/second' AS SELECT 'second' AS result; - -statement ok -CREATE OR REPLACE ROUTE ring_fail GET '/ring/fail' AS SELECT * FROM query($sql); - -# Access-log output is disabled, but the ring still records every request. -query I -SELECT * FROM quackapi_serve(29561, host := '127.0.0.1', access_log := false); ----- -http://127.0.0.1:29561 - -query I -SELECT quackapi_fetch('http://127.0.0.1:29561/ring/first').status; ----- -200 - -query I -SELECT quackapi_fetch('http://127.0.0.1:29561/ring/missing').status; ----- -404 - -query I -SELECT quackapi_fetch('http://127.0.0.1:29561/ring/second').status; ----- -200 - -query II -SELECT status, path FROM quackapi_requests() ORDER BY received_at; ----- -200 /ring/first -404 /ring/missing -200 /ring/second - -query I -SELECT * FROM quackapi_stop(29561); ----- -Stopped quackapi server on port 29561 - -# Opt-in request headers and query are retained in the ring after redaction. -query I -SELECT * FROM quackapi_serve(29566, host := '127.0.0.1', access_log := false, - log_headers := 'authorization,x-tenant-id', log_query := true); ----- -http://127.0.0.1:29566 - -query I -SELECT (quackapi_fetch( - 'http://127.0.0.1:29566/ring/first?token=abc&q=x', - MAP {'Authorization': 'Bearer do-not-store', 'x-tenant-id': 'tenant-42'} -)).status; ----- -200 - -# The outbound client may reorder query parameters, so assert on content, not order. -query IIIII -SELECT headers['authorization'], headers['x-tenant-id'], - contains(query, 'token=[REDACTED]'), contains(query, 'q=x'), contains(query, 'abc') -FROM quackapi_requests(29566); ----- -[REDACTED] tenant-42 true true false - -query I -SELECT * FROM quackapi_stop(29566); ----- -Stopped quackapi server on port 29566 - -# Defaults retain neither optional request field. -query I -SELECT * FROM quackapi_serve(29567, host := '127.0.0.1', access_log := false); ----- -http://127.0.0.1:29567 - -query I -SELECT status FROM quackapi_fetch('http://127.0.0.1:29567/ring/second'); ----- -200 - -query II -SELECT headers IS NULL, query IS NULL FROM quackapi_requests(29567); ----- -true true - -query I -SELECT * FROM quackapi_stop(29567); ----- -Stopped quackapi server on port 29567 - -# The ring keeps only the newest two records, in oldest-first order. -query I -SELECT * FROM quackapi_serve(29562, host := '127.0.0.1', access_log := false, request_ring := 2); ----- -http://127.0.0.1:29562 - -statement ok -SELECT quackapi_fetch('http://127.0.0.1:29562/ring/two-one').status; - -statement ok -SELECT quackapi_fetch('http://127.0.0.1:29562/ring/two-two').status; - -statement ok -SELECT quackapi_fetch('http://127.0.0.1:29562/ring/two-three').status; - -query I -SELECT count(*) FROM quackapi_requests(29562); ----- -2 - -query I -SELECT string_agg(path, ',' ORDER BY received_at) FROM quackapi_requests(29562); ----- -/ring/two-two,/ring/two-three - -query I -SELECT * FROM quackapi_stop(29562); ----- -Stopped quackapi server on port 29562 - -# A zero-sized ring does not retain requests. -query I -SELECT * FROM quackapi_serve(29563, host := '127.0.0.1', access_log := false, request_ring := 0); ----- -http://127.0.0.1:29563 - -statement ok -SELECT quackapi_fetch('http://127.0.0.1:29563/ring/zero-one').status; - -statement ok -SELECT quackapi_fetch('http://127.0.0.1:29563/ring/zero-two').status; - -query I -SELECT count(*) FROM quackapi_requests(29563); ----- -0 - -query I -SELECT * FROM quackapi_stop(29563); ----- -Stopped quackapi server on port 29563 - -# The SET option applies when the named parameter is omitted. -statement ok -SET quackapi_request_ring = 1; - -query I -SELECT * FROM quackapi_serve(29565, host := '127.0.0.1', access_log := false); ----- -http://127.0.0.1:29565 - -statement ok -SELECT quackapi_fetch('http://127.0.0.1:29565/ring/setting-one').status; - -statement ok -SELECT quackapi_fetch('http://127.0.0.1:29565/ring/setting-two').status; - -query I -SELECT count(*) FROM quackapi_requests(29565); ----- -1 - -query I -SELECT * FROM quackapi_stop(29565); ----- -Stopped quackapi server on port 29565 - -statement ok -RESET quackapi_request_ring; - -# Failed requests remain queryable for SQL error filtering. -query I -SELECT * FROM quackapi_serve(29564, host := '127.0.0.1', access_log := false); ----- -http://127.0.0.1:29564 - -query I -SELECT quackapi_fetch('http://127.0.0.1:29564/ring/fail?sql=SELECT%20nope%20FROM%20nowhere').status; ----- -500 - -query II -SELECT status, path FROM quackapi_requests(29564) WHERE status >= 500; ----- -500 /ring/fail - -query I -SELECT * FROM quackapi_stop(29564); ----- -Stopped quackapi server on port 29564 - -# The ring is allocated at serve time, so oversized values are rejected. -statement error -SELECT * FROM quackapi_serve(29569, host := '127.0.0.1', request_ring := 1000001); ----- -request_ring must be between 0 and 1000000 diff --git a/test/sql/quackapi_resource_limits.test b/test/sql/quackapi_resource_limits.test index e181f72..7926807 100644 --- a/test/sql/quackapi_resource_limits.test +++ b/test/sql/quackapi_resource_limits.test @@ -4,6 +4,8 @@ require quackapi +require json + statement ok CREATE ROUTE resource_large GET '/resource-large' AS SELECT repeat('x', 4096) AS text; diff --git a/test/sql/quackapi_route_errors.test b/test/sql/quackapi_route_errors.test index 89e8f5b..6d5d471 100644 --- a/test/sql/quackapi_route_errors.test +++ b/test/sql/quackapi_route_errors.test @@ -4,6 +4,8 @@ require quackapi +require json + query I CREATE OR REPLACE ROUTE sql POST '/sql' AS SELECT * FROM query($sql); ---- @@ -29,12 +31,12 @@ query I CREATE TABLE quackapi_sql_empty (x INTEGER); ---- -# Dynamic SQL is validated before delegation so parser errors cannot become 200 [] +# SQL errors raised by an ordinary registered route remain server errors. query II SELECT status, contains(decode(body), 'SELEC') FROM quackapi_request('POST', '/sql', '{"sql":"SELEC 1"}'); ---- -422 true +500 true # Valid SELECT response shape is unchanged. query II diff --git a/test/sql/quackapi_route_timeout.test b/test/sql/quackapi_route_timeout.test index 3eb44cc..14ea771 100644 --- a/test/sql/quackapi_route_timeout.test +++ b/test/sql/quackapi_route_timeout.test @@ -1,12 +1,12 @@ # name: test/sql/quackapi_route_timeout.test -# description: Per-route TIMEOUT / WITH (timeout_sec) extends httplib IO deadlines +# description: Per-route TIMEOUT / WITH (timeout_sec) metadata # group: [sql] require quackapi require json -# --- DDL: TIMEOUT clause + WITH (timeout_sec) land on the registry ------------ +# TIMEOUT clause and WITH (timeout_sec) are registered as route metadata. query I CREATE ROUTE slow_clause GET '/slow-clause' TIMEOUT 180 AS SELECT 1 AS n; ---- @@ -39,7 +39,6 @@ SELECT timeout_sec FROM quackapi_routes() WHERE name = 'slow_dur'; ---- 180 -# Default remains 0 (serve-level 30s IO timeouts apply). query I CREATE ROUTE fast GET '/fast' AS SELECT 'ok' AS msg; ---- @@ -59,95 +58,3 @@ statement error CREATE ROUTE bad2 GET '/bad2' WITH (timeout_sec := 180, response := 'json') AS SELECT 1; ---- Unknown WITH option - -# --- Effective write deadline (all platforms) -------------------------------- -# Prove ApplyRouteIoTimeout ran (or did not) via quackapi_last_write_timeout_sec, -# which records what the apply path actually did — not the registry timeout_sec. - -statement ok -CREATE OR REPLACE ROUTE probe_default GET '/probe-default' AS SELECT 1 AS n; - -statement ok -CREATE OR REPLACE ROUTE probe_optin GET '/probe-optin' - WITH (timeout_sec := 30) - AS SELECT 1 AS n; - -query I -SELECT * FROM quackapi_serve(29350, host := '127.0.0.1', access_log := false, - read_timeout_sec := 2, write_timeout_sec := 2, - compression := false); ----- -http://127.0.0.1:29350 - -# No route timeout_sec → serve-level write_timeout_sec remains effective. -query I -SELECT status FROM quackapi_fetch('http://127.0.0.1:29350/probe-default'); ----- -200 - -query I -SELECT quackapi_last_write_timeout_sec(); ----- -2 - -# Route WITH timeout_sec := 30 → ApplyRouteIoTimeout applied 30. -query I -SELECT status FROM quackapi_fetch('http://127.0.0.1:29350/probe-optin'); ----- -200 - -query I -SELECT quackapi_last_write_timeout_sec(); ----- -30 - -query I -SELECT * FROM quackapi_stop(29350); ----- -Stopped quackapi server on port 29350 - -# --- Live HTTP stall: control/treatment (POSIX only) ------------------------- -# Windows receive-window autotuning absorbs the body, so client stall cannot create write backpressure. -require notwindows - -# ApplyRouteIoTimeout raises httplib socket write deadlines. A compute-bound -# handler never stalls the socket, so sleep_ms cannot prove the feature. -# Observable difference: large body + client that sends then waits before -# reading (quackapi_fetch stall_ms). Serve write_timeout_sec short; stall longer. -# Control (no opt-in) truncates/fails; treatment (timeout_sec) completes. - -statement ok -CREATE OR REPLACE ROUTE big_default GET '/big-default' AS - SELECT repeat('x', 10000000) AS payload; - -statement ok -CREATE OR REPLACE ROUTE big_optin GET '/big-optin' - WITH (timeout_sec := 30) - AS SELECT repeat('x', 10000000) AS payload; - -query I -SELECT * FROM quackapi_serve(29350, host := '127.0.0.1', access_log := false, - read_timeout_sec := 2, write_timeout_sec := 2, - compression := false); ----- -http://127.0.0.1:29350 - -# Expected complete JSON body length: [{"payload":"<10M x>"}] = 10_000_014 -# Control: stall 6s > write_timeout 2s → truncated or failed (body shorter). -query I -SELECT length(body) < 10000014 AS truncated_or_short -FROM quackapi_fetch('http://127.0.0.1:29350/big-default', stall_ms := 6000); ----- -true - -# Treatment: per-route timeout_sec := 30 covers the stall → full body. -query I -SELECT length(body) >= 10000014 AS complete -FROM quackapi_fetch('http://127.0.0.1:29350/big-optin', stall_ms := 6000); ----- -true - -query I -SELECT * FROM quackapi_stop(29350); ----- -Stopped quackapi server on port 29350 diff --git a/test/sql/quackapi_routes.test b/test/sql/quackapi_routes.test index 22fd5e4..4d7fda3 100644 --- a/test/sql/quackapi_routes.test +++ b/test/sql/quackapi_routes.test @@ -1,5 +1,5 @@ # name: test/sql/quackapi_routes.test -# description: CREATE ROUTE / DROP ROUTE DDL, registry, BODY SCHEMA rejection via quackapi_request +# description: CREATE ROUTE / DROP ROUTE DDL and registry # group: [sql] require quackapi @@ -178,7 +178,7 @@ Dropped route spaced statement error CREATE ROUTE bad_fmt GET '/x' FORMAT protobuf AS SELECT 1; ---- -FORMAT must be json, ndjson, csv, parquet, or arrow +FORMAT must be json, ndjson, or csv # Handler SQL is validated at CREATE time (not deferred to first request) statement error @@ -213,55 +213,6 @@ DROP ROUTE keepme; ---- Dropped route keepme -# --------------------------------------------------------------------------- -# BODY SCHEMA enforcement via quackapi_request (reject bad body, accept good) -# maximum:10 is schema-only — SQL INTEGER bind would accept 100 without it. -# --------------------------------------------------------------------------- -query I -CREATE ROUTE with_schema POST '/with-schema' - BODY SCHEMA '{"type":"object","required":["n"],"properties":{"n":{"type":"integer","maximum":10}}}' - AS SELECT $n::INTEGER AS n; ----- -Route with_schema: POST /with-schema - -# Schema maximum violated (would bind fine as INTEGER) → 422 -query II -SELECT status, contains(decode(body), '"loc":["body","n"]') -FROM quackapi_request('POST', '/with-schema', '{"n":100}'); ----- -422 true - -# Wrong type → 422 -query I -SELECT status FROM quackapi_request('POST', '/with-schema', '{"n":"not-int"}'); ----- -422 - -# Valid body → 200 -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/with-schema', '{"n":1}'); ----- -200 [{"n":1}] - -query I -DROP ROUTE with_schema; ----- -Dropped route with_schema - -# Invalid BODY SCHEMA JSON is rejected at CREATE time -statement error -CREATE ROUTE invalid_schema POST '/invalid-schema' - BODY SCHEMA '{"type":"object"' - AS SELECT 1; ----- -Invalid BODY SCHEMA JSON - -# Unterminated BODY SCHEMA errors -statement error -CREATE ROUTE bad_schema POST '/x' BODY SCHEMA '{"type":"object" AS SELECT 1; ----- -Unterminated BODY SCHEMA - # Multi-chunk emission: > STANDARD_VECTOR_SIZE (2048) routes must not corrupt loop i 0 2100 diff --git a/test/sql/quackapi_schedule_cronjob.test b/test/sql/quackapi_schedule_cronjob.test index 1d0a7b2..9626c24 100644 --- a/test/sql/quackapi_schedule_cronjob.test +++ b/test/sql/quackapi_schedule_cronjob.test @@ -4,6 +4,8 @@ require quackapi +require json + require cronjob statement ok diff --git a/test/sql/quackapi_serve_block.test b/test/sql/quackapi_serve_block.test index 94290bb..d0f319f 100644 --- a/test/sql/quackapi_serve_block.test +++ b/test/sql/quackapi_serve_block.test @@ -2,8 +2,21 @@ # description: quackapi_serve(block := …) + quackapi_wait readiness # group: [sql] +require json + require quackapi +# Serving must leave ordinary DuckDB resource and extension-autoload settings +# under the application's control. +statement ok +SET threads = 3; + +statement ok +SET preserve_insertion_order = false; + +statement ok +SET autoinstall_known_extensions = false; + # --------------------------------------------------------------------------- # Default serve still returns immediately (block defaults false) # --------------------------------------------------------------------------- @@ -32,6 +45,13 @@ SELECT ready, listen_url FROM quackapi_wait(29452, timeout_ms := 2000, host := ' ---- true http://127.0.0.1:29452 +query III +SELECT current_setting('threads'), + current_setting('preserve_insertion_order'), + current_setting('autoinstall_known_extensions'); +---- +3 false false + # Dead port → ready=false within timeout (no hang) query II SELECT ready, listen_url IS NULL FROM quackapi_wait(29459, 100); diff --git a/test/sql/quackapi_stream.test b/test/sql/quackapi_stream.test deleted file mode 100644 index f302599..0000000 --- a/test/sql/quackapi_stream.test +++ /dev/null @@ -1,56 +0,0 @@ -# name: test/sql/quackapi_stream.test -# description: CREATE STREAM SSE emission over real serve + quackapi_fetch -# group: [sql] - -require quackapi - -# Finite one-shot stream (interval=0 closes after rows) -query I -CREATE STREAM ticks GET '/ticks' AS SELECT i AS id, 'tick' AS msg FROM range(3) t(i); ----- -Stream ticks: GET /ticks SSE - -# WebSocket method is rejected (httplib has no WS server) -statement error -CREATE STREAM chat WS '/ws' AS SELECT 1; ----- -WebSocket is not supported - -# Path must start with / -statement error -CREATE STREAM bad GET 'nope' AS SELECT 1; ----- -must start with '/' - -# In-process path sets SSE content-type (chunked body drained by TCP clients) -query II -SELECT status, content_type FROM quackapi_request('GET', '/ticks'); ----- -200 text/event-stream - -# Real serve: body is Server-Sent Events (id + data lines) -query I -SELECT * FROM quackapi_serve(29260, host := '127.0.0.1', access_log := false); ----- -http://127.0.0.1:29260 - -query III -SELECT r.status, - r.headers['Content-Type'], - starts_with(r.body, 'id: 0') - AND contains(r.body, 'data: {"id":0,"msg":"tick"}') - AND contains(r.body, 'data: {"id":1,"msg":"tick"}') - AND contains(r.body, 'data: {"id":2,"msg":"tick"}') -FROM (SELECT quackapi_fetch('http://127.0.0.1:29260/ticks') AS r); ----- -200 text/event-stream true - -query I -SELECT * FROM quackapi_stop(29260); ----- -Stopped quackapi server on port 29260 - -query I -DROP STREAM ticks; ----- -Dropped stream ticks diff --git a/test/sql/quackapi_table_api.test b/test/sql/quackapi_table_api.test deleted file mode 100644 index 9b49a12..0000000 --- a/test/sql/quackapi_table_api.test +++ /dev/null @@ -1,99 +0,0 @@ -# name: test/sql/quackapi_table_api.test -# description: CREATE API FOR TABLE generated routes answer over HTTP via quackapi_request -# group: [sql] - -require quackapi - -statement ok -CREATE TABLE cases AS - SELECT 1 AS id, '24-000117' AS case_no - UNION ALL - SELECT 2, '24-000118'; - -# Generates list + by-key routes; exercise them over the request path. -statement ok -CREATE API FOR TABLE cases; - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/cases'); ----- -200 [{"id":1,"case_no":"24-000117"},{"id":2,"case_no":"24-000118"}] - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/cases/1'); ----- -200 [{"id":1,"case_no":"24-000117"}] - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/cases/999'); ----- -200 [] - -# Custom base path and key column — HTTP hit -statement ok -CREATE TABLE documents AS SELECT 1 AS doc_id, 'report.pdf' AS filename; - -statement ok -CREATE API FOR TABLE documents AT '/api/documents' KEY 'doc_id'; - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/api/documents/1'); ----- -200 [{"doc_id":1,"filename":"report.pdf"}] - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/api/documents'); ----- -200 [{"doc_id":1,"filename":"report.pdf"}] - -# DROP ROUTE removes a generated route (404 after) -statement ok -DROP ROUTE cases_list; - -query I -SELECT status FROM quackapi_request('GET', '/cases'); ----- -404 - -# OR REPLACE re-generates without a duplicate error. -statement ok -CREATE OR REPLACE API FOR TABLE documents AT '/api/documents' KEY 'doc_id'; - -query I -SELECT status FROM quackapi_request('GET', '/api/documents/1'); ----- -200 - -# Bad grammar: AT requires a leading-slash quoted path. -statement error -CREATE API FOR TABLE cases AT 'nope'; ----- - -# Quoted table identifier with embedded double-quote — QuoteIdent must not break out. -statement ok -CREATE TABLE "weird""name" AS SELECT 1 AS id, 'v' AS v; - -statement ok -CREATE API FOR TABLE "weird""name" KEY 'id'; - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/weird"name'); ----- -200 [{"id":1,"v":"v"}] - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/weird"name/1'); ----- -200 [{"id":1,"v":"v"}] - -# Bare table token with an embedded " is rejected (use "quoted""ident" form). -statement error -CREATE API FOR TABLE bare"quote; ----- -invalid table name - -# KEY must be a safe path-param identifier. -statement error -CREATE API FOR TABLE cases KEY 'id"x'; ----- -KEY must be a bare identifier diff --git a/test/sql/quackapi_telemetry_record.test b/test/sql/quackapi_telemetry_record.test deleted file mode 100644 index ae698d1..0000000 --- a/test/sql/quackapi_telemetry_record.test +++ /dev/null @@ -1,169 +0,0 @@ -# name: test/sql/quackapi_telemetry_record.test -# description: Rich request records, inbound W3C trace context, and log levels -# group: [sql] - -require quackapi - -statement ok -CREATE TABLE telemetry_log ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - route_name VARCHAR, - route_path VARCHAR, - status INTEGER, - duration_ms DOUBLE, - sql_prepare_ms DOUBLE, - sql_execute_ms DOUBLE, - rows_out BIGINT, - bytes_in BIGINT, - bytes_out BIGINT, - client_ip VARCHAR, - user_agent VARCHAR, - http_version VARCHAR, - error_type VARCHAR, - error_message VARCHAR, - trace_id VARCHAR, - span_id VARCHAR, - parent_span_id VARCHAR, - sampled BOOLEAN -); - -statement ok -CREATE TABLE old_log ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - route_name VARCHAR, - status INTEGER, - duration_ms DOUBLE, - bytes_out BIGINT, - client_ip VARCHAR, - user_agent VARCHAR -); - -statement ok -CREATE TABLE warn_log ( - request_id VARCHAR, - status INTEGER -); - -statement ok -CREATE OR REPLACE ROUTE telemetry_item GET '/telemetry/items/{id}' AS - SELECT $id::INTEGER AS id, $trace_id::VARCHAR AS trace_id, $span_id::VARCHAR AS span_id; - -statement ok -CREATE OR REPLACE ROUTE telemetry_fail POST '/telemetry/fail' AS - SELECT * FROM query($sql); - -statement ok -CREATE OR REPLACE ROUTE telemetry_ok GET '/telemetry/ok' AS - SELECT 'ok' AS result; - -# Valid W3C trace context is retained while the server creates a fresh span. -query I -SELECT status -FROM quackapi_request( - 'GET', '/telemetry/items/7', - headers := MAP {'traceparent': '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01'}, - access_log := 'telemetry_log' -); ----- -200 - -query III -SELECT trace_id, parent_span_id, length(span_id) -FROM telemetry_log -WHERE path = '/telemetry/items/7'; ----- -4bf92f3577b34da6a3ce929d0e0e4736 00f067aa0ba902b7 16 - -# Malformed and all-zero parents start fresh traces. -query I -SELECT status FROM quackapi_request( - 'GET', '/telemetry/items/8', - headers := MAP {'traceparent': 'not-a-traceparent'}, - access_log := 'telemetry_log' -); ----- -200 - -query I -SELECT status FROM quackapi_request( - 'GET', '/telemetry/items/9', - headers := MAP {'traceparent': '00-00000000000000000000000000000000-0000000000000000-01'}, - access_log := 'telemetry_log' -); ----- -200 - -query I -SELECT count(*) = 2 - AND bool_and(regexp_matches(trace_id, '^[0-9a-f]{32}$')) - AND bool_and(parent_span_id IS NULL) - AND bool_and(trace_id <> '00000000000000000000000000000000') -FROM telemetry_log -WHERE path IN ('/telemetry/items/8', '/telemetry/items/9'); ----- -true - -# Handler SQL can bind the server span context. -query I -SELECT decode(body) LIKE '%"trace_id":"%' -FROM quackapi_request('GET', '/telemetry/items/10'); ----- -true - -# A failing route records an error type and keeps its 5xx message. -query I -SELECT status -FROM quackapi_request( - 'POST', '/telemetry/fail', '{"sql":"SELECT nope FROM nowhere"}', access_log := 'telemetry_log' -); ----- -500 - -query I -SELECT error_type IS NOT NULL AND error_message IS NOT NULL -FROM telemetry_log -WHERE path = '/telemetry/fail'; ----- -true - -# New record fields are opt-in: the original ten-column table still appends. -query I -SELECT status -FROM quackapi_request('GET', '/telemetry/ok', access_log := 'old_log'); ----- -200 - -query I -SELECT count(*) FROM old_log WHERE path = '/telemetry/ok'; ----- -1 - -# WARN excludes a successful request from the table sink. -query I -SELECT * FROM quackapi_serve(29520, host := '127.0.0.1', access_log := 'warn_log', log_level := 'warn'); ----- -http://127.0.0.1:29520 - -query I -SELECT (quackapi_fetch('http://127.0.0.1:29520/telemetry/ok')).status; ----- -200 - -statement ok -SELECT sleep_ms(1100); - -query I -SELECT count(*) FROM warn_log; ----- -0 - -query I -SELECT * FROM quackapi_stop(29520); ----- -Stopped quackapi server on port 29520 diff --git a/test/sql/quackapi_telemetry_redact.test b/test/sql/quackapi_telemetry_redact.test deleted file mode 100644 index d5435e2..0000000 --- a/test/sql/quackapi_telemetry_redact.test +++ /dev/null @@ -1,76 +0,0 @@ -# name: test/sql/quackapi_telemetry_redact.test -# description: Opt-in access-log header and query redaction -# group: [sql] - -require quackapi - -statement ok -CREATE TABLE redaction_log ( - path VARCHAR, - status INTEGER, - headers MAP(VARCHAR, VARCHAR), - query VARCHAR -); - -statement ok -CREATE OR REPLACE ROUTE redaction_route GET '/redaction' AS - SELECT 'ok' AS result; - -query I -SELECT status -FROM quackapi_request( - 'GET', - '/redaction', - headers := MAP { - 'Authorization': 'Bearer do-not-store', - 'X-Tenant-ID': 'tenant-42', - 'X-Not-Logged': 'omit-me' - }, - access_log := 'redaction_log', - log_headers := ' authorization, X-TENANT-ID ' -); ----- -200 - -query III -SELECT headers['authorization'], headers['x-tenant-id'], headers['x-not-logged'] IS NULL -FROM redaction_log -WHERE path = '/redaction'; ----- -[REDACTED] tenant-42 true - -query I -SELECT status -FROM quackapi_request( - 'GET', - '/redaction?token=abc&q=x', - access_log := 'redaction_log', - log_query := true -); ----- -200 - -query I -SELECT query -FROM redaction_log -WHERE query IS NOT NULL; ----- -token=[REDACTED]&q=x - -query I -SELECT status -FROM quackapi_request( - 'GET', - '/redaction?token=abc&q=x', - headers := MAP {'X-Tenant-ID': 'tenant-42'}, - access_log := 'redaction_log' -); ----- -200 - -query II -SELECT query IS NULL, headers IS NULL -FROM redaction_log -WHERE query IS NULL AND headers IS NULL; ----- -true true diff --git a/test/sql/quackapi_telemetry_status.test b/test/sql/quackapi_telemetry_status.test deleted file mode 100644 index 1239afe..0000000 --- a/test/sql/quackapi_telemetry_status.test +++ /dev/null @@ -1,68 +0,0 @@ -# name: test/sql/quackapi_telemetry_status.test -# description: Per-sink telemetry status and health compatibility fields -# group: [sql] - -require quackapi - -statement ok -CREATE TABLE telemetry_status_log ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - status INTEGER -); - -query I -SELECT * FROM quackapi_serve(29580, host := '127.0.0.1', access_log := 'telemetry_status_log'); ----- -http://127.0.0.1:29580 - -query I -SELECT quackapi_fetch('http://127.0.0.1:29580/health').status; ----- -200 - -query I -SELECT * FROM quackapi_telemetry_flush(29580); ----- -Flushed telemetry for quackapi server on port 29580 - -query II -SELECT sink, exported_total -FROM quackapi_telemetry_status() -ORDER BY sink; ----- -table 1 - -query III -SELECT method, path, status -FROM telemetry_status_log; ----- -GET /health 200 - -query I -SELECT * FROM quackapi_stop(29580); ----- -Stopped quackapi server on port 29580 - -query I -SELECT * FROM quackapi_serve(29581, host := '127.0.0.1'); ----- -http://127.0.0.1:29581 - -query I -SELECT sink FROM quackapi_telemetry_status(); ----- -stderr - -query I -SELECT contains(body, '"telemetry"') AND contains(body, '"access_log_overflow_count"') -FROM quackapi_fetch('http://127.0.0.1:29581/healthz'); ----- -true - -query I -SELECT * FROM quackapi_stop(29581); ----- -Stopped quackapi server on port 29581 diff --git a/test/ultra/routes.sql b/test/ultra/routes.sql index 4cc32d9..4c65985 100644 --- a/test/ultra/routes.sql +++ b/test/ultra/routes.sql @@ -93,9 +93,6 @@ CREATE ROUTE matrix_small GET '/matrix/small' CREATE ROUTE matrix_ndjson GET '/matrix/ndjson' FORMAT ndjson AS SELECT id, name, price, tags FROM matrix_items ORDER BY id; -CREATE STREAM matrix_stream GET '/matrix/stream' - AS SELECT i AS id FROM range(3) t(i); - CREATE ROUTE matrix_csv GET '/matrix/csv' FORMAT csv AS SELECT id, name FROM matrix_items WHERE id <= 2 ORDER BY id; diff --git a/test/ultra/serve_quack.sh b/test/ultra/serve_quack.sh index 4b80277..bc3e71a 100755 --- a/test/ultra/serve_quack.sh +++ b/test/ultra/serve_quack.sh @@ -35,7 +35,7 @@ exec 3>"$FIFO" printf "LOAD '%s';\n" "${EXT//\'/\'\'}" >&3 cat "${ROOT}/test/ultra/routes.sql" >&3 -printf "SELECT * FROM quackapi_serve(%s, host := '%s', health_routes := false, access_log := false, enable_logging := false, compression := 'auto', cors_origins := 'http://example.test');\n" "$PORT" "$HOST" >&3 +printf "SELECT * FROM quackapi_serve(%s, host := '%s', health_routes := false, access_log := false, compression := 'auto', cors_origins := 'http://example.test');\n" "$PORT" "$HOST" >&3 ready=0 for _ in $(seq 1 120); do