From 7f52a4e640ed6920bfe055e833bff1cea32d7cfe Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 11:58:01 -0700 Subject: [PATCH 01/93] refactor(server): extract binding, encoding, and logging seams --- src/include/quackapi_logging.hpp | 9 + src/include/quackapi_request_binding.hpp | 31 + src/include/quackapi_response_encoding.hpp | 33 + src/quackapi_logging.cpp | 38 ++ src/quackapi_request_binding.cpp | 217 ++++++ src/quackapi_response_encoding.cpp | 519 ++++++++++++++ src/quackapi_server.cpp | 753 +-------------------- 7 files changed, 854 insertions(+), 746 deletions(-) create mode 100644 src/include/quackapi_logging.hpp create mode 100644 src/include/quackapi_request_binding.hpp create mode 100644 src/include/quackapi_response_encoding.hpp create mode 100644 src/quackapi_logging.cpp create mode 100644 src/quackapi_request_binding.cpp create mode 100644 src/quackapi_response_encoding.cpp diff --git a/src/include/quackapi_logging.hpp b/src/include/quackapi_logging.hpp new file mode 100644 index 0000000..040f17b --- /dev/null +++ b/src/include/quackapi_logging.hpp @@ -0,0 +1,9 @@ +#pragma once + +namespace duckdb { + +struct QuackapiRequestRecord; + +void EmitAccessLogStderr(const QuackapiRequestRecord &entry); + +} // namespace duckdb diff --git a/src/include/quackapi_request_binding.hpp b/src/include/quackapi_request_binding.hpp new file mode 100644 index 0000000..8d7588d --- /dev/null +++ b/src/include/quackapi_request_binding.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include "duckdb/common/case_insensitive_map.hpp" +#include "duckdb/common/types.hpp" +#include "httplib.hpp" + +namespace duckdb { + +//! Preserve whether a JSON member was an actual JSON null. A string value of +//! "null" remains a string and must not silently become SQL NULL. +struct JsonBodyField { + string value; + bool explicit_null; + + JsonBodyField() : explicit_null(false) { + } + JsonBodyField(string value_p, bool explicit_null_p) : value(std::move(value_p)), explicit_null(explicit_null_p) { + } +}; + +bool IsJsonMediaType(const string &media); +bool IsFormUrlEncodedMediaType(const string &media); +bool IsMultipartMediaType(const string &media); +bool IsBodyMethod(const string &method); +void ParseFormUrlEncoded(const string &body, case_insensitive_map_t &out); +bool TryExtractFlatJsonObject(const string &raw_body, case_insensitive_map_t &fields); +void ParseQueryStringIntoParams(const string &query, duckdb_httplib::Params ¶ms); + +} // namespace duckdb diff --git a/src/include/quackapi_response_encoding.hpp b/src/include/quackapi_response_encoding.hpp new file mode 100644 index 0000000..b01a803 --- /dev/null +++ b/src/include/quackapi_response_encoding.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include "duckdb/common/types.hpp" +#include "duckdb/common/types/value.hpp" +#include "httplib.hpp" + +namespace duckdb { + +class Connection; +struct QuackapiRoute; + +enum class ResponseMode { JSON, HTML, TEXT }; +enum class BodyFormat { JSON, NDJSON, CSV, PARQUET, ARROW }; + +string ValueToJson(const Value &value); +bool IsSpecialResponseColumn(const string &name); +void SetJson(duckdb_httplib::Response &res, int status, const string &body); +void SetInternalError(duckdb_httplib::Response &res, const string &server_side_detail); +ResponseMode ResponseModeFor(const vector &names); +BodyFormat ResolveBodyFormat(const QuackapiRoute &route, const duckdb_httplib::Request &req); +string EmptyResultBody(const QuackapiRoute &route); +string SerializeRowsJsonArray(const vector &names, const vector &data_cols, + const vector> &rows); +string SerializeRowJsonObject(const vector &names, const vector &data_cols, const vector &cols); +string SerializeRowsNdjson(const vector &names, const vector &data_cols, + const vector> &rows); +string SerializeRowsCsv(const vector &names, const vector &data_cols, const vector> &rows); +string SerializeRowsParquet(Connection &con, const vector &names, const vector &types, + const vector &data_cols, const vector> &rows); +string SerializeRowsArrow(Connection &con, const vector &names, const vector &types, + const vector &data_cols, const vector> &rows); + +} // namespace duckdb diff --git a/src/quackapi_logging.cpp b/src/quackapi_logging.cpp new file mode 100644 index 0000000..af1d71f --- /dev/null +++ b/src/quackapi_logging.cpp @@ -0,0 +1,38 @@ +#include "quackapi_logging.hpp" + +#include + +#include "quackapi_server.hpp" +#include "quackapi_util.hpp" + +namespace duckdb { +string JsonEscape(const string &value) { + return QuackapiJsonEscape(value); +} + +void EmitAccessLogStderr(const QuackapiRequestRecord &entry) { + string path_esc; + path_esc.reserve(entry.path.size() + 8); + for (unsigned char c : entry.path) { + if (c == '"' || c == '\\') { + path_esc += '\\'; + path_esc += static_cast(c); + } else if (c < 0x20) { + char buf[8]; + snprintf(buf, sizeof(buf), "\\u%04x", c); + path_esc += buf; + } else { + path_esc += static_cast(c); + } + } + fprintf(stderr, + "{\"type\":\"access\",\"method\":\"%s\",\"path\":\"%s\",\"status\":%d," + "\"latency_ms\":%.3f,\"request_id\":\"%s\",\"bytes\":%llu,\"client_ip\":\"%s\"," + "\"user_agent\":\"%s\",\"route_path\":\"%s\",\"trace_id\":\"%s\"}\n", + JsonEscape(entry.method).c_str(), path_esc.c_str(), entry.status, entry.duration_ms, + JsonEscape(entry.request_id).c_str(), (unsigned long long)entry.bytes_out, + JsonEscape(entry.client_ip).c_str(), JsonEscape(entry.user_agent).c_str(), + JsonEscape(entry.route_path).c_str(), JsonEscape(entry.trace_id).c_str()); +} + +} // namespace duckdb diff --git a/src/quackapi_request_binding.cpp b/src/quackapi_request_binding.cpp new file mode 100644 index 0000000..1646da6 --- /dev/null +++ b/src/quackapi_request_binding.cpp @@ -0,0 +1,217 @@ +#include "quackapi_request_binding.hpp" + +#include "duckdb/common/string_util.hpp" + +namespace duckdb { +bool IsJsonMediaType(const string &media) { + return media == "application/json" || StringUtil::EndsWith(media, "+json"); +} + +bool IsFormUrlEncodedMediaType(const string &media) { + return media == "application/x-www-form-urlencoded"; +} + +bool IsMultipartMediaType(const string &media) { + return StringUtil::StartsWith(media, "multipart/form-data"); +} + +bool IsBodyMethod(const string &method) { + return method == "POST" || method == "PUT" || method == "PATCH"; +} + +//! application/x-www-form-urlencoded → key/value map (last wins). +void ParseFormUrlEncoded(const string &body, case_insensitive_map_t &out) { + idx_t i = 0; + while (i < body.size()) { + idx_t amp = body.find('&', i); + if (amp == string::npos) { + amp = body.size(); + } + string pair = body.substr(i, amp - i); + if (!pair.empty()) { + auto eq = pair.find('='); + string key, val; + if (eq == string::npos) { + key = duckdb_httplib::decode_query_component(pair, true); + } else { + key = duckdb_httplib::decode_query_component(pair.substr(0, eq), true); + val = duckdb_httplib::decode_query_component(pair.substr(eq + 1), true); + } + if (!key.empty()) { + out[key] = val; + } + } + i = amp + 1; + } +} +//! Fast path: flat JSON object of string/number/bool/null scalars (no DuckDB round-trip). +//! Returns true on success. false → caller may fall back to SQL extract or error. +bool TryExtractFlatJsonObject(const string &raw_body, case_insensitive_map_t &fields) { + const char *s = raw_body.c_str(); + const char *end = s + raw_body.size(); + auto skip_ws = [&]() { + while (s < end && (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')) { + s++; + } + }; + auto is_digit = [](char c) { + return c >= '0' && c <= '9'; + }; + // This path deliberately accepts only unescaped JSON strings. Escapes require + // full JSON decoding (in particular, UTF-16 surrogate handling), so leave them + // to DuckDB instead of attempting a partial implementation here. + auto parse_plain_string = [&](string &out) { + if (s >= end || *s != '"') { + return false; + } + s++; + while (s < end && *s != '"') { + const auto ch = static_cast(*s); + if (ch < 0x20 || *s == '\\') { + return false; + } + out.push_back(*s++); + } + if (s >= end) { + return false; + } + s++; + return true; + }; + auto parse_number = [&]() { + const char *start = s; + if (s < end && *s == '-') { + s++; + } + if (s >= end) { + return false; + } + if (*s == '0') { + s++; + if (s < end && is_digit(*s)) { + return false; + } + } else if (*s >= '1' && *s <= '9') { + do { + s++; + } while (s < end && is_digit(*s)); + } else { + return false; + } + if (s < end && *s == '.') { + s++; + if (s >= end || !is_digit(*s)) { + return false; + } + do { + s++; + } while (s < end && is_digit(*s)); + } + if (s < end && (*s == 'e' || *s == 'E')) { + s++; + if (s < end && (*s == '+' || *s == '-')) { + s++; + } + if (s >= end || !is_digit(*s)) { + return false; + } + do { + s++; + } while (s < end && is_digit(*s)); + } + return s > start; + }; + skip_ws(); + if (s >= end || *s != '{') { + return false; + } + s++; + fields.clear(); + skip_ws(); + if (s < end && *s == '}') { + s++; + skip_ws(); + return s == end; + } + while (s < end) { + skip_ws(); + string key; + if (!parse_plain_string(key)) { + return false; + } + skip_ws(); + if (s >= end || *s != ':') { + return false; + } + s++; + skip_ws(); + if (s >= end) { + return false; + } + string val; + bool explicit_null = false; + if (*s == '"') { + if (!parse_plain_string(val)) { + return false; + } + } else if (*s == 'n' && s + 4 <= end && string(s, 4) == "null") { + explicit_null = true; + s += 4; + } else if (*s == 't' && s + 4 <= end && string(s, 4) == "true") { + val = "true"; + s += 4; + } else if (*s == 'f' && s + 5 <= end && string(s, 5) == "false") { + val = "false"; + s += 5; + } else if (*s == '-' || (*s >= '0' && *s <= '9')) { + const char *start = s; + if (!parse_number()) { + return false; + } + val.assign(start, s - start); + } else { + // nested object/array — not flat; fall back + return false; + } + fields[key] = {val, explicit_null}; + skip_ws(); + if (s < end && *s == ',') { + s++; + continue; + } + if (s < end && *s == '}') { + s++; + skip_ws(); + return s == end; + } + return false; + } + return false; +} +//! Parse application/x-www-form-urlencoded query into httplib Params. +void ParseQueryStringIntoParams(const string &query, duckdb_httplib::Params ¶ms) { + if (query.empty()) { + return; + } + idx_t start = 0; + while (start < query.size()) { + idx_t amp = query.find('&', start); + if (amp == string::npos) { + amp = query.size(); + } + string pair = query.substr(start, amp - start); + start = amp + 1; + if (pair.empty()) { + continue; + } + auto eq = pair.find('='); + if (eq == string::npos) { + params.emplace(duckdb_httplib::decode_query_component(pair, true), string()); + } else { + params.emplace(duckdb_httplib::decode_query_component(pair.substr(0, eq), true), + duckdb_httplib::decode_query_component(pair.substr(eq + 1), true)); + } + } +} + +} // namespace duckdb diff --git a/src/quackapi_response_encoding.cpp b/src/quackapi_response_encoding.cpp new file mode 100644 index 0000000..0797308 --- /dev/null +++ b/src/quackapi_response_encoding.cpp @@ -0,0 +1,519 @@ +#include "quackapi_response_encoding.hpp" + +#include +#include +#include + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/file_open_flags.hpp" +#include "duckdb/common/file_system.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/uuid.hpp" +#include "duckdb/main/appender.hpp" +#include "duckdb/main/connection.hpp" +#include "duckdb/parser/keyword_helper.hpp" +#include "quackapi_state.hpp" +#include "quackapi_util.hpp" + +namespace duckdb { +namespace { + +string TrimMediaToken(string value) { + StringUtil::Trim(value); + return value; +} + +} // namespace + +//! Render a DuckDB Value as JSON. The database typed the column — the JSON +//! representation follows the type, not string-formatting heuristics. +string ValueToJson(const Value &value) { + if (value.IsNull()) { + return "null"; + } + auto &type = value.type(); + switch (type.id()) { + case LogicalTypeId::BOOLEAN: + return value.GetValue() ? "true" : "false"; + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::HUGEINT: + case LogicalTypeId::UTINYINT: + case LogicalTypeId::USMALLINT: + case LogicalTypeId::UINTEGER: + case LogicalTypeId::UBIGINT: + case LogicalTypeId::DECIMAL: + return value.ToString(); + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: { + auto d = value.GetValue(); + if (std::isnan(d) || std::isinf(d)) { + // JSON has no NaN/Infinity — mirror FastAPI/ujson and emit null + return "null"; + } + return value.ToString(); + } + case LogicalTypeId::LIST: { + auto &children = ListValue::GetChildren(value); + string result = "["; + for (idx_t i = 0; i < children.size(); i++) { + if (i > 0) { + result += ","; + } + result += ValueToJson(children[i]); + } + result += "]"; + return result; + } + case LogicalTypeId::STRUCT: { + auto &children = StructValue::GetChildren(value); + auto &child_types = StructType::GetChildTypes(type); + string result = "{"; + for (idx_t i = 0; i < children.size(); i++) { + if (i > 0) { + result += ","; + } + result += "\"" + QuackapiJsonEscape(child_types[i].first) + "\":" + ValueToJson(children[i]); + } + result += "}"; + return result; + } + default: + return "\"" + QuackapiJsonEscape(value.ToString()) + "\""; + } +} + +//! True if column name is a special response control column (stripped from body). +bool IsSpecialResponseColumn(const string &name) { + auto lower = StringUtil::Lower(name); + return lower == "location" || lower == "set_cookie" || lower == "set-cookie"; +} + +void SetJson(duckdb_httplib::Response &res, int status, const string &body) { + res.status = status; + res.set_content(body, "application/json"); +} + +//! Response mode inferred from the single output column's name — the database +//! names the payload. `html` -> text/html, `text` -> text/plain; anything else +//! serializes as row data (json / ndjson / csv / parquet / arrow via FORMAT + Accept). + +ResponseMode ResponseModeFor(const vector &names) { + if (names.size() != 1) { + return ResponseMode::JSON; + } + auto lower = StringUtil::Lower(names[0]); + if (lower == "html") { + return ResponseMode::HTML; + } + if (lower == "text") { + return ResponseMode::TEXT; + } + return ResponseMode::JSON; +} + +//! Wire format for multi-row data responses (not html/text column modes). + +BodyFormat ParseBodyFormatToken(const string &token) { + auto t = StringUtil::Lower(token); + if (t == "ndjson") { + return BodyFormat::NDJSON; + } + if (t == "csv") { + return BodyFormat::CSV; + } + if (t == "parquet") { + return BodyFormat::PARQUET; + } + if (t == "arrow" || t == "arrows") { + return BodyFormat::ARROW; + } + return BodyFormat::JSON; +} + +//! Map Accept media-type (no parameters) to BodyFormat. Returns false if unknown. +bool MediaTypeToBodyFormat(const string &media, BodyFormat &out) { + auto m = StringUtil::Lower(media); + // strip parameters already; still trim + m = TrimMediaToken(m); + if (m == "application/x-ndjson" || m == "application/jsonl" || m == "application/ndjson") { + out = BodyFormat::NDJSON; + return true; + } + if (m == "text/csv" || m == "application/csv") { + out = BodyFormat::CSV; + return true; + } + if (m == "application/vnd.apache.parquet" || m == "application/parquet") { + out = BodyFormat::PARQUET; + return true; + } + // Arrow IPC stream (what nanoarrow FORMAT ARROWS emits) or file. + if (m == "application/vnd.apache.arrow.stream" || m == "application/vnd.apache.arrow.file" || + m == "application/x-apache-arrow-stream" || m == "application/arrow") { + out = BodyFormat::ARROW; + return true; + } + if (m == "application/json" || m == "text/json" || m == "*/*") { + out = BodyFormat::JSON; + return true; + } + return false; +} + +//! Parse one Accept token's q-value (default 1.0). +double ParseAcceptQ(const string &token) { + auto parts = StringUtil::Split(token, ';'); + if (parts.size() < 2) { + return 1.0; + } + for (idx_t i = 1; i < parts.size(); i++) { + auto p = TrimMediaToken(parts[i]); + auto pl = StringUtil::Lower(p); + if (StringUtil::StartsWith(pl, "q=")) { + return atof(p.c_str() + 2); + } + } + return 1.0; +} + +//! When route FORMAT is default/json, honor Accept for ndjson/csv/parquet/arrow/json. +//! Explicit route FORMAT (ndjson|csv|parquet|arrow) always wins. +BodyFormat ResolveBodyFormat(const QuackapiRoute &route, const duckdb_httplib::Request &req) { + auto route_fmt = StringUtil::Lower(route.response_format.empty() ? "json" : route.response_format); + if (route_fmt == "ndjson" || route_fmt == "csv" || route_fmt == "parquet" || route_fmt == "arrow" || + route_fmt == "arrows") { + return ParseBodyFormatToken(route_fmt); + } + // Default / explicit json: negotiate Accept when present. + auto it = req.headers.find("Accept"); + if (it == req.headers.end() || it->second.empty()) { + return BodyFormat::JSON; + } + // Pick the highest-q recognized type among json / ndjson / csv / parquet / arrow. + BodyFormat best = BodyFormat::JSON; + double best_q = -1.0; + bool saw_recognized = false; + auto tokens = StringUtil::Split(it->second, ','); + for (auto &tok : tokens) { + auto t = TrimMediaToken(tok); + if (t.empty()) { + continue; + } + double q = ParseAcceptQ(t); + if (q <= 0.0) { + continue; + } + auto semi = t.find(';'); + string media = semi == string::npos ? t : TrimMediaToken(t.substr(0, semi)); + BodyFormat f; + if (!MediaTypeToBodyFormat(media, f)) { + continue; + } + // Prefer more specific non-wildcard when q ties: ndjson/csv/parquet/arrow over */* json. + bool better = !saw_recognized || q > best_q; + if (!better && q == best_q) { + // Prefer non-json when client listed it at same q after json — first highest wins + // unless current best is from */* and this is concrete. + if (best == BodyFormat::JSON && f != BodyFormat::JSON) { + better = true; + } + } + if (better) { + best = f; + best_q = q; + saw_recognized = true; + } + } + return saw_recognized ? best : BodyFormat::JSON; +} + +//! RFC 4180-style CSV field escape (quote when needed). +string CsvEscapeField(const string &input) { + bool need_quote = false; + for (unsigned char c : input) { + if (c == '"' || c == ',' || c == '\n' || c == '\r') { + need_quote = true; + break; + } + } + if (!need_quote) { + return input; + } + string out; + out.reserve(input.size() + 2); + out += '"'; + for (unsigned char c : input) { + if (c == '"') { + out += "\"\""; + } else { + out += static_cast(c); + } + } + out += '"'; + return out; +} + +//! Cell text for CSV: null → empty; scalars ToString; nested → JSON text. +string ValueToCsvCell(const Value &value) { + if (value.IsNull()) { + return ""; + } + auto id = value.type().id(); + if (id == LogicalTypeId::LIST || id == LogicalTypeId::STRUCT || id == LogicalTypeId::MAP) { + return ValueToJson(value); + } + return value.ToString(); +} + +string SerializeRowsJsonArray(const vector &names, const vector &data_cols, + const vector> &rows) { + string body = "["; + bool first_row = true; + for (auto &cols : rows) { + if (!first_row) { + body += ","; + } + first_row = false; + body += "{"; + bool first_col = true; + for (auto col : data_cols) { + if (!first_col) { + body += ","; + } + first_col = false; + body += "\"" + QuackapiJsonEscape(names[col]) + "\":" + ValueToJson(cols[col]); + } + body += "}"; + } + body += "]"; + return body; +} + +//! Single-row JSON object (ENVELOPE object). Caller guarantees rows.size() == 1. +string SerializeRowJsonObject(const vector &names, const vector &data_cols, const vector &cols) { + string body = "{"; + bool first_col = true; + for (auto col : data_cols) { + if (!first_col) { + body += ","; + } + first_col = false; + body += "\"" + QuackapiJsonEscape(names[col]) + "\":" + ValueToJson(cols[col]); + } + body += "}"; + return body; +} + +string EmptyResultBody(const QuackapiRoute &route) { + if (!route.empty_body.empty()) { + return route.empty_body; + } + return "{\"detail\":\"Not Found\"}"; +} + +string SerializeRowsNdjson(const vector &names, const vector &data_cols, + const vector> &rows) { + string body; + for (auto &cols : rows) { + body += "{"; + bool first_col = true; + for (auto col : data_cols) { + if (!first_col) { + body += ","; + } + first_col = false; + body += "\"" + QuackapiJsonEscape(names[col]) + "\":" + ValueToJson(cols[col]); + } + body += "}\n"; + } + return body; +} + +string SerializeRowsCsv(const vector &names, const vector &data_cols, + const vector> &rows) { + string body; + // header + for (idx_t i = 0; i < data_cols.size(); i++) { + if (i > 0) { + body += ","; + } + body += CsvEscapeField(names[data_cols[i]]); + } + body += "\n"; + for (auto &cols : rows) { + for (idx_t i = 0; i < data_cols.size(); i++) { + if (i > 0) { + body += ","; + } + auto col = data_cols[i]; + string cell; + if (col < cols.size()) { + cell = ValueToCsvCell(cols[col]); + } + body += CsvEscapeField(cell); + } + body += "\n"; + } + return body; +} + +//! Shared: TEMP table + Appender fill for binary serdes (parquet / arrow). +void FillTempTableForSerdes(Connection &con, const string &table, const vector &names, + const vector &types, const vector &data_cols, + const vector> &rows, const char *label) { + string create_sql = "CREATE TEMP TABLE " + table + " ("; + for (idx_t i = 0; i < data_cols.size(); i++) { + if (i > 0) { + create_sql += ", "; + } + auto c = data_cols[i]; + create_sql += KeywordHelper::WriteOptionallyQuoted(names[c]); + create_sql += " "; + create_sql += types[c].ToString(); + } + create_sql += ")"; + auto create_res = con.Query(create_sql); + if (create_res->HasError()) { + throw InvalidInputException("%s serialize CREATE: %s", label, create_res->GetError()); + } + + if (!rows.empty()) { + Appender appender(con, table); + for (auto &cols : rows) { + appender.BeginRow(); + for (auto c : data_cols) { + if (c < cols.size()) { + appender.Append(cols[c]); + } else { + appender.Append(Value()); + } + } + appender.EndRow(); + } + appender.Close(); + } +} + +//! OS temp dir + basename for parquet/arrow scratch files. Never hardcode /tmp +//! (Windows CI has no /tmp). Paths use forward slashes for SQL COPY literals. +string SerdesTempFilePath(FileSystem &fs, const string &filename) { + string dir; +#if defined(_WIN32) || defined(WIN32) + const char *t = std::getenv("TEMP"); + if (!t || !t[0]) { + t = std::getenv("TMP"); + } + dir = (t && t[0]) ? string(t) : string("."); +#else + const char *t = std::getenv("TMPDIR"); + dir = (t && t[0]) ? string(t) : string("/tmp"); +#endif + string path = fs.JoinPath(dir, filename); + return StringUtil::Replace(path, "\\", "/"); +} + +string SqlQuotePath(const string &path) { + return StringUtil::Replace(path, "'", "''"); +} + +//! Read path bytes then delete the file. On open failure, best-effort remove. +string ReadAndRemoveFileBytes(Connection &con, const string &path) { + auto &fs = FileSystem::GetFileSystem(*con.context); + unique_ptr handle; + try { + handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ); + } catch (...) { + fs.TryRemoveFile(path); + throw; + } + idx_t file_size = handle->GetFileSize(); + string body; + if (file_size > 0) { + // mutable buffer — data_ptr_cast rejects const char* from string::data() + vector buf(file_size); + handle->Read(buf.data(), file_size); + body.assign(reinterpret_cast(buf.data()), file_size); + } + handle->Close(); + fs.TryRemoveFile(path); + return body; +} + +//! Serialize result rows as a Parquet file body (magic "PAR1"). Uses a unique +//! TEMP table + COPY TO (FORMAT PARQUET) + read bytes, then cleans up. +string SerializeRowsParquet(Connection &con, const vector &names, const vector &types, + const vector &data_cols, const vector> &rows) { + if (data_cols.empty()) { + throw InvalidInputException("parquet serialize: no data columns"); + } + auto &fs = FileSystem::GetFileSystem(*con.context); + string uid = StringUtil::Replace(UUID::ToString(UUIDv7::GenerateRandomUUID()), "-", ""); + string table = "qa_parquet_" + uid; + string path = SerdesTempFilePath(fs, table + ".parquet"); + + FillTempTableForSerdes(con, table, names, types, data_cols, rows, "parquet"); + + auto copy_res = con.Query("COPY " + table + " TO '" + SqlQuotePath(path) + "' (FORMAT PARQUET)"); + if (copy_res->HasError()) { + con.Query("DROP TABLE IF EXISTS " + table); + fs.TryRemoveFile(path); + throw InvalidInputException("parquet serialize COPY: %s", copy_res->GetError()); + } + con.Query("DROP TABLE IF EXISTS " + table); + return ReadAndRemoveFileBytes(con, path); +} + +//! Ensure community nanoarrow is available (LOAD, else INSTALL FROM community). +void EnsureNanoarrowLoaded(Connection &con) { + auto load = con.Query("LOAD nanoarrow"); + if (!load->HasError()) { + return; + } + auto inst = con.Query("INSTALL nanoarrow FROM community"); + if (inst->HasError()) { + throw InvalidInputException( + "arrow serialize: nanoarrow not available (%s). INSTALL nanoarrow FROM community and retry.", + inst->GetError()); + } + load = con.Query("LOAD nanoarrow"); + if (load->HasError()) { + throw InvalidInputException("arrow serialize: could not LOAD nanoarrow: %s", load->GetError()); + } +} + +//! Serialize result rows as Arrow IPC stream bytes (nanoarrow FORMAT ARROWS). +//! Magic is 0xFFFFFFFF stream continuation (not ARROW1 file). Content-Type: +//! application/vnd.apache.arrow.stream. Same TEMP+COPY+read pattern as parquet. +string SerializeRowsArrow(Connection &con, const vector &names, const vector &types, + const vector &data_cols, const vector> &rows) { + if (data_cols.empty()) { + throw InvalidInputException("arrow serialize: no data columns"); + } + EnsureNanoarrowLoaded(con); + auto &fs = FileSystem::GetFileSystem(*con.context); + string uid = StringUtil::Replace(UUID::ToString(UUIDv7::GenerateRandomUUID()), "-", ""); + string table = "qa_arrow_" + uid; + string path = SerdesTempFilePath(fs, table + ".arrows"); + + FillTempTableForSerdes(con, table, names, types, data_cols, rows, "arrow"); + + auto copy_res = con.Query("COPY " + table + " TO '" + SqlQuotePath(path) + "' (FORMAT ARROWS)"); + if (copy_res->HasError()) { + con.Query("DROP TABLE IF EXISTS " + table); + fs.TryRemoveFile(path); + throw InvalidInputException("arrow serialize COPY: %s", copy_res->GetError()); + } + con.Query("DROP TABLE IF EXISTS " + table); + return ReadAndRemoveFileBytes(con, path); +} + +void SetInternalError(duckdb_httplib::Response &res, const string &server_side_detail) { + // Return DuckDB's diagnostic so clients can diagnose route execution failures. + fprintf(stderr, "quackapi: internal error: %s\n", server_side_detail.c_str()); + SetJson(res, 500, "{\"detail\":\"" + QuackapiJsonEscape(server_side_detail) + "\"}"); +} + +} // namespace duckdb diff --git a/src/quackapi_server.cpp b/src/quackapi_server.cpp index 386c26c..0ffcdc7 100644 --- a/src/quackapi_server.cpp +++ b/src/quackapi_server.cpp @@ -1,5 +1,9 @@ #include "quackapi_server.hpp" +#include "quackapi_logging.hpp" +#include "quackapi_request_binding.hpp" +#include "quackapi_response_encoding.hpp" + #include #include #include @@ -283,66 +287,6 @@ bool CompressGzip(const string &input, string &output) { } } -//! Render a DuckDB Value as JSON. The database typed the column — the JSON -//! representation follows the type, not string-formatting heuristics. -string ValueToJson(const Value &value) { - if (value.IsNull()) { - return "null"; - } - auto &type = value.type(); - switch (type.id()) { - case LogicalTypeId::BOOLEAN: - return value.GetValue() ? "true" : "false"; - case LogicalTypeId::TINYINT: - case LogicalTypeId::SMALLINT: - case LogicalTypeId::INTEGER: - case LogicalTypeId::BIGINT: - case LogicalTypeId::HUGEINT: - case LogicalTypeId::UTINYINT: - case LogicalTypeId::USMALLINT: - case LogicalTypeId::UINTEGER: - case LogicalTypeId::UBIGINT: - case LogicalTypeId::DECIMAL: - return value.ToString(); - case LogicalTypeId::FLOAT: - case LogicalTypeId::DOUBLE: { - auto d = value.GetValue(); - if (std::isnan(d) || std::isinf(d)) { - // JSON has no NaN/Infinity — mirror FastAPI/ujson and emit null - return "null"; - } - return value.ToString(); - } - case LogicalTypeId::LIST: { - auto &children = ListValue::GetChildren(value); - string result = "["; - for (idx_t i = 0; i < children.size(); i++) { - if (i > 0) { - result += ","; - } - result += ValueToJson(children[i]); - } - result += "]"; - return result; - } - case LogicalTypeId::STRUCT: { - auto &children = StructValue::GetChildren(value); - auto &child_types = StructType::GetChildTypes(type); - string result = "{"; - for (idx_t i = 0; i < children.size(); i++) { - if (i > 0) { - result += ","; - } - result += "\"" + QuackapiJsonEscape(child_types[i].first) + "\":" + ValueToJson(children[i]); - } - result += "}"; - return result; - } - default: - return "\"" + QuackapiJsonEscape(value.ToString()) + "\""; - } -} - //! FastAPI-shaped validation error body (loc = [kind, name]). string ValidationErrorJson(const string &loc_kind, const string ¶m_name, const string &msg, const string &type) { return "{\"detail\":[{\"loc\":[\"" + QuackapiJsonEscape(loc_kind) + "\",\"" + QuackapiJsonEscape(param_name) + @@ -378,7 +322,7 @@ string PairsToJsonObject(const PairRange &pairs) { //! The json_schema extension includes an RFC 6901 pointer in failures (for //! example, "At /items/0/qty: ..."). Surface it as FastAPI's typed loc array //! instead of losing the field path behind a synthetic _schema member. -string ValidationErrorJsonSchema(const string &msg, const string &type, const string &raw_body) { +string ValidationErrorJsonSchema(Connection &con, const string &msg, const string &type, const string &raw_body) { const string markers[] = {"At /", "at /"}; for (auto &marker : markers) { auto start_marker = msg.find(marker); @@ -401,7 +345,7 @@ string ValidationErrorJsonSchema(const string &msg, const string &type, const st } if (end > start) { return QuackapiValidationErrorsJson( - {{QuackapiValidationBodyPointerLoc(msg.substr(start, end - start), raw_body), msg, type}}); + {{QuackapiValidationBodyPointerLoc(con, msg.substr(start, end - start), raw_body), msg, type}}); } } return ValidationErrorJson("body", "_schema", msg, type); @@ -409,14 +353,6 @@ string ValidationErrorJsonSchema(const string &msg, const string &type, const st //! Preserve whether a JSON member was an actual JSON null. A string value of //! "null" remains a string and must not silently become SQL NULL. -struct JsonBodyField { - string value; - bool explicit_null; - JsonBodyField() : explicit_null(false) { - } - JsonBodyField(string value_p, bool explicit_null_p) : value(std::move(value_p)), explicit_null(explicit_null_p) { - } -}; //! Media type from Content-Type (strip parameters; lowercased). string ContentTypeMedia(const case_insensitive_map_t &headers) { @@ -433,193 +369,6 @@ string ContentTypeMedia(const case_insensitive_map_t &headers) { return StringUtil::Lower(ct); } -bool IsJsonMediaType(const string &media) { - return media == "application/json" || StringUtil::EndsWith(media, "+json"); -} - -bool IsFormUrlEncodedMediaType(const string &media) { - return media == "application/x-www-form-urlencoded"; -} - -bool IsMultipartMediaType(const string &media) { - return StringUtil::StartsWith(media, "multipart/form-data"); -} - -bool IsBodyMethod(const string &method) { - return method == "POST" || method == "PUT" || method == "PATCH"; -} - -//! application/x-www-form-urlencoded → key/value map (last wins). -void ParseFormUrlEncoded(const string &body, case_insensitive_map_t &out) { - idx_t i = 0; - while (i < body.size()) { - idx_t amp = body.find('&', i); - if (amp == string::npos) { - amp = body.size(); - } - string pair = body.substr(i, amp - i); - if (!pair.empty()) { - auto eq = pair.find('='); - string key, val; - if (eq == string::npos) { - key = duckdb_httplib::decode_query_component(pair, true); - } else { - key = duckdb_httplib::decode_query_component(pair.substr(0, eq), true); - val = duckdb_httplib::decode_query_component(pair.substr(eq + 1), true); - } - if (!key.empty()) { - out[key] = val; - } - } - i = amp + 1; - } -} - -//! Fast path: flat JSON object of string/number/bool/null scalars (no DuckDB round-trip). -//! Returns true on success. false → caller may fall back to SQL extract or error. -bool TryExtractFlatJsonObject(const string &raw_body, case_insensitive_map_t &fields) { - const char *s = raw_body.c_str(); - const char *end = s + raw_body.size(); - auto skip_ws = [&]() { - while (s < end && (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')) { - s++; - } - }; - auto is_digit = [](char c) { - return c >= '0' && c <= '9'; - }; - // This path deliberately accepts only unescaped JSON strings. Escapes require - // full JSON decoding (in particular, UTF-16 surrogate handling), so leave them - // to DuckDB instead of attempting a partial implementation here. - auto parse_plain_string = [&](string &out) { - if (s >= end || *s != '"') { - return false; - } - s++; - while (s < end && *s != '"') { - const auto ch = static_cast(*s); - if (ch < 0x20 || *s == '\\') { - return false; - } - out.push_back(*s++); - } - if (s >= end) { - return false; - } - s++; - return true; - }; - auto parse_number = [&]() { - const char *start = s; - if (s < end && *s == '-') { - s++; - } - if (s >= end) { - return false; - } - if (*s == '0') { - s++; - if (s < end && is_digit(*s)) { - return false; - } - } else if (*s >= '1' && *s <= '9') { - do { - s++; - } while (s < end && is_digit(*s)); - } else { - return false; - } - if (s < end && *s == '.') { - s++; - if (s >= end || !is_digit(*s)) { - return false; - } - do { - s++; - } while (s < end && is_digit(*s)); - } - if (s < end && (*s == 'e' || *s == 'E')) { - s++; - if (s < end && (*s == '+' || *s == '-')) { - s++; - } - if (s >= end || !is_digit(*s)) { - return false; - } - do { - s++; - } while (s < end && is_digit(*s)); - } - return s > start; - }; - skip_ws(); - if (s >= end || *s != '{') { - return false; - } - s++; - fields.clear(); - skip_ws(); - if (s < end && *s == '}') { - s++; - skip_ws(); - return s == end; - } - while (s < end) { - skip_ws(); - string key; - if (!parse_plain_string(key)) { - return false; - } - skip_ws(); - if (s >= end || *s != ':') { - return false; - } - s++; - skip_ws(); - if (s >= end) { - return false; - } - string val; - bool explicit_null = false; - if (*s == '"') { - if (!parse_plain_string(val)) { - return false; - } - } else if (*s == 'n' && s + 4 <= end && string(s, 4) == "null") { - explicit_null = true; - s += 4; - } else if (*s == 't' && s + 4 <= end && string(s, 4) == "true") { - val = "true"; - s += 4; - } else if (*s == 'f' && s + 5 <= end && string(s, 5) == "false") { - val = "false"; - s += 5; - } else if (*s == '-' || (*s >= '0' && *s <= '9')) { - const char *start = s; - if (!parse_number()) { - return false; - } - val.assign(start, s - start); - } else { - // nested object/array — not flat; fall back - return false; - } - fields[key] = {val, explicit_null}; - skip_ws(); - if (s < end && *s == ',') { - s++; - continue; - } - if (s < end && *s == '}') { - s++; - skip_ws(); - return s == end; - } - return false; - } - return false; -} - //! Extract top-level JSON object fields as string values for binding. //! On invalid JSON sets err_json and returns false. Arrays/non-objects → model_attributes_type. bool ExtractJsonBodyFields(Connection &con, const string &raw_body, case_insensitive_map_t &fields, @@ -741,7 +490,7 @@ bool ValidateBodySchema(Connection &con, const string &schema, const string &raw msg = msg.substr(0, 200); } } - err_json = ValidationErrorJsonSchema(msg, "value_error", raw_body); + err_json = ValidationErrorJsonSchema(con, msg, "value_error", raw_body); return false; } @@ -1247,439 +996,6 @@ bool FindHeaderValue(const case_insensitive_map_t &headers, const string return false; } -//! True if column name is a special response control column (stripped from body). -bool IsSpecialResponseColumn(const string &name) { - auto lower = StringUtil::Lower(name); - return lower == "location" || lower == "set_cookie" || lower == "set-cookie"; -} - -void SetJson(duckdb_httplib::Response &res, int status, const string &body) { - res.status = status; - res.set_content(body, "application/json"); -} - -//! Response mode inferred from the single output column's name — the database -//! names the payload. `html` -> text/html, `text` -> text/plain; anything else -//! serializes as row data (json / ndjson / csv / parquet / arrow via FORMAT + Accept). -enum class ResponseMode { JSON, HTML, TEXT }; - -ResponseMode ResponseModeFor(const vector &names) { - if (names.size() != 1) { - return ResponseMode::JSON; - } - auto lower = StringUtil::Lower(names[0]); - if (lower == "html") { - return ResponseMode::HTML; - } - if (lower == "text") { - return ResponseMode::TEXT; - } - return ResponseMode::JSON; -} - -//! Wire format for multi-row data responses (not html/text column modes). -enum class BodyFormat { JSON, NDJSON, CSV, PARQUET, ARROW }; - -BodyFormat ParseBodyFormatToken(const string &token) { - auto t = StringUtil::Lower(token); - if (t == "ndjson") { - return BodyFormat::NDJSON; - } - if (t == "csv") { - return BodyFormat::CSV; - } - if (t == "parquet") { - return BodyFormat::PARQUET; - } - if (t == "arrow" || t == "arrows") { - return BodyFormat::ARROW; - } - return BodyFormat::JSON; -} - -//! Map Accept media-type (no parameters) to BodyFormat. Returns false if unknown. -bool MediaTypeToBodyFormat(const string &media, BodyFormat &out) { - auto m = StringUtil::Lower(media); - // strip parameters already; still trim - m = QuackapiTrim(m); - if (m == "application/x-ndjson" || m == "application/jsonl" || m == "application/ndjson") { - out = BodyFormat::NDJSON; - return true; - } - if (m == "text/csv" || m == "application/csv") { - out = BodyFormat::CSV; - return true; - } - if (m == "application/vnd.apache.parquet" || m == "application/parquet") { - out = BodyFormat::PARQUET; - return true; - } - // Arrow IPC stream (what nanoarrow FORMAT ARROWS emits) or file. - if (m == "application/vnd.apache.arrow.stream" || m == "application/vnd.apache.arrow.file" || - m == "application/x-apache-arrow-stream" || m == "application/arrow") { - out = BodyFormat::ARROW; - return true; - } - if (m == "application/json" || m == "text/json" || m == "*/*") { - out = BodyFormat::JSON; - return true; - } - return false; -} - -//! Parse one Accept token's q-value (default 1.0). -double ParseAcceptQ(const string &token) { - auto parts = StringUtil::Split(token, ';'); - if (parts.size() < 2) { - return 1.0; - } - for (idx_t i = 1; i < parts.size(); i++) { - auto p = QuackapiTrim(parts[i]); - auto pl = StringUtil::Lower(p); - if (StringUtil::StartsWith(pl, "q=")) { - return atof(p.c_str() + 2); - } - } - return 1.0; -} - -//! When route FORMAT is default/json, honor Accept for ndjson/csv/parquet/arrow/json. -//! Explicit route FORMAT (ndjson|csv|parquet|arrow) always wins. -BodyFormat ResolveBodyFormat(const QuackapiRoute &route, const duckdb_httplib::Request &req) { - auto route_fmt = StringUtil::Lower(route.response_format.empty() ? "json" : route.response_format); - if (route_fmt == "ndjson" || route_fmt == "csv" || route_fmt == "parquet" || route_fmt == "arrow" || - route_fmt == "arrows") { - return ParseBodyFormatToken(route_fmt); - } - // Default / explicit json: negotiate Accept when present. - auto it = req.headers.find("Accept"); - if (it == req.headers.end() || it->second.empty()) { - return BodyFormat::JSON; - } - // Pick the highest-q recognized type among json / ndjson / csv / parquet / arrow. - BodyFormat best = BodyFormat::JSON; - double best_q = -1.0; - bool saw_recognized = false; - auto tokens = StringUtil::Split(it->second, ','); - for (auto &tok : tokens) { - auto t = QuackapiTrim(tok); - if (t.empty()) { - continue; - } - double q = ParseAcceptQ(t); - if (q <= 0.0) { - continue; - } - auto semi = t.find(';'); - string media = semi == string::npos ? t : QuackapiTrim(t.substr(0, semi)); - BodyFormat f; - if (!MediaTypeToBodyFormat(media, f)) { - continue; - } - // Prefer more specific non-wildcard when q ties: ndjson/csv/parquet/arrow over */* json. - bool better = !saw_recognized || q > best_q; - if (!better && q == best_q) { - // Prefer non-json when client listed it at same q after json — first highest wins - // unless current best is from */* and this is concrete. - if (best == BodyFormat::JSON && f != BodyFormat::JSON) { - better = true; - } - } - if (better) { - best = f; - best_q = q; - saw_recognized = true; - } - } - return saw_recognized ? best : BodyFormat::JSON; -} - -//! RFC 4180-style CSV field escape (quote when needed). -string CsvEscapeField(const string &input) { - bool need_quote = false; - for (unsigned char c : input) { - if (c == '"' || c == ',' || c == '\n' || c == '\r') { - need_quote = true; - break; - } - } - if (!need_quote) { - return input; - } - string out; - out.reserve(input.size() + 2); - out += '"'; - for (unsigned char c : input) { - if (c == '"') { - out += "\"\""; - } else { - out += static_cast(c); - } - } - out += '"'; - return out; -} - -//! Cell text for CSV: null → empty; scalars ToString; nested → JSON text. -string ValueToCsvCell(const Value &value) { - if (value.IsNull()) { - return ""; - } - auto id = value.type().id(); - if (id == LogicalTypeId::LIST || id == LogicalTypeId::STRUCT || id == LogicalTypeId::MAP) { - return ValueToJson(value); - } - return value.ToString(); -} - -string SerializeRowsJsonArray(const vector &names, const vector &data_cols, - const vector> &rows) { - string body = "["; - bool first_row = true; - for (auto &cols : rows) { - if (!first_row) { - body += ","; - } - first_row = false; - body += "{"; - bool first_col = true; - for (auto col : data_cols) { - if (!first_col) { - body += ","; - } - first_col = false; - body += "\"" + QuackapiJsonEscape(names[col]) + "\":" + ValueToJson(cols[col]); - } - body += "}"; - } - body += "]"; - return body; -} - -//! Single-row JSON object (ENVELOPE object). Caller guarantees rows.size() == 1. -string SerializeRowJsonObject(const vector &names, const vector &data_cols, const vector &cols) { - string body = "{"; - bool first_col = true; - for (auto col : data_cols) { - if (!first_col) { - body += ","; - } - first_col = false; - body += "\"" + QuackapiJsonEscape(names[col]) + "\":" + ValueToJson(cols[col]); - } - body += "}"; - return body; -} - -string EmptyResultBody(const QuackapiRoute &route) { - if (!route.empty_body.empty()) { - return route.empty_body; - } - return "{\"detail\":\"Not Found\"}"; -} - -string SerializeRowsNdjson(const vector &names, const vector &data_cols, - const vector> &rows) { - string body; - for (auto &cols : rows) { - body += "{"; - bool first_col = true; - for (auto col : data_cols) { - if (!first_col) { - body += ","; - } - first_col = false; - body += "\"" + QuackapiJsonEscape(names[col]) + "\":" + ValueToJson(cols[col]); - } - body += "}\n"; - } - return body; -} - -string SerializeRowsCsv(const vector &names, const vector &data_cols, - const vector> &rows) { - string body; - // header - for (idx_t i = 0; i < data_cols.size(); i++) { - if (i > 0) { - body += ","; - } - body += CsvEscapeField(names[data_cols[i]]); - } - body += "\n"; - for (auto &cols : rows) { - for (idx_t i = 0; i < data_cols.size(); i++) { - if (i > 0) { - body += ","; - } - auto col = data_cols[i]; - string cell; - if (col < cols.size()) { - cell = ValueToCsvCell(cols[col]); - } - body += CsvEscapeField(cell); - } - body += "\n"; - } - return body; -} - -//! Shared: TEMP table + Appender fill for binary serdes (parquet / arrow). -void FillTempTableForSerdes(Connection &con, const string &table, const vector &names, - const vector &types, const vector &data_cols, - const vector> &rows, const char *label) { - string create_sql = "CREATE TEMP TABLE " + table + " ("; - for (idx_t i = 0; i < data_cols.size(); i++) { - if (i > 0) { - create_sql += ", "; - } - auto c = data_cols[i]; - create_sql += KeywordHelper::WriteOptionallyQuoted(names[c]); - create_sql += " "; - create_sql += types[c].ToString(); - } - create_sql += ")"; - auto create_res = con.Query(create_sql); - if (create_res->HasError()) { - throw InvalidInputException("%s serialize CREATE: %s", label, create_res->GetError()); - } - - if (!rows.empty()) { - Appender appender(con, table); - for (auto &cols : rows) { - appender.BeginRow(); - for (auto c : data_cols) { - if (c < cols.size()) { - appender.Append(cols[c]); - } else { - appender.Append(Value()); - } - } - appender.EndRow(); - } - appender.Close(); - } -} - -//! OS temp dir + basename for parquet/arrow scratch files. Never hardcode /tmp -//! (Windows CI has no /tmp). Paths use forward slashes for SQL COPY literals. -string SerdesTempFilePath(FileSystem &fs, const string &filename) { - string dir; -#if defined(_WIN32) || defined(WIN32) - const char *t = std::getenv("TEMP"); - if (!t || !t[0]) { - t = std::getenv("TMP"); - } - dir = (t && t[0]) ? string(t) : string("."); -#else - const char *t = std::getenv("TMPDIR"); - dir = (t && t[0]) ? string(t) : string("/tmp"); -#endif - string path = fs.JoinPath(dir, filename); - return StringUtil::Replace(path, "\\", "/"); -} - -string SqlQuotePath(const string &path) { - return StringUtil::Replace(path, "'", "''"); -} - -//! Read path bytes then delete the file. On open failure, best-effort remove. -string ReadAndRemoveFileBytes(Connection &con, const string &path) { - auto &fs = FileSystem::GetFileSystem(*con.context); - unique_ptr handle; - try { - handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ); - } catch (...) { - fs.TryRemoveFile(path); - throw; - } - idx_t file_size = handle->GetFileSize(); - string body; - if (file_size > 0) { - // mutable buffer — data_ptr_cast rejects const char* from string::data() - vector buf(file_size); - handle->Read(buf.data(), file_size); - body.assign(reinterpret_cast(buf.data()), file_size); - } - handle->Close(); - fs.TryRemoveFile(path); - return body; -} - -//! Serialize result rows as a Parquet file body (magic "PAR1"). Uses a unique -//! TEMP table + COPY TO (FORMAT PARQUET) + read bytes, then cleans up. -string SerializeRowsParquet(Connection &con, const vector &names, const vector &types, - const vector &data_cols, const vector> &rows) { - if (data_cols.empty()) { - throw InvalidInputException("parquet serialize: no data columns"); - } - auto &fs = FileSystem::GetFileSystem(*con.context); - string uid = StringUtil::Replace(UUID::ToString(UUIDv7::GenerateRandomUUID()), "-", ""); - string table = "qa_parquet_" + uid; - string path = SerdesTempFilePath(fs, table + ".parquet"); - - FillTempTableForSerdes(con, table, names, types, data_cols, rows, "parquet"); - - auto copy_res = con.Query("COPY " + table + " TO '" + SqlQuotePath(path) + "' (FORMAT PARQUET)"); - if (copy_res->HasError()) { - con.Query("DROP TABLE IF EXISTS " + table); - fs.TryRemoveFile(path); - throw InvalidInputException("parquet serialize COPY: %s", copy_res->GetError()); - } - con.Query("DROP TABLE IF EXISTS " + table); - return ReadAndRemoveFileBytes(con, path); -} - -//! Ensure community nanoarrow is available (LOAD, else INSTALL FROM community). -void EnsureNanoarrowLoaded(Connection &con) { - auto load = con.Query("LOAD nanoarrow"); - if (!load->HasError()) { - return; - } - auto inst = con.Query("INSTALL nanoarrow FROM community"); - if (inst->HasError()) { - throw InvalidInputException( - "arrow serialize: nanoarrow not available (%s). INSTALL nanoarrow FROM community and retry.", - inst->GetError()); - } - load = con.Query("LOAD nanoarrow"); - if (load->HasError()) { - throw InvalidInputException("arrow serialize: could not LOAD nanoarrow: %s", load->GetError()); - } -} - -//! Serialize result rows as Arrow IPC stream bytes (nanoarrow FORMAT ARROWS). -//! Magic is 0xFFFFFFFF stream continuation (not ARROW1 file). Content-Type: -//! application/vnd.apache.arrow.stream. Same TEMP+COPY+read pattern as parquet. -string SerializeRowsArrow(Connection &con, const vector &names, const vector &types, - const vector &data_cols, const vector> &rows) { - if (data_cols.empty()) { - throw InvalidInputException("arrow serialize: no data columns"); - } - EnsureNanoarrowLoaded(con); - auto &fs = FileSystem::GetFileSystem(*con.context); - string uid = StringUtil::Replace(UUID::ToString(UUIDv7::GenerateRandomUUID()), "-", ""); - string table = "qa_arrow_" + uid; - string path = SerdesTempFilePath(fs, table + ".arrows"); - - FillTempTableForSerdes(con, table, names, types, data_cols, rows, "arrow"); - - auto copy_res = con.Query("COPY " + table + " TO '" + SqlQuotePath(path) + "' (FORMAT ARROWS)"); - if (copy_res->HasError()) { - con.Query("DROP TABLE IF EXISTS " + table); - fs.TryRemoveFile(path); - throw InvalidInputException("arrow serialize COPY: %s", copy_res->GetError()); - } - con.Query("DROP TABLE IF EXISTS " + table); - return ReadAndRemoveFileBytes(con, path); -} - -void SetInternalError(duckdb_httplib::Response &res, const string &server_side_detail) { - // Return DuckDB's diagnostic so clients can diagnose route execution failures. - fprintf(stderr, "quackapi: internal error: %s\n", server_side_detail.c_str()); - SetJson(res, 500, "{\"detail\":\"" + QuackapiJsonEscape(server_side_detail) + "\"}"); -} - //! Collect request headers into a case-insensitive map (first value wins). case_insensitive_map_t CollectHeaders(const duckdb_httplib::Request &req) { case_insensitive_map_t headers; @@ -1875,35 +1191,6 @@ static unique_ptr AccessLogAppender(Connection &con, const vector(con, parts[0], parts[1], parts[2]); } -static string JsonEscape(const string &value) { - return QuackapiJsonEscape(value); -} - -static void EmitAccessLogStderr(const QuackapiRequestRecord &entry) { - string path_esc; - path_esc.reserve(entry.path.size() + 8); - for (unsigned char c : entry.path) { - if (c == '"' || c == '\\') { - path_esc += '\\'; - path_esc += static_cast(c); - } else if (c < 0x20) { - char buf[8]; - snprintf(buf, sizeof(buf), "\\u%04x", c); - path_esc += buf; - } else { - path_esc += static_cast(c); - } - } - fprintf(stderr, - "{\"type\":\"access\",\"method\":\"%s\",\"path\":\"%s\",\"status\":%d," - "\"latency_ms\":%.3f,\"request_id\":\"%s\",\"bytes\":%llu,\"client_ip\":\"%s\"," - "\"user_agent\":\"%s\",\"route_path\":\"%s\",\"trace_id\":\"%s\"}\n", - JsonEscape(entry.method).c_str(), path_esc.c_str(), entry.status, entry.duration_ms, - JsonEscape(entry.request_id).c_str(), (unsigned long long)entry.bytes_out, - JsonEscape(entry.client_ip).c_str(), JsonEscape(entry.user_agent).c_str(), - JsonEscape(entry.route_path).c_str(), JsonEscape(entry.trace_id).c_str()); -} - static constexpr const char *QUACKAPI_STATIC_LOG_MARKER = "X-Quackapi-Static-Access-Log"; static constexpr const char *QUACKAPI_STATIC_LOG_STARTED = "X-Quackapi-Static-Started"; static constexpr const char *QUACKAPI_STATIC_LOG_RECEIVED = "X-Quackapi-Static-Received"; @@ -2331,32 +1618,6 @@ void QuackapiHttpServer::Dispatch(const duckdb_httplib::Request &req, duckdb_htt HandleRequest(req, res); } -//! Parse application/x-www-form-urlencoded query into httplib Params. -void ParseQueryStringIntoParams(const string &query, duckdb_httplib::Params ¶ms) { - if (query.empty()) { - return; - } - idx_t start = 0; - while (start < query.size()) { - idx_t amp = query.find('&', start); - if (amp == string::npos) { - amp = query.size(); - } - string pair = query.substr(start, amp - start); - start = amp + 1; - if (pair.empty()) { - continue; - } - auto eq = pair.find('='); - if (eq == string::npos) { - params.emplace(duckdb_httplib::decode_query_component(pair, true), string()); - } else { - params.emplace(duckdb_httplib::decode_query_component(pair.substr(0, eq), true), - duckdb_httplib::decode_query_component(pair.substr(eq + 1), true)); - } - } -} - void QuackapiInProcessRequest(DatabaseInstance &db, const string &method, const string &path_in, const string &body, int &status_out, string &body_out, string &content_type_out, const unordered_map *req_headers, From caa5b42c673124a34f3c261543adf6c7e1169755 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 11:56:50 -0700 Subject: [PATCH 02/93] Split quackapi POST boundaries --- CMakeLists.txt | 3 + src/include/quackapi_http_fetch.hpp | 5 ++ src/include/quackapi_post.hpp | 14 +++++ src/include/quackapi_post_executor.hpp | 23 ++++++++ src/include/quackapi_post_provider.hpp | 19 +++++++ src/quackapi_http_fetch.cpp | 72 +++++------------------- src/quackapi_post.cpp | 78 ++++++++++++++++++++++++++ src/quackapi_post_executor.cpp | 71 +++++++++++++++++++++++ src/quackapi_post_provider.cpp | 11 ++++ 9 files changed, 238 insertions(+), 58 deletions(-) create mode 100644 src/include/quackapi_post.hpp create mode 100644 src/include/quackapi_post_executor.hpp create mode 100644 src/include/quackapi_post_provider.hpp create mode 100644 src/quackapi_post.cpp create mode 100644 src/quackapi_post_executor.cpp create mode 100644 src/quackapi_post_provider.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c96f34..fbf8f73 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,9 @@ set(EXTENSION_SOURCES src/quackapi_ddl.cpp src/quackapi_auth.cpp src/quackapi_http_fetch.cpp + src/quackapi_post.cpp + src/quackapi_post_executor.cpp + src/quackapi_post_provider.cpp src/quackapi_limits.cpp src/quackapi_middleware.cpp src/quackapi_table_api.cpp diff --git a/src/include/quackapi_http_fetch.hpp b/src/include/quackapi_http_fetch.hpp index 1642b05..a371ce9 100644 --- a/src/include/quackapi_http_fetch.hpp +++ b/src/include/quackapi_http_fetch.hpp @@ -84,6 +84,11 @@ struct QuackapiHttpFetch { static void ResetPool(); }; +//! Shared SQL conversion helpers for the fetch and POST scalar functions. +LogicalType QuackapiHttpFetchResultType(); +Value QuackapiHttpFetchResultValue(const QuackapiHttpFetchResult &result); +unordered_map QuackapiHttpHeadersFromValue(const Value &value); + //! quackapi_fetch / quackapi_post / quackapi_http_pool / quackapi_parallel_fetch. void RegisterQuackapiHttpFetchFunctions(ExtensionLoader &loader); diff --git a/src/include/quackapi_post.hpp b/src/include/quackapi_post.hpp new file mode 100644 index 0000000..d2ef6d3 --- /dev/null +++ b/src/include/quackapi_post.hpp @@ -0,0 +1,14 @@ +//===----------------------------------------------------------------------===// +// quackapi_post.hpp +// +// SQL registration boundary for quackapi_post. +//===----------------------------------------------------------------------===// +#pragma once + +#include "duckdb/main/extension/extension_loader.hpp" + +namespace duckdb { + +void RegisterQuackapiPostFunctions(ExtensionLoader &loader); + +} // namespace duckdb diff --git a/src/include/quackapi_post_executor.hpp b/src/include/quackapi_post_executor.hpp new file mode 100644 index 0000000..db2c270 --- /dev/null +++ b/src/include/quackapi_post_executor.hpp @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// quackapi_post_executor.hpp +// +// Scheduling boundary for one scalar-function input chunk. This phase keeps +// the existing per-chunk worker behavior; the boundary allows the following +// executor change without altering POST registration or provider code. +//===----------------------------------------------------------------------===// +#pragma once + +#include "quackapi_http_fetch.hpp" + +#include + +namespace duckdb { + +using QuackapiPostOperation = std::function; + +struct QuackapiPostExecutor { + static void Execute(idx_t count, const vector &skip, vector &results, + const QuackapiPostOperation &perform); +}; + +} // namespace duckdb diff --git a/src/include/quackapi_post_provider.hpp b/src/include/quackapi_post_provider.hpp new file mode 100644 index 0000000..1c3d44f --- /dev/null +++ b/src/include/quackapi_post_provider.hpp @@ -0,0 +1,19 @@ +//===----------------------------------------------------------------------===// +// quackapi_post_provider.hpp +// +// Provider boundary for outbound POST. The implementation currently preserves +// the existing HTTP/HTTPS transports; later provider work can replace it +// without changing the SQL surface or scheduler. +//===----------------------------------------------------------------------===// +#pragma once + +#include "quackapi_http_fetch.hpp" + +namespace duckdb { + +struct QuackapiPostProvider { + static QuackapiHttpFetchResult Post(DatabaseInstance &db, const string &url, const string &body, + const string &content_type, const unordered_map &extra_headers); +}; + +} // namespace duckdb diff --git a/src/quackapi_http_fetch.cpp b/src/quackapi_http_fetch.cpp index 78628f2..6e6e53f 100644 --- a/src/quackapi_http_fetch.cpp +++ b/src/quackapi_http_fetch.cpp @@ -1,4 +1,5 @@ #include "quackapi_http_fetch.hpp" +#include "quackapi_post.hpp" #include "quackapi_thread_join.hpp" #include "quackapi_limits.hpp" @@ -690,47 +691,6 @@ void FetchScalar(DataChunk &args, ExpressionState &state, Vector &result) { } } -void PostScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - const auto count = args.size(); - result.SetVectorType(VectorType::FLAT_VECTOR); - - vector skip(count, false); - vector urls(count); - vector bodies(count); - vector content_types(count); - vector> headers(count); - for (idx_t i = 0; i < count; i++) { - auto url = args.data[0].GetValue(i); - if (url.IsNull()) { - skip[i] = true; - continue; - } - urls[i] = url.ToString(); - auto body = args.data[1].GetValue(i); - bodies[i] = body.IsNull() ? string() : body.ToString(); - content_types[i] = "application/json"; - if (args.ColumnCount() > 2) { - auto ct = args.data[2].GetValue(i); - if (!ct.IsNull()) { - content_types[i] = ct.ToString(); - } - } - if (args.ColumnCount() > 3) { - headers[i] = HeadersFromValue(args.data[3].GetValue(i)); - } - } - - vector out(count); - FanOutRequests(count, skip, out, [&](idx_t i) { - return QuackapiHttpFetch::Post(db, urls[i], bodies[i], content_types[i], headers[i]); - }); - - for (idx_t i = 0; i < count; i++) { - result.SetValue(i, skip[i] ? Value(FetchResultType()) : ToValue(out[i])); - } -} - struct PoolGlobalState : public GlobalTableFunctionState { vector rows; idx_t offset = 0; @@ -856,6 +816,18 @@ void ParallelFetchExec(ClientContext &, TableFunctionInput &data_p, DataChunk &o } // namespace +LogicalType QuackapiHttpFetchResultType() { + return FetchResultType(); +} + +Value QuackapiHttpFetchResultValue(const QuackapiHttpFetchResult &result) { + return ToValue(result); +} + +unordered_map QuackapiHttpHeadersFromValue(const Value &value) { + return HeadersFromValue(value); +} + void RegisterQuackapiHttpFetchFunctions(ExtensionLoader &loader) { const auto header_type = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); const auto result_type = FetchResultType(); @@ -926,23 +898,7 @@ void RegisterQuackapiHttpFetchFunctions(ExtensionLoader &loader) { fetch_tf.named_parameters["stall_ms"] = LogicalType::INTEGER; loader.RegisterFunction(fetch_tf); - ScalarFunctionSet post_set("quackapi_post"); - ScalarFunction post2("quackapi_post", {LogicalType::VARCHAR, LogicalType::VARCHAR}, result_type, PostScalar); - post2.stability = FunctionStability::VOLATILE; - ScalarFunction post2j("quackapi_post", {LogicalType::VARCHAR, LogicalType::JSON()}, result_type, PostScalar); - post2j.stability = FunctionStability::VOLATILE; - ScalarFunction post3("quackapi_post", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, - result_type, PostScalar); - post3.stability = FunctionStability::VOLATILE; - ScalarFunction post4("quackapi_post", - {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, header_type}, result_type, - PostScalar); - post4.stability = FunctionStability::VOLATILE; - post_set.AddFunction(post2); - post_set.AddFunction(post2j); - post_set.AddFunction(post3); - post_set.AddFunction(post4); - loader.RegisterFunction(post_set); + RegisterQuackapiPostFunctions(loader); loader.RegisterFunction(TableFunction("quackapi_http_pool", {}, PoolExec, PoolBind, PoolInit)); diff --git a/src/quackapi_post.cpp b/src/quackapi_post.cpp new file mode 100644 index 0000000..213c01d --- /dev/null +++ b/src/quackapi_post.cpp @@ -0,0 +1,78 @@ +#include "quackapi_post.hpp" +#include "quackapi_post_executor.hpp" +#include "quackapi_post_provider.hpp" + +#include "duckdb/function/scalar_function.hpp" + +namespace duckdb { + +namespace { + +void PostScalar(DataChunk &args, ExpressionState &state, Vector &result) { + auto &db = *state.GetContext().db; + const auto count = args.size(); + result.SetVectorType(VectorType::FLAT_VECTOR); + + vector skip(count, false); + vector urls(count); + vector bodies(count); + vector content_types(count); + vector> headers(count); + for (idx_t i = 0; i < count; i++) { + auto url = args.data[0].GetValue(i); + if (url.IsNull()) { + skip[i] = true; + continue; + } + urls[i] = url.ToString(); + auto body = args.data[1].GetValue(i); + bodies[i] = body.IsNull() ? string() : body.ToString(); + content_types[i] = "application/json"; + if (args.ColumnCount() > 2) { + auto content_type = args.data[2].GetValue(i); + if (!content_type.IsNull()) { + content_types[i] = content_type.ToString(); + } + } + if (args.ColumnCount() > 3) { + headers[i] = QuackapiHttpHeadersFromValue(args.data[3].GetValue(i)); + } + } + + vector responses(count); + QuackapiPostExecutor::Execute(count, skip, responses, [&](idx_t i) { + return QuackapiPostProvider::Post(db, urls[i], bodies[i], content_types[i], headers[i]); + }); + + for (idx_t i = 0; i < count; i++) { + result.SetValue(i, skip[i] ? Value(QuackapiHttpFetchResultType()) : QuackapiHttpFetchResultValue(responses[i])); + } +} + +} // namespace + +void RegisterQuackapiPostFunctions(ExtensionLoader &loader) { + const auto header_type = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); + const auto result_type = QuackapiHttpFetchResultType(); + + // POST changes remote state, so all overloads must stay volatile. + ScalarFunctionSet post_set("quackapi_post"); + ScalarFunction post2("quackapi_post", {LogicalType::VARCHAR, LogicalType::VARCHAR}, result_type, PostScalar); + post2.stability = FunctionStability::VOLATILE; + ScalarFunction post2j("quackapi_post", {LogicalType::VARCHAR, LogicalType::JSON()}, result_type, PostScalar); + post2j.stability = FunctionStability::VOLATILE; + ScalarFunction post3("quackapi_post", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, + result_type, PostScalar); + post3.stability = FunctionStability::VOLATILE; + ScalarFunction post4("quackapi_post", + {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, header_type}, result_type, + PostScalar); + post4.stability = FunctionStability::VOLATILE; + post_set.AddFunction(post2); + post_set.AddFunction(post2j); + post_set.AddFunction(post3); + post_set.AddFunction(post4); + loader.RegisterFunction(post_set); +} + +} // namespace duckdb diff --git a/src/quackapi_post_executor.cpp b/src/quackapi_post_executor.cpp new file mode 100644 index 0000000..80dae0f --- /dev/null +++ b/src/quackapi_post_executor.cpp @@ -0,0 +1,71 @@ +#include "quackapi_post_executor.hpp" + +#include "quackapi_limits.hpp" +#include "quackapi_thread_join.hpp" + +#include +#include + +#include "duckdb/common/exception.hpp" + +namespace duckdb { + +namespace { + +constexpr idx_t MAX_POST_FANOUT = 32; + +} // namespace + +void QuackapiPostExecutor::Execute(idx_t count, const vector &skip, vector &results, + const QuackapiPostOperation &perform) { + const auto deadline = QuackapiCurrentDeadline(); + auto run_one = [&](idx_t i) { + QuackapiSetCurrentDeadline(deadline); + try { + results[i] = perform(i); + } catch (std::exception &e) { + results[i].request_error = e.what(); + } catch (...) { + results[i].request_error = "unknown error during outbound HTTP request"; + } + }; + + idx_t active = 0; + for (idx_t i = 0; i < count; i++) { + if (!skip[i]) { + active++; + } + } + if (active == 0) { + return; + } + if (active == 1) { + for (idx_t i = 0; i < count; i++) { + if (!skip[i]) { + run_one(i); + } + } + return; + } + + const idx_t n_threads = active < MAX_POST_FANOUT ? active : MAX_POST_FANOUT; + std::atomic cursor {0}; + vector threads; + threads.reserve(n_threads); + JoinOutboundThreads> join_threads {threads}; + for (idx_t t = 0; t < n_threads; t++) { + threads.emplace_back([&]() { + for (;;) { + const idx_t i = cursor.fetch_add(1); + if (i >= count) { + return; + } + if (!skip[i]) { + run_one(i); + } + } + }); + } +} + +} // namespace duckdb diff --git a/src/quackapi_post_provider.cpp b/src/quackapi_post_provider.cpp new file mode 100644 index 0000000..8133072 --- /dev/null +++ b/src/quackapi_post_provider.cpp @@ -0,0 +1,11 @@ +#include "quackapi_post_provider.hpp" + +namespace duckdb { + +QuackapiHttpFetchResult QuackapiPostProvider::Post(DatabaseInstance &db, const string &url, const string &body, + const string &content_type, + const unordered_map &extra_headers) { + return QuackapiHttpFetch::Post(db, url, body, content_type, extra_headers); +} + +} // namespace duckdb From 0b2e1a71474523890e96e92a06d2e17027411f21 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 11:57:38 -0700 Subject: [PATCH 03/93] Replace validation JSON parser with DuckDB JSON --- src/include/quackapi_validation.hpp | 6 +- src/quackapi_openapi.cpp | 4 +- src/quackapi_validation.cpp | 379 ++++++++-------------------- 3 files changed, 106 insertions(+), 283 deletions(-) diff --git a/src/include/quackapi_validation.hpp b/src/include/quackapi_validation.hpp index a42bf17..4ebebfc 100644 --- a/src/include/quackapi_validation.hpp +++ b/src/include/quackapi_validation.hpp @@ -4,6 +4,8 @@ namespace duckdb { +class Connection; + //! One FastAPI-compatible request-validation issue. loc_json is deliberately a //! JSON array so body paths may contain array indexes as numbers. struct QuackapiValidationIssue { @@ -20,11 +22,11 @@ string QuackapiValidationErrorsJson(const vector &issue //! body location, e.g. /items/0/sku -> ["body","items",0,"sku"]. json_body is //! the validated request payload and distinguishes a numeric object key from //! an array index. -string QuackapiValidationBodyPointerLoc(const string &pointer, const string &json_body); +string QuackapiValidationBodyPointerLoc(Connection &con, const string &pointer, const string &json_body); //! Convert DuckDB's json_transform structure syntax into the compatible //! OpenAPI schema fragment. The declaration remains the runtime source of //! truth; this only projects its native STRUCT/LIST/scalar shape for clients. -string QuackapiDuckdbTransformToOpenApi(const string &body_type); +string QuackapiDuckdbTransformToOpenApi(Connection &con, const string &body_type); } // namespace duckdb diff --git a/src/quackapi_openapi.cpp b/src/quackapi_openapi.cpp index a2b3fcc..da6016b 100644 --- a/src/quackapi_openapi.cpp +++ b/src/quackapi_openapi.cpp @@ -314,12 +314,12 @@ string BuildOpenApiDocument(DatabaseInstance &db, const string &server_url) { string body_schema; if (!route.body_schema.empty() && !route.body_type.empty()) { body_schema = "{\"allOf\":[" + route.body_schema + "," + - QuackapiDuckdbTransformToOpenApi(route.body_type) + + QuackapiDuckdbTransformToOpenApi(con, route.body_type) + "],\"x-duckdb-json-transform\":" + JsonString(route.body_type) + "}"; } else if (!route.body_schema.empty()) { body_schema = route.body_schema; } else { - body_schema = QuackapiDuckdbTransformToOpenApi(route.body_type); + body_schema = QuackapiDuckdbTransformToOpenApi(con, route.body_type); if (body_schema.size() > 1 && body_schema.back() == '}') { body_schema.pop_back(); body_schema += ",\"x-duckdb-json-transform\":" + JsonString(route.body_type) + "}"; diff --git a/src/quackapi_validation.cpp b/src/quackapi_validation.cpp index 0bc1b3c..24f482f 100644 --- a/src/quackapi_validation.cpp +++ b/src/quackapi_validation.cpp @@ -1,5 +1,8 @@ #include "quackapi_validation.hpp" +#include "duckdb/common/unordered_map.hpp" +#include "duckdb/main/connection.hpp" + #include "quackapi_util.hpp" namespace duckdb { @@ -10,269 +13,84 @@ enum class JsonNodeKind : uint8_t { NIL, BOOL, NUMBER, STRING, ARRAY, OBJECT }; struct JsonNode { JsonNodeKind kind = JsonNodeKind::NIL; + idx_t id = 0; + string key; string scalar; - vector children; - vector> members; }; -//! Small, dependency-free JSON reader used only for declared structures and -//! error locations. DuckDB still owns request parsing and type validation. -class JsonReader { -public: - explicit JsonReader(const string &input_p) : input(input_p) { - } - - bool ParseDocument(JsonNode &result) { - SkipWhitespace(); - if (!ParseValue(result, 0)) { - return false; - } - SkipWhitespace(); - return position == input.size(); - } - -private: - static constexpr idx_t MAX_NESTING_DEPTH = 128; - const string &input; - idx_t position = 0; +struct JsonTree { + vector nodes; + unordered_map> children_by_parent; +}; - void SkipWhitespace() { - while (position < input.size() && (input[position] == ' ' || input[position] == '\n' || - input[position] == '\r' || input[position] == '\t')) { - position++; - } - } +bool IsMissingJsonExtension(const string &error) { + return error.find("json extension") != string::npos; +} - bool Consume(char expected) { - SkipWhitespace(); - if (position >= input.size() || input[position] != expected) { - return false; - } - position++; - return true; +JsonNodeKind JsonTypeToKind(const string &type) { + if (type == "OBJECT") { + return JsonNodeKind::OBJECT; } - - static int HexValue(char c) { - if (c >= '0' && c <= '9') { - return c - '0'; - } - if (c >= 'a' && c <= 'f') { - return 10 + c - 'a'; - } - if (c >= 'A' && c <= 'F') { - return 10 + c - 'A'; - } - return -1; + if (type == "ARRAY") { + return JsonNodeKind::ARRAY; } - - static void AppendUtf8(string &target, uint32_t codepoint) { - if (codepoint <= 0x7f) { - target += char(codepoint); - } else if (codepoint <= 0x7ff) { - target += char(0xc0 | (codepoint >> 6)); - target += char(0x80 | (codepoint & 0x3f)); - } else if (codepoint <= 0xffff) { - target += char(0xe0 | (codepoint >> 12)); - target += char(0x80 | ((codepoint >> 6) & 0x3f)); - target += char(0x80 | (codepoint & 0x3f)); - } else { - target += char(0xf0 | (codepoint >> 18)); - target += char(0x80 | ((codepoint >> 12) & 0x3f)); - target += char(0x80 | ((codepoint >> 6) & 0x3f)); - target += char(0x80 | (codepoint & 0x3f)); - } + if (type == "VARCHAR") { + return JsonNodeKind::STRING; } - - bool ParseUnicodeEscape(uint32_t &codepoint) { - if (position + 4 > input.size()) { - return false; - } - codepoint = 0; - for (idx_t i = 0; i < 4; i++) { - auto hex = HexValue(input[position++]); - if (hex < 0) { - return false; - } - codepoint = (codepoint << 4) | uint32_t(hex); - } - return true; + if (type == "BOOLEAN") { + return JsonNodeKind::BOOL; } - - bool ParseString(string &result) { - if (!Consume('"')) { - return false; - } - result.clear(); - while (position < input.size()) { - auto c = input[position++]; - if (c == '"') { - return true; - } - if (uint8_t(c) < 0x20) { - return false; - } - if (c != '\\') { - result += c; - continue; - } - if (position >= input.size()) { - return false; - } - auto escaped = input[position++]; - switch (escaped) { - case '"': - case '\\': - case '/': - result += escaped; - break; - case 'b': - result += '\b'; - break; - case 'f': - result += '\f'; - break; - case 'n': - result += '\n'; - break; - case 'r': - result += '\r'; - break; - case 't': - result += '\t'; - break; - case 'u': { - uint32_t codepoint; - if (!ParseUnicodeEscape(codepoint)) { - return false; - } - if (codepoint >= 0xd800 && codepoint <= 0xdbff && position + 6 <= input.size() && - input[position] == '\\' && input[position + 1] == 'u') { - position += 2; - uint32_t low; - if (!ParseUnicodeEscape(low) || low < 0xdc00 || low > 0xdfff) { - return false; - } - codepoint = 0x10000 + ((codepoint - 0xd800) << 10) + (low - 0xdc00); - } - AppendUtf8(result, codepoint); - break; - } - default: - return false; - } - } - return false; + if (type == "TINYINT" || type == "SMALLINT" || type == "INTEGER" || type == "BIGINT" || type == "HUGEINT" || + type == "UTINYINT" || type == "USMALLINT" || type == "UINTEGER" || type == "UBIGINT" || type == "UHUGEINT" || + type == "DECIMAL" || type == "FLOAT" || type == "DOUBLE") { + return JsonNodeKind::NUMBER; } + return JsonNodeKind::NIL; +} - bool ParseValue(JsonNode &result, idx_t depth) { - SkipWhitespace(); - if (position >= input.size()) { - return false; - } - auto c = input[position]; - if (c == '"') { - result.kind = JsonNodeKind::STRING; - return ParseString(result.scalar); - } - if (c == '{') { - return depth < MAX_NESTING_DEPTH && ParseObject(result, depth); - } - if (c == '[') { - return depth < MAX_NESTING_DEPTH && ParseArray(result, depth); - } - if (input.compare(position, 4, "true") == 0 || input.compare(position, 5, "false") == 0) { - result.kind = JsonNodeKind::BOOL; - position += input.compare(position, 4, "true") == 0 ? 4 : 5; - return true; - } - if (input.compare(position, 4, "null") == 0) { - result.kind = JsonNodeKind::NIL; - position += 4; - return true; - } - if (c == '-' || (c >= '0' && c <= '9')) { - auto start = position++; - while (position < input.size()) { - auto number_char = input[position]; - if (!((number_char >= '0' && number_char <= '9') || number_char == '.' || number_char == 'e' || - number_char == 'E' || number_char == '+' || number_char == '-')) { - break; - } - position++; - } - result.kind = JsonNodeKind::NUMBER; - result.scalar = input.substr(start, position - start); - return true; - } +//! Materialize one DuckDB-native JSON traversal. json_tree owns parsing, +//! escaping, Unicode handling, and node classification; this code only stores +//! its rows so callers can walk the resulting tree without reparsing input. +bool ReadJsonTree(Connection &con, const string &json, JsonTree &tree) { + const string sql = "SELECT id, parent, key, type, CASE WHEN type = 'VARCHAR' THEN json_extract_string(atom, '$') " + "ELSE atom::VARCHAR END " + "FROM json_tree(?::JSON) ORDER BY id"; + auto result = con.Query(sql, Value(json)); + if (result->HasError() && IsMissingJsonExtension(result->GetError())) { + auto load = con.Query("LOAD json"); + if (!load->HasError()) { + result = con.Query(sql, Value(json)); + } + } + if (result->HasError()) { return false; } - - bool ParseArray(JsonNode &result, idx_t depth) { - if (!Consume('[')) { - return false; - } - result.kind = JsonNodeKind::ARRAY; - result.children.clear(); - SkipWhitespace(); - if (position < input.size() && input[position] == ']') { - position++; - return true; - } - while (true) { - JsonNode child; - if (!ParseValue(child, depth + 1)) { - return false; - } - result.children.push_back(std::move(child)); - SkipWhitespace(); - if (position >= input.size()) { - return false; - } - if (input[position] == ']') { - position++; - return true; - } - if (input[position++] != ',') { - return false; - } - } - } - - bool ParseObject(JsonNode &result, idx_t depth) { - if (!Consume('{')) { - return false; - } - result.kind = JsonNodeKind::OBJECT; - result.members.clear(); - SkipWhitespace(); - if (position < input.size() && input[position] == '}') { - position++; - return true; - } - while (true) { - string key; - if (!ParseString(key) || !Consume(':')) { - return false; - } - JsonNode value; - if (!ParseValue(value, depth + 1)) { - return false; + auto chunk = result->Fetch(); + while (chunk && chunk->size() > 0) { + for (idx_t row = 0; row < chunk->size(); row++) { + JsonNode node; + node.id = chunk->GetValue(0, row).GetValue(); + auto parent = chunk->GetValue(1, row); + auto key = chunk->GetValue(2, row); + if (!key.IsNull()) { + node.key = key.GetValue(); } - result.members.emplace_back(std::move(key), std::move(value)); - SkipWhitespace(); - if (position >= input.size()) { - return false; + auto type = chunk->GetValue(3, row).GetValue(); + node.kind = JsonTypeToKind(type); + auto scalar = chunk->GetValue(4, row); + if (!scalar.IsNull()) { + node.scalar = scalar.GetValue(); } - if (input[position] == '}') { - position++; - return true; - } - if (input[position++] != ',') { - return false; + const auto index = tree.nodes.size(); + if (!parent.IsNull()) { + tree.children_by_parent[parent.GetValue()].push_back(index); } + tree.nodes.push_back(std::move(node)); } + chunk = result->Fetch(); } -}; + return !tree.nodes.empty(); +} bool IsCanonicalArrayIndex(const string &segment) { if (segment.empty() || (segment.size() > 1 && segment[0] == '0')) { @@ -307,25 +125,18 @@ string DecodeJsonPointerSegment(const string &segment) { return result; } -const JsonNode *FindMember(const JsonNode &node, const string &name) { - for (auto &member : node.members) { - if (member.first == name) { - return &member.second; - } +const JsonNode *FindChild(const JsonTree &tree, const JsonNode &parent, const string &key) { + auto children = tree.children_by_parent.find(parent.id); + if (children == tree.children_by_parent.end()) { + return nullptr; } - return nullptr; -} - -idx_t ParseIndex(const string &segment) { - idx_t index = 0; - for (auto c : segment) { - auto digit = idx_t(c - '0'); - if (index > (idx_t(-1) - digit) / 10) { - return idx_t(-1); + for (auto child_index : children->second) { + const auto &child = tree.nodes[child_index]; + if (child.key == key) { + return &child; } - index = index * 10 + digit; } - return index; + return nullptr; } string TransformScalarToOpenApi(const string &declared_type) { @@ -366,21 +177,32 @@ string TransformScalarToOpenApi(const string &declared_type) { return "{\"type\":\"string\"}"; } -string TransformNodeToOpenApi(const JsonNode &node) { +string TransformNodeToOpenApi(const JsonTree &tree, idx_t node_index) { + const auto &node = tree.nodes[node_index]; switch (node.kind) { case JsonNodeKind::STRING: return TransformScalarToOpenApi(node.scalar); - case JsonNodeKind::ARRAY: + case JsonNodeKind::ARRAY: { + auto children = tree.children_by_parent.find(node.id); return "{\"type\":\"array\",\"items\":" + - (node.children.empty() ? string("{}") : TransformNodeToOpenApi(node.children[0])) + "}"; + (children == tree.children_by_parent.end() || children->second.empty() + ? string("{}") + : TransformNodeToOpenApi(tree, children->second[0])) + + "}"; + } case JsonNodeKind::OBJECT: { string result = "{\"type\":\"object\",\"properties\":{"; - for (idx_t i = 0; i < node.members.size(); i++) { - if (i > 0) { - result += ","; + auto children = tree.children_by_parent.find(node.id); + if (children != tree.children_by_parent.end()) { + bool first = true; + for (auto child_index : children->second) { + const auto &child = tree.nodes[child_index]; + if (!first) { + result += ","; + } + first = false; + result += "\"" + QuackapiJsonEscape(child.key) + "\":" + TransformNodeToOpenApi(tree, child_index); } - result += "\"" + QuackapiJsonEscape(node.members[i].first) + - "\":" + TransformNodeToOpenApi(node.members[i].second); } return result + "}}"; } @@ -391,13 +213,13 @@ string TransformNodeToOpenApi(const JsonNode &node) { } // namespace -string QuackapiValidationBodyPointerLoc(const string &pointer, const string &json_body) { +string QuackapiValidationBodyPointerLoc(Connection &con, const string &pointer, const string &json_body) { string result = "[\"body\""; if (pointer.empty()) { return result + "]"; } - JsonNode root; - const JsonNode *current = JsonReader(json_body).ParseDocument(root) ? &root : nullptr; + JsonTree tree; + const JsonNode *current = ReadJsonTree(con, json_body, tree) ? &tree.nodes[0] : nullptr; idx_t start = pointer.size() > 0 && pointer[0] == '/' ? 1 : 0; while (true) { auto end = pointer.find('/', start); @@ -406,11 +228,10 @@ string QuackapiValidationBodyPointerLoc(const string &pointer, const string &jso result += ","; if (current && current->kind == JsonNodeKind::ARRAY && IsCanonicalArrayIndex(segment)) { result += segment; - auto index = ParseIndex(segment); - current = index < current->children.size() ? ¤t->children[index] : nullptr; + current = FindChild(tree, *current, segment); } else { result += "\"" + QuackapiJsonEscape(segment) + "\""; - current = current && current->kind == JsonNodeKind::OBJECT ? FindMember(*current, segment) : nullptr; + current = current && current->kind == JsonNodeKind::OBJECT ? FindChild(tree, *current, segment) : nullptr; } if (end == string::npos) { break; @@ -420,12 +241,12 @@ string QuackapiValidationBodyPointerLoc(const string &pointer, const string &jso return result + "]"; } -string QuackapiDuckdbTransformToOpenApi(const string &body_type) { - JsonNode structure; - if (!JsonReader(body_type).ParseDocument(structure)) { +string QuackapiDuckdbTransformToOpenApi(Connection &con, const string &body_type) { + JsonTree tree; + if (!ReadJsonTree(con, body_type, tree)) { return "{}"; } - return TransformNodeToOpenApi(structure); + return TransformNodeToOpenApi(tree, 0); } string QuackapiValidationErrorsJson(const vector &issues) { From 14abd7253c7a814e2783ab1e707d7f3275c3b0b2 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 11:58:20 -0700 Subject: [PATCH 04/93] build: register extracted server boundaries --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index fbf8f73..1c94828 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,9 @@ set(EXTENSION_SOURCES src/quackapi_extension.cpp src/quackapi_registry.cpp src/quackapi_server.cpp + src/quackapi_request_binding.cpp + src/quackapi_response_encoding.cpp + src/quackapi_logging.cpp src/quackapi_validation.cpp src/quackapi_batteries.cpp src/quackapi_ddl.cpp From c94a1efa6b1f5b06fa6412f26e7d6adcc314cbc3 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:02:54 -0700 Subject: [PATCH 05/93] feat(encoding): add bounded native JSON chunk encoder --- src/include/quackapi_response_encoding.hpp | 43 ++++++ src/quackapi_response_encoding.cpp | 158 +++++++++++++++++++++ 2 files changed, 201 insertions(+) diff --git a/src/include/quackapi_response_encoding.hpp b/src/include/quackapi_response_encoding.hpp index b01a803..b41f542 100644 --- a/src/include/quackapi_response_encoding.hpp +++ b/src/include/quackapi_response_encoding.hpp @@ -3,14 +3,57 @@ #include "duckdb/common/types.hpp" #include "duckdb/common/types/value.hpp" #include "httplib.hpp" +#include namespace duckdb { class Connection; struct QuackapiRoute; +class DataChunk; enum class ResponseMode { JSON, HTML, TEXT }; enum class BodyFormat { JSON, NDJSON, CSV, PARQUET, ARROW }; +enum class JsonRowMode { ARRAY, OBJECT, NDJSON }; + +//! Bounded accumulator shared by streaming encoders. Each append is atomic: +//! once the next append would exceed the limit, it is rejected and Data() is +//! empty so callers cannot accidentally send a truncated response. +class QuackapiResponseWriter { +public: + explicit QuackapiResponseWriter(idx_t max_bytes); + bool Append(const char *data, idx_t length); + bool Append(const string &data); + idx_t BytesWritten() const; + bool ExceededLimit() const; + const string &Data() const; + +private: + idx_t max_bytes; + idx_t bytes_written; + bool exceeded_limit; + string data; +}; + +//! Vectorized JSON row encoder. A native DuckDB json_object expression is +//! bound once for the result schema and evaluated against each result chunk. +class QuackapiJsonChunkEncoder { +public: + QuackapiJsonChunkEncoder(Connection &con, const vector &names, const vector &types, + const vector &data_cols, JsonRowMode mode, QuackapiResponseWriter &writer); + ~QuackapiJsonChunkEncoder(); + QuackapiJsonChunkEncoder(QuackapiJsonChunkEncoder &&) noexcept; + QuackapiJsonChunkEncoder &operator=(QuackapiJsonChunkEncoder &&) noexcept; + QuackapiJsonChunkEncoder(const QuackapiJsonChunkEncoder &) = delete; + QuackapiJsonChunkEncoder &operator=(const QuackapiJsonChunkEncoder &) = delete; + + void AppendChunk(DataChunk &chunk); + idx_t RowCount() const; + void Finish(); + +private: + class State; + unique_ptr state; +}; string ValueToJson(const Value &value); bool IsSpecialResponseColumn(const string &name); diff --git a/src/quackapi_response_encoding.cpp b/src/quackapi_response_encoding.cpp index 0797308..b83b543 100644 --- a/src/quackapi_response_encoding.cpp +++ b/src/quackapi_response_encoding.cpp @@ -9,15 +9,24 @@ #include "duckdb/common/file_system.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/common/types/uuid.hpp" +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/function/function_binder.hpp" #include "duckdb/main/appender.hpp" #include "duckdb/main/connection.hpp" #include "duckdb/parser/keyword_helper.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_reference_expression.hpp" #include "quackapi_state.hpp" #include "quackapi_util.hpp" namespace duckdb { namespace { +const string &EmptyString() { + static const string empty; + return empty; +} + string TrimMediaToken(string value) { StringUtil::Trim(value); return value; @@ -25,6 +34,155 @@ string TrimMediaToken(string value) { } // namespace +QuackapiResponseWriter::QuackapiResponseWriter(idx_t max_bytes) + : max_bytes(max_bytes), bytes_written(0), exceeded_limit(false) { +} + +bool QuackapiResponseWriter::Append(const char *append_data, idx_t length) { + if (exceeded_limit) { + return false; + } + if (length > max_bytes - bytes_written) { + exceeded_limit = true; + return false; + } + if (length) { + data.append(append_data, length); + } + bytes_written += length; + return true; +} + +bool QuackapiResponseWriter::Append(const string &append_data) { + return Append(append_data.data(), append_data.size()); +} + +idx_t QuackapiResponseWriter::BytesWritten() const { + return bytes_written; +} + +bool QuackapiResponseWriter::ExceededLimit() const { + return exceeded_limit; +} + +const string &QuackapiResponseWriter::Data() const { + return exceeded_limit ? EmptyString() : data; +} + +class QuackapiJsonChunkEncoder::State { +public: + State(Connection &con, const vector &names, const vector &types, + const vector &data_cols, JsonRowMode mode, QuackapiResponseWriter &writer) + : mode(mode), writer(writer), row_count(0), encoded_rows(0), finished(false) { + if (names.size() != types.size()) { + throw InvalidInputException("JSON encoder: result name/type count mismatch"); + } + // json_object supplies DuckDB's own recursive serializer for every column + // type (including MAP, STRUCT, LIST, and JSON), rather than Value::ToString. + auto load = con.Query("LOAD json"); + if (load->HasError()) { + throw InvalidInputException("JSON encoder: could not LOAD json: %s", load->GetError()); + } + vector> args; + for (auto col : data_cols) { + if (col >= names.size()) { + throw InvalidInputException("JSON encoder: data column index is out of bounds"); + } + args.push_back(make_uniq(Value(names[col]))); + args.push_back(make_uniq(types[col], UnsafeNumericCast(col))); + } + FunctionBinder binder(*con.context); + ErrorData error; + expression = binder.BindScalarFunction(DEFAULT_SCHEMA, "json_object", std::move(args), error); + if (!expression) { + throw InvalidInputException("JSON encoder: could not bind json_object: %s", error.Message()); + } + executor = make_uniq(*con.context, *expression); + if (mode == JsonRowMode::ARRAY) { + writer.Append("["); + } + } + + void AppendChunk(DataChunk &chunk) { + if (finished) { + throw InternalException("JSON encoder: cannot append after Finish"); + } + row_count += chunk.size(); + if (writer.ExceededLimit() || chunk.size() == 0 || (mode == JsonRowMode::OBJECT && encoded_rows > 0)) { + return; + } + Vector rows(LogicalType::JSON(), chunk.size()); + executor->ExecuteExpression(chunk, rows); + UnifiedVectorFormat format; + rows.ToUnifiedFormat(chunk.size(), format); + auto data = format.GetData(); + for (idx_t i = 0; i < chunk.size(); i++) { + if (mode == JsonRowMode::ARRAY && encoded_rows > 0 && !writer.Append(",")) { + return; + } + auto index = format.sel->get_index(i); + if (!format.validity.RowIsValid(index)) { + throw InternalException("JSON encoder: json_object returned NULL"); + } + auto row = data[index].GetString(); + if (!writer.Append(row.data(), row.size())) { + return; + } + if (mode == JsonRowMode::NDJSON && !writer.Append("\n")) { + return; + } + encoded_rows++; + if (mode == JsonRowMode::OBJECT) { + return; + } + } + } + + void Finish() { + if (finished) { + return; + } + finished = true; + if (mode == JsonRowMode::ARRAY) { + writer.Append("]"); + } else if (mode == JsonRowMode::OBJECT && encoded_rows == 0) { + writer.Append("null"); + } + } + + JsonRowMode mode; + QuackapiResponseWriter &writer; + idx_t row_count; + idx_t encoded_rows; + bool finished; + unique_ptr expression; + unique_ptr executor; +}; + +QuackapiJsonChunkEncoder::QuackapiJsonChunkEncoder(Connection &con, const vector &names, + const vector &types, const vector &data_cols, + JsonRowMode mode, QuackapiResponseWriter &writer) + : state(make_uniq(con, names, types, data_cols, mode, writer)) { +} + +QuackapiJsonChunkEncoder::~QuackapiJsonChunkEncoder() = default; + +QuackapiJsonChunkEncoder::QuackapiJsonChunkEncoder(QuackapiJsonChunkEncoder &&) noexcept = default; + +QuackapiJsonChunkEncoder &QuackapiJsonChunkEncoder::operator=(QuackapiJsonChunkEncoder &&) noexcept = default; + +void QuackapiJsonChunkEncoder::AppendChunk(DataChunk &chunk) { + state->AppendChunk(chunk); +} + +idx_t QuackapiJsonChunkEncoder::RowCount() const { + return state->row_count; +} + +void QuackapiJsonChunkEncoder::Finish() { + state->Finish(); +} + //! Render a DuckDB Value as JSON. The database typed the column — the JSON //! representation follows the type, not string-formatting heuristics. string ValueToJson(const Value &value) { From 91bc6bbb6a22636d02c31b5f4e16d9dedb36504d Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:03:45 -0700 Subject: [PATCH 06/93] Bound quackapi POST execution per database --- src/include/quackapi_post_executor.hpp | 39 ++- src/quackapi_post.cpp | 35 +-- src/quackapi_post_executor.cpp | 321 +++++++++++++++++++++---- 3 files changed, 332 insertions(+), 63 deletions(-) diff --git a/src/include/quackapi_post_executor.hpp b/src/include/quackapi_post_executor.hpp index db2c270..201381e 100644 --- a/src/include/quackapi_post_executor.hpp +++ b/src/include/quackapi_post_executor.hpp @@ -1,23 +1,46 @@ //===----------------------------------------------------------------------===// // quackapi_post_executor.hpp // -// Scheduling boundary for one scalar-function input chunk. This phase keeps -// the existing per-chunk worker behavior; the boundary allows the following -// executor change without altering POST registration or provider code. +// Database-owned bounded execution for outbound POST requests. //===----------------------------------------------------------------------===// #pragma once #include "quackapi_http_fetch.hpp" +#include "duckdb/storage/object_cache.hpp" -#include +#include +#include namespace duckdb { -using QuackapiPostOperation = std::function; +struct QuackapiPostWorkerState { + virtual ~QuackapiPostWorkerState() = default; +}; + +struct QuackapiPostRequest { + string url; + string body; + string content_type; + unordered_map headers; + shared_ptr> cancelled; +}; + +class ClientContext; +class DatabaseInstance; + +class QuackapiPostExecutor : public ObjectCacheEntry { +public: + explicit QuackapiPostExecutor(DatabaseInstance &db); + ~QuackapiPostExecutor() override; + string GetObjectType() override; + optional_idx GetEstimatedCacheMemory() const override; + static vector Execute(ClientContext &context, vector requests, + const vector &skip); + static string ObjectType(); -struct QuackapiPostExecutor { - static void Execute(idx_t count, const vector &skip, vector &results, - const QuackapiPostOperation &perform); +private: + class Impl; + unique_ptr impl; }; } // namespace duckdb diff --git a/src/quackapi_post.cpp b/src/quackapi_post.cpp index 213c01d..3d296c3 100644 --- a/src/quackapi_post.cpp +++ b/src/quackapi_post.cpp @@ -3,46 +3,48 @@ #include "quackapi_post_provider.hpp" #include "duckdb/function/scalar_function.hpp" +#include "duckdb/main/config.hpp" +#include "duckdb/main/database.hpp" namespace duckdb { namespace { +void ValidatePostConcurrency(ClientContext &, SetScope, Value &value) { + if (value.GetValue() < 0) { + throw InvalidInputException("quackapi_post_concurrency must be 0 or a positive integer"); + } +} + void PostScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; + auto &context = state.GetContext(); const auto count = args.size(); result.SetVectorType(VectorType::FLAT_VECTOR); vector skip(count, false); - vector urls(count); - vector bodies(count); - vector content_types(count); - vector> headers(count); + vector requests(count); for (idx_t i = 0; i < count; i++) { auto url = args.data[0].GetValue(i); if (url.IsNull()) { skip[i] = true; continue; } - urls[i] = url.ToString(); + requests[i].url = url.ToString(); auto body = args.data[1].GetValue(i); - bodies[i] = body.IsNull() ? string() : body.ToString(); - content_types[i] = "application/json"; + requests[i].body = body.IsNull() ? string() : body.ToString(); + requests[i].content_type = "application/json"; if (args.ColumnCount() > 2) { auto content_type = args.data[2].GetValue(i); if (!content_type.IsNull()) { - content_types[i] = content_type.ToString(); + requests[i].content_type = content_type.ToString(); } } if (args.ColumnCount() > 3) { - headers[i] = QuackapiHttpHeadersFromValue(args.data[3].GetValue(i)); + requests[i].headers = QuackapiHttpHeadersFromValue(args.data[3].GetValue(i)); } } - vector responses(count); - QuackapiPostExecutor::Execute(count, skip, responses, [&](idx_t i) { - return QuackapiPostProvider::Post(db, urls[i], bodies[i], content_types[i], headers[i]); - }); + auto responses = QuackapiPostExecutor::Execute(context, std::move(requests), skip); for (idx_t i = 0; i < count; i++) { result.SetValue(i, skip[i] ? Value(QuackapiHttpFetchResultType()) : QuackapiHttpFetchResultValue(responses[i])); @@ -52,6 +54,11 @@ void PostScalar(DataChunk &args, ExpressionState &state, Vector &result) { } // namespace void RegisterQuackapiPostFunctions(ExtensionLoader &loader) { + auto &config = DBConfig::GetConfig(loader.GetDatabaseInstance()); + config.AddExtensionOption("quackapi_post_concurrency", + "Maximum concurrent outbound quackapi_post requests for this database. " + "0 uses the database thread count (minimum 1).", + LogicalType::BIGINT, Value::BIGINT(0), ValidatePostConcurrency, SetScope::GLOBAL); const auto header_type = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); const auto result_type = QuackapiHttpFetchResultType(); diff --git a/src/quackapi_post_executor.cpp b/src/quackapi_post_executor.cpp index 80dae0f..f711d11 100644 --- a/src/quackapi_post_executor.cpp +++ b/src/quackapi_post_executor.cpp @@ -1,71 +1,310 @@ #include "quackapi_post_executor.hpp" #include "quackapi_limits.hpp" -#include "quackapi_thread_join.hpp" - -#include -#include +#include "quackapi_post_provider.hpp" #include "duckdb/common/exception.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/main/database.hpp" + +#include +#include +#include +#include namespace duckdb { namespace { -constexpr idx_t MAX_POST_FANOUT = 32; +constexpr auto POST_CONCURRENCY_SETTING = "quackapi_post_concurrency"; + +struct PostBatch { + vector results; + vector>> cancellation_flags; + idx_t completed = 0; + idx_t accepted = 0; + std::atomic cancelled {false}; + std::atomic interrupted {false}; + std::chrono::steady_clock::time_point deadline; + std::mutex mutex; + std::condition_variable cv; +}; + +struct PostJob { + shared_ptr batch; + idx_t row; + QuackapiPostRequest request; +}; + +bool IsExpired(std::chrono::steady_clock::time_point deadline) { + return deadline != std::chrono::steady_clock::time_point::max() && std::chrono::steady_clock::now() >= deadline; +} + +void CheckCaller(ClientContext &context, std::chrono::steady_clock::time_point deadline) { + if (context.IsInterrupted() || IsExpired(deadline)) { + throw InterruptException(); + } +} + +idx_t EffectiveConcurrency(DatabaseInstance &db) { + Value setting; + auto found = db.TryGetCurrentSetting(POST_CONCURRENCY_SETTING, setting); + if (found) { + const auto configured = setting.GetValue(); + if (configured > 0) { + return NumericCast(configured); + } + } + return MaxValue(1, db.NumberOfThreads()); +} } // namespace -void QuackapiPostExecutor::Execute(idx_t count, const vector &skip, vector &results, - const QuackapiPostOperation &perform) { - const auto deadline = QuackapiCurrentDeadline(); - auto run_one = [&](idx_t i) { - QuackapiSetCurrentDeadline(deadline); +class QuackapiPostExecutor::Impl { +public: + explicit Impl(DatabaseInstance &db_p) : db(db_p) { + } + + ~Impl() { + { + std::lock_guard guard(lock); + stopping = true; + } + queue_cv.notify_all(); + space_cv.notify_all(); + for (auto &worker : workers) { + if (worker.joinable()) { + worker.join(); + } + } + } + + vector Execute(ClientContext &context, vector requests, + const vector &skip) { + const auto deadline = QuackapiCurrentDeadline(); + auto batch = make_shared_ptr(); + batch->results.resize(requests.size()); + batch->cancellation_flags.resize(requests.size()); + batch->deadline = deadline; + for (idx_t i = 0; i < requests.size(); i++) { + if (!skip[i]) { + requests[i].cancelled = make_shared_ptr>(false); + batch->cancellation_flags[i] = requests[i].cancelled; + } + } + const auto concurrency = EffectiveConcurrency(db); + const auto queue_capacity = + concurrency > NumericLimits::Maximum() / 2 ? NumericLimits::Maximum() : concurrency * 2; + try { - results[i] = perform(i); - } catch (std::exception &e) { - results[i].request_error = e.what(); - } catch (...) { - results[i].request_error = "unknown error during outbound HTTP request"; + for (idx_t i = 0; i < requests.size(); i++) { + if (skip[i]) { + continue; + } + CheckCaller(context, deadline); + auto job = make_shared_ptr(); + job->batch = batch; + job->row = i; + job->request = std::move(requests[i]); + + std::unique_lock guard(lock); + while (!stopping && queue.size() >= queue_capacity) { + CheckCaller(context, deadline); + space_cv.wait_for(guard, std::chrono::milliseconds(10)); + } + CheckCaller(context, deadline); + if (stopping || batch->cancelled.load()) { + throw InterruptException(); + } + EnsureWorkers(concurrency); + queue.push_back(std::move(job)); + { + std::lock_guard batch_guard(batch->mutex); + batch->accepted++; + } + queue_cv.notify_all(); + } + } catch (InterruptException &) { + CancelBatch(batch); + DrainBatch(batch, Accepted(batch)); + throw; } - }; - idx_t active = 0; - for (idx_t i = 0; i < count; i++) { - if (!skip[i]) { - active++; + const auto expected = requests.size() - CountSkipped(skip); + while (true) { + { + std::lock_guard guard(batch->mutex); + if (batch->completed >= expected) { + break; + } + } + if (context.IsInterrupted() || IsExpired(deadline) || batch->interrupted.load()) { + CancelBatch(batch); + } + std::unique_lock guard(batch->mutex); + batch->cv.wait_for(guard, std::chrono::milliseconds(10)); } + if (batch->interrupted.load() || context.IsInterrupted() || IsExpired(deadline)) { + throw InterruptException(); + } + return std::move(batch->results); } - if (active == 0) { - return; + +private: + static idx_t Accepted(const shared_ptr &batch) { + std::lock_guard guard(batch->mutex); + return batch->accepted; } - if (active == 1) { - for (idx_t i = 0; i < count; i++) { - if (!skip[i]) { - run_one(i); + + static idx_t CountSkipped(const vector &skip) { + idx_t count = 0; + for (auto value : skip) { + count += value ? 1 : 0; + } + return count; + } + + void EnsureWorkers(idx_t desired) { + while (workers.size() < desired) { + workers.emplace_back([this]() { WorkerLoop(); }); + } + } + + void CancelQueued(const shared_ptr &batch) { + std::lock_guard guard(lock); + for (auto it = queue.begin(); it != queue.end();) { + if ((*it)->batch == batch) { + (*it)->request.cancelled->store(true); + { + std::lock_guard batch_guard(batch->mutex); + batch->completed++; + } + it = queue.erase(it); + } else { + ++it; } } - return; + space_cv.notify_all(); + batch->cv.notify_all(); } - const idx_t n_threads = active < MAX_POST_FANOUT ? active : MAX_POST_FANOUT; - std::atomic cursor {0}; - vector threads; - threads.reserve(n_threads); - JoinOutboundThreads> join_threads {threads}; - for (idx_t t = 0; t < n_threads; t++) { - threads.emplace_back([&]() { - for (;;) { - const idx_t i = cursor.fetch_add(1); - if (i >= count) { - return; + void CancelBatch(const shared_ptr &batch) { + batch->cancelled.store(true); + for (const auto &flag : batch->cancellation_flags) { + if (flag) { + flag->store(true); + } + } + CancelQueued(batch); + } + + void DrainBatch(const shared_ptr &batch, idx_t expected) { + std::unique_lock guard(batch->mutex); + batch->cv.wait(guard, [&]() { return batch->completed >= expected; }); + } + + void WorkerLoop() { + unique_ptr state; + try { + state = QuackapiPostProvider::CreateWorkerState(db); + } catch (...) { + // Each task receives a concrete error below if worker state setup failed. + } + for (;;) { + shared_ptr job; + { + std::unique_lock guard(lock); + queue_cv.wait(guard, + [&]() { return stopping || (!queue.empty() && active < EffectiveConcurrency(db)); }); + if (stopping && queue.empty()) { + break; + } + job = std::move(queue.front()); + queue.pop_front(); + active++; + space_cv.notify_all(); + } + + auto &batch = *job->batch; + bool skip_job; + { + std::lock_guard guard(batch.mutex); + skip_job = batch.cancelled.load() || job->request.cancelled->load() || IsExpired(batch.deadline); + } + QuackapiHttpFetchResult response; + if (!skip_job) { + const auto old_deadline = QuackapiCurrentDeadline(); + QuackapiSetCurrentDeadline(batch.deadline); + try { + if (!state) { + throw IOException("failed to initialize outbound POST worker"); + } + if (job->request.cancelled->load() || IsExpired(batch.deadline)) { + throw InterruptException(); + } + response = QuackapiPostProvider::Post(db, *state, job->request); + } catch (InterruptException &) { + batch.cancelled.store(true); + batch.interrupted.store(true); + } catch (std::exception &e) { + response.request_error = e.what(); + } catch (...) { + response.request_error = "unknown error during outbound HTTP request"; } - if (!skip[i]) { - run_one(i); + QuackapiSetCurrentDeadline(old_deadline); + } + { + std::lock_guard guard(batch.mutex); + if (!skip_job && !batch.cancelled.load()) { + batch.results[job->row] = std::move(response); } + batch.completed++; + } + { + std::lock_guard guard(lock); + active--; } - }); + queue_cv.notify_all(); + batch.cv.notify_all(); + if (batch.cancelled.load()) { + CancelQueued(job->batch); + } + } + state.reset(); } + + DatabaseInstance &db; + std::mutex lock; + std::condition_variable queue_cv; + std::condition_variable space_cv; + std::deque> queue; + vector workers; + idx_t active = 0; + bool stopping = false; +}; + +QuackapiPostExecutor::QuackapiPostExecutor(DatabaseInstance &db) : impl(make_uniq(db)) { +} + +QuackapiPostExecutor::~QuackapiPostExecutor() = default; + +string QuackapiPostExecutor::ObjectType() { + return "quackapi_post_executor"; +} + +string QuackapiPostExecutor::GetObjectType() { + return ObjectType(); +} + +optional_idx QuackapiPostExecutor::GetEstimatedCacheMemory() const { + return optional_idx(); +} + +vector +QuackapiPostExecutor::Execute(ClientContext &context, vector requests, const vector &skip) { + auto &db = DatabaseInstance::GetDatabase(context); + auto executor = db.GetObjectCache().GetOrCreateWithTypePrefix("database", db); + return executor->impl->Execute(context, std::move(requests), skip); } } // namespace duckdb From 400459daead4d7962beba748577676f8765a40fb Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:05:21 -0700 Subject: [PATCH 07/93] Route outbound POST through HTTPUtil providers --- src/include/quackapi_post_provider.hpp | 13 +- src/quackapi_post_provider.cpp | 179 ++++++++++++++++++++++++- 2 files changed, 183 insertions(+), 9 deletions(-) diff --git a/src/include/quackapi_post_provider.hpp b/src/include/quackapi_post_provider.hpp index 1c3d44f..a5a347f 100644 --- a/src/include/quackapi_post_provider.hpp +++ b/src/include/quackapi_post_provider.hpp @@ -1,9 +1,8 @@ //===----------------------------------------------------------------------===// // quackapi_post_provider.hpp // -// Provider boundary for outbound POST. The implementation currently preserves -// the existing HTTP/HTTPS transports; later provider work can replace it -// without changing the SQL surface or scheduler. +// HTTPUtil-backed outbound POST provider. Each executor worker owns its +// provider client state; clients are never shared across workers. //===----------------------------------------------------------------------===// #pragma once @@ -11,9 +10,13 @@ namespace duckdb { +class QuackapiPostWorkerState; +struct QuackapiPostRequest; + struct QuackapiPostProvider { - static QuackapiHttpFetchResult Post(DatabaseInstance &db, const string &url, const string &body, - const string &content_type, const unordered_map &extra_headers); + static unique_ptr CreateWorkerState(DatabaseInstance &db); + static QuackapiHttpFetchResult Post(DatabaseInstance &db, QuackapiPostWorkerState &state, + const QuackapiPostRequest &request); }; } // namespace duckdb diff --git a/src/quackapi_post_provider.cpp b/src/quackapi_post_provider.cpp index 8133072..213f7e9 100644 --- a/src/quackapi_post_provider.cpp +++ b/src/quackapi_post_provider.cpp @@ -1,11 +1,182 @@ #include "quackapi_post_provider.hpp" +#include "quackapi_post_executor.hpp" +#include "quackapi_limits.hpp" + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/string_util.hpp" + +#include + namespace duckdb { -QuackapiHttpFetchResult QuackapiPostProvider::Post(DatabaseInstance &db, const string &url, const string &body, - const string &content_type, - const unordered_map &extra_headers) { - return QuackapiHttpFetch::Post(db, url, body, content_type, extra_headers); +namespace { + +constexpr int64_t OUTBOUND_TIMEOUT_FALLBACK_MS = 600 * 1000; + +//! Connection settings that must match before a worker may reuse its provider +//! client. Request headers and timeouts stay request-local. +struct ClientConfig { + string provider_name; + string params_type; + string origin; + string http_proxy; + idx_t http_proxy_port = 0; + string http_proxy_username; + string http_proxy_password; + bool keep_alive = true; + bool follow_location = true; + bool override_verify_ssl = false; + bool verify_ssl = true; + + bool Matches(const HTTPUtil &util, const HTTPParams ¶ms, const string &candidate_origin) const { + return provider_name == util.GetName() && params_type == typeid(params).name() && origin == candidate_origin && + http_proxy == params.http_proxy && http_proxy_port == params.http_proxy_port && + http_proxy_username == params.http_proxy_username && http_proxy_password == params.http_proxy_password && + keep_alive == params.keep_alive && follow_location == params.follow_location && + override_verify_ssl == params.override_verify_ssl && verify_ssl == params.verify_ssl; + } + + void Set(const HTTPUtil &util, const HTTPParams ¶ms, const string &candidate_origin) { + provider_name = util.GetName(); + params_type = typeid(params).name(); + origin = candidate_origin; + http_proxy = params.http_proxy; + http_proxy_port = params.http_proxy_port; + http_proxy_username = params.http_proxy_username; + http_proxy_password = params.http_proxy_password; + keep_alive = params.keep_alive; + follow_location = params.follow_location; + override_verify_ssl = params.override_verify_ssl; + verify_ssl = params.verify_ssl; + } +}; + +struct WorkerState : public QuackapiPostWorkerState { + unique_ptr client; + HTTPUtil *client_util = nullptr; + ClientConfig client_config; + bool has_client_config = false; + + void Reset() { + client.reset(); + client_util = nullptr; + has_client_config = false; + } +}; + +void InsertExtraHeaders(HTTPHeaders &headers, const unordered_map &extra_headers) { + for (auto &header : extra_headers) { + headers.Insert(header.first, header.second); + } +} + +QuackapiHttpFetchResult FromResponse(unique_ptr response) { + QuackapiHttpFetchResult result; + if (!response) { + result.request_error = "null HTTP response"; + return result; + } + result.status = response->status; + result.body = std::move(response->body); + result.reason = std::move(response->reason); + result.success = response->Success(); + result.request_error = response->GetRequestError(); + result.headers = std::move(response->headers); + return result; +} + +bool IsCancelled(const QuackapiPostRequest &request) { + if (request.cancelled && request.cancelled->load()) { + return true; + } + return std::chrono::steady_clock::now() >= QuackapiCurrentDeadline(); +} + +void ThrowIfCancelled(const QuackapiPostRequest &request) { + if (IsCancelled(request)) { + throw InterruptException(); + } +} + +string UnsupportedProviderMessage(const string &provider_name) { + return StringUtil::Format("quackapi_post requires an explicitly loaded HTTPUtil provider that supports POST. " + "Load curl_httpfs (recommended) or httpfs, then retry. Active HTTPUtil is '%s'.", + provider_name); +} + +void DiscardClient(WorkerState &state) { + state.Reset(); +} + +} // namespace + +unique_ptr QuackapiPostProvider::CreateWorkerState(DatabaseInstance &) { + return make_uniq(); +} + +QuackapiHttpFetchResult QuackapiPostProvider::Post(DatabaseInstance &db, QuackapiPostWorkerState &state_p, + const QuackapiPostRequest &request) { + ThrowIfCancelled(request); + auto &state = static_cast(state_p); + auto &http_util = HTTPUtil::Get(db); + const auto provider_name = http_util.GetName(); + if (provider_name == "Built-In") { + throw InvalidConfigurationException(UnsupportedProviderMessage(provider_name)); + } + + string path; + string origin; + HTTPUtil::DecomposeURL(request.url, path, origin); + auto params = http_util.InitializeParameters(db, request.url); + params->keep_alive = true; + params->retries = 0; // POST is not safe to replay after an ambiguous transport failure. + const auto timeout_ms = QuackapiRemainingTimeoutMillis(OUTBOUND_TIMEOUT_FALLBACK_MS); + params->timeout = static_cast(timeout_ms / 1000); + params->timeout_usec = static_cast((timeout_ms % 1000) * 1000); + + const bool reused = state.client && state.client_util == &http_util && state.has_client_config && + state.client_config.Matches(http_util, *params, origin); + if (!reused) { + DiscardClient(state); + state.client = http_util.InitializeClient(*params, origin); + if (!state.client) { + throw InvalidConfigurationException(UnsupportedProviderMessage(provider_name)); + } + state.client_util = &http_util; + state.client_config.Set(http_util, *params, origin); + state.has_client_config = true; + } else { + // Timeouts are request-specific. Reapply them to a worker-reused client. + state.client->Initialize(*params); + } + + HTTPHeaders headers(db); + if (!request.content_type.empty()) { + headers.Insert("Content-Type", request.content_type); + } else if (StringUtil::StartsWith(StringUtil::Lower(request.url), "http://")) { + // Preserve the previous plain-HTTP empty-content-type default. + headers.Insert("Content-Type", "application/octet-stream"); + } + InsertExtraHeaders(headers, request.headers); + PostRequestInfo info(request.url, headers, *params, const_data_ptr_cast(request.body.data()), request.body.size()); + info.try_request = true; + + // The caller can be interrupted while this worker is being scheduled or + // while its provider client is prepared. Never send after that signal. + ThrowIfCancelled(request); + QuackapiHttpFetchResult result; + try { + result = FromResponse(http_util.Request(info, state.client)); + } catch (NotImplementedException &) { + DiscardClient(state); + throw InvalidConfigurationException(UnsupportedProviderMessage(provider_name)); + } + result.reused_connection = reused; + if (!result.request_error.empty()) { + DiscardClient(state); + } + return result; } } // namespace duckdb From f86dfd78e93bb4ff88e8d0a40284a81758ccdfd9 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:05:47 -0700 Subject: [PATCH 08/93] feat(encoding): add bounded native CSV chunk writer --- src/include/quackapi_response_encoding.hpp | 20 +++ src/quackapi_response_encoding.cpp | 140 +++++++++++++++++++++ 2 files changed, 160 insertions(+) diff --git a/src/include/quackapi_response_encoding.hpp b/src/include/quackapi_response_encoding.hpp index b41f542..934f174 100644 --- a/src/include/quackapi_response_encoding.hpp +++ b/src/include/quackapi_response_encoding.hpp @@ -55,6 +55,26 @@ class QuackapiJsonChunkEncoder { unique_ptr state; }; +//! DuckDB-native CSV writer for result chunks. All output, including the +//! header and row terminators, flows through the same bounded response writer. +class QuackapiCsvChunkEncoder { +public: + QuackapiCsvChunkEncoder(Connection &con, const vector &names, const vector &types, + const vector &data_cols, QuackapiResponseWriter &writer); + ~QuackapiCsvChunkEncoder(); + QuackapiCsvChunkEncoder(QuackapiCsvChunkEncoder &&) noexcept; + QuackapiCsvChunkEncoder &operator=(QuackapiCsvChunkEncoder &&) noexcept; + QuackapiCsvChunkEncoder(const QuackapiCsvChunkEncoder &) = delete; + QuackapiCsvChunkEncoder &operator=(const QuackapiCsvChunkEncoder &) = delete; + + void AppendChunk(DataChunk &chunk); + void Finish(); + +private: + class State; + unique_ptr state; +}; + string ValueToJson(const Value &value); bool IsSpecialResponseColumn(const string &name); void SetJson(duckdb_httplib::Response &res, int status, const string &body); diff --git a/src/quackapi_response_encoding.cpp b/src/quackapi_response_encoding.cpp index b83b543..ab9678c 100644 --- a/src/quackapi_response_encoding.cpp +++ b/src/quackapi_response_encoding.cpp @@ -5,8 +5,10 @@ #include #include "duckdb/common/exception.hpp" +#include "duckdb/common/csv_writer.hpp" #include "duckdb/common/file_open_flags.hpp" #include "duckdb/common/file_system.hpp" +#include "duckdb/common/serializer/write_stream.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/common/types/uuid.hpp" #include "duckdb/execution/expression_executor.hpp" @@ -15,6 +17,7 @@ #include "duckdb/main/connection.hpp" #include "duckdb/parser/keyword_helper.hpp" #include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_cast_expression.hpp" #include "duckdb/planner/expression/bound_reference_expression.hpp" #include "quackapi_state.hpp" #include "quackapi_util.hpp" @@ -27,6 +30,24 @@ const string &EmptyString() { return empty; } +class BoundedResponseStream : public WriteStream { +public: + explicit BoundedResponseStream(QuackapiResponseWriter &writer) : writer(writer) { + } + + void WriteData(const_data_ptr_t buffer, idx_t write_size) override { + writer.Append(reinterpret_cast(buffer), write_size); + } + +private: + QuackapiResponseWriter &writer; +}; + +bool IsNestedCsvCell(const LogicalType &type) { + auto id = type.id(); + return id == LogicalTypeId::LIST || id == LogicalTypeId::MAP || id == LogicalTypeId::STRUCT; +} + string TrimMediaToken(string value) { StringUtil::Trim(value); return value; @@ -183,6 +204,125 @@ void QuackapiJsonChunkEncoder::Finish() { state->Finish(); } +class QuackapiCsvChunkEncoder::State { +public: + State(Connection &con, const vector &names, const vector &types, + const vector &data_cols, QuackapiResponseWriter &response_writer) + : response_writer(response_writer), stream(response_writer), + writer(stream, SelectNames(names, data_cols), false), writer_state(*con.context, 8192), executor(nullptr), + finished(false) { + if (names.size() != types.size()) { + throw InvalidInputException("CSV encoder: result name/type count mismatch"); + } + bool needs_json = false; + for (auto col : data_cols) { + if (col >= names.size()) { + throw InvalidInputException("CSV encoder: data column index is out of bounds"); + } + needs_json = needs_json || IsNestedCsvCell(types[col]); + } + if (needs_json) { + auto load = con.Query("LOAD json"); + if (load->HasError()) { + throw InvalidInputException("CSV encoder: could not LOAD json: %s", load->GetError()); + } + } + for (auto col : data_cols) { + unique_ptr cell = + make_uniq(types[col], UnsafeNumericCast(col)); + if (IsNestedCsvCell(types[col])) { + vector> args; + args.push_back(std::move(cell)); + FunctionBinder binder(*con.context); + ErrorData error; + cell = binder.BindScalarFunction(DEFAULT_SCHEMA, "to_json", std::move(args), error); + if (!cell) { + throw InvalidInputException("CSV encoder: could not bind to_json: %s", error.Message()); + } + } + casts.push_back(BoundCastExpression::AddCastToType(*con.context, std::move(cell), LogicalType::VARCHAR)); + output_types.push_back(LogicalType::VARCHAR); + } + if (!casts.empty()) { + executor = make_uniq(*con.context, casts); + } + writer.options.null_str = {""}; + writer.options.dialect_options.header = CSVOption(true, true); + writer.writer_options.newline_writing_mode = CSVNewLineMode::WRITE_AFTER; + writer.Initialize(); + writer.WriteRawString(writer.writer_options.newline); + } + + static vector SelectNames(const vector &names, const vector &data_cols) { + vector result; + result.reserve(data_cols.size()); + for (auto col : data_cols) { + if (col >= names.size()) { + throw InvalidInputException("CSV encoder: data column index is out of bounds"); + } + result.push_back(names[col]); + } + return result; + } + + void AppendChunk(DataChunk &chunk) { + if (finished) { + throw InternalException("CSV encoder: cannot append after Finish"); + } + if (chunk.size() == 0) { + return; + } + if (!executor) { + for (idx_t i = 0; i < chunk.size(); i++) { + response_writer.Append("\n"); + } + return; + } + DataChunk output; + output.Initialize(executor->GetContext(), output_types); + executor->Execute(chunk, output); + writer.WriteChunk(output, writer_state); + writer.Flush(writer_state); + } + + void Finish() { + if (finished) { + return; + } + finished = true; + writer.Flush(writer_state); + } + + QuackapiResponseWriter &response_writer; + BoundedResponseStream stream; + CSVWriter writer; + CSVWriterState writer_state; + vector> casts; + vector output_types; + unique_ptr executor; + bool finished; +}; + +QuackapiCsvChunkEncoder::QuackapiCsvChunkEncoder(Connection &con, const vector &names, + const vector &types, const vector &data_cols, + QuackapiResponseWriter &writer) + : state(make_uniq(con, names, types, data_cols, writer)) { +} + +QuackapiCsvChunkEncoder::~QuackapiCsvChunkEncoder() = default; + +QuackapiCsvChunkEncoder::QuackapiCsvChunkEncoder(QuackapiCsvChunkEncoder &&) noexcept = default; + +QuackapiCsvChunkEncoder &QuackapiCsvChunkEncoder::operator=(QuackapiCsvChunkEncoder &&) noexcept = default; + +void QuackapiCsvChunkEncoder::AppendChunk(DataChunk &chunk) { + state->AppendChunk(chunk); +} + +void QuackapiCsvChunkEncoder::Finish() { + state->Finish(); +} + //! Render a DuckDB Value as JSON. The database typed the column — the JSON //! representation follows the type, not string-formatting heuristics. string ValueToJson(const Value &value) { From 16f7c030d36185958665ae97c80354e4b9489d5d Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:04:52 -0700 Subject: [PATCH 09/93] Remove dedicated queue and policy modules --- docs/QUEUE.md | 80 -- docs/guide/policies.md | 141 -- docs/guide/queue.md | 155 --- src/include/quackapi_policy.hpp | 43 - src/include/quackapi_queue.hpp | 28 - src/quackapi_policy.cpp | 1371 ------------------- src/quackapi_queue.cpp | 1035 -------------- test/sql/fixtures/dequeue_conflict_repro.md | 15 - test/sql/quackapi_policy.test | 535 -------- test/sql/quackapi_queue.test | 353 ----- 10 files changed, 3756 deletions(-) delete mode 100644 docs/QUEUE.md delete mode 100644 docs/guide/policies.md delete mode 100644 docs/guide/queue.md delete mode 100644 src/include/quackapi_policy.hpp delete mode 100644 src/include/quackapi_queue.hpp delete mode 100644 src/quackapi_policy.cpp delete mode 100644 src/quackapi_queue.cpp delete mode 100644 test/sql/fixtures/dequeue_conflict_repro.md delete mode 100644 test/sql/quackapi_policy.test delete mode 100644 test/sql/quackapi_queue.test diff --git a/docs/QUEUE.md b/docs/QUEUE.md deleted file mode 100644 index dc69dd8..0000000 --- a/docs/QUEUE.md +++ /dev/null @@ -1,80 +0,0 @@ -# quackapi durable job queue - -Broker-less, single-file job queue built into quackapi. Your queue is a table — -no Redis, no NATS required. - -## Surface - -```sql -CREATE [OR REPLACE] QUEUE - [WITH ( - max_attempts=, -- default 3; after this, job → dead - visibility_timeout='30s'|30, -- claim lease (s/m/h or seconds) - backoff_base_seconds= -- default 2; nack delay = base^attempts (0 = immediate) - )]; - -DROP QUEUE ; - -quackapi_enqueue(queue VARCHAR, payload VARCHAR|JSON [, max_attempts INTEGER]) → BIGINT -quackapi_dequeue(queue VARCHAR [, n INTEGER]) → TABLE(id, queue, payload, status, attempts, max_attempts, visible_at, last_error, delivery_generation) -quackapi_ack(queue VARCHAR, job_id BIGINT, delivery_generation BIGINT) → BOOLEAN -quackapi_nack(queue VARCHAR, job_id BIGINT, delivery_generation BIGINT [, requeue BOOLEAN [, error VARCHAR]]) → VARCHAR -- pending, dead, or stale -quackapi_renew(queue VARCHAR, job_id BIGINT, delivery_generation BIGINT) → BOOLEAN -quackapi_queues() → TABLE(name, depth, in_flight, dead, max_attempts, visibility_timeout_sec, backoff_base_sec) -``` - -## Storage - -- **Jobs** live in `quackapi_jobs` (normal DuckDB table + `quackapi_job_seq`). - Survives process restart in the user's `.db` file. -- **Queue options** live on the `DatabaseInstance` (same lifecycle as routes — - re-run `CREATE QUEUE` after reopen). -- Payload is stored as JSON **text** (`VARCHAR`) so `LOAD json` is not required - at runtime. Cast to `JSON` when the json extension is loaded. - -## Semantics - -| Op | Behavior | -|----|----------| -| enqueue | `status='pending'`, `visible_at=now()` | -| dequeue | Atomic `UPDATE … WHERE id=(SELECT … LIMIT 1) RETURNING`: `status='running'`, `attempts++`, `delivery_generation++`, `visible_at=now()+visibility_timeout` | -| redelivery | Each expired delivery is recovered independently. A timed-out final attempt becomes `dead`; earlier attempts become claimable again. | -| ack | `status='done'` only for the supplied generation while its lease is active; `false` means stale, expired, or already completed. | -| nack (requeue) | `pending` + exponential backoff, or `dead` if `attempts >= max_attempts`; returns `stale` when the supplied generation no longer owns the delivery. | -| nack (requeue=false) | `dead` immediately | -| renew | Extends the configured visibility timeout only for the supplied generation while its current lease is active. Returns `false` after expiry or supersession. | - -The generation fences every delivery transition. A worker that times out and is -superseded cannot acknowledge, retry, or renew the newer delivery. Queue -transitions serialize on a per-database lock shared by static and loadable -extension copies; DuckDB remains single-writer per database file. For multi-node -workers use a real broker (Redis / NATS). - -## Delivery-generation migration - -This release adds the persisted `delivery_generation` column to existing -`quackapi_jobs` tables automatically. The old two-argument `quackapi_ack` and -old `quackapi_nack` forms have been removed because they could let a stale worker -change a redelivered job. Update workers to carry the generation returned by -`quackapi_dequeue` through every delivery transition. - -## Worker = compose `cronjob` - -Do **not** expect a C++ thread pool in quackapi. Schedule SQL: - -```sql -INSTALL cronjob FROM community; -LOAD cronjob; - -SELECT cron($$ - SELECT quackapi_ack('emails', id, delivery_generation) - FROM quackapi_dequeue('emails', 1) -$$, '*/1 * * * * *'); -``` - -Or drain from an HTTP route (`POST /drain`) that runs the same SELECT. - -## Redis - -The community Redis extension remains the "I already run Redis" option -(LPUSH-style). Native table-queue is the zero-dependency default. diff --git a/docs/guide/policies.md b/docs/guide/policies.md deleted file mode 100644 index 9a80262..0000000 --- a/docs/guide/policies.md +++ /dev/null @@ -1,141 +0,0 @@ -# Row access & masking policies - -Policies are a security leapfrog over vanilla FastAPI: **claims-keyed row filters** and **column masks** expressed in SQL, bound to tables, enforced on HTTP handlers that touch those tables. - -All examples run against `build/release/duckdb -unsigned` with `LOAD quackapi;`. - ---- - -## The idea - -1. Authenticate with JWT (or API key). -2. Claims bind as `$claims_tenant`, `$claims_role`, `$claims_sub`, … -3. A **row access policy** is a boolean expression over table columns + claims. -4. A **masking policy** rewrites a column value (placeholder `val` = original). -5. `ALTER TABLE` binds policies to tables/columns. -6. Routes that `SELECT` from bound tables get filters/masks automatically. - -Unauthenticated access to a policied table fails **closed** (HTTP **403**). - ---- - -## Row access policy - -```sql -CREATE TABLE pol_orders ( - id INTEGER, - tenant_id VARCHAR, - amount INTEGER -); - -INSERT INTO pol_orders VALUES - (1, 'acme', 100), - (2, 'beta', 200), - (3, 'acme', 150); - -CREATE AUTH jwt_pol AS JWT ( SECRET 'policy-http-secret' ); - -CREATE ROW ACCESS POLICY tenant_isolation - AS (tenant_id VARCHAR) RETURNS BOOLEAN - USING (tenant_id = $claims_tenant OR $claims_role = 'admin'); - -ALTER TABLE pol_orders - ADD ROW ACCESS POLICY tenant_isolation ON (tenant_id); - -CREATE ROUTE list_orders GET '/orders' REQUIRE jwt_pol AS -SELECT id, tenant_id, amount FROM pol_orders ORDER BY id; -``` - -JWT payload for tenant `acme`, role `user` → only acme rows: - -```sh -curl http://127.0.0.1:8000/orders -H "Authorization: Bearer $JWT_ACME" -# [{"id":1,"tenant_id":"acme","amount":100},{"id":3,"tenant_id":"acme","amount":150}] -# HTTP 200 -``` - -Admin role (`$claims_role = 'admin'`) sees all tenants. -Missing JWT on a `REQUIRE` route → **401** (auth runs before policy). -Public route on a policied table without claims → **403** “Policy denies…”. - ---- - -## Masking policy - -```sql -CREATE TABLE pol_users ( - id INTEGER, - email VARCHAR, - tenant_id VARCHAR -); - -CREATE MASKING POLICY mask_email ON VARCHAR - USING (CASE WHEN $claims_role = 'admin' THEN val ELSE '***' END); - -ALTER TABLE pol_users - ADD ROW ACCESS POLICY tenant_isolation ON (tenant_id); - -ALTER TABLE pol_users - MODIFY COLUMN email SET MASKING POLICY mask_email; - -CREATE ROUTE list_users GET '/users' REQUIRE jwt_pol AS -SELECT id, email, tenant_id FROM pol_users ORDER BY id; -``` - -| Role | `email` column | -|------|----------------| -| `user` | `***` | -| `admin` | real address | - -`val` is the original column value inside the masking expression. - ---- - -## Inspect - -```sql -SELECT name, kind, signature, expression, bound_table, bound_columns -FROM quackapi_policies(); -``` - -| kind | Meaning | -|------|---------| -| `ROW_ACCESS` | Row filter | -| `MASKING` | Column rewrite | - ---- - -## Grammar - -```sql -CREATE [OR REPLACE] ROW ACCESS POLICY - AS ( [] [, ...]) RETURNS BOOLEAN - USING (); - -CREATE [OR REPLACE] MASKING POLICY - ON - USING (); - -ALTER TABLE ADD ROW ACCESS POLICY ON (); -ALTER TABLE MODIFY COLUMN SET MASKING POLICY ; - -DROP ROW ACCESS POLICY ; -DROP MASKING POLICY ; -``` - -Names must be unique across both policy kinds. - ---- - -## Design notes - -- Policies are **helpers**, not a full multi-tenant RBAC product. -- Prefer JWT claims you control (`tenant`, `role`) over trusting query params. -- Combine with [CREATE GROUP](groups.md) so every `/api/v1/*` route requires the same auth scheme. - ---- - -## Next - -- [Static files](static-files.md) -- [OpenAPI](openapi.md) diff --git a/docs/guide/queue.md b/docs/guide/queue.md deleted file mode 100644 index 418c4ed..0000000 --- a/docs/guide/queue.md +++ /dev/null @@ -1,155 +0,0 @@ -# CREATE QUEUE — background jobs without Redis - -Jobs live in an ordinary DuckDB table: **`quackapi_jobs`**. No external broker. Enqueue from a route, drain with a worker route (or scheduled SQL). - -All examples run against `build/release/duckdb -unsigned` with `LOAD quackapi;`. - ---- - -## Create a queue - -```sql -CREATE QUEUE default WITH ( - max_attempts=3, - visibility_timeout='30s', - backoff_base_seconds=2 -); -``` - -| Option | Default | Meaning | -|--------|---------|---------| -| `max_attempts` | 3 | After this many claims, nack → `dead` | -| `visibility_timeout` | 30s | Lease length while `running` (`'30s'`, `'5m'`, or bare seconds) | -| `backoff_base_seconds` | 2 | Nack delay ≈ base^attempts (`0` = immediate) | - -```sql -SELECT name, depth, in_flight, dead, - max_attempts, visibility_timeout_sec, backoff_base_sec -FROM quackapi_queues(); -``` - ---- - -## HTTP: enqueue + drain worker - -```sql -CREATE QUEUE default WITH (max_attempts=3, visibility_timeout='30s'); - -CREATE ROUTE enqueue POST '/jobs' STATUS 201 - PARAM payload VARCHAR - AS -SELECT quackapi_enqueue('default', $payload::VARCHAR) AS job_id; - --- Worker tick: claim up to 10 jobs and ack each in one SELECT -CREATE ROUTE drain POST '/drain' AS -SELECT id AS job_id, - 'processed:' || payload AS result, - quackapi_ack('default', id, delivery_generation) AS acked -FROM quackapi_dequeue('default', 10); - -CREATE ROUTE results GET '/results' AS -SELECT id AS job_id, payload, status -FROM quackapi_jobs -WHERE status = 'done' -ORDER BY id; - -CREATE ROUTE stats GET '/stats' AS -SELECT name, depth, in_flight, dead FROM quackapi_queues(); -``` - -Live run: - -```sh -curl -X POST http://127.0.0.1:8000/jobs \ - -H 'Content-Type: application/json' \ - -d '{"payload":"{\"task\":\"email\"}"}' -# [{"job_id":1}] -# HTTP 201 - -curl http://127.0.0.1:8000/stats -# [{"name":"default","depth":1,"in_flight":0,"dead":0}] - -# httplib wants a body on POST — send empty JSON -curl -X POST http://127.0.0.1:8000/drain \ - -H 'Content-Type: application/json' -d '{}' -# [{"job_id":1,"result":"processed:{\"task\":\"email\"}","acked":true}] - -curl http://127.0.0.1:8000/results -# [{"job_id":1,"payload":"{\"task\":\"email\"}","status":"done"}] -``` - ---- - -## SQL API (no HTTP) - -```sql -SELECT quackapi_enqueue('default', '{"task":"send_email"}'); --- → job_id (BIGINT) - -SELECT id, payload, status, attempts -FROM quackapi_dequeue('default', 10); --- claims: status=running, attempts++, visibility lease - -SELECT quackapi_ack('default', 1, 1); --- true if job was running with delivery_generation 1 - -SELECT quackapi_nack('default', 1, 1, true, 'try_again'); --- 'pending' (retry) or 'dead' at max_attempts - -SELECT quackapi_nack('default', 1, 1, false, 'no_retry'); --- straight to 'dead' - -SELECT quackapi_renew('default', 1, 1); --- true if the active lease was renewed -``` - -Payload is stored as **VARCHAR** (JSON text). You may also pass `JSON` when the json extension is loaded. - -Per-job max attempts override: - -```sql -SELECT quackapi_enqueue('default', '{"z":1}', 1); -``` - ---- - -## Semantics - -| Op | Behavior | -|----|----------| -| enqueue | `pending`, visible now | -| dequeue | Exclusive claim → `running` + lease | -| redelivery | `running` rows with expired lease are claimable again | -| ack | `done` | -| nack (requeue) | `pending` + backoff, or `dead` at max_attempts | -| nack (no requeue) | `dead` immediately | - -**Lifecycle note:** queue *options* live on the database instance (re-run `CREATE QUEUE` after reopen). Job *rows* in `quackapi_jobs` are durable in your `.db` file. - -Concurrency: DuckDB is single-writer per file. Claims serialize on the write lock — correct for one process. For multi-node workers, use an external broker. - ---- - -## Worker options - -1. **HTTP drain route** (above) — cron or a second process POSTs `/drain`. -2. **Scheduled SQL** — community `cronjob` extension (or any external scheduler) running the same `SELECT … FROM quackapi_dequeue … ack`. -3. **Interactive shell** — run dequeue/ack by hand while developing. - -There is no built-in C++ thread pool. The queue is storage + claim primitives; **you** schedule the worker. - ---- - -## Drop - -```sql -DROP QUEUE default; --- registry entry removed; quackapi_jobs rows for that queue remain on disk -``` - ---- - -## Next - -- [CREATE STREAM](stream.md) — push events while jobs run -- Deeper notes: [QUEUE.md](../QUEUE.md) diff --git a/src/include/quackapi_policy.hpp b/src/include/quackapi_policy.hpp deleted file mode 100644 index 17e7542..0000000 --- a/src/include/quackapi_policy.hpp +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once - -#include "duckdb/function/table_function.hpp" -#include "duckdb/parser/parser_extension.hpp" - -namespace duckdb { - -class DatabaseInstance; -class ExtensionLoader; - -//! CREATE / DROP ROW ACCESS POLICY, CREATE / DROP MASKING POLICY, -//! ALTER TABLE … ADD/DROP ROW ACCESS POLICY, ALTER TABLE … SET/UNSET MASKING POLICY. -class PolicyDdlParserExtension : public ParserExtension { -public: - PolicyDdlParserExtension(); -}; - -TableFunction GetApplyPolicyFunction(); -TableFunction GetQuackapiPoliciesFunction(); - -//! Rewrite handler SQL so parsed references to policy-bound base tables become -//! secure subqueries (row filter + column masks). Uses policy expressions as-is -//! ($claims_* remain named parameters for the caller's verified-claims binder). -//! -//! When policies are configured, sources whose base-table identity cannot be -//! safely rewritten (for example views and table macros) fail closed. For an -//! unauthenticated access `deny_unauthenticated` is set; otherwise -//! `policy_error` describes the rejected source for server-side logging only. -string RewriteHandlerWithPolicies(DatabaseInstance &db, const string &handler_sql, bool authenticated, - bool &deny_unauthenticated, string &policy_error); - -//! Compatibility wrapper for callers that only need the unauthenticated result. -inline string RewriteHandlerWithPolicies(DatabaseInstance &db, const string &handler_sql, bool authenticated, - bool &deny_unauthenticated) { - string policy_error; - return RewriteHandlerWithPolicies(db, handler_sql, authenticated, deny_unauthenticated, policy_error); -} - -//! True when parsed handler SQL references a policy-bound table or an indirect -//! source that must be rejected while policies are active. -bool HandlerTouchesPoliciedTable(DatabaseInstance &db, const string &handler_sql); - -} // namespace duckdb diff --git a/src/include/quackapi_queue.hpp b/src/include/quackapi_queue.hpp deleted file mode 100644 index 72b8ab5..0000000 --- a/src/include/quackapi_queue.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/extension/extension_loader.hpp" -#include "duckdb/parser/parser_extension.hpp" - -namespace duckdb { - -//! CREATE [OR REPLACE] QUEUE / DROP QUEUE syntax. -//! Grammar: -//! CREATE [OR REPLACE] QUEUE -//! [WITH (max_attempts=, visibility_timeout='30s'|30, backoff_base_seconds=)] -//! DROP QUEUE -class QueueDdlParserExtension : public ParserExtension { -public: - QueueDdlParserExtension(); -}; - -//! Internal apply target for planned CREATE/DROP QUEUE. -TableFunction GetApplyQueueFunction(); - -//! Register quackapi_enqueue / dequeue / ack / nack / queues functions. -void RegisterQuackapiQueueFunctions(ExtensionLoader &loader); - -//! Ensure the durable quackapi_jobs table + sequence exist (idempotent). -void EnsureQuackapiJobsTable(DatabaseInstance &db); - -} // namespace duckdb diff --git a/src/quackapi_policy.cpp b/src/quackapi_policy.cpp deleted file mode 100644 index 35d85da..0000000 --- a/src/quackapi_policy.cpp +++ /dev/null @@ -1,1371 +0,0 @@ -#include "quackapi_policy.hpp" - -#include - -#include "duckdb/common/exception.hpp" -#include "duckdb/common/string_util.hpp" -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/main/connection.hpp" -#include "duckdb/parser/parser.hpp" -#include "duckdb/parser/parser_extension.hpp" -#include "duckdb/parser/parsed_expression_iterator.hpp" -#include "duckdb/parser/expression/subquery_expression.hpp" -#include "duckdb/parser/query_node/select_node.hpp" -#include "duckdb/parser/query_node/set_operation_node.hpp" -#include "duckdb/parser/statement/select_statement.hpp" -#include "duckdb/parser/tableref/list.hpp" - -#include "quackapi_state.hpp" -#include "quackapi_util.hpp" - -namespace duckdb { - -namespace { - -bool IsIdentStart(char c) { - return StringUtil::CharacterIsAlpha(c) || c == '_'; -} - -bool IsIdentChar(char c) { - return StringUtil::CharacterIsAlpha(c) || StringUtil::CharacterIsDigit(c) || c == '_'; -} - -//! First whitespace-delimited token (space/tab/newline all count as boundaries). -string NextToken(const string &s, idx_t &consumed) { - idx_t i = 0; - while (i < s.size() && !StringUtil::CharacterIsSpace(s[i])) { - i++; - } - consumed = i; - return s.substr(0, i); -} - -//! Extract a parenthesized span starting at rest[0] == '('. Balanced parens; -//! ignores parens inside single-quoted strings ('' escape). -bool ExtractParenGroup(const string &rest, idx_t start, string &inner, idx_t &end_out) { - if (start >= rest.size() || rest[start] != '(') { - return false; - } - int depth = 0; - bool in_str = false; - for (idx_t i = start; i < rest.size(); i++) { - char c = rest[i]; - if (in_str) { - if (c == '\'') { - if (i + 1 < rest.size() && rest[i + 1] == '\'') { - i++; // escaped quote - } else { - in_str = false; - } - } - continue; - } - if (c == '\'') { - in_str = true; - continue; - } - if (c == '(') { - depth++; - } else if (c == ')') { - depth--; - if (depth == 0) { - inner = rest.substr(start + 1, i - start - 1); - end_out = i + 1; - return true; - } - } - } - return false; -} - -//! Parse "col [TYPE], col [TYPE], ..." into names + optional types. -bool ParseColList(const string &inner, vector &names, vector &types, string &err) { - names.clear(); - types.clear(); - string s = QuackapiTrim(inner); - if (s.empty()) { - err = "column list must not be empty"; - return false; - } - idx_t i = 0; - while (i < s.size()) { - while (i < s.size() && (StringUtil::CharacterIsSpace(s[i]) || s[i] == ',')) { - i++; - } - if (i >= s.size()) { - break; - } - if (!IsIdentStart(s[i])) { - err = "expected column name in list"; - return false; - } - idx_t start = i; - i++; - while (i < s.size() && IsIdentChar(s[i])) { - i++; - } - string col = s.substr(start, i - start); - while (i < s.size() && StringUtil::CharacterIsSpace(s[i])) { - i++; - } - string typ; - if (i < s.size() && IsIdentStart(s[i])) { - idx_t ts = i; - i++; - while (i < s.size() && IsIdentChar(s[i])) { - i++; - } - typ = StringUtil::Upper(s.substr(ts, i - ts)); - if (typ == "INT") { - typ = "INTEGER"; - } else if (typ == "BOOL") { - typ = "BOOLEAN"; - } else if (typ == "TEXT" || typ == "STRING") { - typ = "VARCHAR"; - } else if (typ == "REAL") { - typ = "FLOAT"; - } - } - names.push_back(col); - types.push_back(typ); - } - if (names.empty()) { - err = "column list must not be empty"; - return false; - } - return true; -} - -string JoinComma(const vector &parts) { - string out; - for (idx_t i = 0; i < parts.size(); i++) { - if (i > 0) { - out += ", "; - } - out += parts[i]; - } - return out; -} - -//! Replace whole-word identifier `val` with `replacement` (masking policy body). -string SubstituteValPlaceholder(const string &expr, const string &replacement) { - string out; - out.reserve(expr.size() + 8); - bool in_str = false; - for (idx_t i = 0; i < expr.size();) { - char c = expr[i]; - if (in_str) { - out += c; - if (c == '\'') { - if (i + 1 < expr.size() && expr[i + 1] == '\'') { - out += expr[i + 1]; - i += 2; - continue; - } - in_str = false; - } - i++; - continue; - } - if (c == '\'') { - in_str = true; - out += c; - i++; - continue; - } - if (IsIdentStart(c)) { - idx_t j = i + 1; - while (j < expr.size() && IsIdentChar(expr[j])) { - j++; - } - string tok = expr.substr(i, j - i); - if (StringUtil::Lower(tok) == "val") { - out += replacement; - } else { - out += tok; - } - i = j; - continue; - } - out += c; - i++; - } - return out; -} - -// `physical_relation` is built from parsed catalog/schema/table identifiers. It -// is deliberately not reconstructed from route text, so an attached catalog or -// quoted identifier cannot silently target a different object. -string BuildSecureSubqueryQuoted(const string &physical_relation, const QuackapiRowAccessPolicy *rap_for_where, - const string &where_expr_or_empty, - const vector> &masked_cols) { - string where_sql; - if (!where_expr_or_empty.empty()) { - where_sql = " WHERE (" + where_expr_or_empty + ")"; - } - (void)rap_for_where; - if (masked_cols.empty()) { - return "SELECT * FROM " + physical_relation + where_sql; - } - string excludes; - string extras; - for (idx_t i = 0; i < masked_cols.size(); i++) { - if (i > 0) { - excludes += ", "; - extras += ", "; - } - // Quote masked column names in EXCLUDE / AS - string col_esc; - for (char c : masked_cols[i].first) { - if (c == '"') { - col_esc += "\"\""; - } else { - col_esc += c; - } - } - string colq = "\"" + col_esc + "\""; - excludes += colq; - extras += "(" + masked_cols[i].second + ") AS " + colq; - } - return "SELECT * EXCLUDE (" + excludes + "), " + extras + " FROM " + physical_relation + where_sql; -} - -//===--------------------------------------------------------------------===// -// DDL parse data -//===--------------------------------------------------------------------===// - -struct PolicyDdlParseData : public ParserExtensionParseData { - //! CREATE_ROW | CREATE_MASK | DROP_ROW | DROP_MASK | BIND_ROW | UNBIND_ROW | BIND_MASK | UNBIND_MASK - string action; - bool or_replace = false; - string name; - string value_type; // masking ON - vector arg_columns; // RAP signature / bind ON cols / mask column - vector arg_types; - string expression; - string table_name; - string column_name; // masking bind - - unique_ptr Copy() const override { - auto copy = make_uniq(); - copy->action = action; - copy->or_replace = or_replace; - copy->name = name; - copy->value_type = value_type; - copy->arg_columns = arg_columns; - copy->arg_types = arg_types; - copy->expression = expression; - copy->table_name = table_name; - copy->column_name = column_name; - return std::move(copy); - } - string ToString() const override { - return action + " POLICY " + name; - } -}; - -ParserExtensionParseResult PolicyDdlParse(ParserExtensionInfo *, const string &query) { - auto q = QuackapiTrim(query); - auto upper = StringUtil::Upper(q); - - // ---- DROP ROW ACCESS POLICY / DROP MASKING POLICY ---- - if (StringUtil::StartsWith(upper, "DROP ROW ACCESS POLICY ")) { - auto name = QuackapiTrim(q.substr(23)); - if (name.empty() || name.find(' ') != string::npos) { - return ParserExtensionParseResult("DROP ROW ACCESS POLICY expects a single policy name"); - } - auto data = make_uniq(); - data->action = "DROP_ROW"; - data->name = name; - return ParserExtensionParseResult(std::move(data)); - } - if (StringUtil::StartsWith(upper, "DROP MASKING POLICY ")) { - auto name = QuackapiTrim(q.substr(20)); - if (name.empty() || name.find(' ') != string::npos) { - return ParserExtensionParseResult("DROP MASKING POLICY expects a single policy name"); - } - auto data = make_uniq(); - data->action = "DROP_MASK"; - data->name = name; - return ParserExtensionParseResult(std::move(data)); - } - - // ---- CREATE [OR REPLACE] ROW ACCESS POLICY ---- - bool or_replace = false; - idx_t pos = 0; - bool is_create_row = false; - bool is_create_mask = false; - if (StringUtil::StartsWith(upper, "CREATE OR REPLACE ROW ACCESS POLICY ")) { - pos = 36; - or_replace = true; - is_create_row = true; - } else if (StringUtil::StartsWith(upper, "CREATE ROW ACCESS POLICY ")) { - pos = 25; - is_create_row = true; - } else if (StringUtil::StartsWith(upper, "CREATE OR REPLACE MASKING POLICY ")) { - pos = 33; - or_replace = true; - is_create_mask = true; - } else if (StringUtil::StartsWith(upper, "CREATE MASKING POLICY ")) { - pos = 22; - is_create_mask = true; - } - - if (is_create_row) { - auto rest = QuackapiTrim(q.substr(pos)); - idx_t name_len = 0; - auto name = NextToken(rest, name_len); - if (name.empty() || name_len >= rest.size()) { - return ParserExtensionParseResult( - "CREATE ROW ACCESS POLICY AS () RETURNS BOOLEAN USING ()"); - } - rest = QuackapiTrim(rest.substr(name_len)); - auto ru = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(ru, "AS")) { - return ParserExtensionParseResult("Expected AS () after policy name"); - } - rest = QuackapiTrim(rest.substr(2)); - if (rest.empty() || rest[0] != '(') { - return ParserExtensionParseResult("Expected AS ()"); - } - string col_inner; - idx_t after_cols = 0; - if (!ExtractParenGroup(rest, 0, col_inner, after_cols)) { - return ParserExtensionParseResult("Unterminated column list in AS (...)"); - } - vector cols, types; - string err; - if (!ParseColList(col_inner, cols, types, err)) { - return ParserExtensionParseResult("ROW ACCESS POLICY AS (...): " + err); - } - rest = QuackapiTrim(rest.substr(after_cols)); - ru = StringUtil::Upper(rest); - // RETURNS BOOLEAN - if (!StringUtil::StartsWith(ru, "RETURNS")) { - return ParserExtensionParseResult("Expected RETURNS BOOLEAN after AS (...)"); - } - rest = QuackapiTrim(rest.substr(7)); - ru = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(ru, "BOOLEAN")) { - return ParserExtensionParseResult("ROW ACCESS POLICY must RETURNS BOOLEAN"); - } - rest = QuackapiTrim(rest.substr(7)); - ru = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(ru, "USING")) { - return ParserExtensionParseResult("Expected USING () after RETURNS BOOLEAN"); - } - rest = QuackapiTrim(rest.substr(5)); - if (rest.empty() || rest[0] != '(') { - return ParserExtensionParseResult("Expected USING ()"); - } - string expr; - idx_t after_expr = 0; - if (!ExtractParenGroup(rest, 0, expr, after_expr)) { - return ParserExtensionParseResult("Unterminated USING (...) expression"); - } - expr = QuackapiTrim(expr); - if (expr.empty()) { - return ParserExtensionParseResult("USING expression must not be empty"); - } - rest = QuackapiTrim(rest.substr(after_expr)); - if (!rest.empty()) { - return ParserExtensionParseResult("Unexpected tokens after USING (...)"); - } - auto data = make_uniq(); - data->action = "CREATE_ROW"; - data->or_replace = or_replace; - data->name = name; - data->arg_columns = std::move(cols); - data->arg_types = std::move(types); - data->expression = expr; - return ParserExtensionParseResult(std::move(data)); - } - - if (is_create_mask) { - auto rest = QuackapiTrim(q.substr(pos)); - idx_t name_len = 0; - auto name = NextToken(rest, name_len); - if (name.empty() || name_len >= rest.size()) { - return ParserExtensionParseResult("CREATE MASKING POLICY ON USING ()"); - } - rest = QuackapiTrim(rest.substr(name_len)); - auto ru = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(ru, "ON")) { - return ParserExtensionParseResult("Expected ON after masking policy name"); - } - // ON may be followed by space or newline - if (rest.size() == 2 || !StringUtil::CharacterIsSpace(rest[2])) { - // "ON" alone without trailing space after token — still need type next - if (rest.size() == 2) { - return ParserExtensionParseResult("Expected ON USING ()"); - } - } - rest = QuackapiTrim(rest.substr(2)); - idx_t type_len = 0; - auto vtype = StringUtil::Upper(NextToken(rest, type_len)); - if (vtype.empty() || type_len >= rest.size()) { - return ParserExtensionParseResult("Expected ON USING ()"); - } - if (vtype == "INT") { - vtype = "INTEGER"; - } else if (vtype == "BOOL") { - vtype = "BOOLEAN"; - } else if (vtype == "TEXT" || vtype == "STRING") { - vtype = "VARCHAR"; - } else if (vtype == "REAL") { - vtype = "FLOAT"; - } - rest = QuackapiTrim(rest.substr(type_len)); - ru = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(ru, "USING")) { - return ParserExtensionParseResult("Expected USING () after ON "); - } - rest = QuackapiTrim(rest.substr(5)); - if (rest.empty() || rest[0] != '(') { - return ParserExtensionParseResult("Expected USING ()"); - } - string expr; - idx_t after_expr = 0; - if (!ExtractParenGroup(rest, 0, expr, after_expr)) { - return ParserExtensionParseResult("Unterminated USING (...) expression"); - } - expr = QuackapiTrim(expr); - if (expr.empty()) { - return ParserExtensionParseResult("USING expression must not be empty"); - } - rest = QuackapiTrim(rest.substr(after_expr)); - if (!rest.empty()) { - return ParserExtensionParseResult("Unexpected tokens after USING (...)"); - } - auto data = make_uniq(); - data->action = "CREATE_MASK"; - data->or_replace = or_replace; - data->name = name; - data->value_type = vtype; - data->expression = expr; - return ParserExtensionParseResult(std::move(data)); - } - - // ---- ALTER TABLE … ---- - if (StringUtil::StartsWith(upper, "ALTER TABLE ")) { - auto rest = QuackapiTrim(q.substr(12)); - // table name - string table; - if (rest.empty()) { - return ParserExtensionParseResult(); - } - if (rest[0] == '"') { - // quoted - idx_t i = 1; - string result; - while (i < rest.size()) { - if (rest[i] == '"') { - if (i + 1 < rest.size() && rest[i + 1] == '"') { - result += '"'; - i += 2; - continue; - } - table = result; - rest = QuackapiTrim(rest.substr(i + 1)); - break; - } - result += rest[i]; - i++; - } - if (table.empty()) { - return ParserExtensionParseResult(); - } - } else { - idx_t i = 0; - while (i < rest.size() && IsIdentChar(rest[i])) { - i++; - } - if (i == 0) { - return ParserExtensionParseResult(); - } - table = rest.substr(0, i); - rest = QuackapiTrim(rest.substr(i)); - } - auto ru = StringUtil::Upper(rest); - - // ADD [OR REPLACE] ROW ACCESS POLICY

ON () - bool add_or_replace = false; - if (StringUtil::StartsWith(ru, "ADD OR REPLACE ROW ACCESS POLICY ")) { - rest = QuackapiTrim(rest.substr(33)); - add_or_replace = true; - } else if (StringUtil::StartsWith(ru, "ADD ROW ACCESS POLICY ")) { - rest = QuackapiTrim(rest.substr(22)); - } else if (StringUtil::StartsWith(ru, "DROP ROW ACCESS POLICY ")) { - auto pname = QuackapiTrim(rest.substr(23)); - if (pname.empty() || pname.find(' ') != string::npos) { - return ParserExtensionParseResult("ALTER TABLE … DROP ROW ACCESS POLICY expects a policy name"); - } - auto data = make_uniq(); - data->action = "UNBIND_ROW"; - data->table_name = table; - data->name = pname; - return ParserExtensionParseResult(std::move(data)); - } else if (StringUtil::StartsWith(ru, "MODIFY COLUMN ") || StringUtil::StartsWith(ru, "ALTER COLUMN ")) { - idx_t skip = StringUtil::StartsWith(ru, "MODIFY COLUMN ") ? 14 : 13; - rest = QuackapiTrim(rest.substr(skip)); - // column name - string col; - idx_t ci = 0; - if (rest.empty()) { - return ParserExtensionParseResult("ALTER TABLE … COLUMN expects a column name"); - } - if (rest[0] == '"') { - return ParserExtensionParseResult("Quoted column names in SET MASKING POLICY not yet supported"); - } - while (ci < rest.size() && IsIdentChar(rest[ci])) { - ci++; - } - col = rest.substr(0, ci); - rest = QuackapiTrim(rest.substr(ci)); - ru = StringUtil::Upper(rest); - if (StringUtil::StartsWith(ru, "SET MASKING POLICY ")) { - auto pname = QuackapiTrim(rest.substr(19)); - if (pname.empty() || pname.find(' ') != string::npos) { - return ParserExtensionParseResult("SET MASKING POLICY expects a policy name"); - } - auto data = make_uniq(); - data->action = "BIND_MASK"; - data->or_replace = true; // SET replaces - data->table_name = table; - data->column_name = col; - data->name = pname; - return ParserExtensionParseResult(std::move(data)); - } - if (StringUtil::StartsWith(ru, "UNSET MASKING POLICY") || - StringUtil::StartsWith(ru, "DROP MASKING POLICY")) { - auto data = make_uniq(); - data->action = "UNBIND_MASK"; - data->table_name = table; - data->column_name = col; - return ParserExtensionParseResult(std::move(data)); - } - // Not our ALTER COLUMN form — leave to core. - return ParserExtensionParseResult(); - } else { - // Not a policy ALTER — leave to DuckDB core. - return ParserExtensionParseResult(); - } - - // ADD ROW ACCESS POLICY path continues - idx_t pname_len = 0; - string pname = NextToken(rest, pname_len); - if (pname.empty()) { - return ParserExtensionParseResult("ADD ROW ACCESS POLICY expects a policy name"); - } - rest = pname_len >= rest.size() ? string() : QuackapiTrim(rest.substr(pname_len)); - ru = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(ru, "ON")) { - return ParserExtensionParseResult("Expected ON () after policy name"); - } - rest = QuackapiTrim(rest.substr(2)); - if (rest.empty() || rest[0] != '(') { - return ParserExtensionParseResult("Expected ON ()"); - } - string col_inner; - idx_t after = 0; - if (!ExtractParenGroup(rest, 0, col_inner, after)) { - return ParserExtensionParseResult("Unterminated ON (...) column list"); - } - vector cols, types; - string err; - if (!ParseColList(col_inner, cols, types, err)) { - return ParserExtensionParseResult("ON (...): " + err); - } - rest = QuackapiTrim(rest.substr(after)); - if (!rest.empty()) { - return ParserExtensionParseResult("Unexpected tokens after ON (...)"); - } - auto data = make_uniq(); - data->action = "BIND_ROW"; - data->or_replace = add_or_replace; - data->table_name = table; - data->name = pname; - data->arg_columns = std::move(cols); - return ParserExtensionParseResult(std::move(data)); - } - - return ParserExtensionParseResult(); -} - -//===--------------------------------------------------------------------===// -// Apply table function -//===--------------------------------------------------------------------===// - -struct ApplyPolicyBindData : public TableFunctionData { - string action; - bool or_replace = false; - string name; - string value_type; - string arg_columns_csv; // comma-separated - string arg_types_csv; - string expression; - string table_name; - string column_name; - bool finished = false; -}; - -vector SplitCsv(const string &csv) { - vector out; - if (csv.empty()) { - return out; - } - idx_t i = 0; - while (i < csv.size()) { - idx_t j = i; - while (j < csv.size() && csv[j] != ',') { - j++; - } - out.push_back(QuackapiTrim(csv.substr(i, j - i))); - i = j < csv.size() ? j + 1 : j; - } - return out; -} - -unique_ptr ApplyPolicyBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - bind_data->action = input.inputs[0].GetValue(); - bind_data->or_replace = input.inputs[1].GetValue(); - bind_data->name = input.inputs[2].GetValue(); - bind_data->value_type = input.inputs[3].GetValue(); - bind_data->arg_columns_csv = input.inputs[4].GetValue(); - bind_data->arg_types_csv = input.inputs[5].GetValue(); - bind_data->expression = input.inputs[6].GetValue(); - bind_data->table_name = input.inputs[7].GetValue(); - bind_data->column_name = input.inputs[8].GetValue(); - BindStatusColumn(return_types, names); - return std::move(bind_data); -} - -void ApplyPolicyExec(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &bind_data = data_p.bind_data->CastNoConst(); - if (bind_data.finished) { - return; - } - auto &state = QuackapiState::Get(*context.db); - string message; - auto cols = SplitCsv(bind_data.arg_columns_csv); - auto types = SplitCsv(bind_data.arg_types_csv); - - if (bind_data.action == "CREATE_ROW") { - QuackapiRowAccessPolicy p; - p.name = bind_data.name; - p.arg_columns = cols; - p.arg_types = types; - while (p.arg_types.size() < p.arg_columns.size()) { - p.arg_types.push_back(""); - } - p.expression = bind_data.expression; - state.AddRowAccessPolicy(p, bind_data.or_replace); - message = StringUtil::Format("Row access policy %s", p.name); - } else if (bind_data.action == "CREATE_MASK") { - QuackapiMaskingPolicy p; - p.name = bind_data.name; - p.value_type = bind_data.value_type; - p.expression = bind_data.expression; - state.AddMaskingPolicy(p, bind_data.or_replace); - message = StringUtil::Format("Masking policy %s ON %s", p.name, p.value_type); - } else if (bind_data.action == "DROP_ROW") { - if (!state.DropRowAccessPolicy(bind_data.name)) { - throw InvalidInputException("Row access policy \"%s\" does not exist", bind_data.name); - } - message = StringUtil::Format("Dropped row access policy %s", bind_data.name); - } else if (bind_data.action == "DROP_MASK") { - if (!state.DropMaskingPolicy(bind_data.name)) { - throw InvalidInputException("Masking policy \"%s\" does not exist", bind_data.name); - } - message = StringUtil::Format("Dropped masking policy %s", bind_data.name); - } else if (bind_data.action == "BIND_ROW") { - QuackapiRowAccessBinding b; - b.table_name = bind_data.table_name; - b.policy_name = bind_data.name; - b.columns = cols; - state.BindRowAccessPolicy(b, bind_data.or_replace); - message = StringUtil::Format("Bound row access policy %s on %s", b.policy_name, b.table_name); - } else if (bind_data.action == "UNBIND_ROW") { - if (!state.UnbindRowAccessPolicy(bind_data.table_name, bind_data.name)) { - throw InvalidInputException("Table \"%s\" has no row access policy \"%s\"", bind_data.table_name, - bind_data.name); - } - message = StringUtil::Format("Dropped row access policy %s from %s", bind_data.name, bind_data.table_name); - } else if (bind_data.action == "BIND_MASK") { - QuackapiMaskingBinding b; - b.table_name = bind_data.table_name; - b.column_name = bind_data.column_name; - b.policy_name = bind_data.name; - state.BindMaskingPolicy(b, bind_data.or_replace); - message = StringUtil::Format("Set masking policy %s on %s.%s", b.policy_name, b.table_name, b.column_name); - } else if (bind_data.action == "UNBIND_MASK") { - if (!state.UnbindMaskingPolicy(bind_data.table_name, bind_data.column_name)) { - throw InvalidInputException("Column \"%s\".\"%s\" has no masking policy", bind_data.table_name, - bind_data.column_name); - } - message = StringUtil::Format("Unset masking policy on %s.%s", bind_data.table_name, bind_data.column_name); - } else { - throw InvalidInputException("Unknown policy action \"%s\"", bind_data.action); - } - - EmitOneShotStatus(output, bind_data.finished, message); -} - -TableFunction MakeApplyPolicyFunction() { - // action, or_replace, name, value_type, arg_columns_csv, arg_types_csv, expression, table, column - return MakeApplyDdlFunction("quackapi_apply_policy", - {LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::VARCHAR}, - ApplyPolicyExec, ApplyPolicyBind); -} - -ParserExtensionPlanResult PolicyDdlPlan(ParserExtensionInfo *, ClientContext &, - unique_ptr parse_data) { - auto &data = static_cast(*parse_data); - ParserExtensionPlanResult result; - result.function = MakeApplyPolicyFunction(); - result.parameters.push_back(Value(data.action)); - result.parameters.push_back(Value::BOOLEAN(data.or_replace)); - result.parameters.push_back(Value(data.name)); - result.parameters.push_back(Value(data.value_type)); - result.parameters.push_back(Value(JoinComma(data.arg_columns))); - result.parameters.push_back(Value(JoinComma(data.arg_types))); - result.parameters.push_back(Value(data.expression)); - result.parameters.push_back(Value(data.table_name)); - result.parameters.push_back(Value(data.column_name)); - FinishDdlPlan(result); - return result; -} - -//===--------------------------------------------------------------------===// -// quackapi_policies() -//===--------------------------------------------------------------------===// - -struct PolicyRow { - string name; - string kind; - string signature; - string expression; - string bound_table; - string bound_columns; -}; - -struct PoliciesBindData : public TableFunctionData {}; - -struct PoliciesGlobalState : public GlobalTableFunctionState { - vector rows; - idx_t offset = 0; -}; - -unique_ptr PoliciesBind(ClientContext &, TableFunctionBindInput &, vector &return_types, - vector &names) { - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("name"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("kind"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("signature"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("expression"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("bound_table"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("bound_columns"); - return make_uniq(); -} - -unique_ptr PoliciesInit(ClientContext &context, TableFunctionInitInput &) { - auto st = make_uniq(); - auto &state = QuackapiState::Get(*context.db); - auto raps = state.SnapshotRowAccessPolicies(); - auto masks = state.SnapshotMaskingPolicies(); - auto rap_binds = state.SnapshotRowAccessBindings(); - auto mask_binds = state.SnapshotMaskingBindings(); - - for (auto &p : raps) { - string sig = "("; - for (idx_t i = 0; i < p.arg_columns.size(); i++) { - if (i > 0) { - sig += ", "; - } - sig += p.arg_columns[i]; - if (i < p.arg_types.size() && !p.arg_types[i].empty()) { - sig += " "; - sig += p.arg_types[i]; - } - } - sig += ") RETURNS BOOLEAN"; - bool any = false; - for (auto &b : rap_binds) { - if (b.policy_name == p.name) { - PolicyRow row; - row.name = p.name; - row.kind = "ROW_ACCESS"; - row.signature = sig; - row.expression = p.expression; - row.bound_table = b.table_name; - row.bound_columns = JoinComma(b.columns); - st->rows.push_back(std::move(row)); - any = true; - } - } - if (!any) { - PolicyRow row; - row.name = p.name; - row.kind = "ROW_ACCESS"; - row.signature = sig; - row.expression = p.expression; - st->rows.push_back(std::move(row)); - } - } - for (auto &p : masks) { - string sig = "ON " + p.value_type; - bool any = false; - for (auto &b : mask_binds) { - if (b.policy_name == p.name) { - PolicyRow row; - row.name = p.name; - row.kind = "MASKING"; - row.signature = sig; - row.expression = p.expression; - row.bound_table = b.table_name; - row.bound_columns = b.column_name; - st->rows.push_back(std::move(row)); - any = true; - } - } - if (!any) { - PolicyRow row; - row.name = p.name; - row.kind = "MASKING"; - row.signature = sig; - row.expression = p.expression; - st->rows.push_back(std::move(row)); - } - } - return std::move(st); -} - -void PoliciesExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.rows.size() && row < STANDARD_VECTOR_SIZE) { - auto &r = state.rows[state.offset]; - output.SetValue(0, row, Value(r.name)); - output.SetValue(1, row, Value(r.kind)); - output.SetValue(2, row, Value(r.signature)); - output.SetValue(3, row, Value(r.expression)); - output.SetValue(4, row, Value(r.bound_table)); - output.SetValue(5, row, Value(r.bound_columns)); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -} // namespace - -//===--------------------------------------------------------------------===// -// Enforcement rewrite -//===--------------------------------------------------------------------===// - -//! Parsed identity of a catalog relation. Policy DDL historically stores a -//! string name, so turn that name into the same AST representation used for a -//! route before comparing it to a route reference. -struct PolicyTableIdentity { - string catalog; - string schema; - string table; -}; - -string QuoteCatalogIdentifier(const string &ident) { - string out = "\""; - for (auto c : ident) { - if (c == '"') { - out += "\"\""; - } else { - out += c; - } - } - return out + "\""; -} - -bool SameCatalogIdentifier(const string &left, const string &right) { - return StringUtil::Lower(left) == StringUtil::Lower(right); -} - -string CurrentCatalog(Connection &con) { - auto result = con.Query("SELECT current_database()"); - if (result->HasError()) { - return string(); - } - auto chunk = result->Fetch(); - if (!chunk || chunk->size() == 0 || chunk->GetValue(0, 0).IsNull()) { - return string(); - } - return chunk->GetValue(0, 0).ToString(); -} - -PolicyTableIdentity IdentityFromBaseRef(const BaseTableRef &ref, const string &default_catalog) { - PolicyTableIdentity identity; - identity.catalog = ref.catalog_name.empty() ? default_catalog : ref.catalog_name; - identity.schema = ref.schema_name.empty() ? "main" : ref.schema_name; - identity.table = ref.table_name; - return identity; -} - -bool ParsePolicyTableIdentity(const string &table_name, const string &default_catalog, PolicyTableIdentity &identity) { - try { - Parser parser; - parser.ParseQuery("SELECT * FROM " + table_name); - if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { - return false; - } - auto &select = parser.statements[0]->Cast(); - if (!select.node || select.node->type != QueryNodeType::SELECT_NODE) { - return false; - } - auto &node = select.node->Cast(); - if (!node.from_table || node.from_table->type != TableReferenceType::BASE_TABLE) { - return false; - } - identity = IdentityFromBaseRef(node.from_table->Cast(), default_catalog); - return !identity.table.empty(); - } catch (...) { - return false; - } -} - -bool SamePolicyTableIdentity(const PolicyTableIdentity &left, const PolicyTableIdentity &right) { - return SameCatalogIdentifier(left.catalog, right.catalog) && SameCatalogIdentifier(left.schema, right.schema) && - SameCatalogIdentifier(left.table, right.table); -} - -string QuotePolicyTableIdentity(const PolicyTableIdentity &identity) { - return QuoteCatalogIdentifier(identity.catalog) + "." + QuoteCatalogIdentifier(identity.schema) + "." + - QuoteCatalogIdentifier(identity.table); -} - -//! Replace a policy signature argument without touching SQL string literals. -string ReplacePolicyIdentifier(const string &expr, const string &from, const string &to) { - string out; - bool in_str = false; - for (idx_t i = 0; i < expr.size();) { - if (in_str) { - out += expr[i]; - if (expr[i] == '\'') { - if (i + 1 < expr.size() && expr[i + 1] == '\'') { - out += expr[i + 1]; - i += 2; - continue; - } - in_str = false; - } - i++; - continue; - } - if (expr[i] == '\'') { - in_str = true; - out += expr[i++]; - continue; - } - if (IsIdentStart(expr[i])) { - idx_t end = i + 1; - while (end < expr.size() && IsIdentChar(expr[end])) { - end++; - } - auto token = expr.substr(i, end - i); - out += SameCatalogIdentifier(token, from) ? to : token; - i = end; - continue; - } - out += expr[i++]; - } - return out; -} - -struct PolicyRewriteContext { - DatabaseInstance &db; - Connection con; - const vector &rap_bindings; - const vector &mask_bindings; - bool authenticated; - string default_catalog; - bool deny_unauthenticated = false; - string error; - unordered_map view_cache; - - PolicyRewriteContext(DatabaseInstance &db_p, const vector &rap_bindings_p, - const vector &mask_bindings_p, bool authenticated_p) - : db(db_p), con(db_p), rap_bindings(rap_bindings_p), mask_bindings(mask_bindings_p), - authenticated(authenticated_p), default_catalog(CurrentCatalog(con)) { - } - - void Reject(const string &reason) { - if (error.empty()) { - error = reason; - } - if (!authenticated) { - deny_unauthenticated = true; - } - } -}; - -bool IsCatalogView(PolicyRewriteContext &ctx, const PolicyTableIdentity &identity, bool &is_view) { - auto key = StringUtil::Lower(identity.catalog) + "\x1f" + StringUtil::Lower(identity.schema) + "\x1f" + - StringUtil::Lower(identity.table); - auto cached = ctx.view_cache.find(key); - if (cached != ctx.view_cache.end()) { - is_view = cached->second; - return true; - } - auto result = ctx.con.Query( - "SELECT 1 FROM duckdb_views() WHERE database_name = ? AND schema_name = ? AND view_name = ? LIMIT 1", - Value(identity.catalog), Value(identity.schema), Value(identity.table)); - if (result->HasError()) { - return false; - } - auto chunk = result->Fetch(); - is_view = chunk && chunk->size() > 0; - ctx.view_cache.emplace(std::move(key), is_view); - return true; -} - -bool ParseSecureSubquery(const string &sql, unique_ptr &out) { - try { - Parser parser; - parser.ParseQuery(sql); - if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { - return false; - } - out.reset(static_cast(parser.statements[0].release())); - return true; - } catch (...) { - return false; - } -} - -bool RewritePolicyQueryNode(QueryNode &node, PolicyRewriteContext &ctx); -bool RewritePolicySelect(SelectStatement &statement, PolicyRewriteContext &ctx); - -//! Policy-bearing relations can occur in scalar/EXISTS/IN subqueries inside a -//! SELECT list, predicate, join condition, or modifier — not only in FROM. -//! Walk parsed expressions recursively so those reads receive the same AST -//! rewrite as top-level relation sources. -bool RewritePolicyExpression(ParsedExpression &expression, PolicyRewriteContext &ctx) { - if (!ctx.error.empty()) { - return false; - } - try { - if (expression.GetExpressionClass() == ExpressionClass::SUBQUERY) { - auto &subquery = expression.Cast(); - if (!subquery.subquery || !RewritePolicySelect(*subquery.subquery, ctx)) { - if (ctx.error.empty()) { - ctx.Reject("policy enforcement could not inspect a nested subquery"); - } - return false; - } - } - bool rewritten = true; - ParsedExpressionIterator::EnumerateChildren(expression, [&](ParsedExpression &child) { - if (rewritten && !RewritePolicyExpression(child, ctx)) { - rewritten = false; - } - }); - return rewritten && ctx.error.empty(); - } catch (...) { - ctx.Reject("policy enforcement could not inspect a nested expression"); - return false; - } -} - -bool RewritePolicyModifiers(QueryNode &node, PolicyRewriteContext &ctx) { - bool rewritten = true; - try { - ParsedExpressionIterator::EnumerateQueryNodeModifiers(node, [&](unique_ptr &expression) { - if (rewritten && expression && !RewritePolicyExpression(*expression, ctx)) { - rewritten = false; - } - }); - } catch (...) { - ctx.Reject("policy enforcement could not inspect a query modifier"); - return false; - } - return rewritten && ctx.error.empty(); -} - -bool RewritePolicySelect(SelectStatement &statement, PolicyRewriteContext &ctx) { - if (!statement.node) { - ctx.Reject("policy enforcement could not inspect an empty SELECT statement"); - return false; - } - return RewritePolicyQueryNode(*statement.node, ctx); -} - -bool RewritePolicyTableRef(unique_ptr &ref, PolicyRewriteContext &ctx) { - if (!ref || !ctx.error.empty()) { - return false; - } - switch (ref->type) { - case TableReferenceType::BASE_TABLE: { - auto &base = ref->Cast(); - auto identity = IdentityFromBaseRef(base, ctx.default_catalog); - bool is_view = false; - if (!IsCatalogView(ctx, identity, is_view)) { - ctx.Reject("policy enforcement could not resolve catalog object identity"); - return false; - } - if (is_view) { - // A view can hide a protected relation. Do not infer dependencies from - // view SQL text: reject the indirect source until it has a bound rewrite. - ctx.Reject("policy enforcement rejects indirect view reads"); - return false; - } - - const QuackapiRowAccessBinding *rap_binding = nullptr; - const QuackapiMaskingBinding *first_mask = nullptr; - for (auto &binding : ctx.rap_bindings) { - PolicyTableIdentity bound; - if (!ParsePolicyTableIdentity(binding.table_name, ctx.default_catalog, bound)) { - ctx.Reject("policy binding has an invalid catalog identity"); - return false; - } - if (SamePolicyTableIdentity(identity, bound)) { - rap_binding = &binding; - break; - } - } - for (auto &binding : ctx.mask_bindings) { - PolicyTableIdentity bound; - if (!ParsePolicyTableIdentity(binding.table_name, ctx.default_catalog, bound)) { - ctx.Reject("policy binding has an invalid catalog identity"); - return false; - } - if (SamePolicyTableIdentity(identity, bound)) { - first_mask = &binding; - break; - } - } - if (!rap_binding && !first_mask) { - return true; - } - if (!ctx.authenticated) { - ctx.deny_unauthenticated = true; - return false; - } - - auto &state = QuackapiState::Get(ctx.db); - QuackapiRowAccessPolicy rap_storage; - const QuackapiRowAccessPolicy *rap = nullptr; - string where_expr; - if (rap_binding) { - if (!state.GetRowAccessPolicy(rap_binding->policy_name, rap_storage)) { - ctx.Reject("row-access policy binding references a missing policy"); - return false; - } - where_expr = rap_storage.expression; - for (idx_t i = 0; i < rap_storage.arg_columns.size() && i < rap_binding->columns.size(); i++) { - if (!SameCatalogIdentifier(rap_storage.arg_columns[i], rap_binding->columns[i])) { - where_expr = - ReplacePolicyIdentifier(where_expr, rap_storage.arg_columns[i], rap_binding->columns[i]); - } - } - rap = &rap_storage; - } - - vector> masked; - for (auto &binding : ctx.mask_bindings) { - PolicyTableIdentity bound; - if (!ParsePolicyTableIdentity(binding.table_name, ctx.default_catalog, bound)) { - ctx.Reject("policy binding has an invalid catalog identity"); - return false; - } - if (!SamePolicyTableIdentity(identity, bound)) { - continue; - } - QuackapiMaskingPolicy mask; - if (!state.GetMaskingPolicy(binding.policy_name, mask)) { - ctx.Reject("masking policy binding references a missing policy"); - return false; - } - masked.emplace_back(binding.column_name, - SubstituteValPlaceholder(mask.expression, QuoteCatalogIdentifier(binding.column_name))); - } - - unique_ptr secure_query; - if (!ParseSecureSubquery(BuildSecureSubqueryQuoted(QuotePolicyTableIdentity(identity), rap, where_expr, masked), - secure_query)) { - ctx.Reject("policy enforcement could not construct a secure subquery"); - return false; - } - string alias = ref->alias.empty() ? base.table_name : ref->alias; - auto replacement = make_uniq(std::move(secure_query), alias); - ref->CopyProperties(*replacement); - replacement->alias = alias; - ref = std::move(replacement); - return true; - } - case TableReferenceType::SUBQUERY: - return RewritePolicySelect(*ref->Cast().subquery, ctx); - case TableReferenceType::JOIN: { - auto &join = ref->Cast(); - if (!RewritePolicyTableRef(join.left, ctx) || !RewritePolicyTableRef(join.right, ctx)) { - return false; - } - return !join.condition || RewritePolicyExpression(*join.condition, ctx); - } - case TableReferenceType::PIVOT: { - auto &pivot = ref->Cast(); - if (!RewritePolicyTableRef(pivot.source, ctx)) { - return false; - } - for (auto &aggregate : pivot.aggregates) { - if (aggregate && !RewritePolicyExpression(*aggregate, ctx)) { - return false; - } - } - return true; - } - case TableReferenceType::TABLE_FUNCTION: - ctx.Reject("policy enforcement rejects table functions and table macros"); - return false; - case TableReferenceType::CTE: - case TableReferenceType::EMPTY_FROM: - return true; - case TableReferenceType::EXPRESSION_LIST: { - auto &values = ref->Cast(); - for (auto &row : values.values) { - for (auto &expression : row) { - if (expression && !RewritePolicyExpression(*expression, ctx)) { - return false; - } - } - } - return true; - } - default: - ctx.Reject("policy enforcement rejects an unsupported relation source"); - return false; - } -} - -bool RewritePolicyQueryNode(QueryNode &node, PolicyRewriteContext &ctx) { - for (auto &entry : node.cte_map.map) { - if (!entry.second || !entry.second->query || !RewritePolicySelect(*entry.second->query, ctx)) { - ctx.Reject("policy enforcement could not inspect a common table expression"); - return false; - } - } - switch (node.type) { - case QueryNodeType::SELECT_NODE: { - auto &select = node.Cast(); - if (select.from_table && !RewritePolicyTableRef(select.from_table, ctx)) { - return false; - } - for (auto &expression : select.select_list) { - if (expression && !RewritePolicyExpression(*expression, ctx)) { - return false; - } - } - for (auto &expression : select.groups.group_expressions) { - if (expression && !RewritePolicyExpression(*expression, ctx)) { - return false; - } - } - if ((select.where_clause && !RewritePolicyExpression(*select.where_clause, ctx)) || - (select.having && !RewritePolicyExpression(*select.having, ctx)) || - (select.qualify && !RewritePolicyExpression(*select.qualify, ctx))) { - return false; - } - return RewritePolicyModifiers(node, ctx); - } - case QueryNodeType::SET_OPERATION_NODE: { - auto &setop = node.Cast(); - for (auto &child : setop.children) { - if (!child || !RewritePolicyQueryNode(*child, ctx)) { - return false; - } - } - return RewritePolicyModifiers(node, ctx); - } - default: - ctx.Reject("policy enforcement rejects an unsupported query form"); - return false; - } -} - -//! Parse and rewrite every supported SELECT shape. Callers receive an explicit -//! error when a protected relation cannot be inspected safely. -bool RewriteHandlerWithPoliciesAst(DatabaseInstance &db, const string &handler_sql, bool authenticated, - bool &deny_unauthenticated, string &policy_error, string &rewritten_sql) { - deny_unauthenticated = false; - policy_error.clear(); - auto &state = QuackapiState::Get(db); - auto rap_bindings = state.SnapshotRowAccessBindings(); - auto mask_bindings = state.SnapshotMaskingBindings(); - if (rap_bindings.empty() && mask_bindings.empty()) { - rewritten_sql = handler_sql; - return true; - } - PolicyRewriteContext ctx(db, rap_bindings, mask_bindings, authenticated); - try { - Parser parser; - parser.ParseQuery(handler_sql); - if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) { - ctx.Reject("policy enforcement supports one SELECT statement per handler"); - } else { - auto &select = parser.statements[0]->Cast(); - RewritePolicySelect(select, ctx); - if (ctx.error.empty() && !ctx.deny_unauthenticated) { - rewritten_sql = select.ToString(); - } - } - } catch (...) { - ctx.Reject("policy enforcement could not parse handler SQL"); - } - deny_unauthenticated = ctx.deny_unauthenticated; - policy_error = ctx.error; - if (!policy_error.empty() && !authenticated) { - deny_unauthenticated = true; - } - if (policy_error.empty() && !deny_unauthenticated) { - return true; - } - rewritten_sql = handler_sql; - return true; -} - -bool HandlerTouchesPoliciedTable(DatabaseInstance &db, const string &handler_sql) { - bool deny_unauthenticated = false; - string policy_error; - string rewritten; - RewriteHandlerWithPoliciesAst(db, handler_sql, false, deny_unauthenticated, policy_error, rewritten); - return deny_unauthenticated || !policy_error.empty(); -} - -string RewriteHandlerWithPolicies(DatabaseInstance &db, const string &handler_sql, bool authenticated, - bool &deny_unauthenticated, string &policy_error) { - string parsed_rewrite; - RewriteHandlerWithPoliciesAst(db, handler_sql, authenticated, deny_unauthenticated, policy_error, parsed_rewrite); - return parsed_rewrite; -} - -PolicyDdlParserExtension::PolicyDdlParserExtension() { - parse_function = PolicyDdlParse; - plan_function = PolicyDdlPlan; -} - -TableFunction GetApplyPolicyFunction() { - return MakeApplyPolicyFunction(); -} - -TableFunction GetQuackapiPoliciesFunction() { - return TableFunction("quackapi_policies", {}, PoliciesExec, PoliciesBind, PoliciesInit); -} - -} // namespace duckdb diff --git a/src/quackapi_queue.cpp b/src/quackapi_queue.cpp deleted file mode 100644 index b50b736..0000000 --- a/src/quackapi_queue.cpp +++ /dev/null @@ -1,1035 +0,0 @@ -#include "quackapi_queue.hpp" - -#include "duckdb/common/exception.hpp" -#include "duckdb/common/string_util.hpp" -#include "duckdb/common/types.hpp" -#include "duckdb/common/types/timestamp.hpp" -#include "duckdb/common/types/vector.hpp" -#include "duckdb/common/unordered_map.hpp" -#include "duckdb/function/scalar_function.hpp" -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/main/connection.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/main/extension/extension_loader.hpp" -#include "duckdb/parser/parser_extension.hpp" - -#include "quackapi_state.hpp" - -#include -#include -#include "quackapi_util.hpp" - -namespace duckdb { - -namespace { - -//===--------------------------------------------------------------------===// -// SQL helpers -//===--------------------------------------------------------------------===// - -//! Escape a string for use as a single-quoted SQL literal. -string SqlQuote(const string &s) { - string out = "'"; - for (char c : s) { - if (c == '\'') { - out += "''"; - } else { - out += c; - } - } - out += "'"; - return out; -} - -void CheckQuery(unique_ptr &res, const string &ctx) { - if (res->HasError()) { - throw InvalidInputException("quackapi queue %s: %s", ctx, res->GetError()); - } -} - -void RunSQL(DatabaseInstance &db, const string &sql, const string &ctx) { - Connection con(db); - auto res = con.Query(sql); - CheckQuery(res, ctx); -} - -} // namespace - -void EnsureQuackapiJobsTable(DatabaseInstance &db) { - // One statement per Query — DuckDB does not accept multi-statement strings here. - RunSQL(db, "CREATE SEQUENCE IF NOT EXISTS quackapi_job_seq START 1", "create sequence"); - // payload is VARCHAR holding JSON text — avoids a hard runtime dep on LOAD - // json while still accepting payload::JSON at the function boundary (cast to - // text on insert). Users can CAST(payload AS JSON) when the json extension - // is loaded. - RunSQL(db, - "CREATE TABLE IF NOT EXISTS quackapi_jobs (" - "id BIGINT PRIMARY KEY, " - "queue VARCHAR NOT NULL, " - "payload VARCHAR NOT NULL, " - "status VARCHAR NOT NULL DEFAULT 'pending', " - "attempts INTEGER NOT NULL DEFAULT 0, " - "max_attempts INTEGER NOT NULL DEFAULT 3, " - "visible_at TIMESTAMP NOT NULL, " - "created_at TIMESTAMP NOT NULL, " - "updated_at TIMESTAMP NOT NULL, " - "last_error VARCHAR, " - "delivery_generation BIGINT NOT NULL DEFAULT 0, " - "worker_id VARCHAR" - ")", - "create table"); - // Existing durable queue tables predate delivery fencing. The generation is - // intentionally persisted with the job: a process restart must not make a - // stale worker's completion valid again. - RunSQL(db, - "ALTER TABLE quackapi_jobs " - // DuckDB cannot ADD COLUMN with a NOT NULL constraint. New tables use - // the constraint above; old durable tables are safely backfilled here. - "ADD COLUMN IF NOT EXISTS delivery_generation BIGINT DEFAULT 0", - "migrate delivery generation"); - RunSQL(db, "UPDATE quackapi_jobs SET delivery_generation = 0 WHERE delivery_generation IS NULL", - "backfill delivery generation"); - RunSQL(db, - "CREATE INDEX IF NOT EXISTS idx_quackapi_jobs_ready " - "ON quackapi_jobs (queue, status, visible_at, id)", - "create index"); -} - -namespace { - -//===--------------------------------------------------------------------===// -// Duration parse: 30 | '30' | '30s' | '5m' | '1h' -//===--------------------------------------------------------------------===// - -bool ParseDurationSeconds(const string &raw, int32_t &out_sec, string &err) { - auto s = QuackapiTrim(raw); - if (s.empty()) { - err = "empty duration"; - return false; - } - // Strip surrounding quotes if present. - if (s.size() >= 2 && s.front() == '\'' && s.back() == '\'') { - s = s.substr(1, s.size() - 2); - } - if (s.empty()) { - err = "empty duration"; - return false; - } - idx_t i = 0; - bool neg = false; - if (s[i] == '-') { - neg = true; - i++; - } - if (i >= s.size() || !StringUtil::CharacterIsDigit(s[i])) { - err = "duration must start with a number"; - return false; - } - int64_t num = 0; - while (i < s.size() && StringUtil::CharacterIsDigit(s[i])) { - num = num * 10 + (s[i] - '0'); - if (num > 86400LL * 365) { - err = "duration too large"; - return false; - } - i++; - } - int64_t mult = 1; - if (i < s.size()) { - auto unit = StringUtil::Lower(s.substr(i)); - if (unit == "s" || unit == "sec" || unit == "secs" || unit == "second" || unit == "seconds") { - mult = 1; - } else if (unit == "m" || unit == "min" || unit == "mins" || unit == "minute" || unit == "minutes") { - mult = 60; - } else if (unit == "h" || unit == "hr" || unit == "hrs" || unit == "hour" || unit == "hours") { - mult = 3600; - } else { - err = "unknown duration unit \"" + unit + "\" — use s, m, or h"; - return false; - } - } - int64_t sec = num * mult; - if (neg) { - sec = -sec; - } - if (sec < 0) { - err = "duration must be non-negative"; - return false; - } - if (sec > 86400LL * 7) { - err = "visibility_timeout max is 7 days"; - return false; - } - out_sec = static_cast(sec); - return true; -} - -//===--------------------------------------------------------------------===// -// CREATE / DROP QUEUE parser -//===--------------------------------------------------------------------===// - -struct QueueDdlParseData : public ParserExtensionParseData { - string action; // CREATE / DROP - bool or_replace = false; - QuackapiQueue queue; - - unique_ptr Copy() const override { - auto copy = make_uniq(); - copy->action = action; - copy->or_replace = or_replace; - copy->queue = queue; - return std::move(copy); - } - string ToString() const override { - return action + " QUEUE " + queue.name; - } -}; - -//! Grammar: -//! CREATE [OR REPLACE] QUEUE -//! [WITH ( max_attempts= , visibility_timeout='30s'|30 , backoff_base_seconds= )] -//! DROP QUEUE -ParserExtensionParseResult QueueDdlParse(ParserExtensionInfo *, const string &query) { - auto q = QuackapiTrim(query); - auto upper = StringUtil::Upper(q); - - bool or_replace = false; - idx_t pos; - if (StringUtil::StartsWith(upper, "CREATE QUEUE ")) { - pos = 13; - } else if (StringUtil::StartsWith(upper, "CREATE OR REPLACE QUEUE ")) { - pos = 24; - or_replace = true; - } else if (StringUtil::StartsWith(upper, "DROP QUEUE ")) { - auto name = QuackapiTrim(q.substr(11)); - if (name.empty() || name.find(' ') != string::npos) { - return ParserExtensionParseResult("DROP QUEUE expects a single queue name"); - } - auto data = make_uniq(); - data->action = "DROP"; - data->queue.name = name; - return ParserExtensionParseResult(std::move(data)); - } else { - return ParserExtensionParseResult(); - } - - auto rest = QuackapiTrim(q.substr(pos)); - if (rest.empty()) { - return ParserExtensionParseResult("CREATE QUEUE expects a queue name"); - } - - // — bare identifier up to space or end or WITH - idx_t name_end = 0; - while (name_end < rest.size() && !StringUtil::CharacterIsSpace(rest[name_end]) && rest[name_end] != '(') { - name_end++; - } - auto name = rest.substr(0, name_end); - if (name.empty()) { - return ParserExtensionParseResult("CREATE QUEUE expects a queue name"); - } - // Reject path-like names - for (char c : name) { - if (!(StringUtil::CharacterIsAlpha(c) || StringUtil::CharacterIsDigit(c) || c == '_' || c == '-')) { - return ParserExtensionParseResult("Queue name must be an identifier ([A-Za-z0-9_-]+)"); - } - } - rest = QuackapiTrim(rest.substr(name_end)); - auto rest_upper = StringUtil::Upper(rest); - - QuackapiQueue queue; - queue.name = name; - // defaults already set on QuackapiQueue - - // optional WITH ( ... ) - if (!rest.empty()) { - if (!(StringUtil::StartsWith(rest_upper, "WITH") && - (rest.size() == 4 || StringUtil::CharacterIsSpace(rest[4]) || rest[4] == '('))) { - return ParserExtensionParseResult("Expected WITH (...) after queue name, or end of statement"); - } - rest = QuackapiTrim(rest.substr(4)); - if (rest.empty() || rest[0] != '(') { - return ParserExtensionParseResult("WITH expects a parenthesized option list"); - } - auto close = rest.find(')'); - if (close == string::npos) { - return ParserExtensionParseResult("Unterminated WITH ( ... ) options"); - } - auto opts = QuackapiTrim(rest.substr(1, close - 1)); - rest = QuackapiTrim(rest.substr(close + 1)); - if (!rest.empty()) { - return ParserExtensionParseResult("Unexpected tokens after WITH options"); - } - - // Parse comma-separated key=value pairs - idx_t oi = 0; - while (oi < opts.size()) { - while (oi < opts.size() && (StringUtil::CharacterIsSpace(opts[oi]) || opts[oi] == ',')) { - oi++; - } - if (oi >= opts.size()) { - break; - } - // key - idx_t key_start = oi; - while (oi < opts.size() && (StringUtil::CharacterIsAlpha(opts[oi]) || - StringUtil::CharacterIsDigit(opts[oi]) || opts[oi] == '_')) { - oi++; - } - auto key = StringUtil::Lower(opts.substr(key_start, oi - key_start)); - while (oi < opts.size() && StringUtil::CharacterIsSpace(opts[oi])) { - oi++; - } - if (oi >= opts.size() || opts[oi] != '=') { - return ParserExtensionParseResult("WITH options: expected key=value (got key \"" + key + "\")"); - } - oi++; // = - while (oi < opts.size() && StringUtil::CharacterIsSpace(opts[oi])) { - oi++; - } - if (oi >= opts.size()) { - return ParserExtensionParseResult("WITH options: missing value for \"" + key + "\""); - } - // value: quoted string or bare token up to comma - string val; - if (opts[oi] == '\'') { - idx_t j = oi + 1; - string raw; - while (j < opts.size()) { - if (opts[j] == '\'') { - if (j + 1 < opts.size() && opts[j + 1] == '\'') { - raw += '\''; - j += 2; - continue; - } - break; - } - raw += opts[j]; - j++; - } - if (j >= opts.size() || opts[j] != '\'') { - return ParserExtensionParseResult("Unterminated quoted value for \"" + key + "\""); - } - val = raw; - oi = j + 1; - } else { - idx_t j = oi; - while (j < opts.size() && opts[j] != ',' && !StringUtil::CharacterIsSpace(opts[j])) { - j++; - } - val = opts.substr(oi, j - oi); - oi = j; - } - - if (key == "max_attempts") { - int v = atoi(val.c_str()); - if (v < 1 || v > 1000) { - return ParserExtensionParseResult("max_attempts must be between 1 and 1000"); - } - queue.max_attempts = v; - } else if (key == "visibility_timeout" || key == "visibility_timeout_sec") { - int32_t sec = 0; - string err; - if (!ParseDurationSeconds(val, sec, err)) { - return ParserExtensionParseResult("visibility_timeout: " + err); - } - if (sec < 1) { - return ParserExtensionParseResult("visibility_timeout must be at least 1 second"); - } - queue.visibility_timeout_sec = sec; - } else if (key == "backoff_base_seconds" || key == "backoff_base") { - int v = atoi(val.c_str()); - if (v < 0 || v > 3600) { - return ParserExtensionParseResult("backoff_base_seconds must be between 0 and 3600"); - } - queue.backoff_base_sec = v; - } else { - return ParserExtensionParseResult( - "Unknown QUEUE option \"" + key + - "\" — expected max_attempts, visibility_timeout, backoff_base_seconds"); - } - } - } - - auto data = make_uniq(); - data->action = "CREATE"; - data->or_replace = or_replace; - data->queue = queue; - return ParserExtensionParseResult(std::move(data)); -} - -struct ApplyQueueBindData : public TableFunctionData { - string action; - bool or_replace = false; - string name; - int32_t max_attempts = 3; - int32_t visibility_timeout_sec = 30; - int32_t backoff_base_sec = 2; - bool finished = false; -}; - -unique_ptr ApplyQueueBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - bind_data->action = input.inputs[0].GetValue(); - bind_data->or_replace = input.inputs[1].GetValue(); - bind_data->name = input.inputs[2].GetValue(); - bind_data->max_attempts = input.inputs[3].GetValue(); - bind_data->visibility_timeout_sec = input.inputs[4].GetValue(); - bind_data->backoff_base_sec = input.inputs[5].GetValue(); - BindStatusColumn(return_types, names); - return std::move(bind_data); -} - -void ApplyQueueExec(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &bind_data = data_p.bind_data->CastNoConst(); - if (bind_data.finished) { - return; - } - auto &state = QuackapiState::Get(*context.db); - string message; - if (bind_data.action == "CREATE") { - // Durable jobs table lives in the user's catalog / .db file. - EnsureQuackapiJobsTable(*context.db); - QuackapiQueue q; - q.name = bind_data.name; - q.max_attempts = bind_data.max_attempts; - q.visibility_timeout_sec = bind_data.visibility_timeout_sec; - q.backoff_base_sec = bind_data.backoff_base_sec; - state.AddQueue(q, bind_data.or_replace); - message = StringUtil::Format("Queue %s: max_attempts=%d visibility_timeout=%ds", q.name, q.max_attempts, - q.visibility_timeout_sec); - } else { - if (state.DropQueue(bind_data.name)) { - message = StringUtil::Format("Dropped queue %s", bind_data.name); - } else { - throw InvalidInputException("Queue \"%s\" does not exist", bind_data.name); - } - } - EmitOneShotStatus(output, bind_data.finished, message); -} - -TableFunction MakeApplyQueueFunction() { - return MakeApplyDdlFunction("quackapi_apply_queue", - {LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::VARCHAR, LogicalType::INTEGER, - LogicalType::INTEGER, LogicalType::INTEGER}, - ApplyQueueExec, ApplyQueueBind); -} - -ParserExtensionPlanResult QueueDdlPlan(ParserExtensionInfo *, ClientContext &, - unique_ptr parse_data) { - auto &data = static_cast(*parse_data); - ParserExtensionPlanResult result; - result.function = MakeApplyQueueFunction(); - result.parameters.push_back(Value(data.action)); - result.parameters.push_back(Value::BOOLEAN(data.or_replace)); - result.parameters.push_back(Value(data.queue.name)); - result.parameters.push_back(Value::INTEGER(data.queue.max_attempts)); - result.parameters.push_back(Value::INTEGER(data.queue.visibility_timeout_sec)); - result.parameters.push_back(Value::INTEGER(data.queue.backoff_base_sec)); - FinishDdlPlan(result); - return result; -} - -//===--------------------------------------------------------------------===// -// Lookup helper -//===--------------------------------------------------------------------===// - -QuackapiQueue RequireQueue(DatabaseInstance &db, const string &name) { - QuackapiQueue q; - if (!QuackapiState::Get(db).GetQueue(name, q)) { - throw InvalidInputException("Queue \"%s\" does not exist — CREATE QUEUE first", name); - } - return q; -} - -//===--------------------------------------------------------------------===// -// quackapi_enqueue(queue, payload [, max_attempts]) → job_id BIGINT -//===--------------------------------------------------------------------===// - -int64_t EnqueueJob(DatabaseInstance &db, const string &queue_name, const string &payload_json, - int32_t max_attempts_override) { - auto q = RequireQueue(db, queue_name); - EnsureQuackapiJobsTable(db); - int32_t max_att = max_attempts_override > 0 ? max_attempts_override : q.max_attempts; - if (max_att < 1) { - throw InvalidInputException("quackapi_enqueue: max_attempts must be >= 1"); - } - // Store payload as JSON text (VARCHAR). Callers may pass JSON or VARCHAR. - string sql = - StringUtil::Format("INSERT INTO quackapi_jobs " - "(id, queue, payload, status, attempts, max_attempts, visible_at, created_at, updated_at) " - "SELECT nextval('quackapi_job_seq'), %s, %s, 'pending', 0, %d, " - "now()::TIMESTAMP, now()::TIMESTAMP, now()::TIMESTAMP " - "RETURNING id", - SqlQuote(queue_name), SqlQuote(payload_json), max_att); - Connection con(db); - auto res = con.Query(sql); - CheckQuery(res, "enqueue"); - if (res->RowCount() == 0) { - throw InternalException("quackapi_enqueue: INSERT RETURNING produced no row"); - } - return res->GetValue(0, 0).GetValue(); -} - -//! enqueue(queue, payload) / enqueue(queue, payload, max_attempts) — one body. -//! Optional 3rd arg via args.ColumnCount() (same pattern as nack). -void EnqueueScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - UnifiedVectorFormat qdata, pdata, mdata; - args.data[0].ToUnifiedFormat(args.size(), qdata); - args.data[1].ToUnifiedFormat(args.size(), pdata); - const bool has_max = args.ColumnCount() >= 3; - if (has_max) { - args.data[2].ToUnifiedFormat(args.size(), mdata); - } - result.SetVectorType(VectorType::FLAT_VECTOR); - auto out = FlatVector::GetData(result); - auto &validity = FlatVector::Validity(result); - for (idx_t i = 0; i < args.size(); i++) { - auto qi = qdata.sel->get_index(i); - auto pi = pdata.sel->get_index(i); - if (!qdata.validity.RowIsValid(qi) || !pdata.validity.RowIsValid(pi)) { - validity.SetInvalid(i); - continue; - } - auto qn = UnifiedVectorFormat::GetData(qdata)[qi].GetString(); - auto payload = UnifiedVectorFormat::GetData(pdata)[pi].GetString(); - int32_t max_att = 0; - if (has_max) { - auto mi = mdata.sel->get_index(i); - if (mdata.validity.RowIsValid(mi)) { - max_att = UnifiedVectorFormat::GetData(mdata)[mi]; - } - } - out[i] = EnqueueJob(db, qn, payload, max_att); - } -} - -//===--------------------------------------------------------------------===// -// quackapi_dequeue(queue [, n]) table function — claim jobs -//===--------------------------------------------------------------------===// - -struct DequeueBindData : public TableFunctionData { - string queue_name; - int32_t n = 1; -}; - -struct DequeueGlobalState : public GlobalTableFunctionState { - vector> rows; // pre-claimed rows - idx_t offset = 0; -}; - -unique_ptr DequeueBind(ClientContext &, TableFunctionBindInput &input, vector &return_types, - vector &names) { - auto bind_data = make_uniq(); - bind_data->queue_name = input.inputs[0].GetValue(); - if (input.inputs.size() > 1 && !input.inputs[1].IsNull()) { - bind_data->n = input.inputs[1].GetValue(); - } - if (bind_data->n < 1) { - throw InvalidInputException("quackapi_dequeue: n must be >= 1"); - } - if (bind_data->n > 1000) { - throw InvalidInputException("quackapi_dequeue: n max is 1000"); - } - return_types = {LogicalType::BIGINT, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::VARCHAR, LogicalType::INTEGER, LogicalType::INTEGER, - LogicalType::TIMESTAMP, LogicalType::VARCHAR, LogicalType::BIGINT}; - names = {"id", "queue", "payload", - "status", "attempts", "max_attempts", - "visible_at", "last_error", "delivery_generation"}; - return std::move(bind_data); -} - -//! Recover every expired delivery in a queue. This deliberately does not use -//! a queue-wide "live lease" gate: one active job must never delay recovery of -//! another job whose lease already expired. -void RecoverExpiredJobs(Connection &con, const string &queue_name) { - // A legacy database can contain an exhausted pending row from the old expiry - // path. Dead-letter it before claiming so an upgrade cannot create attempt - // max_attempts + 1. - string exhausted_pending_sql = - StringUtil::Format("UPDATE quackapi_jobs SET status = 'dead', visible_at = now()::TIMESTAMP, " - "updated_at = now()::TIMESTAMP, worker_id = NULL, " - "last_error = COALESCE(last_error, 'attempt limit reached before delivery') " - "WHERE queue = %s AND status = 'pending' AND attempts >= max_attempts", - SqlQuote(queue_name)); - auto exhausted_pending = con.Query(exhausted_pending_sql); - CheckQuery(exhausted_pending, "dequeue exhausted-pending sweep"); - - // A timed-out final delivery is terminal. Other timed-out deliveries become - // pending atomically with the state transition and can be claimed below. - string exhausted_running_sql = - StringUtil::Format("UPDATE quackapi_jobs SET status = 'dead', visible_at = now()::TIMESTAMP, " - "updated_at = now()::TIMESTAMP, worker_id = NULL, " - "last_error = 'visibility lease expired after maximum attempts' " - "WHERE queue = %s AND status = 'running' " - "AND visible_at <= now()::TIMESTAMP AND attempts >= max_attempts", - SqlQuote(queue_name)); - auto exhausted_running = con.Query(exhausted_running_sql); - CheckQuery(exhausted_running, "dequeue exhausted-running sweep"); - - string retry_sql = StringUtil::Format("UPDATE quackapi_jobs SET status = 'pending', visible_at = now()::TIMESTAMP, " - "updated_at = now()::TIMESTAMP, worker_id = NULL " - "WHERE queue = %s AND status = 'running' " - "AND visible_at <= now()::TIMESTAMP AND attempts < max_attempts", - SqlQuote(queue_name)); - auto retry = con.Query(retry_sql); - CheckQuery(retry, "dequeue expiry sweep"); -} - -unique_ptr ClaimPendingJob(Connection &con, const string &queue_name, - int32_t visibility_timeout_sec) { - string sql = - StringUtil::Format("UPDATE quackapi_jobs SET " - "status = 'running', " - "attempts = attempts + 1, " - "delivery_generation = delivery_generation + 1, " - "visible_at = now()::TIMESTAMP + to_seconds(%d), " - "updated_at = now()::TIMESTAMP, " - "worker_id = NULL " - "WHERE id = (" - " SELECT id FROM quackapi_jobs " - " WHERE queue = %s " - " AND status = 'pending' " - " AND attempts < max_attempts " - " AND visible_at <= now()::TIMESTAMP " - " ORDER BY id LIMIT 1" - ") " - "AND status = 'pending' " - "RETURNING id, queue, payload, status, attempts, max_attempts, visible_at, last_error, " - "delivery_generation", - visibility_timeout_sec, SqlQuote(queue_name)); - unique_ptr res; - for (int attempt = 0; attempt < 8; attempt++) { - res = con.Query(sql); - if (!res->HasError()) { - return res; - } - const string &err = res->GetError(); - // Concurrent claim on the same tuple — retry; do not surface 422 Conflict. - if (StringUtil::Contains(err, "Conflict")) { - continue; - } - CheckQuery(res, "dequeue"); - } - CheckQuery(res, "dequeue"); - return res; -} - -unique_ptr DequeueInit(ClientContext &context, TableFunctionInitInput &input) { - auto &bind = input.bind_data->Cast(); - auto state = make_uniq(); - auto q = RequireQueue(*context.db, bind.queue_name); - EnsureQuackapiJobsTable(*context.db); - - // The gate lives on QuackapiState (one per DatabaseInstance) so static and - // loadable extension copies serialize claims, expiry recovery, and delivery - // transitions through the same mutex. - auto &qa = QuackapiState::Get(*context.db); - std::lock_guard claim_lock(qa.DequeueClaimMutex()); - - Connection con(*context.db); - for (int32_t i = 0; i < bind.n; i++) { - RecoverExpiredJobs(con, bind.queue_name); - auto res = ClaimPendingJob(con, bind.queue_name, q.visibility_timeout_sec); - if (res->RowCount() == 0) { - break; - } - vector row; - for (idx_t c = 0; c < 9; c++) { - row.push_back(res->GetValue(c, 0)); - } - state->rows.push_back(std::move(row)); - } - return std::move(state); -} - -void DequeueExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.rows.size() && row < STANDARD_VECTOR_SIZE) { - auto &r = state.rows[state.offset]; - for (idx_t c = 0; c < 9; c++) { - output.SetValue(c, row, r[c]); - } - row++; - state.offset++; - } - output.SetCardinality(row); -} - -//===--------------------------------------------------------------------===// -// quackapi_ack(queue, job_id, delivery_generation) → BOOLEAN -//===--------------------------------------------------------------------===// - -bool AckJob(DatabaseInstance &db, const string &queue_name, int64_t job_id, int64_t delivery_generation) { - RequireQueue(db, queue_name); - auto &qa = QuackapiState::Get(db); - std::lock_guard lock(qa.DequeueClaimMutex()); - EnsureQuackapiJobsTable(db); - string sql = StringUtil::Format( - "UPDATE quackapi_jobs SET status = 'done', updated_at = now()::TIMESTAMP, worker_id = NULL " - "WHERE id = %lld AND queue = %s AND status = 'running' " - "AND delivery_generation = %lld AND visible_at > now()::TIMESTAMP " - "RETURNING id", - static_cast(job_id), SqlQuote(queue_name), static_cast(delivery_generation)); - Connection con(db); - auto res = con.Query(sql); - CheckQuery(res, "ack"); - return res->RowCount() > 0; -} - -void AckScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - UnifiedVectorFormat qdata, idata, gdata; - args.data[0].ToUnifiedFormat(args.size(), qdata); - args.data[1].ToUnifiedFormat(args.size(), idata); - args.data[2].ToUnifiedFormat(args.size(), gdata); - result.SetVectorType(VectorType::FLAT_VECTOR); - auto out = FlatVector::GetData(result); - auto &validity = FlatVector::Validity(result); - for (idx_t i = 0; i < args.size(); i++) { - auto qi = qdata.sel->get_index(i); - auto ii = idata.sel->get_index(i); - auto gi = gdata.sel->get_index(i); - if (!qdata.validity.RowIsValid(qi) || !idata.validity.RowIsValid(ii) || !gdata.validity.RowIsValid(gi)) { - validity.SetInvalid(i); - continue; - } - auto qn = UnifiedVectorFormat::GetData(qdata)[qi].GetString(); - auto jid = UnifiedVectorFormat::GetData(idata)[ii]; - auto generation = UnifiedVectorFormat::GetData(gdata)[gi]; - out[i] = AckJob(db, qn, jid, generation); - } -} - -//===--------------------------------------------------------------------===// -// quackapi_nack(queue, job_id, delivery_generation [, requeue [, error]]) → VARCHAR status -//===--------------------------------------------------------------------===// - -string NackJob(DatabaseInstance &db, const string &queue_name, int64_t job_id, int64_t delivery_generation, - bool requeue, const string &error) { - auto q = RequireQueue(db, queue_name); - auto &qa = QuackapiState::Get(db); - std::lock_guard lock(qa.DequeueClaimMutex()); - EnsureQuackapiJobsTable(db); - - // The shared delivery lock keeps this read and conditional transition one - // queue operation. The generation + unexpired-lease predicate is repeated in - // the UPDATE so the ownership proof remains explicit in the SQL itself. - string read_sql = StringUtil::Format("SELECT attempts, max_attempts FROM quackapi_jobs " - "WHERE id = %lld AND queue = %s AND status = 'running' " - "AND delivery_generation = %lld AND visible_at > now()::TIMESTAMP", - static_cast(job_id), SqlQuote(queue_name), - static_cast(delivery_generation)); - Connection con(db); - auto read = con.Query(read_sql); - CheckQuery(read, "nack read"); - if (read->RowCount() == 0) { - return "stale"; - } - int32_t attempts = read->GetValue(0, 0).GetValue(); - int32_t max_att = read->GetValue(1, 0).GetValue(); - - bool to_dead = !requeue || attempts >= max_att; - int32_t backoff = 0; - if (!to_dead) { - // exponential: base^min(attempts, 6). backoff_base_sec=0 means immediate retry. - if (q.backoff_base_sec > 0) { - int exp = attempts; - if (exp > 6) { - exp = 6; - } - if (exp < 0) { - exp = 0; - } - double secs = std::pow(static_cast(q.backoff_base_sec), exp); - if (secs > 3600.0) { - secs = 3600.0; - } - backoff = static_cast(secs); - } - } - - string sql; - if (to_dead) { - sql = StringUtil::Format("UPDATE quackapi_jobs SET status = 'dead', " - "visible_at = now()::TIMESTAMP, " - "last_error = %s, updated_at = now()::TIMESTAMP, worker_id = NULL " - "WHERE id = %lld AND queue = %s AND status = 'running' " - "AND delivery_generation = %lld AND visible_at > now()::TIMESTAMP " - "RETURNING status", - SqlQuote(error.empty() ? "nack" : error), static_cast(job_id), - SqlQuote(queue_name), static_cast(delivery_generation)); - } else { - sql = StringUtil::Format("UPDATE quackapi_jobs SET status = 'pending', " - "visible_at = now()::TIMESTAMP + to_seconds(%d), " - "last_error = %s, updated_at = now()::TIMESTAMP, worker_id = NULL " - "WHERE id = %lld AND queue = %s AND status = 'running' " - "AND delivery_generation = %lld AND visible_at > now()::TIMESTAMP " - "RETURNING status", - backoff, SqlQuote(error.empty() ? "nack" : error), static_cast(job_id), - SqlQuote(queue_name), static_cast(delivery_generation)); - } - auto res = con.Query(sql); - CheckQuery(res, "nack"); - if (res->RowCount() == 0) { - return "stale"; - } - return res->GetValue(0, 0).GetValue(); -} - -//! nack(queue, job_id, delivery_generation [, requeue [, error]]) — one body, optional args by arity. -void NackScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - UnifiedVectorFormat qdata, idata, gdata, rdata, edata; - args.data[0].ToUnifiedFormat(args.size(), qdata); - args.data[1].ToUnifiedFormat(args.size(), idata); - args.data[2].ToUnifiedFormat(args.size(), gdata); - const bool has_requeue = args.ColumnCount() >= 4; - const bool has_error = args.ColumnCount() >= 5; - if (has_requeue) { - args.data[3].ToUnifiedFormat(args.size(), rdata); - } - if (has_error) { - args.data[4].ToUnifiedFormat(args.size(), edata); - } - result.SetVectorType(VectorType::FLAT_VECTOR); - auto out = FlatVector::GetData(result); - auto &validity = FlatVector::Validity(result); - for (idx_t i = 0; i < args.size(); i++) { - auto qi = qdata.sel->get_index(i); - auto ii = idata.sel->get_index(i); - auto gi = gdata.sel->get_index(i); - if (!qdata.validity.RowIsValid(qi) || !idata.validity.RowIsValid(ii) || !gdata.validity.RowIsValid(gi)) { - validity.SetInvalid(i); - continue; - } - auto qn = UnifiedVectorFormat::GetData(qdata)[qi].GetString(); - auto jid = UnifiedVectorFormat::GetData(idata)[ii]; - auto generation = UnifiedVectorFormat::GetData(gdata)[gi]; - bool requeue = true; - if (has_requeue) { - auto ri = rdata.sel->get_index(i); - if (rdata.validity.RowIsValid(ri)) { - requeue = UnifiedVectorFormat::GetData(rdata)[ri]; - } - } - string err = "nack"; - if (has_error) { - auto ei = edata.sel->get_index(i); - if (edata.validity.RowIsValid(ei)) { - err = UnifiedVectorFormat::GetData(edata)[ei].GetString(); - } - } - auto st = NackJob(db, qn, jid, generation, requeue, err); - out[i] = StringVector::AddString(result, st); - } -} - -//===--------------------------------------------------------------------===// -// quackapi_renew(queue, job_id, delivery_generation) → BOOLEAN -//===--------------------------------------------------------------------===// - -bool RenewJob(DatabaseInstance &db, const string &queue_name, int64_t job_id, int64_t delivery_generation) { - auto q = RequireQueue(db, queue_name); - auto &qa = QuackapiState::Get(db); - std::lock_guard lock(qa.DequeueClaimMutex()); - EnsureQuackapiJobsTable(db); - string sql = StringUtil::Format("UPDATE quackapi_jobs SET visible_at = now()::TIMESTAMP + to_seconds(%d), " - "updated_at = now()::TIMESTAMP " - "WHERE id = %lld AND queue = %s AND status = 'running' " - "AND delivery_generation = %lld AND visible_at > now()::TIMESTAMP " - "RETURNING id", - q.visibility_timeout_sec, static_cast(job_id), SqlQuote(queue_name), - static_cast(delivery_generation)); - Connection con(db); - auto res = con.Query(sql); - CheckQuery(res, "renew"); - return res->RowCount() > 0; -} - -void RenewScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - UnifiedVectorFormat qdata, idata, gdata; - args.data[0].ToUnifiedFormat(args.size(), qdata); - args.data[1].ToUnifiedFormat(args.size(), idata); - args.data[2].ToUnifiedFormat(args.size(), gdata); - result.SetVectorType(VectorType::FLAT_VECTOR); - auto out = FlatVector::GetData(result); - auto &validity = FlatVector::Validity(result); - for (idx_t i = 0; i < args.size(); i++) { - auto qi = qdata.sel->get_index(i); - auto ii = idata.sel->get_index(i); - auto gi = gdata.sel->get_index(i); - if (!qdata.validity.RowIsValid(qi) || !idata.validity.RowIsValid(ii) || !gdata.validity.RowIsValid(gi)) { - validity.SetInvalid(i); - continue; - } - auto qn = UnifiedVectorFormat::GetData(qdata)[qi].GetString(); - auto jid = UnifiedVectorFormat::GetData(idata)[ii]; - auto generation = UnifiedVectorFormat::GetData(gdata)[gi]; - out[i] = RenewJob(db, qn, jid, generation); - } -} - -//===--------------------------------------------------------------------===// -// quackapi_queues() — name, depth, in_flight, dead (+ options) -//===--------------------------------------------------------------------===// - -struct QueuesBindData : public TableFunctionData {}; - -struct QueuesGlobalState : public GlobalTableFunctionState { - vector queues; - // Parallel stats vectors indexed same as queues - vector depth; - vector in_flight; - vector dead; - idx_t offset = 0; -}; - -unique_ptr QueuesBind(ClientContext &, TableFunctionBindInput &, vector &return_types, - vector &names) { - return_types = {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT, LogicalType::BIGINT, - LogicalType::INTEGER, LogicalType::INTEGER, LogicalType::INTEGER}; - names = {"name", "depth", "in_flight", "dead", "max_attempts", "visibility_timeout_sec", "backoff_base_sec"}; - return make_uniq(); -} - -unique_ptr QueuesInit(ClientContext &context, TableFunctionInitInput &) { - auto state = make_uniq(); - state->queues = QuackapiState::Get(*context.db).SnapshotQueues(); - state->depth.assign(state->queues.size(), 0); - state->in_flight.assign(state->queues.size(), 0); - state->dead.assign(state->queues.size(), 0); - - // Jobs table may not exist yet if no CREATE QUEUE ran (empty registry). - if (state->queues.empty()) { - return std::move(state); - } - try { - EnsureQuackapiJobsTable(*context.db); - } catch (...) { - return std::move(state); - } - Connection con(*context.db); - // depth = claimable (pending OR running with expired lease) - // in_flight = running with active lease - // dead = dead - auto res = - con.Query("SELECT queue, " - "count(*) FILTER (WHERE status = 'pending' OR " - " (status = 'running' AND visible_at <= now()::TIMESTAMP))::BIGINT AS depth, " - "count(*) FILTER (WHERE status = 'running' AND visible_at > now()::TIMESTAMP)::BIGINT AS in_flight, " - "count(*) FILTER (WHERE status = 'dead')::BIGINT AS dead " - "FROM quackapi_jobs GROUP BY queue"); - if (res->HasError()) { - return std::move(state); - } - unordered_map idx; - for (idx_t i = 0; i < state->queues.size(); i++) { - idx[state->queues[i].name] = i; - } - for (idx_t r = 0; r < res->RowCount(); r++) { - auto qn = res->GetValue(0, r).GetValue(); - auto it = idx.find(qn); - if (it == idx.end()) { - continue; // orphaned jobs from dropped queue — ignore for this view - } - state->depth[it->second] = res->GetValue(1, r).GetValue(); - state->in_flight[it->second] = res->GetValue(2, r).GetValue(); - state->dead[it->second] = res->GetValue(3, r).GetValue(); - } - return std::move(state); -} - -void QueuesExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.queues.size() && row < STANDARD_VECTOR_SIZE) { - auto &q = state.queues[state.offset]; - output.SetValue(0, row, Value(q.name)); - output.SetValue(1, row, Value::BIGINT(state.depth[state.offset])); - output.SetValue(2, row, Value::BIGINT(state.in_flight[state.offset])); - output.SetValue(3, row, Value::BIGINT(state.dead[state.offset])); - output.SetValue(4, row, Value::INTEGER(q.max_attempts)); - output.SetValue(5, row, Value::INTEGER(q.visibility_timeout_sec)); - output.SetValue(6, row, Value::INTEGER(q.backoff_base_sec)); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -} // namespace - -//===--------------------------------------------------------------------===// -// Public registration -//===--------------------------------------------------------------------===// - -QueueDdlParserExtension::QueueDdlParserExtension() { - parse_function = QueueDdlParse; - plan_function = QueueDdlPlan; -} - -TableFunction GetApplyQueueFunction() { - return MakeApplyQueueFunction(); -} - -void RegisterQuackapiQueueFunctions(ExtensionLoader &loader) { - // enqueue(queue, payload) / enqueue(queue, payload, max_attempts) - ScalarFunctionSet enqueue_set("quackapi_enqueue"); - enqueue_set.AddFunction(ScalarFunction("quackapi_enqueue", {LogicalType::VARCHAR, LogicalType::VARCHAR}, - LogicalType::BIGINT, EnqueueScalar)); - enqueue_set.AddFunction(ScalarFunction("quackapi_enqueue", - {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::INTEGER}, - LogicalType::BIGINT, EnqueueScalar)); - // JSON payload overload (JSON is a logical type alias of VARCHAR storage) - enqueue_set.AddFunction(ScalarFunction("quackapi_enqueue", {LogicalType::VARCHAR, LogicalType::JSON()}, - LogicalType::BIGINT, EnqueueScalar)); - enqueue_set.AddFunction(ScalarFunction("quackapi_enqueue", - {LogicalType::VARCHAR, LogicalType::JSON(), LogicalType::INTEGER}, - LogicalType::BIGINT, EnqueueScalar)); - loader.RegisterFunction(enqueue_set); - - // dequeue(queue) / dequeue(queue, n) - TableFunctionSet dequeue_set("quackapi_dequeue"); - TableFunction dequeue1("quackapi_dequeue", {LogicalType::VARCHAR}, DequeueExec, DequeueBind, DequeueInit); - TableFunction dequeue2("quackapi_dequeue", {LogicalType::VARCHAR, LogicalType::INTEGER}, DequeueExec, DequeueBind, - DequeueInit); - dequeue_set.AddFunction(dequeue1); - dequeue_set.AddFunction(dequeue2); - loader.RegisterFunction(dequeue_set); - - // Delivery ownership is mandatory. The retired two-argument form could let - // an expired worker complete a newer delivery of the same job. - loader.RegisterFunction(ScalarFunction("quackapi_ack", - {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT}, - LogicalType::BOOLEAN, AckScalar)); - - // nack(queue, job_id, delivery_generation [, requeue [, error]]) - ScalarFunctionSet nack_set("quackapi_nack"); - nack_set.AddFunction(ScalarFunction("quackapi_nack", - {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT}, - LogicalType::VARCHAR, NackScalar)); - nack_set.AddFunction(ScalarFunction( - "quackapi_nack", {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT, LogicalType::BOOLEAN}, - LogicalType::VARCHAR, NackScalar)); - nack_set.AddFunction(ScalarFunction( - "quackapi_nack", - {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT, LogicalType::BOOLEAN, LogicalType::VARCHAR}, - LogicalType::VARCHAR, NackScalar)); - loader.RegisterFunction(nack_set); - - // renew(queue, job_id, delivery_generation) → bool - loader.RegisterFunction(ScalarFunction("quackapi_renew", - {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT}, - LogicalType::BOOLEAN, RenewScalar)); - - // queues() inspection - loader.RegisterFunction(TableFunction("quackapi_queues", {}, QueuesExec, QueuesBind, QueuesInit)); -} - -} // namespace duckdb diff --git a/test/sql/fixtures/dequeue_conflict_repro.md b/test/sql/fixtures/dequeue_conflict_repro.md deleted file mode 100644 index b7011bc..0000000 --- a/test/sql/fixtures/dequeue_conflict_repro.md +++ /dev/null @@ -1,15 +0,0 @@ -# Dequeue claim race (not runnable in single-threaded SQLLogic) - -**Where:** `src/quackapi_queue.cpp` — `DequeueInit` UPDATE…WHERE id=(SELECT…LIMIT 1) without re-checking claim predicates on the outer UPDATE. - -**Repro (5 lines):** - -```text -1. LOAD quackapi; CREATE QUEUE q; SELECT quackapi_enqueue('q','only-one'); -2. CREATE ROUTE dq GET '/dq' AS SELECT id,payload FROM quackapi_dequeue('q',1); -3. SELECT * FROM quackapi_serve(PORT, worker_threads:=8); -4. Parallel: curl PORT/dq ×8 -5. Observe: one 200 with the job; losers → 422 "Conflict on tuple deletion!" (should be empty 200 []) -``` - -**Observable wrong behaviour:** concurrent claimants raise TransactionContext conflicts (surfaced as 422) instead of a clean exclusive claim / empty result for losers. diff --git a/test/sql/quackapi_policy.test b/test/sql/quackapi_policy.test deleted file mode 100644 index 580708c..0000000 --- a/test/sql/quackapi_policy.test +++ /dev/null @@ -1,535 +0,0 @@ -# name: test/sql/quackapi_policy.test -# description: ROW ACCESS / MASKING enforcement via quackapi_request (claims-keyed) -# group: [sql] - -require quackapi - -require json - -# JWTs in test/sql/fixtures/jwt_pol_*.txt — SECRET 'policy-test-secret' (HS256). -# Loaded via SET VARIABLE so signature bytes that contain '--' are not SQL comments. -# require json: JWT claims parse uses json_* inside quackapi_verify_auth. - -# --------------------------------------------------------------------------- -# DDL setup — policies, binds, seed, routes -# --------------------------------------------------------------------------- -statement ok -CREATE TABLE policy_orders (id INTEGER, tenant_id VARCHAR, amount INTEGER); - -statement ok -CREATE TABLE policy_users (id INTEGER, email VARCHAR, tenant_id VARCHAR); - -statement ok -INSERT INTO policy_orders VALUES - (1, 'acme', 100), - (2, 'beta', 200), - (3, 'acme', 150); - -statement ok -INSERT INTO policy_users VALUES - (1, 'alice@acme.test', 'acme'), - (2, 'bob@beta.test', 'beta'); - -query I -CREATE AUTH jwt_pol AS JWT ( SECRET 'policy-test-secret' ); ----- -Auth jwt_pol: JWT_HS256 - -query I -CREATE ROW ACCESS POLICY tenant_isolation - AS (tenant_id VARCHAR) RETURNS BOOLEAN - USING (tenant_id = $claims_tenant OR $claims_role = 'admin'); ----- -Row access policy tenant_isolation - -query I -CREATE MASKING POLICY mask_email ON VARCHAR - USING (CASE WHEN $claims_role = 'admin' THEN val ELSE '***' END); ----- -Masking policy mask_email ON VARCHAR - -# Duplicate without OR REPLACE errors -statement error -CREATE ROW ACCESS POLICY tenant_isolation - AS (tenant_id VARCHAR) RETURNS BOOLEAN - USING (true); ----- -already exists - -# Name collision across kinds errors -statement error -CREATE MASKING POLICY tenant_isolation ON VARCHAR USING (val); ----- -already used by a row access policy - -query I -ALTER TABLE policy_orders ADD ROW ACCESS POLICY tenant_isolation ON (tenant_id); ----- -Bound row access policy tenant_isolation on policy_orders - -query I -ALTER TABLE policy_users ADD ROW ACCESS POLICY tenant_isolation ON (tenant_id); ----- -Bound row access policy tenant_isolation on policy_users - -query I -ALTER TABLE policy_users MODIFY COLUMN email SET MASKING POLICY mask_email; ----- -Set masking policy mask_email on policy_users.email - -query I -CREATE ROUTE list_orders GET '/orders' REQUIRE jwt_pol AS - SELECT id, tenant_id, amount FROM policy_orders ORDER BY id; ----- -Route list_orders: GET /orders - -query I -CREATE ROUTE list_users GET '/users' REQUIRE jwt_pol AS - SELECT id, email, tenant_id FROM policy_users ORDER BY id; ----- -Route list_users: GET /users - -# Public route on policied table — CREATE ok; request must fail closed (403) -query I -CREATE ROUTE public_orders GET '/public_orders' AS - SELECT id, tenant_id FROM policy_orders; ----- -Route public_orders: GET /public_orders - -# Parsed policy enforcement covers schema-qualified base tables. Views and -# table macros are deliberately rejected while a bound expansion is unavailable, -# rather than leaking their protected dependencies. -statement ok -CREATE VIEW policy_orders_view AS SELECT * FROM policy_orders; - -statement ok -CREATE MACRO policy_orders_macro() AS TABLE SELECT * FROM policy_orders; - -query I -CREATE ROUTE list_orders_qualified GET '/orders-qualified' REQUIRE jwt_pol AS - SELECT id, tenant_id, amount FROM main.policy_orders ORDER BY id; ----- -Route list_orders_qualified: GET /orders-qualified - -query I -CREATE ROUTE public_orders_view GET '/public_orders_view' AS - SELECT id, tenant_id FROM policy_orders_view; ----- -Route public_orders_view: GET /public_orders_view - -query I -CREATE ROUTE public_orders_macro GET '/public_orders_macro' AS - SELECT id, tenant_id FROM policy_orders_macro(); ----- -Route public_orders_macro: GET /public_orders_macro - -# Nested scalar expressions and CTEs must be walked by the policy rewriter; -# a relation hidden outside the top-level FROM is not a safe bypass. The CTE -# deliberately shadows the bound table name while its body uses a qualified -# alias for the actual catalog table. -query I -CREATE ROUTE public_users_scalar GET '/public_users_scalar' AS - SELECT (SELECT email FROM policy_users ORDER BY id LIMIT 1) AS email; ----- -Route public_users_scalar: GET /public_users_scalar - -query I -CREATE ROUTE list_users_scalar GET '/users-scalar' REQUIRE jwt_pol AS - SELECT (SELECT email FROM policy_users WHERE tenant_id = $claims_tenant ORDER BY id LIMIT 1) AS email; ----- -Route list_users_scalar: GET /users-scalar - -query I -CREATE ROUTE list_users_cte GET '/users-cte' REQUIRE jwt_pol AS - WITH policy_users AS ( - SELECT p.id, p.email, p.tenant_id FROM main.policy_users AS p - ) - SELECT u.email FROM policy_users AS u ORDER BY u.id; ----- -Route list_users_cte: GET /users-cte - -# Table macros remain an indirect relation source for all callers until a -# bound expansion exists; authenticated credentials do not make it safe. -query I -CREATE ROUTE auth_orders_macro GET '/auth_orders_macro' REQUIRE jwt_pol AS - SELECT id, tenant_id FROM policy_orders_macro(); ----- -Route auth_orders_macro: GET /auth_orders_macro - -# Policy enforcement is read-only today. Reject protected-table mutations -# rather than executing an update without a row-level write policy. -query I -CREATE ROUTE mutate_orders POST '/mutate_orders' REQUIRE jwt_pol AS - UPDATE policy_orders SET amount = 0 WHERE id = 1 RETURNING id; ----- -Route mutate_orders: POST /mutate_orders - -query I -CREATE STREAM policy_orders_sse GET '/policy-orders-sse' AS - SELECT id, tenant_id FROM policy_orders; ----- -Stream policy_orders_sse: GET /policy-orders-sse SSE - -query I -CREATE GRAPHQL ROUTE policy_orders_gql POST '/orders-gql' FROM policy_orders REQUIRE jwt_pol; ----- -GraphQL route policy_orders_gql: POST /orders-gql FROM policy_orders REQUIRE jwt_pol LIMIT 100 - -query I -CREATE GRAPHQL ROUTE policy_users_gql POST '/users-gql' FROM policy_users REQUIRE jwt_pol; ----- -GraphQL route policy_users_gql: POST /users-gql FROM policy_users REQUIRE jwt_pol LIMIT 100 - -# This registration must never turn a policy-protected table into a public -# built-in GraphQL endpoint. -query I -CREATE GRAPHQL FOR TABLE policy_orders; ----- -GraphQL registered: policy_orders (allowlist mode on POST /graphql) - -statement ok -SET VARIABLE jwt_acme_user = ( - SELECT rtrim(content, E'\n\r ') FROM read_text('test/sql/fixtures/jwt_pol_acme_user.txt') -); - -statement ok -SET VARIABLE jwt_acme_admin = ( - SELECT rtrim(content, E'\n\r ') FROM read_text('test/sql/fixtures/jwt_pol_acme_admin.txt') -); - -statement ok -SET VARIABLE jwt_beta_user = ( - SELECT rtrim(content, E'\n\r ') FROM read_text('test/sql/fixtures/jwt_pol_beta_user.txt') -); - -# --------------------------------------------------------------------------- -# Row access policy — filters rows by claim (via quackapi_request) -# --------------------------------------------------------------------------- -# acme user → only acme rows -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/orders', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 [{"id":1,"tenant_id":"acme","amount":100},{"id":3,"tenant_id":"acme","amount":150}] - -# beta user → only beta rows -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/orders', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_beta_user')} -); ----- -200 [{"id":2,"tenant_id":"beta","amount":200}] - -# admin → all tenants -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/orders', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_admin')} -); ----- -200 [{"id":1,"tenant_id":"acme","amount":100},{"id":2,"tenant_id":"beta","amount":200},{"id":3,"tenant_id":"acme","amount":150}] - -# A qualified direct table reference is rewritten from the parsed base-table -# identity, not a bare-name text match. -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/orders-qualified', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 [{"id":1,"tenant_id":"acme","amount":100},{"id":3,"tenant_id":"acme","amount":150}] - -# --------------------------------------------------------------------------- -# Masking policy — column masked unless claims permit -# --------------------------------------------------------------------------- -# acme user → email masked; RAP also limits to acme row -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/users', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 [{"id":1,"email":"***","tenant_id":"acme"}] - -# admin → email unmasked; sees all tenants -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/users', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_admin')} -); ----- -200 [{"id":1,"email":"alice@acme.test","tenant_id":"acme"},{"id":2,"email":"bob@beta.test","tenant_id":"beta"}] - -# Scalar subqueries and CTE bodies are parsed and rewritten recursively. The -# first verifies authenticated RAP/masking; the second verifies a public -# scalar source fails closed rather than reading the underlying table. -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/users-scalar', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 [{"email":"***"}] - -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/users-cte', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 [{"email":"***"}] - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/public_users_scalar'); ----- -403 {"detail":"Policy denies unauthenticated access"} - -# Indirect table macros and protected-table writes are denied even after JWT -# verification. Confirm the rejected mutation left the table unchanged. -query II -SELECT status, decode(body) -FROM quackapi_request( - 'GET', '/auth_orders_macro', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -403 {"detail":"Policy denies unauthenticated access"} - -query II -SELECT status, decode(body) -FROM quackapi_request( - 'POST', '/mutate_orders', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -403 {"detail":"Policy denies unauthenticated access"} - -query I -SELECT amount FROM policy_orders WHERE id = 1; ----- -100 - -# --------------------------------------------------------------------------- -# Fail closed — unauthenticated request to policied table → 403 (not open) -# --------------------------------------------------------------------------- -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/public_orders'); ----- -403 {"detail":"Policy denies unauthenticated access"} - -# Views and table macros are indirect sources. They fail closed instead of -# bypassing a policy bound to policy_orders. -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/public_orders_view'); ----- -403 {"detail":"Policy denies unauthenticated access"} - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/public_orders_macro'); ----- -403 {"detail":"Policy denies unauthenticated access"} - -# Streams have no auth/claims contract yet, so a protected source must not -# become an unauthenticated SSE bypass. -query I -SELECT status FROM quackapi_request('GET', '/policy-orders-sse'); ----- -403 - -# REQUIRE without credentials still 401 (auth before policy) -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/orders'); ----- -401 {"detail":"Not authenticated"} - -# Named GraphQL applies the same RAP/masking expansion and verified claims as -# REST. Built-in GraphQL remains unauthenticated and therefore cannot read a -# registered protected table. -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/orders-gql', - '{"query":"{ policy_orders { id tenant_id amount } }"}', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 {"data":{"policy_orders":[{"id":1,"tenant_id":"acme","amount":100},{"id":3,"tenant_id":"acme","amount":150}]}} - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/users-gql', - '{"query":"{ policy_users { id email tenant_id } }"}', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 {"data":{"policy_users":[{"id":1,"email":"***","tenant_id":"acme"}]}} - -query I -SELECT status FROM quackapi_request( - 'POST', '/orders-gql', - '{"query":"{ policy_orders { id } }"}' -); ----- -401 - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ policy_orders { id } }"}' -); ----- -200 {"errors":[{"message":"access denied by table policy"}]} - -# Schema discovery uses the same named-route auth gate and does not expose a -# registered protected table through the unauthenticated built-in schema. -query I -SELECT status FROM quackapi_request('GET', '/orders-gql/schema'); ----- -401 - -query I -SELECT status FROM quackapi_request( - 'GET', '/orders-gql/schema', - headers := MAP {'Authorization': 'Bearer ' || getvariable('jwt_acme_user')} -); ----- -200 - -query II -SELECT status, contains(decode(body), 'policy_orders') -FROM quackapi_request('GET', '/graphql/schema'); ----- -200 false - -# --------------------------------------------------------------------------- -# Bad grammar / missing policy -# --------------------------------------------------------------------------- -statement error -CREATE ROW ACCESS POLICY bad AS (x) USING (true); ----- -RETURNS BOOLEAN - -statement error -CREATE MASKING POLICY bad ON VARCHAR AS (val); ----- -USING - -statement error -ALTER TABLE policy_orders ADD ROW ACCESS POLICY nope ON (tenant_id); ----- -does not exist - -# --------------------------------------------------------------------------- -# Cleanup -# --------------------------------------------------------------------------- -query I -DROP GRAPHQL ROUTE policy_orders_gql; ----- -Dropped GraphQL route policy_orders_gql - -query I -DROP GRAPHQL ROUTE policy_users_gql; ----- -Dropped GraphQL route policy_users_gql - -query I -DROP GRAPHQL FOR TABLE policy_orders; ----- -GraphQL unregistered: policy_orders (allowlist empty → catalog closed) - -query I -DROP STREAM policy_orders_sse; ----- -Dropped stream policy_orders_sse - -query I -DROP ROUTE list_orders; ----- -Dropped route list_orders - -query I -DROP ROUTE list_users; ----- -Dropped route list_users - -query I -DROP ROUTE public_orders; ----- -Dropped route public_orders - -query I -DROP ROUTE list_orders_qualified; ----- -Dropped route list_orders_qualified - -query I -DROP ROUTE public_orders_view; ----- -Dropped route public_orders_view - -query I -DROP ROUTE public_orders_macro; ----- -Dropped route public_orders_macro - -query I -DROP ROUTE public_users_scalar; ----- -Dropped route public_users_scalar - -query I -DROP ROUTE list_users_scalar; ----- -Dropped route list_users_scalar - -query I -DROP ROUTE list_users_cte; ----- -Dropped route list_users_cte - -query I -DROP ROUTE auth_orders_macro; ----- -Dropped route auth_orders_macro - -query I -DROP ROUTE mutate_orders; ----- -Dropped route mutate_orders - -query I -DROP AUTH jwt_pol; ----- -Dropped auth jwt_pol - -query I -DROP MASKING POLICY mask_email; ----- -Dropped masking policy mask_email - -query I -DROP ROW ACCESS POLICY tenant_isolation; ----- -Dropped row access policy tenant_isolation - -statement ok -DROP MACRO policy_orders_macro; - -statement ok -DROP VIEW policy_orders_view; - -statement ok -DROP TABLE policy_orders; - -statement ok -DROP TABLE policy_users; diff --git a/test/sql/quackapi_queue.test b/test/sql/quackapi_queue.test deleted file mode 100644 index 860c5ac..0000000 --- a/test/sql/quackapi_queue.test +++ /dev/null @@ -1,353 +0,0 @@ -# name: test/sql/quackapi_queue.test -# description: Queue delivery semantics via quackapi_request / serve+fetch -# group: [sql] - -require quackapi - -require json - -# --------------------------------------------------------------------------- -# Setup: durable queue + HTTP routes (enqueue / claim / ack / nack / stats) -# --------------------------------------------------------------------------- -query I -CREATE QUEUE jobs WITH (max_attempts=3, visibility_timeout='1s', backoff_base_seconds=0); ----- -Queue jobs: max_attempts=3 visibility_timeout=1s - -statement ok -CREATE ROUTE q_enq POST '/q/enq' AS - SELECT quackapi_enqueue('jobs', $payload::VARCHAR) AS job_id; - -# $t probe defeats zero-param static JSON body cache on this branch -statement ok -CREATE ROUTE q_claim GET '/q/claim' AS - SELECT id, payload, status, attempts, delivery_generation FROM quackapi_dequeue('jobs', 1) - WHERE $t::INT >= 0; - -statement ok -CREATE ROUTE q_ack POST '/q/ack' AS - SELECT quackapi_ack('jobs', $id::BIGINT, $delivery_generation::BIGINT) AS ok; - -statement ok -CREATE ROUTE q_nack POST '/q/nack' AS - SELECT quackapi_nack('jobs', $id::BIGINT, $delivery_generation::BIGINT, - $requeue::BOOLEAN, $error::VARCHAR) AS status; - -statement ok -CREATE ROUTE q_stats GET '/q/stats' AS - SELECT name, depth, in_flight, dead FROM quackapi_queues() - WHERE name = 'jobs' AND $t::INT >= 0; - -# --------------------------------------------------------------------------- -# HTTP enqueue → depth; claim leases; second claim empty (visibility hold) -# --------------------------------------------------------------------------- -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/q/enq', '{"payload":"{\"task\":\"a\"}"}'); ----- -200 [{"job_id":1}] - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/stats?t=1'); ----- -[{"name":"jobs","depth":1,"in_flight":0,"dead":0}] - -query II -SELECT status, - json_extract(decode(body)::JSON, '$[0].id')::BIGINT AS id -FROM quackapi_request('GET', '/q/claim?t=1'); ----- -200 1 - -# Leased — invisible to a second dequeue -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/claim?t=2'); ----- -[] - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/stats?t=2'); ----- -[{"name":"jobs","depth":0,"in_flight":1,"dead":0}] - -# Visibility timeout lapses → redelivery (attempts increment) -statement ok -SELECT sleep_ms(1100); - -query III -SELECT status, - json_extract(decode(body)::JSON, '$[0].id')::BIGINT, - json_extract(decode(body)::JSON, '$[0].attempts')::INTEGER -FROM quackapi_request('GET', '/q/claim?t=3'); ----- -200 1 2 - -# --------------------------------------------------------------------------- -# ack removes permanently — no redelivery after visibility would have lapsed -# --------------------------------------------------------------------------- -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/q/ack', '{"id":1,"delivery_generation":2}'); ----- -200 [{"ok":true}] - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/claim?t=4'); ----- -[] - -statement ok -SELECT sleep_ms(1100); - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/claim?t=5'); ----- -[] - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/stats?t=3'); ----- -[{"name":"jobs","depth":0,"in_flight":0,"dead":0}] - -# --------------------------------------------------------------------------- -# nack returns the message for redelivery -# --------------------------------------------------------------------------- -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/q/enq', '{"payload":"{\"task\":\"b\"}"}'); ----- -200 [{"job_id":2}] - -query I -SELECT json_extract(decode(body)::JSON, '$[0].id')::BIGINT FROM quackapi_request('GET', '/q/claim?t=6'); ----- -2 - -query II -SELECT status, decode(body) -FROM quackapi_request('POST', '/q/nack', '{"id":2,"delivery_generation":1,"requeue":true,"error":"try_again"}'); ----- -200 [{"status":"pending"}] - -query III -SELECT status, - json_extract(decode(body)::JSON, '$[0].id')::BIGINT, - json_extract(decode(body)::JSON, '$[0].attempts')::INTEGER -FROM quackapi_request('GET', '/q/claim?t=7'); ----- -200 2 2 - -# nack without requeue → dead; not redelivered -query II -SELECT status, decode(body) -FROM quackapi_request('POST', '/q/nack', '{"id":2,"delivery_generation":2,"requeue":false,"error":"no_retry"}'); ----- -200 [{"status":"dead"}] - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/claim?t=8'); ----- -[] - -query I -SELECT decode(body) FROM quackapi_request('GET', '/q/stats?t=4'); ----- -[{"name":"jobs","depth":0,"in_flight":0,"dead":1}] - -# --------------------------------------------------------------------------- -# Concurrent dequeue: one message → exactly one winner across parallel GETs -# -# Uses its own queue with a visibility timeout far longer than the storm can -# take. The `jobs` queue above is deliberately 1s so the tests before this one -# can exercise lease lapse + redelivery — but redelivery after an expired lease -# is CORRECT queue behavior, so reusing `jobs` here made this test assert -# "the 8-request storm finishes within 1s", a wall-clock property CI cannot -# guarantee. On a loaded runner the lease lapsed mid-storm and a second -# claimant legitimately won job 3 (attempts 1 then 2 -> winner_ids [3,3]). -# A long lease isolates the property actually under test: exclusive claim. -# --------------------------------------------------------------------------- -query I -CREATE QUEUE race WITH (max_attempts=3, visibility_timeout='300s', backoff_base_seconds=0); ----- -Queue race: max_attempts=3 visibility_timeout=300s - -statement ok -CREATE ROUTE q_rclaim GET '/q/rclaim' AS - SELECT id, payload, status, attempts, delivery_generation FROM quackapi_dequeue('race', 1) - WHERE $t::INT >= 0; - -statement ok -CREATE ROUTE q_renq POST '/q/renq' AS - SELECT quackapi_enqueue('race', $payload::VARCHAR) AS job_id; - -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/q/renq', '{"payload":"{\"task\":\"race\"}"}'); ----- -200 [{"job_id":3}] - -query I -SELECT * FROM quackapi_serve(29270, host := '127.0.0.1', access_log := false, worker_threads := 8); ----- -http://127.0.0.1:29270 - -# Exactly one winner with job 3; losers empty []; every attempt 200 with no error. -query III -SELECT list(json_extract(body::JSON, '$[0].id')::BIGINT) - FILTER (WHERE status = 200 AND body != '[]') AS winner_ids, - bool_and(status = 200 AND error IS NULL) AS all_ok, - len(list(idx) FILTER (WHERE body = '[]')) AS empty_n -FROM quackapi_parallel_fetch('http://127.0.0.1:29270/q/rclaim?t=9', 8); ----- -[3] true 7 - -query I -SELECT * FROM quackapi_stop(29270); ----- -Stopped quackapi server on port 29270 - -# --------------------------------------------------------------------------- -# Delivery generations fence stale workers; renew only extends the active lease -# --------------------------------------------------------------------------- -query I -CREATE QUEUE fence WITH (max_attempts=3, visibility_timeout='1s', backoff_base_seconds=0); ----- -Queue fence: max_attempts=3 visibility_timeout=1s - -query I -SELECT quackapi_enqueue('fence', '{"task":"fence"}'); ----- -4 - -query II -SELECT id, delivery_generation FROM quackapi_dequeue('fence'); ----- -4 1 - -statement ok -SELECT sleep_ms(1100); - -query III -SELECT id, attempts, delivery_generation FROM quackapi_dequeue('fence'); ----- -4 2 2 - -# The first worker cannot mutate the redelivered job in any direction. -query I -SELECT quackapi_ack('fence', 4, 1); ----- -false - -query I -SELECT quackapi_nack('fence', 4, 1, true, 'old worker'); ----- -stale - -query I -SELECT quackapi_renew('fence', 4, 1); ----- -false - -query III -SELECT status, attempts, delivery_generation FROM quackapi_jobs WHERE id = 4; ----- -running 2 2 - -# The current receipt can renew the lease and remains exclusively owned beyond -# its original expiry; a duplicate ack has the defined false result. -statement ok -SELECT sleep_ms(300); - -query I -SELECT quackapi_renew('fence', 4, 2); ----- -true - -statement ok -SELECT sleep_ms(800); - -query I -SELECT count(*) FROM quackapi_dequeue('fence'); ----- -0 - -query I -SELECT quackapi_ack('fence', 4, 2); ----- -true - -query I -SELECT quackapi_ack('fence', 4, 2); ----- -false - -# --------------------------------------------------------------------------- -# Lease expiry obeys max_attempts instead of creating an unbounded retry loop -# --------------------------------------------------------------------------- -query I -CREATE QUEUE expiry WITH (max_attempts=2, visibility_timeout='1s', backoff_base_seconds=0); ----- -Queue expiry: max_attempts=2 visibility_timeout=1s - -query I -SELECT quackapi_enqueue('expiry', '{"task":"timeout"}'); ----- -5 - -query III -SELECT id, attempts, delivery_generation FROM quackapi_dequeue('expiry'); ----- -5 1 1 - -statement ok -SELECT sleep_ms(1100); - -query III -SELECT id, attempts, delivery_generation FROM quackapi_dequeue('expiry'); ----- -5 2 2 - -statement ok -SELECT sleep_ms(1100); - -query I -SELECT count(*) FROM quackapi_dequeue('expiry'); ----- -0 - -query II -SELECT status, attempts FROM quackapi_jobs WHERE id = 5; ----- -dead 2 - -# --------------------------------------------------------------------------- -# A live lease cannot starve recovery of another expired job in the queue -# --------------------------------------------------------------------------- -query I -CREATE QUEUE recovery WITH (max_attempts=3, visibility_timeout='300s', backoff_base_seconds=0); ----- -Queue recovery: max_attempts=3 visibility_timeout=300s - -query I -SELECT quackapi_enqueue('recovery', '{"task":"live"}'); ----- -6 - -query I -SELECT quackapi_enqueue('recovery', '{"task":"expired"}'); ----- -7 - -query I -SELECT id FROM quackapi_dequeue('recovery'); ----- -6 - -# Simulate a second process's timed-out delivery while job 6 has a valid local -# lease. This was previously suppressed by the queue-wide live-lease gate. -statement ok -UPDATE quackapi_jobs -SET status = 'running', attempts = 1, delivery_generation = 1, - visible_at = now()::TIMESTAMP - to_seconds(1) -WHERE id = 7; - -query III -SELECT id, attempts, delivery_generation FROM quackapi_dequeue('recovery'); ----- -7 2 2 From 9aaef1241e8ff7cb8263acbedcbf2c60e5d7f013 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:08:29 -0700 Subject: [PATCH 10/93] Remove queue and policy shared integrations --- CMakeLists.txt | 2 - src/include/quackapi_state.hpp | 97 ------------- src/quackapi_extension.cpp | 13 +- src/quackapi_graphql.cpp | 14 -- src/quackapi_middleware.cpp | 18 +-- src/quackapi_registry.cpp | 247 --------------------------------- src/quackapi_server.cpp | 26 +--- 7 files changed, 5 insertions(+), 412 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c94828..27e86ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,8 +35,6 @@ set(EXTENSION_SOURCES src/quackapi_table_api.cpp src/quackapi_openapi.cpp src/quackapi_graphql.cpp - src/quackapi_queue.cpp - src/quackapi_policy.cpp src/quackapi_stream.cpp src/quackapi_from_x.cpp src/quackapi_util.cpp diff --git a/src/include/quackapi_state.hpp b/src/include/quackapi_state.hpp index 9bd4eeb..3695796 100644 --- a/src/include/quackapi_state.hpp +++ b/src/include/quackapi_state.hpp @@ -76,20 +76,6 @@ struct QuackapiParamSpec { idx_t max_length = 0; }; -//! One durable job queue registered via CREATE QUEUE. -//! Jobs live in the shared `quackapi_jobs` table (survives restart in the .db -//! file). Queue options live on the DatabaseInstance (like routes) and must be -//! re-declared after reopen — same lifecycle as CREATE ROUTE. -struct QuackapiQueue { - string name; - //! Attempts before a job is moved to status='dead' (dead-letter). - int32_t max_attempts = 3; - //! Lease duration in seconds after a successful claim (visibility timeout). - int32_t visibility_timeout_sec = 30; - //! Base delay in seconds for exponential nack backoff: base^min(attempts,6). - int32_t backoff_base_sec = 2; -}; - //! One registered route group (FastAPI APIRouter-style prefix + default auth). //! Expanded into routes at CREATE ROUTE time — runtime registry stays flat. struct QuackapiGroup { @@ -171,42 +157,6 @@ struct QuackapiRoute { string schedule; }; -//! Row-access policy: predicate over table columns + $claims_* (JWT/auth claims). -//! CREATE ROW ACCESS POLICY name AS (col [type], ...) RETURNS BOOLEAN USING (expr) -struct QuackapiRowAccessPolicy { - string name; - //! Argument column names from the AS (...) signature (order preserved). - vector arg_columns; - //! Optional type names parallel to arg_columns (may be empty strings). - vector arg_types; - //! Boolean SQL expression; may reference arg column names and $claims_*. - string expression; -}; - -//! Dynamic data masking policy: rewrite a column value from claims context. -//! CREATE MASKING POLICY name ON USING (expr) — `val` is the column value. -struct QuackapiMaskingPolicy { - string name; - string value_type; // e.g. VARCHAR, INTEGER - //! SQL expression returning the same type; use identifier `val` for the raw value. - string expression; -}; - -//! Table binding for a row-access policy (ALTER TABLE … ADD ROW ACCESS POLICY). -struct QuackapiRowAccessBinding { - string table_name; - string policy_name; - //! Table columns mapped onto the policy signature (same arity as policy args). - vector columns; -}; - -//! Column binding for a masking policy (ALTER TABLE … SET MASKING POLICY). -struct QuackapiMaskingBinding { - string table_name; - string column_name; - string policy_name; -}; - //! Transport for CREATE STREAM. SSE is first-class on cpp-httplib; WebSocket is //! not available on the bundled httplib (no Upgrade/WS API) — WS is rejected at DDL. enum class QuackapiStreamTransport : uint8_t { @@ -290,20 +240,6 @@ class QuackapiState : public ObjectCacheEntry { //! Snapshot of stored keys for an auth (hashes only — never raw keys). vector SnapshotApiKeys(const string &auth_name); - //! CREATE [OR REPLACE] QUEUE. Throws on duplicate name unless or_replace. - void AddQueue(const QuackapiQueue &queue, bool or_replace); - //! DROP QUEUE. Returns false if no such queue. Does not delete job rows. - bool DropQueue(const string &name); - //! Lookup by name. Returns false if not registered. - bool GetQueue(const string &name, QuackapiQueue &out); - vector SnapshotQueues(); - //! Shared queue delivery fence for this database. It serializes dequeue, - //! expiry recovery, acknowledgement, retry, and lease renewal across static - //! and loadable copies of the extension. - std::mutex &DequeueClaimMutex() { - return dequeue_claim_mutex; - } - //! Start serving on host:port. Throws if a server already listens there. void StartServer(DatabaseInstance &db, const string &host, int port, const QuackapiServeOptions &opts); //! Stop the server on port (any host). Returns false if none. @@ -326,26 +262,6 @@ class QuackapiState : public ObjectCacheEntry { //! A port is required when more than one server is running. bool FlushTelemetry(int port = 0); - // --- Row access + masking policies (JWT/claims keyed, not DB roles) --- - void AddRowAccessPolicy(const QuackapiRowAccessPolicy &policy, bool or_replace); - bool DropRowAccessPolicy(const string &name); - bool GetRowAccessPolicy(const string &name, QuackapiRowAccessPolicy &out); - vector SnapshotRowAccessPolicies(); - - void AddMaskingPolicy(const QuackapiMaskingPolicy &policy, bool or_replace); - bool DropMaskingPolicy(const string &name); - bool GetMaskingPolicy(const string &name, QuackapiMaskingPolicy &out); - vector SnapshotMaskingPolicies(); - - //! Bind / unbind row-access policy on a table. Throws if policy missing or arity mismatch. - void BindRowAccessPolicy(const QuackapiRowAccessBinding &binding, bool or_replace); - bool UnbindRowAccessPolicy(const string &table_name, const string &policy_name); - vector SnapshotRowAccessBindings(); - - void BindMaskingPolicy(const QuackapiMaskingBinding &binding, bool or_replace); - bool UnbindMaskingPolicy(const string &table_name, const string &column_name); - vector SnapshotMaskingBindings(); - // --- Thin GraphQL v0 table allowlist (built-in POST /graphql) --- // Empty allowlist = open mode (all main-schema tables, v0 default). // Non-empty = only registered tables appear in /graphql and /graphql/schema. @@ -410,19 +326,6 @@ class QuackapiState : public ObjectCacheEntry { vector auths; vector api_keys; - std::mutex queues_mutex; - vector queues; - //! Serialize queue delivery transitions for this database. Function-local - //! statics are unsafe: static+loadable extension copies would each get an - //! independent mutex, allowing concurrent HTTP workers to bypass the fence. - std::mutex dequeue_claim_mutex; - - std::mutex policies_mutex; - vector row_access_policies; - vector masking_policies; - vector row_access_bindings; - vector masking_bindings; - //! GraphQL allowlist (case-sensitive catalog names, like duckdb_tables). std::mutex graphql_mutex; vector graphql_tables; diff --git a/src/quackapi_extension.cpp b/src/quackapi_extension.cpp index 3dc3146..4288e2b 100644 --- a/src/quackapi_extension.cpp +++ b/src/quackapi_extension.cpp @@ -24,8 +24,6 @@ #include "quackapi_from_x.hpp" #include "quackapi_graphql.hpp" #include "quackapi_http_fetch.hpp" -#include "quackapi_queue.hpp" -#include "quackapi_policy.hpp" #include "quackapi_server.hpp" #include "quackapi_state.hpp" #include "quackapi_stream.hpp" @@ -1328,9 +1326,6 @@ static void LoadInternal(ExtensionLoader &loader) { loader.RegisterFunction(GetQuackapiAuthsFunction()); loader.RegisterFunction(GetQuackapiAddApiKeyFunction()); - // Row-access + masking policy inspection (claims-keyed, not DB roles). - loader.RegisterFunction(GetQuackapiPoliciesFunction()); - // quack auth bridge: quackapi_authentication / quackapi_authorization / // quackapi_verify_auth — same signatures as quack_check_token / // quack_nop_authorization so they can be SET as quack_*_function. @@ -1353,9 +1348,6 @@ static void LoadInternal(ExtensionLoader &loader) { // quackapi_http_fetch.hpp for why the un-pooled path costs ~27ms a call. RegisterQuackapiHttpFetchFunctions(loader); - // Durable broker-less job queue (CREATE QUEUE + enqueue/dequeue/ack/nack). - // Backing store is the plain quackapi_jobs table; worker = compose cronjob. - RegisterQuackapiQueueFunctions(loader); loader.RegisterFunction(GetApplyMiddlewareFunction()); loader.RegisterFunction(GetQuackapiMiddlewaresFunction()); @@ -1366,16 +1358,13 @@ static void LoadInternal(ExtensionLoader &loader) { // Thin GraphQL allowlist + named route inspection (CREATE GRAPHQL … DDL). RegisterQuackapiGraphqlFunctions(loader); - // CREATE ROUTE / GROUP / AUTH / QUEUE / POLICY / STREAM / API FOR TABLE / + // CREATE ROUTE / GROUP / AUTH / STREAM / API FOR TABLE / // GRAPHQL FOR TABLE / GRAPHQL ROUTE — all first-class nouns registered. ExtensionCallbackManager::Get(db).Register(RouteDdlParserExtension()); ExtensionCallbackManager::Get(db).Register(GroupDdlParserExtension()); ExtensionCallbackManager::Get(db).Register(AuthDdlParserExtension()); ExtensionCallbackManager::Get(db).Register(TableApiDdlParserExtension()); - ExtensionCallbackManager::Get(db).Register(QueueDdlParserExtension()); ExtensionCallbackManager::Get(db).Register(MiddlewareDdlParserExtension()); - // CREATE ROW ACCESS / MASKING POLICY + ALTER TABLE policy bind - ExtensionCallbackManager::Get(db).Register(PolicyDdlParserExtension()); // CREATE STREAM (SSE) ExtensionCallbackManager::Get(db).Register(StreamDdlParserExtension()); // CREATE / DROP GRAPHQL FOR TABLE (+ DROP GRAPHQL ALL) + GRAPHQL ROUTE diff --git a/src/quackapi_graphql.cpp b/src/quackapi_graphql.cpp index c4a13ce..6e8d353 100644 --- a/src/quackapi_graphql.cpp +++ b/src/quackapi_graphql.cpp @@ -14,7 +14,6 @@ #include "duckdb/parser/parser_extension.hpp" #include "quackapi_limits.hpp" -#include "quackapi_policy.hpp" #include "quackapi_state.hpp" #include "quackapi_util.hpp" @@ -1135,13 +1134,6 @@ string ExecuteGraphqlQuery(DatabaseInstance &db, const string &query, const Grap source_sql += QuoteIdent(field.columns[c]); } source_sql += " FROM " + QuoteGraphqlObject(object->second) + " LIMIT " + std::to_string(limit); - bool deny_unauthenticated = false; - string policy_error; - source_sql = - RewriteHandlerWithPolicies(db, source_sql, options.authenticated, deny_unauthenticated, policy_error); - if (deny_unauthenticated || !policy_error.empty()) { - return GraphqlError("access denied by table policy"); - } string err; string rows_json = SelectTableJson(con, source_sql, field.columns, options, err); if (!err.empty()) { @@ -1212,12 +1204,6 @@ string BuildGraphqlSchema(DatabaseInstance &db, const GraphqlExecOptions &option if ((route_mode || allowlist) && !TableInList(allowed, table)) { continue; } - // Schema discovery follows the same policy gate as execution. A public - // GraphQL endpoint must not reveal a protected table or view name. - if (!options.authenticated && - HandlerTouchesPoliciedTable(db, "SELECT * FROM " + QuoteGraphqlObject(catalog[table]))) { - continue; - } if (!first_table) { body += ","; } diff --git a/src/quackapi_middleware.cpp b/src/quackapi_middleware.cpp index 8737104..efdf820 100644 --- a/src/quackapi_middleware.cpp +++ b/src/quackapi_middleware.cpp @@ -15,7 +15,6 @@ #include "duckdb/planner/expression/bound_parameter_data.hpp" #include "quackapi_limits.hpp" -#include "quackapi_policy.hpp" #include "quackapi_util.hpp" namespace duckdb { @@ -455,24 +454,11 @@ vector QuackapiMiddlewareRegistry::SnapshotAll() { bool ExecuteQuackapiMiddleware(Connection &connection, DatabaseInstance &db, QuackapiMiddlewarePhase phase, const string &group_name, QuackapiMiddlewareContext &context, bool authenticated, const unordered_map &claims) { + (void)authenticated; try { auto middleware = QuackapiMiddlewareRegistry::Get(db).Snapshot(phase, group_name); for (auto &definition : middleware) { - bool deny_unauthenticated = false; - string policy_error; - auto handler_sql = RewriteHandlerWithPolicies(db, definition.handler_sql, authenticated, - deny_unauthenticated, policy_error); - if (deny_unauthenticated) { - context.status = 401; - context.response_body = "{\"detail\":\"Not authenticated\"}"; - return false; - } - if (!policy_error.empty()) { - SetMiddlewareFailure(context); - return false; - } - - auto prepared = connection.Prepare(handler_sql); + auto prepared = connection.Prepare(definition.handler_sql); if (prepared->HasError()) { SetMiddlewareFailure(context); return false; diff --git a/src/quackapi_registry.cpp b/src/quackapi_registry.cpp index 821b2a3..0605653 100644 --- a/src/quackapi_registry.cpp +++ b/src/quackapi_registry.cpp @@ -334,47 +334,6 @@ vector QuackapiState::SnapshotApiKeys(const string &auth_na return result; } -void QuackapiState::AddQueue(const QuackapiQueue &queue, bool or_replace) { - std::lock_guard lock(queues_mutex); - for (auto it = queues.begin(); it != queues.end(); ++it) { - if (it->name == queue.name) { - if (!or_replace) { - throw InvalidInputException("Queue \"%s\" already exists — use CREATE OR REPLACE QUEUE", queue.name); - } - *it = queue; - return; - } - } - queues.push_back(queue); -} - -bool QuackapiState::DropQueue(const string &name) { - std::lock_guard lock(queues_mutex); - for (auto it = queues.begin(); it != queues.end(); ++it) { - if (it->name == name) { - queues.erase(it); - return true; - } - } - return false; -} - -bool QuackapiState::GetQueue(const string &name, QuackapiQueue &out) { - std::lock_guard lock(queues_mutex); - for (auto &q : queues) { - if (q.name == name) { - out = q; - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotQueues() { - std::lock_guard lock(queues_mutex); - return queues; -} - void QuackapiState::StartServer(DatabaseInstance &db, const string &host, int port, const QuackapiServeOptions &opts) { // Mirrors QuackStorageExtensionInfo::CreateServer (duckdb-quack // src/quack_storage.cpp): lock map → reject duplicate key → construct @@ -544,212 +503,6 @@ bool QuackapiState::FlushTelemetry(int port) { return false; } -//===--------------------------------------------------------------------===// -// Row access + masking policies -//===--------------------------------------------------------------------===// - -void QuackapiState::AddRowAccessPolicy(const QuackapiRowAccessPolicy &policy, bool or_replace) { - std::lock_guard lock(policies_mutex); - for (auto it = row_access_policies.begin(); it != row_access_policies.end(); ++it) { - if (it->name == policy.name) { - if (!or_replace) { - throw InvalidInputException( - "Row access policy \"%s\" already exists — use CREATE OR REPLACE ROW ACCESS POLICY", policy.name); - } - *it = policy; - return; - } - } - // Name must not collide with a masking policy. - for (auto &m : masking_policies) { - if (m.name == policy.name) { - throw InvalidInputException("Policy name \"%s\" is already used by a masking policy", policy.name); - } - } - row_access_policies.push_back(policy); -} - -bool QuackapiState::DropRowAccessPolicy(const string &name) { - std::lock_guard lock(policies_mutex); - bool found = false; - for (auto it = row_access_policies.begin(); it != row_access_policies.end(); ++it) { - if (it->name == name) { - row_access_policies.erase(it); - found = true; - break; - } - } - if (found) { - row_access_bindings.erase( - std::remove_if(row_access_bindings.begin(), row_access_bindings.end(), - [&](const QuackapiRowAccessBinding &b) { return b.policy_name == name; }), - row_access_bindings.end()); - } - return found; -} - -bool QuackapiState::GetRowAccessPolicy(const string &name, QuackapiRowAccessPolicy &out) { - std::lock_guard lock(policies_mutex); - for (auto &p : row_access_policies) { - if (p.name == name) { - out = p; - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotRowAccessPolicies() { - std::lock_guard lock(policies_mutex); - return row_access_policies; -} - -void QuackapiState::AddMaskingPolicy(const QuackapiMaskingPolicy &policy, bool or_replace) { - std::lock_guard lock(policies_mutex); - for (auto it = masking_policies.begin(); it != masking_policies.end(); ++it) { - if (it->name == policy.name) { - if (!or_replace) { - throw InvalidInputException( - "Masking policy \"%s\" already exists — use CREATE OR REPLACE MASKING POLICY", policy.name); - } - *it = policy; - return; - } - } - for (auto &r : row_access_policies) { - if (r.name == policy.name) { - throw InvalidInputException("Policy name \"%s\" is already used by a row access policy", policy.name); - } - } - masking_policies.push_back(policy); -} - -bool QuackapiState::DropMaskingPolicy(const string &name) { - std::lock_guard lock(policies_mutex); - bool found = false; - for (auto it = masking_policies.begin(); it != masking_policies.end(); ++it) { - if (it->name == name) { - masking_policies.erase(it); - found = true; - break; - } - } - if (found) { - masking_bindings.erase(std::remove_if(masking_bindings.begin(), masking_bindings.end(), - [&](const QuackapiMaskingBinding &b) { return b.policy_name == name; }), - masking_bindings.end()); - } - return found; -} - -bool QuackapiState::GetMaskingPolicy(const string &name, QuackapiMaskingPolicy &out) { - std::lock_guard lock(policies_mutex); - for (auto &p : masking_policies) { - if (p.name == name) { - out = p; - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotMaskingPolicies() { - std::lock_guard lock(policies_mutex); - return masking_policies; -} - -void QuackapiState::BindRowAccessPolicy(const QuackapiRowAccessBinding &binding, bool or_replace) { - std::lock_guard lock(policies_mutex); - const QuackapiRowAccessPolicy *pol = nullptr; - for (auto &p : row_access_policies) { - if (p.name == binding.policy_name) { - pol = &p; - break; - } - } - if (!pol) { - throw InvalidInputException("Row access policy \"%s\" does not exist", binding.policy_name); - } - if (binding.columns.size() != pol->arg_columns.size()) { - throw InvalidInputException("Row access policy \"%s\" expects %llu column(s), got %llu", binding.policy_name, - (unsigned long long)pol->arg_columns.size(), - (unsigned long long)binding.columns.size()); - } - // One RAP per table (replace same policy name, or or_replace any). - for (auto it = row_access_bindings.begin(); it != row_access_bindings.end(); ++it) { - if (StringUtil::Lower(it->table_name) == StringUtil::Lower(binding.table_name)) { - if (it->policy_name == binding.policy_name || or_replace) { - *it = binding; - return; - } - throw InvalidInputException( - "Table \"%s\" already has row access policy \"%s\" — DROP it first or use OR REPLACE", - binding.table_name, it->policy_name); - } - } - row_access_bindings.push_back(binding); -} - -bool QuackapiState::UnbindRowAccessPolicy(const string &table_name, const string &policy_name) { - std::lock_guard lock(policies_mutex); - for (auto it = row_access_bindings.begin(); it != row_access_bindings.end(); ++it) { - if (StringUtil::Lower(it->table_name) == StringUtil::Lower(table_name) && - (policy_name.empty() || it->policy_name == policy_name)) { - row_access_bindings.erase(it); - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotRowAccessBindings() { - std::lock_guard lock(policies_mutex); - return row_access_bindings; -} - -void QuackapiState::BindMaskingPolicy(const QuackapiMaskingBinding &binding, bool or_replace) { - std::lock_guard lock(policies_mutex); - bool found = false; - for (auto &p : masking_policies) { - if (p.name == binding.policy_name) { - found = true; - break; - } - } - if (!found) { - throw InvalidInputException("Masking policy \"%s\" does not exist", binding.policy_name); - } - for (auto it = masking_bindings.begin(); it != masking_bindings.end(); ++it) { - if (StringUtil::Lower(it->table_name) == StringUtil::Lower(binding.table_name) && - StringUtil::Lower(it->column_name) == StringUtil::Lower(binding.column_name)) { - if (!or_replace && it->policy_name != binding.policy_name) { - throw InvalidInputException("Column \"%s\".\"%s\" already has masking policy \"%s\"", - binding.table_name, binding.column_name, it->policy_name); - } - *it = binding; - return; - } - } - masking_bindings.push_back(binding); -} - -bool QuackapiState::UnbindMaskingPolicy(const string &table_name, const string &column_name) { - std::lock_guard lock(policies_mutex); - for (auto it = masking_bindings.begin(); it != masking_bindings.end(); ++it) { - if (StringUtil::Lower(it->table_name) == StringUtil::Lower(table_name) && - StringUtil::Lower(it->column_name) == StringUtil::Lower(column_name)) { - masking_bindings.erase(it); - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotMaskingBindings() { - std::lock_guard lock(policies_mutex); - return masking_bindings; -} - void QuackapiState::AddGraphqlTable(const string &table_name, bool or_replace) { if (table_name.empty()) { throw InvalidInputException("CREATE GRAPHQL FOR TABLE expects a non-empty table name"); diff --git a/src/quackapi_server.cpp b/src/quackapi_server.cpp index 0ffcdc7..7780da7 100644 --- a/src/quackapi_server.cpp +++ b/src/quackapi_server.cpp @@ -40,7 +40,6 @@ #include "quackapi_graphql.hpp" #include "quackapi_openapi.hpp" #include "quackapi_pg.hpp" -#include "quackapi_policy.hpp" #include "quackapi_limits.hpp" #include "quackapi_middleware.hpp" #include "quackapi_state.hpp" @@ -2331,7 +2330,6 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd try { GraphqlExecOptions opts; opts.allowed_tables = &gql_route.tables; - opts.authenticated = !gql_route.require_auth.empty() && auth_result.ok; opts.claims = &auth_result.claims; opts.query_timeout_ms = options.query_timeout_ms; opts.max_response_bytes = options.max_response_bytes; @@ -2368,7 +2366,6 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd GraphqlExecOptions opts; opts.allowed_tables = &gql_route.tables; opts.mode = "route"; - opts.authenticated = !gql_route.require_auth.empty() && auth_result.ok; opts.claims = &auth_result.claims; opts.query_timeout_ms = options.query_timeout_ms; opts.max_response_bytes = options.max_response_bytes; @@ -2676,15 +2673,7 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd route_name = stream_match.stream.name; route_path = stream_match.stream.pattern; try { - bool policy_denied = false; - string stream_policy_error; - const auto stream_sql = RewriteHandlerWithPolicies(*db, stream_match.stream.handler_sql, false, - policy_denied, stream_policy_error); - if (policy_denied || !stream_policy_error.empty()) { - SetJson(res, 403, "{\"detail\":\"Policy denies unauthenticated access\"}"); - finish(); - return; - } + const auto &stream_sql = stream_match.stream.handler_sql; auto headers = CollectHeaders(req); // HEAD: headers only — do not install a long-lived provider. if (req.method == "HEAD") { @@ -2935,19 +2924,8 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd } // auth_result.claims is ready for $claims_ binding below. - // ---- ROW ACCESS + MASKING (claims-keyed policies on tables) ---- - // When the handler references a table with RAP/masking bindings, wrap - // table refs as secure subqueries. Unauthenticated requests fail closed. bool authenticated = !match.route.require_auth.empty() && auth_result.ok; - bool deny_unauth = false; - string policy_error; - string handler_sql = - RewriteHandlerWithPolicies(*db, match.route.handler_sql, authenticated, deny_unauth, policy_error); - if (deny_unauth || !policy_error.empty()) { - SetJson(res, 403, "{\"detail\":\"Policy denies unauthenticated access\"}"); - finish(); - return; - } + string handler_sql = match.route.handler_sql; try { // Destroy the connection before leaving the request. Retaining it in TLS From 69c3c4aff40443e43ee666595e010e530830565b Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:12:22 -0700 Subject: [PATCH 11/93] Harden POST executor lifecycle and admission --- src/quackapi_post_executor.cpp | 93 +++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 23 deletions(-) diff --git a/src/quackapi_post_executor.cpp b/src/quackapi_post_executor.cpp index f711d11..ff8ed88 100644 --- a/src/quackapi_post_executor.cpp +++ b/src/quackapi_post_executor.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -58,10 +59,19 @@ idx_t EffectiveConcurrency(DatabaseInstance &db) { return MaxValue(1, db.NumberOfThreads()); } +idx_t WaitingQueueCapacity(idx_t concurrency) { + return concurrency > NumericLimits::Maximum() / 2 ? NumericLimits::Maximum() : concurrency * 2; +} + } // namespace class QuackapiPostExecutor::Impl { public: + struct Worker { + std::thread thread; + shared_ptr active_job; + }; + explicit Impl(DatabaseInstance &db_p) : db(db_p) { } @@ -69,12 +79,30 @@ class QuackapiPostExecutor::Impl { { std::lock_guard guard(lock); stopping = true; + for (auto &job : queue) { + job->request.cancelled->store(true); + job->batch->cancelled.store(true); + job->batch->interrupted.store(true); + std::lock_guard batch_guard(job->batch->mutex); + job->batch->completed++; + job->batch->cv.notify_all(); + } + queue.clear(); + for (auto &worker : workers) { + if (worker->active_job) { + auto &job = *worker->active_job; + job.request.cancelled->store(true); + job.batch->cancelled.store(true); + job.batch->interrupted.store(true); + job.batch->cv.notify_all(); + } + } } queue_cv.notify_all(); space_cv.notify_all(); for (auto &worker : workers) { - if (worker.joinable()) { - worker.join(); + if (worker->thread.joinable()) { + worker->thread.join(); } } } @@ -92,10 +120,6 @@ class QuackapiPostExecutor::Impl { batch->cancellation_flags[i] = requests[i].cancelled; } } - const auto concurrency = EffectiveConcurrency(db); - const auto queue_capacity = - concurrency > NumericLimits::Maximum() / 2 ? NumericLimits::Maximum() : concurrency * 2; - try { for (idx_t i = 0; i < requests.size(); i++) { if (skip[i]) { @@ -108,7 +132,7 @@ class QuackapiPostExecutor::Impl { job->request = std::move(requests[i]); std::unique_lock guard(lock); - while (!stopping && queue.size() >= queue_capacity) { + while (!stopping && queue.size() >= WaitingQueueCapacity(EffectiveConcurrency(db))) { CheckCaller(context, deadline); space_cv.wait_for(guard, std::chrono::milliseconds(10)); } @@ -116,21 +140,27 @@ class QuackapiPostExecutor::Impl { if (stopping || batch->cancelled.load()) { throw InterruptException(); } - EnsureWorkers(concurrency); queue.push_back(std::move(job)); { std::lock_guard batch_guard(batch->mutex); batch->accepted++; } + const auto concurrency = EffectiveConcurrency(db); + EnsureWorkers(std::min(concurrency, active + queue.size())); queue_cv.notify_all(); } } catch (InterruptException &) { CancelBatch(batch); DrainBatch(batch, Accepted(batch)); throw; + } catch (...) { + auto error = std::current_exception(); + CancelBatch(batch); + DrainBatch(batch, Accepted(batch)); + std::rethrow_exception(error); } - const auto expected = requests.size() - CountSkipped(skip); + const auto expected = Accepted(batch); while (true) { { std::lock_guard guard(batch->mutex); @@ -156,17 +186,16 @@ class QuackapiPostExecutor::Impl { return batch->accepted; } - static idx_t CountSkipped(const vector &skip) { - idx_t count = 0; - for (auto value : skip) { - count += value ? 1 : 0; - } - return count; - } - void EnsureWorkers(idx_t desired) { while (workers.size() < desired) { - workers.emplace_back([this]() { WorkerLoop(); }); + workers.emplace_back(make_uniq()); + auto *worker = workers.back().get(); + try { + worker->thread = std::thread([this, worker]() { WorkerLoop(*worker); }); + } catch (...) { + workers.pop_back(); + throw; + } } } @@ -185,6 +214,7 @@ class QuackapiPostExecutor::Impl { } } space_cv.notify_all(); + queue_cv.notify_all(); batch->cv.notify_all(); } @@ -203,7 +233,7 @@ class QuackapiPostExecutor::Impl { batch->cv.wait(guard, [&]() { return batch->completed >= expected; }); } - void WorkerLoop() { + void WorkerLoop(Worker &worker) { unique_ptr state; try { state = QuackapiPostProvider::CreateWorkerState(db); @@ -214,14 +244,30 @@ class QuackapiPostExecutor::Impl { shared_ptr job; { std::unique_lock guard(lock); - queue_cv.wait(guard, - [&]() { return stopping || (!queue.empty() && active < EffectiveConcurrency(db)); }); + for (;;) { + if (stopping && queue.empty()) { + break; + } + if (!stopping && !queue.empty()) { + const auto desired = std::min(EffectiveConcurrency(db), active + queue.size()); + try { + EnsureWorkers(desired); + } catch (...) { + // Existing workers can still drain the queue if growth hits a system limit. + } + } + if (!queue.empty() && (stopping || active < EffectiveConcurrency(db))) { + break; + } + queue_cv.wait_for(guard, std::chrono::milliseconds(10)); + } if (stopping && queue.empty()) { break; } job = std::move(queue.front()); queue.pop_front(); active++; + worker.active_job = job; space_cv.notify_all(); } @@ -263,11 +309,12 @@ class QuackapiPostExecutor::Impl { { std::lock_guard guard(lock); active--; + worker.active_job.reset(); } queue_cv.notify_all(); batch.cv.notify_all(); if (batch.cancelled.load()) { - CancelQueued(job->batch); + CancelBatch(job->batch); } } state.reset(); @@ -278,7 +325,7 @@ class QuackapiPostExecutor::Impl { std::condition_variable queue_cv; std::condition_variable space_cv; std::deque> queue; - vector workers; + vector> workers; idx_t active = 0; bool stopping = false; }; From 727078bbc021d583985d4157982a8d8dfb1d056f Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:12:18 -0700 Subject: [PATCH 12/93] Guard POST client reuse configuration --- src/quackapi_post_provider.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/quackapi_post_provider.cpp b/src/quackapi_post_provider.cpp index 213f7e9..de2d604 100644 --- a/src/quackapi_post_provider.cpp +++ b/src/quackapi_post_provider.cpp @@ -30,8 +30,13 @@ struct ClientConfig { bool verify_ssl = true; bool Matches(const HTTPUtil &util, const HTTPParams ¶ms, const string &candidate_origin) const { - return provider_name == util.GetName() && params_type == typeid(params).name() && origin == candidate_origin && - http_proxy == params.http_proxy && http_proxy_port == params.http_proxy_port && + // The common fields below cannot fingerprint provider-specific parameter + // subclasses. Until a provider exposes a configuration comparison API, + // only reuse the exact base parameter type; replace provider-specialized + // clients for every request rather than risk carrying stale settings. + return typeid(params) == typeid(HTTPParams) && provider_name == util.GetName() && + params_type == typeid(params).name() && origin == candidate_origin && http_proxy == params.http_proxy && + (http_proxy.empty() || http_proxy_port == params.http_proxy_port) && http_proxy_username == params.http_proxy_username && http_proxy_password == params.http_proxy_password && keep_alive == params.keep_alive && follow_location == params.follow_location && override_verify_ssl == params.override_verify_ssl && verify_ssl == params.verify_ssl; @@ -42,7 +47,7 @@ struct ClientConfig { params_type = typeid(params).name(); origin = candidate_origin; http_proxy = params.http_proxy; - http_proxy_port = params.http_proxy_port; + http_proxy_port = http_proxy.empty() ? 0 : params.http_proxy_port; http_proxy_username = params.http_proxy_username; http_proxy_password = params.http_proxy_password; keep_alive = params.keep_alive; From f6447a574d63945214bbded2574fdc0c545089dd Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:13:59 -0700 Subject: [PATCH 13/93] refactor(server): encode response chunks within byte limit --- src/include/quackapi_response_encoding.hpp | 13 +- src/quackapi_response_encoding.cpp | 315 +-------------------- src/quackapi_server.cpp | 236 ++++++++------- test/sql/quackapi_format.test | 23 +- 4 files changed, 158 insertions(+), 429 deletions(-) diff --git a/src/include/quackapi_response_encoding.hpp b/src/include/quackapi_response_encoding.hpp index 934f174..1f0c231 100644 --- a/src/include/quackapi_response_encoding.hpp +++ b/src/include/quackapi_response_encoding.hpp @@ -12,7 +12,7 @@ struct QuackapiRoute; class DataChunk; enum class ResponseMode { JSON, HTML, TEXT }; -enum class BodyFormat { JSON, NDJSON, CSV, PARQUET, ARROW }; +enum class BodyFormat { JSON, NDJSON, CSV }; enum class JsonRowMode { ARRAY, OBJECT, NDJSON }; //! Bounded accumulator shared by streaming encoders. Each append is atomic: @@ -23,6 +23,7 @@ class QuackapiResponseWriter { explicit QuackapiResponseWriter(idx_t max_bytes); bool Append(const char *data, idx_t length); bool Append(const string &data); + bool SetMaxBytes(idx_t max_bytes); idx_t BytesWritten() const; bool ExceededLimit() const; const string &Data() const; @@ -82,15 +83,5 @@ void SetInternalError(duckdb_httplib::Response &res, const string &server_side_d ResponseMode ResponseModeFor(const vector &names); BodyFormat ResolveBodyFormat(const QuackapiRoute &route, const duckdb_httplib::Request &req); string EmptyResultBody(const QuackapiRoute &route); -string SerializeRowsJsonArray(const vector &names, const vector &data_cols, - const vector> &rows); -string SerializeRowJsonObject(const vector &names, const vector &data_cols, const vector &cols); -string SerializeRowsNdjson(const vector &names, const vector &data_cols, - const vector> &rows); -string SerializeRowsCsv(const vector &names, const vector &data_cols, const vector> &rows); -string SerializeRowsParquet(Connection &con, const vector &names, const vector &types, - const vector &data_cols, const vector> &rows); -string SerializeRowsArrow(Connection &con, const vector &names, const vector &types, - const vector &data_cols, const vector> &rows); } // namespace duckdb diff --git a/src/quackapi_response_encoding.cpp b/src/quackapi_response_encoding.cpp index ab9678c..a033d13 100644 --- a/src/quackapi_response_encoding.cpp +++ b/src/quackapi_response_encoding.cpp @@ -6,16 +6,11 @@ #include "duckdb/common/exception.hpp" #include "duckdb/common/csv_writer.hpp" -#include "duckdb/common/file_open_flags.hpp" -#include "duckdb/common/file_system.hpp" #include "duckdb/common/serializer/write_stream.hpp" #include "duckdb/common/string_util.hpp" -#include "duckdb/common/types/uuid.hpp" #include "duckdb/execution/expression_executor.hpp" #include "duckdb/function/function_binder.hpp" -#include "duckdb/main/appender.hpp" #include "duckdb/main/connection.hpp" -#include "duckdb/parser/keyword_helper.hpp" #include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/planner/expression/bound_cast_expression.hpp" #include "duckdb/planner/expression/bound_reference_expression.hpp" @@ -78,6 +73,14 @@ bool QuackapiResponseWriter::Append(const string &append_data) { return Append(append_data.data(), append_data.size()); } +bool QuackapiResponseWriter::SetMaxBytes(idx_t max_bytes) { + this->max_bytes = max_bytes; + if (bytes_written > max_bytes) { + exceeded_limit = true; + } + return !exceeded_limit; +} + idx_t QuackapiResponseWriter::BytesWritten() const { return bytes_written; } @@ -396,7 +399,7 @@ void SetJson(duckdb_httplib::Response &res, int status, const string &body) { //! Response mode inferred from the single output column's name — the database //! names the payload. `html` -> text/html, `text` -> text/plain; anything else -//! serializes as row data (json / ndjson / csv / parquet / arrow via FORMAT + Accept). +//! serializes as row data (JSON / NDJSON / CSV via FORMAT + Accept). ResponseMode ResponseModeFor(const vector &names) { if (names.size() != 1) { @@ -422,12 +425,6 @@ BodyFormat ParseBodyFormatToken(const string &token) { if (t == "csv") { return BodyFormat::CSV; } - if (t == "parquet") { - return BodyFormat::PARQUET; - } - if (t == "arrow" || t == "arrows") { - return BodyFormat::ARROW; - } return BodyFormat::JSON; } @@ -444,16 +441,6 @@ bool MediaTypeToBodyFormat(const string &media, BodyFormat &out) { out = BodyFormat::CSV; return true; } - if (m == "application/vnd.apache.parquet" || m == "application/parquet") { - out = BodyFormat::PARQUET; - return true; - } - // Arrow IPC stream (what nanoarrow FORMAT ARROWS emits) or file. - if (m == "application/vnd.apache.arrow.stream" || m == "application/vnd.apache.arrow.file" || - m == "application/x-apache-arrow-stream" || m == "application/arrow") { - out = BodyFormat::ARROW; - return true; - } if (m == "application/json" || m == "text/json" || m == "*/*") { out = BodyFormat::JSON; return true; @@ -477,12 +464,11 @@ double ParseAcceptQ(const string &token) { return 1.0; } -//! When route FORMAT is default/json, honor Accept for ndjson/csv/parquet/arrow/json. -//! Explicit route FORMAT (ndjson|csv|parquet|arrow) always wins. +//! When route FORMAT is default/json, honor Accept for ndjson/csv/json. +//! Explicit route FORMAT (ndjson|csv) always wins. BodyFormat ResolveBodyFormat(const QuackapiRoute &route, const duckdb_httplib::Request &req) { auto route_fmt = StringUtil::Lower(route.response_format.empty() ? "json" : route.response_format); - if (route_fmt == "ndjson" || route_fmt == "csv" || route_fmt == "parquet" || route_fmt == "arrow" || - route_fmt == "arrows") { + if (route_fmt == "ndjson" || route_fmt == "csv") { return ParseBodyFormatToken(route_fmt); } // Default / explicit json: negotiate Accept when present. @@ -490,7 +476,7 @@ BodyFormat ResolveBodyFormat(const QuackapiRoute &route, const duckdb_httplib::R if (it == req.headers.end() || it->second.empty()) { return BodyFormat::JSON; } - // Pick the highest-q recognized type among json / ndjson / csv / parquet / arrow. + // Pick the highest-q recognized type among json / ndjson / csv. BodyFormat best = BodyFormat::JSON; double best_q = -1.0; bool saw_recognized = false; @@ -510,7 +496,7 @@ BodyFormat ResolveBodyFormat(const QuackapiRoute &route, const duckdb_httplib::R if (!MediaTypeToBodyFormat(media, f)) { continue; } - // Prefer more specific non-wildcard when q ties: ndjson/csv/parquet/arrow over */* json. + // Prefer a concrete non-json format when q ties with JSON. bool better = !saw_recognized || q > best_q; if (!better && q == best_q) { // Prefer non-json when client listed it at same q after json — first highest wins @@ -528,83 +514,6 @@ BodyFormat ResolveBodyFormat(const QuackapiRoute &route, const duckdb_httplib::R return saw_recognized ? best : BodyFormat::JSON; } -//! RFC 4180-style CSV field escape (quote when needed). -string CsvEscapeField(const string &input) { - bool need_quote = false; - for (unsigned char c : input) { - if (c == '"' || c == ',' || c == '\n' || c == '\r') { - need_quote = true; - break; - } - } - if (!need_quote) { - return input; - } - string out; - out.reserve(input.size() + 2); - out += '"'; - for (unsigned char c : input) { - if (c == '"') { - out += "\"\""; - } else { - out += static_cast(c); - } - } - out += '"'; - return out; -} - -//! Cell text for CSV: null → empty; scalars ToString; nested → JSON text. -string ValueToCsvCell(const Value &value) { - if (value.IsNull()) { - return ""; - } - auto id = value.type().id(); - if (id == LogicalTypeId::LIST || id == LogicalTypeId::STRUCT || id == LogicalTypeId::MAP) { - return ValueToJson(value); - } - return value.ToString(); -} - -string SerializeRowsJsonArray(const vector &names, const vector &data_cols, - const vector> &rows) { - string body = "["; - bool first_row = true; - for (auto &cols : rows) { - if (!first_row) { - body += ","; - } - first_row = false; - body += "{"; - bool first_col = true; - for (auto col : data_cols) { - if (!first_col) { - body += ","; - } - first_col = false; - body += "\"" + QuackapiJsonEscape(names[col]) + "\":" + ValueToJson(cols[col]); - } - body += "}"; - } - body += "]"; - return body; -} - -//! Single-row JSON object (ENVELOPE object). Caller guarantees rows.size() == 1. -string SerializeRowJsonObject(const vector &names, const vector &data_cols, const vector &cols) { - string body = "{"; - bool first_col = true; - for (auto col : data_cols) { - if (!first_col) { - body += ","; - } - first_col = false; - body += "\"" + QuackapiJsonEscape(names[col]) + "\":" + ValueToJson(cols[col]); - } - body += "}"; - return body; -} - string EmptyResultBody(const QuackapiRoute &route) { if (!route.empty_body.empty()) { return route.empty_body; @@ -612,202 +521,6 @@ string EmptyResultBody(const QuackapiRoute &route) { return "{\"detail\":\"Not Found\"}"; } -string SerializeRowsNdjson(const vector &names, const vector &data_cols, - const vector> &rows) { - string body; - for (auto &cols : rows) { - body += "{"; - bool first_col = true; - for (auto col : data_cols) { - if (!first_col) { - body += ","; - } - first_col = false; - body += "\"" + QuackapiJsonEscape(names[col]) + "\":" + ValueToJson(cols[col]); - } - body += "}\n"; - } - return body; -} - -string SerializeRowsCsv(const vector &names, const vector &data_cols, - const vector> &rows) { - string body; - // header - for (idx_t i = 0; i < data_cols.size(); i++) { - if (i > 0) { - body += ","; - } - body += CsvEscapeField(names[data_cols[i]]); - } - body += "\n"; - for (auto &cols : rows) { - for (idx_t i = 0; i < data_cols.size(); i++) { - if (i > 0) { - body += ","; - } - auto col = data_cols[i]; - string cell; - if (col < cols.size()) { - cell = ValueToCsvCell(cols[col]); - } - body += CsvEscapeField(cell); - } - body += "\n"; - } - return body; -} - -//! Shared: TEMP table + Appender fill for binary serdes (parquet / arrow). -void FillTempTableForSerdes(Connection &con, const string &table, const vector &names, - const vector &types, const vector &data_cols, - const vector> &rows, const char *label) { - string create_sql = "CREATE TEMP TABLE " + table + " ("; - for (idx_t i = 0; i < data_cols.size(); i++) { - if (i > 0) { - create_sql += ", "; - } - auto c = data_cols[i]; - create_sql += KeywordHelper::WriteOptionallyQuoted(names[c]); - create_sql += " "; - create_sql += types[c].ToString(); - } - create_sql += ")"; - auto create_res = con.Query(create_sql); - if (create_res->HasError()) { - throw InvalidInputException("%s serialize CREATE: %s", label, create_res->GetError()); - } - - if (!rows.empty()) { - Appender appender(con, table); - for (auto &cols : rows) { - appender.BeginRow(); - for (auto c : data_cols) { - if (c < cols.size()) { - appender.Append(cols[c]); - } else { - appender.Append(Value()); - } - } - appender.EndRow(); - } - appender.Close(); - } -} - -//! OS temp dir + basename for parquet/arrow scratch files. Never hardcode /tmp -//! (Windows CI has no /tmp). Paths use forward slashes for SQL COPY literals. -string SerdesTempFilePath(FileSystem &fs, const string &filename) { - string dir; -#if defined(_WIN32) || defined(WIN32) - const char *t = std::getenv("TEMP"); - if (!t || !t[0]) { - t = std::getenv("TMP"); - } - dir = (t && t[0]) ? string(t) : string("."); -#else - const char *t = std::getenv("TMPDIR"); - dir = (t && t[0]) ? string(t) : string("/tmp"); -#endif - string path = fs.JoinPath(dir, filename); - return StringUtil::Replace(path, "\\", "/"); -} - -string SqlQuotePath(const string &path) { - return StringUtil::Replace(path, "'", "''"); -} - -//! Read path bytes then delete the file. On open failure, best-effort remove. -string ReadAndRemoveFileBytes(Connection &con, const string &path) { - auto &fs = FileSystem::GetFileSystem(*con.context); - unique_ptr handle; - try { - handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ); - } catch (...) { - fs.TryRemoveFile(path); - throw; - } - idx_t file_size = handle->GetFileSize(); - string body; - if (file_size > 0) { - // mutable buffer — data_ptr_cast rejects const char* from string::data() - vector buf(file_size); - handle->Read(buf.data(), file_size); - body.assign(reinterpret_cast(buf.data()), file_size); - } - handle->Close(); - fs.TryRemoveFile(path); - return body; -} - -//! Serialize result rows as a Parquet file body (magic "PAR1"). Uses a unique -//! TEMP table + COPY TO (FORMAT PARQUET) + read bytes, then cleans up. -string SerializeRowsParquet(Connection &con, const vector &names, const vector &types, - const vector &data_cols, const vector> &rows) { - if (data_cols.empty()) { - throw InvalidInputException("parquet serialize: no data columns"); - } - auto &fs = FileSystem::GetFileSystem(*con.context); - string uid = StringUtil::Replace(UUID::ToString(UUIDv7::GenerateRandomUUID()), "-", ""); - string table = "qa_parquet_" + uid; - string path = SerdesTempFilePath(fs, table + ".parquet"); - - FillTempTableForSerdes(con, table, names, types, data_cols, rows, "parquet"); - - auto copy_res = con.Query("COPY " + table + " TO '" + SqlQuotePath(path) + "' (FORMAT PARQUET)"); - if (copy_res->HasError()) { - con.Query("DROP TABLE IF EXISTS " + table); - fs.TryRemoveFile(path); - throw InvalidInputException("parquet serialize COPY: %s", copy_res->GetError()); - } - con.Query("DROP TABLE IF EXISTS " + table); - return ReadAndRemoveFileBytes(con, path); -} - -//! Ensure community nanoarrow is available (LOAD, else INSTALL FROM community). -void EnsureNanoarrowLoaded(Connection &con) { - auto load = con.Query("LOAD nanoarrow"); - if (!load->HasError()) { - return; - } - auto inst = con.Query("INSTALL nanoarrow FROM community"); - if (inst->HasError()) { - throw InvalidInputException( - "arrow serialize: nanoarrow not available (%s). INSTALL nanoarrow FROM community and retry.", - inst->GetError()); - } - load = con.Query("LOAD nanoarrow"); - if (load->HasError()) { - throw InvalidInputException("arrow serialize: could not LOAD nanoarrow: %s", load->GetError()); - } -} - -//! Serialize result rows as Arrow IPC stream bytes (nanoarrow FORMAT ARROWS). -//! Magic is 0xFFFFFFFF stream continuation (not ARROW1 file). Content-Type: -//! application/vnd.apache.arrow.stream. Same TEMP+COPY+read pattern as parquet. -string SerializeRowsArrow(Connection &con, const vector &names, const vector &types, - const vector &data_cols, const vector> &rows) { - if (data_cols.empty()) { - throw InvalidInputException("arrow serialize: no data columns"); - } - EnsureNanoarrowLoaded(con); - auto &fs = FileSystem::GetFileSystem(*con.context); - string uid = StringUtil::Replace(UUID::ToString(UUIDv7::GenerateRandomUUID()), "-", ""); - string table = "qa_arrow_" + uid; - string path = SerdesTempFilePath(fs, table + ".arrows"); - - FillTempTableForSerdes(con, table, names, types, data_cols, rows, "arrow"); - - auto copy_res = con.Query("COPY " + table + " TO '" + SqlQuotePath(path) + "' (FORMAT ARROWS)"); - if (copy_res->HasError()) { - con.Query("DROP TABLE IF EXISTS " + table); - fs.TryRemoveFile(path); - throw InvalidInputException("arrow serialize COPY: %s", copy_res->GetError()); - } - con.Query("DROP TABLE IF EXISTS " + table); - return ReadAndRemoveFileBytes(con, path); -} - void SetInternalError(duckdb_httplib::Response &res, const string &server_side_detail) { // Return DuckDB's diagnostic so clients can diagnose route execution failures. fprintf(stderr, "quackapi: internal error: %s\n", server_side_detail.c_str()); diff --git a/src/quackapi_server.cpp b/src/quackapi_server.cpp index 7780da7..88fea3c 100644 --- a/src/quackapi_server.cpp +++ b/src/quackapi_server.cpp @@ -3583,138 +3583,182 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd } } - // Materialize rows once so we can apply headers then serialize body. - vector> rows; - idx_t response_value_bytes = 0; + // HTML/TEXT mode uses the single remaining data column name. + vector data_names; + for (auto c : data_cols) { + data_names.push_back(names[c]); + } + auto mode = ResponseModeFor(data_names); + const bool object_envelope = + StringUtil::Lower(match.route.response_envelope.empty() ? "array" : match.route.response_envelope) == + "object"; + + idx_t max_response_bytes = static_cast(options.max_response_bytes); + QuackapiResponseWriter response_writer(max_response_bytes); + unique_ptr json_encoder; + unique_ptr csv_encoder; + if (!data_cols.empty() && mode == ResponseMode::JSON) { + if (body_format == BodyFormat::CSV) { + csv_encoder = make_uniq(con, names, result->types, data_cols, response_writer); + } else { + JsonRowMode row_mode = JsonRowMode::ARRAY; + if (body_format == BodyFormat::NDJSON) { + row_mode = JsonRowMode::NDJSON; + } else if (object_envelope) { + row_mode = JsonRowMode::OBJECT; + } + json_encoder = make_uniq(con, names, result->types, data_cols, row_mode, + response_writer); + } + } + + string location_value; + vector set_cookie_values; + idx_t header_value_bytes = 0; + idx_t rows_fetched = 0; + bool limit_exceeded = false; while (true) { auto chunk = result->Fetch(); if (!chunk || chunk->size() == 0) { break; } - for (idx_t row = 0; row < chunk->size(); row++) { - vector cols; - cols.resize(chunk->ColumnCount()); - for (idx_t col = 0; col < chunk->ColumnCount(); col++) { - cols[col] = chunk->GetValue(col, row); - response_value_bytes += cols[col].ToString().size(); - if (response_value_bytes > static_cast(options.max_response_bytes)) { - SetJson(res, 507, "{\"detail\":\"Response exceeds configured byte limit\"}"); - finish(); - return; + rows_fetched += chunk->size(); + rows_out = static_cast(rows_fetched); + + // Keep response-control columns outside the body while reading them + // from every chunk, matching the old first-Location/all-Cookies rules. + for (idx_t row = 0; row < chunk->size() && !limit_exceeded; row++) { + if (location_col < chunk->ColumnCount()) { + auto value = chunk->GetValue(location_col, row); + if (!value.IsNull()) { + auto text = value.ToString(); + if (text.size() > max_response_bytes - header_value_bytes) { + limit_exceeded = true; + } else { + header_value_bytes += text.size(); + if (location_value.empty() && !text.empty()) { + location_value = text; + } + } + } + } + if (set_cookie_col < chunk->ColumnCount()) { + auto value = chunk->GetValue(set_cookie_col, row); + if (!value.IsNull()) { + auto text = value.ToString(); + if (text.size() > max_response_bytes - header_value_bytes) { + limit_exceeded = true; + } else { + header_value_bytes += text.size(); + set_cookie_values.push_back(std::move(text)); + } } } - rows.push_back(std::move(cols)); } - } - rows_out = static_cast(rows.size()); + if (limit_exceeded || !response_writer.SetMaxBytes(max_response_bytes - header_value_bytes)) { + limit_exceeded = true; + break; + } - // Apply Location / Set-Cookie from first (or each) row. - string location_value; - vector set_cookie_values; - for (auto &cols : rows) { - if (location_col < cols.size() && !cols[location_col].IsNull() && location_value.empty()) { - location_value = cols[location_col].ToString(); + if (mode == ResponseMode::HTML || mode == ResponseMode::TEXT) { + auto col = data_cols[0]; + auto &vector = chunk->data[col]; + UnifiedVectorFormat format; + vector.ToUnifiedFormat(chunk->size(), format); + const bool raw_string_bytes = result->types[col].id() == LogicalTypeId::BLOB || + result->types[col].id() == LogicalTypeId::VARCHAR || + result->types[col].IsJSONType(); + const string_t *values = nullptr; + if (raw_string_bytes) { + values = format.GetData(); + } + for (idx_t row = 0; row < chunk->size(); row++) { + auto index = format.sel->get_index(row); + if (!format.validity.RowIsValid(index)) { + continue; + } + if (raw_string_bytes) { + auto &value = values[index]; + response_writer.Append(value.GetData(), value.GetSize()); + } else { + auto value = chunk->GetValue(col, row); + auto text = value.ToString(); + response_writer.Append(text); + } + if (response_writer.ExceededLimit()) { + limit_exceeded = true; + break; + } + } + } else if (csv_encoder) { + csv_encoder->AppendChunk(*chunk); + } else if (json_encoder) { + json_encoder->AppendChunk(*chunk); } - if (set_cookie_col < cols.size() && !cols[set_cookie_col].IsNull()) { - set_cookie_values.push_back(cols[set_cookie_col].ToString()); + if (response_writer.ExceededLimit()) { + limit_exceeded = true; + break; } } - if (!location_value.empty()) { - res.set_header("Location", location_value); - } - for (auto &cv : set_cookie_values) { - // httplib Headers is multimap — set_header appends. - res.set_header("Set-Cookie", cv); - } - // HTML/TEXT mode uses the single remaining data column name. - vector data_names; - for (auto c : data_cols) { - data_names.push_back(names[c]); + if (limit_exceeded) { + SetJson(res, 507, "{\"detail\":\"Response exceeds configured byte limit\"}"); + finish(); + return; } - auto mode = ResponseModeFor(data_names); // 0-row EMPTY STATUS: override success STATUS (any FORMAT / html/text). - if (rows.empty() && match.route.empty_status != 0) { + if (rows_fetched == 0 && match.route.empty_status != 0) { SetJson(res, match.route.empty_status, EmptyResultBody(match.route)); finish(); return; } + if (!location_value.empty()) { + res.set_header("Location", location_value); + } + for (auto &cookie : set_cookie_values) { + // httplib Headers is multimap — set_header appends. + res.set_header("Set-Cookie", cookie); + } + // No data columns: empty body (redirect / cookie-only responses). if (data_cols.empty()) { res.status = match.route.status; res.body.clear(); - // Drop Content-Type when there is no body. finish(); return; } - // HTML/TEXT mode: a single data column named `html`/`text` serves its raw - // string value (e.g. SELECT tera_render(...) AS html). Multiple rows are - // concatenated in order, so a query returning fragments streams a page. - // Column-name modes win over FORMAT / Accept. - if (mode != ResponseMode::JSON) { - string body; - idx_t col = data_cols[0]; - for (auto &cols : rows) { - if (col < cols.size() && !cols[col].IsNull()) { - body += cols[col].ToString(); - } - } - res.status = match.route.status; - res.set_content(body, - mode == ResponseMode::HTML ? "text/html; charset=utf-8" : "text/plain; charset=utf-8"); - // Keep body so Content-Length is correct; httplib omits the body for HEAD. + if (json_encoder) { + json_encoder->Finish(); + } + if (csv_encoder) { + csv_encoder->Finish(); + } + if (response_writer.ExceededLimit()) { + SetJson(res, 507, "{\"detail\":\"Response exceeds configured byte limit\"}"); finish(); return; } - const bool object_envelope = - StringUtil::Lower(match.route.response_envelope.empty() ? "array" : match.route.response_envelope) == - "object"; - - // Serialize rows: json array | json object | ndjson | csv | parquet | arrow. - string body; - const char *content_type = "application/json"; - if (body_format == BodyFormat::NDJSON) { - body = SerializeRowsNdjson(names, data_cols, rows); - content_type = "application/x-ndjson"; + if (mode == ResponseMode::HTML || mode == ResponseMode::TEXT) { + res.status = match.route.status; + res.set_content(response_writer.Data(), + mode == ResponseMode::HTML ? "text/html; charset=utf-8" : "text/plain; charset=utf-8"); } else if (body_format == BodyFormat::CSV) { - body = SerializeRowsCsv(names, data_cols, rows); - content_type = "text/csv; charset=utf-8"; - } else if (body_format == BodyFormat::PARQUET) { - body = SerializeRowsParquet(con, names, result->types, data_cols, rows); - content_type = "application/vnd.apache.parquet"; - } else if (body_format == BodyFormat::ARROW) { - body = SerializeRowsArrow(con, names, result->types, data_cols, rows); - // nanoarrow FORMAT ARROWS produces IPC *stream* (0xFFFFFFFF magic). - content_type = "application/vnd.apache.arrow.stream"; - } else if (object_envelope) { - if (rows.size() > 1) { - SetJson(res, 500, - "{\"detail\":\"ENVELOPE object requires exactly one row, got " + std::to_string(rows.size()) + - "\"}"); - finish(); - return; - } - if (rows.empty()) { - // No EMPTY STATUS → JSON null (array default remains []). - body = "null"; - } else { - body = SerializeRowJsonObject(names, data_cols, rows[0]); - } - SetJson(res, match.route.status, body); - finish(); - return; + res.status = match.route.status; + res.set_content(response_writer.Data(), "text/csv; charset=utf-8"); + } else if (body_format == BodyFormat::NDJSON) { + res.status = match.route.status; + res.set_content(response_writer.Data(), "application/x-ndjson"); + } else if (object_envelope && rows_fetched > 1) { + SetJson(res, 500, + "{\"detail\":\"ENVELOPE object requires exactly one row, got " + std::to_string(rows_fetched) + + "\"}"); } else { - body = SerializeRowsJsonArray(names, data_cols, rows); - SetJson(res, match.route.status, body); - finish(); - return; + SetJson(res, match.route.status, response_writer.Data()); } - res.status = match.route.status; - res.set_content(body, content_type); // Keep body so Content-Length is correct; httplib omits the body for HEAD. } catch (std::exception &ex) { SetInternalError(res, ex.what()); diff --git a/test/sql/quackapi_format.test b/test/sql/quackapi_format.test index 3ceda07..8d97463 100644 --- a/test/sql/quackapi_format.test +++ b/test/sql/quackapi_format.test @@ -1,11 +1,9 @@ # name: test/sql/quackapi_format.test -# description: FORMAT json|ndjson|csv|parquet + Accept via quackapi_request +# description: FORMAT json|ndjson|csv + Accept via quackapi_request # group: [sql] require quackapi -require parquet - statement ok CREATE OR REPLACE ROUTE items GET '/items' AS SELECT 1 AS id, 'alice' AS name UNION ALL SELECT 2, 'bob'; @@ -18,19 +16,14 @@ statement ok CREATE OR REPLACE ROUTE items_csv GET '/items_csv' FORMAT csv AS SELECT 1 AS id, 'alice' AS name UNION ALL SELECT 2, 'bob,jr'; -statement ok -CREATE OR REPLACE ROUTE items_parquet GET '/items_parquet' FORMAT parquet AS - SELECT 1 AS id, 'alice' AS name UNION ALL SELECT 2, 'bob'; - query II SELECT name, format FROM quackapi_routes() -WHERE name IN ('items','items_ndjson','items_csv','items_parquet') +WHERE name IN ('items','items_ndjson','items_csv') ORDER BY name; ---- items json items_csv csv items_ndjson ndjson -items_parquet parquet query II SELECT status, content_type FROM quackapi_request('GET', '/items'); @@ -75,15 +68,3 @@ FROM quackapi_request( ); ---- 200 true - -query II -SELECT status, starts_with(content_type, 'application/vnd.apache.parquet') -FROM quackapi_request('GET', '/items_parquet'); ----- -200 true - -# PAR1 magic on parquet body -query I -SELECT decode(body[1:4]) FROM quackapi_request('GET', '/items_parquet'); ----- -PAR1 From 99cf8d3fa74f1ae6412f34244c150d412490c92a Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:06:32 -0700 Subject: [PATCH 14/93] refactor(postgres): remove native libpq path --- src/include/quackapi_pg.hpp | 34 -- src/quackapi_pg.cpp | 617 ------------------------ test/sql/quackapi_pg_prepare_cache.test | 99 ---- 3 files changed, 750 deletions(-) delete mode 100644 src/include/quackapi_pg.hpp delete mode 100644 src/quackapi_pg.cpp delete mode 100644 test/sql/quackapi_pg_prepare_cache.test diff --git a/src/include/quackapi_pg.hpp b/src/include/quackapi_pg.hpp deleted file mode 100644 index 36ffd83..0000000 --- a/src/include/quackapi_pg.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include "duckdb/common/string.hpp" -#include "duckdb/common/types/value.hpp" -#include "duckdb/common/case_insensitive_map.hpp" - -#include -#include - -namespace duckdb { - -//! Thin libpq path for pure-HTTP routes that hit Postgres — same shape as -//! FastAPI+psycopg: bind params, fresh PQexecParams, JSON rows. Bypasses DuckDB -//! ATTACH/scanner for that request (optional; when pg_dsn is set on serve or -//! quackapi_request / SET quackapi_pg_dsn). -//! -//! Whether the native libpq execution path completed, is proven inapplicable, -//! or failed after PostgreSQL transport/execution began. Callers may fall back -//! to DuckDB only for NOT_APPLICABLE: replaying a failed mutating PG command -//! against another backend is unsafe. -enum class QuackapiPgNativeResult : uint8_t { - NOT_APPLICABLE, - SUCCESS, - FAILED, -}; - -//! A bounded native libpq execution. `max_response_bytes` caps JSON assembly -//! while rows arrive in single-row mode. On FAILED, err_out is a sanitized -//! stable category; "PostgreSQL deadline exceeded" maps to HTTP 504. -QuackapiPgNativeResult QuackapiTryPgNative(const string &dsn, const string &handler_sql, - const case_insensitive_map_t> &provided, - idx_t max_response_bytes, string &json_body, string &err_out); - -} // namespace duckdb diff --git a/src/quackapi_pg.cpp b/src/quackapi_pg.cpp deleted file mode 100644 index ff870f8..0000000 --- a/src/quackapi_pg.cpp +++ /dev/null @@ -1,617 +0,0 @@ -#include "quackapi_pg.hpp" - -#include "duckdb/common/string_util.hpp" -#include "quackapi_util.hpp" -#include "quackapi_limits.hpp" -#include "duckdb/parser/parser.hpp" -#include - -#ifndef QUACKAPI_HAS_LIBPQ -// Built without libpq — always fall through to DuckDB. -namespace duckdb { -QuackapiPgNativeResult QuackapiTryPgNative(const string &, const string &, - const case_insensitive_map_t> &, idx_t, string &, - string &) { - return QuackapiPgNativeResult::NOT_APPLICABLE; -} -} // namespace duckdb -#else - -#include - -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#else -#include -#endif - -namespace duckdb { - -namespace { - -// Thread-local connection: one PGconn per HTTP worker thread (psycopg-pool shape). -// -// Do NOT cache PQprepare by SQL alone: the named statement embeds catalog OIDs -// (domains, types, …). DROP + CREATE of the same name leaves a sticky dead OID -// ("cache lookup failed for type") while the SQL text is unchanged. Mirror the -// DuckDB handler path — reuse the connection, execute fresh each request. -struct TlsPg { - string dsn; - PGconn *conn = nullptr; - string err; - //! Cached, rounded-down PostgreSQL statement_timeout. Reusing it avoids a - //! second network round trip on the normal same-budget request path. - int64_t statement_timeout_ms = -1; - ~TlsPg() { - if (conn) { - PQfinish(conn); - conn = nullptr; - } - } - void Reset() { - if (conn) { - PQfinish(conn); - conn = nullptr; - } - statement_timeout_ms = -1; - } - PGconn *Get(const string &want_dsn); -}; - -bool DeadlineExpired() { - auto deadline = QuackapiCurrentDeadline(); - return deadline != std::chrono::steady_clock::time_point::max() && std::chrono::steady_clock::now() >= deadline; -} - -//! Wait on the libpq socket without ever exceeding the HTTP request's shared -//! deadline. This covers connection polling, output flush, and result input. -bool WaitForPgSocket(PGconn *conn, bool want_read, bool want_write, string &err_out) { - const int fd = PQsocket(conn); - if (fd < 0) { - err_out = "PostgreSQL transport failed"; - return false; - } - while (true) { - auto deadline = QuackapiCurrentDeadline(); - int64_t timeout_ms = 30000; -#ifdef _WIN32 - timeval timeout {}; -#endif - if (deadline == std::chrono::steady_clock::time_point::max()) { - // Keep the finite fallback for callers outside an HTTP request. - timeout_ms = 30000; -#ifdef _WIN32 - timeout.tv_sec = 30; -#endif - } else { - auto remaining = - std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()) - .count(); - if (remaining <= 0) { - err_out = "PostgreSQL deadline exceeded"; - return false; - } - timeout_ms = std::max(1, (remaining + 999) / 1000); -#ifdef _WIN32 - timeout.tv_sec = static_cast(remaining / 1000000); - timeout.tv_usec = static_cast(remaining % 1000000); -#endif - } - -#ifdef _WIN32 - fd_set read_fds, write_fds; - FD_ZERO(&read_fds); - FD_ZERO(&write_fds); - if (want_read) { - FD_SET(fd, &read_fds); - } - if (want_write) { - FD_SET(fd, &write_fds); - } - const int rc = select(0, want_read ? &read_fds : nullptr, want_write ? &write_fds : nullptr, nullptr, &timeout); -#else - // poll has no FD_SETSIZE ceiling, unlike select/FD_SET. The HTTP process - // can legitimately hold unrelated descriptors in addition to its fixed - // worker connections, so this must remain safe at high descriptor values. - pollfd poll_fd {}; - poll_fd.fd = fd; - poll_fd.events = static_cast((want_read ? POLLIN : 0) | (want_write ? POLLOUT : 0)); - const int bounded_timeout = static_cast(std::min(timeout_ms, std::numeric_limits::max())); - const int rc = poll(&poll_fd, 1, bounded_timeout); - if (rc > 0 && (poll_fd.revents & (POLLERR | POLLHUP | POLLNVAL))) { - err_out = "PostgreSQL transport failed"; - return false; - } -#endif - if (rc > 0) { - return true; - } - if (rc == 0) { - err_out = "PostgreSQL deadline exceeded"; - return false; - } -#ifndef _WIN32 - if (errno == EINTR) { - continue; - } -#endif - err_out = "PostgreSQL transport failed"; - return false; - } -} - -PGconn *TlsPg::Get(const string &want_dsn) { - if (conn && dsn == want_dsn && PQstatus(conn) == CONNECTION_OK) { - return conn; - } - Reset(); - dsn = want_dsn; - const char *keys[] = {"dbname", nullptr}; - const char *values[] = {dsn.c_str(), nullptr}; - conn = PQconnectStartParams(keys, values, 1); - if (!conn) { - err = "PostgreSQL connection failed"; - return nullptr; - } - while (true) { - auto status = PQconnectPoll(conn); - if (status == PGRES_POLLING_OK) { - if (PQsetnonblocking(conn, 1) == 0) { - return conn; - } - err = "PostgreSQL transport failed"; - Reset(); - return nullptr; - } - if (status == PGRES_POLLING_FAILED) { - err = DeadlineExpired() ? "PostgreSQL deadline exceeded" : "PostgreSQL connection failed"; - Reset(); - return nullptr; - } - const bool wants_read = status == PGRES_POLLING_READING || status == PGRES_POLLING_ACTIVE; - const bool wants_write = status == PGRES_POLLING_WRITING || status == PGRES_POLLING_ACTIVE; - if ((!wants_read && !wants_write) || !WaitForPgSocket(conn, wants_read, wants_write, err)) { - if (err.empty()) { - err = "PostgreSQL transport failed"; - } - Reset(); - return nullptr; - } - } -} - -TlsPg &Tls() { - thread_local TlsPg t; - return t; -} - -// $name or $name::TYPE → collect unique names in order of first appearance. -void CollectNamedParams(const string &sql, const vector &protected_text, vector &names) { - names.clear(); - for (idx_t i = 0; i < sql.size(); i++) { - if (protected_text[i] || sql[i] != '$') { - continue; - } - if (i + 1 >= sql.size() || !((sql[i + 1] >= 'A' && sql[i + 1] <= 'Z') || - (sql[i + 1] >= 'a' && sql[i + 1] <= 'z') || sql[i + 1] == '_')) { - continue; - } - idx_t j = i + 1; - while (j < sql.size() && ((sql[j] >= 'A' && sql[j] <= 'Z') || (sql[j] >= 'a' && sql[j] <= 'z') || - (sql[j] >= '0' && sql[j] <= '9') || sql[j] == '_')) { - j++; - } - string name = sql.substr(i + 1, j - (i + 1)); - // skip ::type - bool seen = false; - for (auto &n : names) { - if (StringUtil::Lower(n) == StringUtil::Lower(name)) { - seen = true; - break; - } - } - if (!seen) { - names.push_back(name); - } - i = j - 1; - } -} - -// DuckDB route SQL → Postgres text suitable for PQexecParams. -// - strip "pg." catalog qualifier -// - $name::TYPE / $name → $1, $2, … (order of first appearance) -// Rejects obvious DuckDB-only surface (postgres_execute, etc.). -bool ToPgSql(const string &in, string &out, vector ¶m_names, string &err) { - // The lexer owns quoting, dollar-quoted bodies and comments. Rewriting - // their contents corrupts SQL literals and can invent phantom parameters. - vector protected_text(in.size(), false); - string executable = in; - auto tokens = Parser::Tokenize(in); - for (idx_t t = 0; t < tokens.size(); t++) { - const auto &token = tokens[t]; - if (token.type == SimplifiedTokenType::SIMPLIFIED_TOKEN_STRING_CONSTANT || - token.type == SimplifiedTokenType::SIMPLIFIED_TOKEN_COMMENT || - (token.start < in.size() && in[token.start] == '"')) { - auto end = t + 1 < tokens.size() ? tokens[t + 1].start : in.size(); - for (idx_t p = token.start; p < end; p++) { - protected_text[p] = true; - executable[p] = ' '; - } - } - } - auto lower = StringUtil::Lower(executable); - if (StringUtil::Contains(lower, "postgres_execute") || StringUtil::Contains(lower, "postgres_query") || - StringUtil::Contains(lower, "postgres_scan")) { - err = "duckdb-only TVF"; - return false; - } - CollectNamedParams(in, protected_text, param_names); - // Map name → $n - case_insensitive_map_t idx; - for (idx_t i = 0; i < param_names.size(); i++) { - idx[param_names[i]] = i + 1; - } - out.clear(); - out.reserve(in.size() + 8); - for (idx_t i = 0; i < in.size(); i++) { - if (protected_text[i]) { - out.push_back(in[i]); - continue; - } - // strip pg. qualifier (case-insensitive) - if ((in[i] == 'p' || in[i] == 'P') && i + 2 < in.size() && (in[i + 1] == 'g' || in[i + 1] == 'G') && - in[i + 2] == '.') { - // word boundary before p - if (i == 0 || !((in[i - 1] >= 'A' && in[i - 1] <= 'Z') || (in[i - 1] >= 'a' && in[i - 1] <= 'z') || - (in[i - 1] >= '0' && in[i - 1] <= '9') || in[i - 1] == '_')) { - i += 2; - continue; - } - } - if (in[i] == '$' && i + 1 < in.size() && - ((in[i + 1] >= 'A' && in[i + 1] <= 'Z') || (in[i + 1] >= 'a' && in[i + 1] <= 'z') || in[i + 1] == '_')) { - idx_t j = i + 1; - while (j < in.size() && ((in[j] >= 'A' && in[j] <= 'Z') || (in[j] >= 'a' && in[j] <= 'z') || - (in[j] >= '0' && in[j] <= '9') || in[j] == '_')) { - j++; - } - string name = in.substr(i + 1, j - (i + 1)); - // Keep explicit casts: they determine parameter and response types. - auto it = idx.find(name); - if (it == idx.end()) { - err = "unmapped param $" + name; - return false; - } - out += "$"; - out += std::to_string(it->second); - i = j - 1; - continue; - } - out.push_back(in[i]); - } - return true; -} - -//! Append while enforcing the configured final JSON body ceiling. This guards -//! each intermediate append instead of discovering an oversized body only -//! after libpq rows have been fully serialized. -bool AppendJson(string &body, const char *data, idx_t length, idx_t max_bytes, string &err_out) { - if (DeadlineExpired()) { - err_out = "PostgreSQL deadline exceeded"; - return false; - } - if (length > max_bytes || body.size() > max_bytes - length) { - err_out = "PostgreSQL response exceeds configured byte limit"; - return false; - } - body.append(data, length); - return true; -} - -bool AppendJsonLiteral(string &body, const char *literal, idx_t max_bytes, string &err_out) { - return AppendJson(body, literal, std::strlen(literal), max_bytes, err_out); -} - -//! Escape directly into the bounded response rather than allocating a second -//! unbounded escaped copy of an individual PostgreSQL field. -bool AppendJsonQuoted(string &body, const char *data, idx_t length, idx_t max_bytes, string &err_out) { - if (!AppendJsonLiteral(body, "\"", max_bytes, err_out)) { - return false; - } - static const char hex[] = "0123456789abcdef"; - for (idx_t i = 0; i < length; i++) { - if ((i & 4095) == 0 && DeadlineExpired()) { - err_out = "PostgreSQL deadline exceeded"; - return false; - } - const auto ch = static_cast(data[i]); - if (ch == '"' || ch == '\\') { - const char escaped[] = {'\\', static_cast(ch)}; - if (!AppendJson(body, escaped, sizeof(escaped), max_bytes, err_out)) { - return false; - } - } else if (ch == '\b') { - if (!AppendJsonLiteral(body, "\\b", max_bytes, err_out)) - return false; - } else if (ch == '\f') { - if (!AppendJsonLiteral(body, "\\f", max_bytes, err_out)) - return false; - } else if (ch == '\n') { - if (!AppendJsonLiteral(body, "\\n", max_bytes, err_out)) - return false; - } else if (ch == '\r') { - if (!AppendJsonLiteral(body, "\\r", max_bytes, err_out)) - return false; - } else if (ch == '\t') { - if (!AppendJsonLiteral(body, "\\t", max_bytes, err_out)) - return false; - } else if (ch < 0x20) { - const char escaped[] = {'\\', 'u', '0', '0', hex[ch >> 4], hex[ch & 0x0F]}; - if (!AppendJson(body, escaped, sizeof(escaped), max_bytes, err_out)) { - return false; - } - } else { - const char plain[] = {static_cast(ch)}; - if (!AppendJson(body, plain, sizeof(plain), max_bytes, err_out)) { - return false; - } - } - } - return AppendJsonLiteral(body, "\"", max_bytes, err_out); -} - -bool EqualsAsciiIgnoreCase(const char *value, idx_t length, const char *expected) { - const auto expected_length = std::strlen(expected); - if (length != expected_length) { - return false; - } - for (idx_t i = 0; i < length; i++) { - auto lhs = static_cast(value[i]); - auto rhs = static_cast(expected[i]); - if (lhs >= 'A' && lhs <= 'Z') - lhs = static_cast(lhs + ('a' - 'A')); - if (rhs >= 'A' && rhs <= 'Z') - rhs = static_cast(rhs + ('a' - 'A')); - if (lhs != rhs) - return false; - } - return true; -} - -bool AppendPgRowsJson(PGresult *res, string &body, bool &first_row, idx_t max_bytes, string &err_out) { - const int rows = PQntuples(res); - const int cols = PQnfields(res); - for (int row = 0; row < rows; row++) { - if (!first_row && !AppendJsonLiteral(body, ",", max_bytes, err_out)) { - return false; - } - first_row = false; - if (!AppendJsonLiteral(body, "{", max_bytes, err_out)) { - return false; - } - for (int col = 0; col < cols; col++) { - if (col && !AppendJsonLiteral(body, ",", max_bytes, err_out)) { - return false; - } - const char *name = PQfname(res, col); - if (!AppendJsonQuoted(body, name ? name : "", name ? std::strlen(name) : 0, max_bytes, err_out) || - !AppendJsonLiteral(body, ":", max_bytes, err_out)) { - return false; - } - if (PQgetisnull(res, row, col)) { - if (!AppendJsonLiteral(body, "null", max_bytes, err_out)) - return false; - continue; - } - const char *value = PQgetvalue(res, row, col); - const idx_t length = static_cast(PQgetlength(res, row, col)); - const auto oid = PQftype(res, col); - const bool numeric = - oid == 20 || oid == 21 || oid == 23 || oid == 26 || oid == 700 || oid == 701 || oid == 1700; - const bool non_finite = EqualsAsciiIgnoreCase(value, length, "nan") || - EqualsAsciiIgnoreCase(value, length, "infinity") || - EqualsAsciiIgnoreCase(value, length, "-infinity"); - if (oid == 16) { - if (!AppendJsonLiteral(body, (length == 1 && (value[0] == 't' || value[0] == 'T')) ? "true" : "false", - max_bytes, err_out)) - return false; - } else if (oid == 114 || oid == 3802 || (numeric && !non_finite)) { - if (!AppendJson(body, value, length, max_bytes, err_out)) - return false; - } else if (!AppendJsonQuoted(body, value, length, max_bytes, err_out)) { - return false; - } - } - if (!AppendJsonLiteral(body, "}", max_bytes, err_out)) { - return false; - } - } - return true; -} - -bool IsPgCancellation(PGresult *result) { - const char *state = PQresultErrorField(result, PG_DIAG_SQLSTATE); - return state && std::strcmp(state, "57014") == 0; -} - -bool FlushPgOutput(PGconn *conn, string &err_out) { - while (true) { - const auto status = PQflush(conn); - if (status == 0) { - return true; - } - if (status < 0) { - err_out = "PostgreSQL transport failed"; - return false; - } - if (!WaitForPgSocket(conn, false, true, err_out)) { - return false; - } - } -} - -//! Consume a single non-row PostgreSQL command without a synchronous PQexec. -bool ReceivePgCommand(PGconn *conn, string &err_out) { - if (!FlushPgOutput(conn, err_out)) { - return false; - } - bool saw_command = false; - while (true) { - if (PQconsumeInput(conn) == 0) { - err_out = "PostgreSQL transport failed"; - return false; - } - while (!PQisBusy(conn)) { - auto *result = PQgetResult(conn); - if (!result) { - if (!saw_command) - err_out = "PostgreSQL execution failed"; - return saw_command; - } - auto status = PQresultStatus(result); - const bool ok = status == PGRES_COMMAND_OK; - const bool canceled = IsPgCancellation(result); - PQclear(result); - if (!ok) { - err_out = - canceled || DeadlineExpired() ? "PostgreSQL deadline exceeded" : "PostgreSQL execution failed"; - return false; - } - saw_command = true; - } - if (!WaitForPgSocket(conn, true, false, err_out)) { - return false; - } - } -} - -bool EnsurePgStatementTimeout(TlsPg &tls, PGconn *conn, string &err_out) { - // Round down so the server-side deadline never extends the shared request - // budget. A stable bucket avoids a SET round trip on common requests. - const auto remaining = QuackapiRemainingTimeoutMillis(30000); - const int64_t target = std::max(1, (remaining / 10) * 10); - if (tls.statement_timeout_ms == target) { - return true; - } - const string sql = "SET statement_timeout = " + std::to_string(target); - if (PQsendQuery(conn, sql.c_str()) == 0 || !ReceivePgCommand(conn, err_out)) { - if (err_out.empty()) - err_out = "PostgreSQL transport failed"; - return false; - } - tls.statement_timeout_ms = target; - return true; -} - -} // namespace - -QuackapiPgNativeResult QuackapiTryPgNative(const string &dsn, const string &handler_sql, - const case_insensitive_map_t> &provided, - idx_t max_response_bytes, string &json_body, string &err_out) { - if (dsn.empty()) { - return QuackapiPgNativeResult::NOT_APPLICABLE; - } - string pg_sql; - vector names; - if (!ToPgSql(handler_sql, pg_sql, names, err_out)) { - // Translation happens before any network operation, so DuckDB may safely - // handle its own syntax/parameter validation when this path cannot. - return QuackapiPgNativeResult::NOT_APPLICABLE; - } - vector vals; - vector storage; - storage.reserve(names.size()); - vals.reserve(names.size()); - for (auto &name : names) { - auto it = provided.find(name); - if (it == provided.end()) { - // Let DuckDB's normal binder produce the usual 422 missing-parameter - // response. PostgreSQL has not observed a command at this point. - return QuackapiPgNativeResult::NOT_APPLICABLE; - } - storage.push_back(it->second.second); - vals.push_back(storage.back().c_str()); - } - auto &tls = Tls(); - PGconn *conn = tls.Get(dsn); - if (!conn) { - err_out = tls.err.empty() ? "PostgreSQL connection failed" : tls.err; - return QuackapiPgNativeResult::FAILED; - } - if (!EnsurePgStatementTimeout(tls, conn, err_out)) { - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - - // Single-row mode means libpq yields each tuple as it arrives. We serialize - // and cap it incrementally instead of allowing PQexecParams to accumulate an - // unbounded full result before the HTTP response limit can be checked. - if (PQsendQueryParams(conn, pg_sql.c_str(), static_cast(vals.size()), nullptr, vals.data(), nullptr, nullptr, - 0) == 0 || - PQsetSingleRowMode(conn) == 0) { - err_out = "PostgreSQL transport failed"; - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - if (!FlushPgOutput(conn, err_out)) { - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - - json_body.clear(); - if (!AppendJsonLiteral(json_body, "[", max_response_bytes, err_out)) { - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - bool first_row = true; - while (true) { - if (PQconsumeInput(conn) == 0) { - err_out = "PostgreSQL transport failed"; - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - while (!PQisBusy(conn)) { - auto *result = PQgetResult(conn); - if (!result) { - if (!AppendJsonLiteral(json_body, "]", max_response_bytes, err_out)) { - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - return QuackapiPgNativeResult::SUCCESS; - } - const auto status = PQresultStatus(result); - if (status == PGRES_SINGLE_TUPLE) { - const bool appended = AppendPgRowsJson(result, json_body, first_row, max_response_bytes, err_out); - PQclear(result); - if (!appended) { - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - continue; - } - const bool ok = status == PGRES_TUPLES_OK || status == PGRES_COMMAND_OK; - const bool canceled = IsPgCancellation(result); - PQclear(result); - if (!ok) { - err_out = - canceled || DeadlineExpired() ? "PostgreSQL deadline exceeded" : "PostgreSQL execution failed"; - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - } - if (!WaitForPgSocket(conn, true, false, err_out)) { - tls.Reset(); - return QuackapiPgNativeResult::FAILED; - } - } -} - -} // namespace duckdb - -#endif // QUACKAPI_HAS_LIBPQ diff --git a/test/sql/quackapi_pg_prepare_cache.test b/test/sql/quackapi_pg_prepare_cache.test deleted file mode 100644 index 6637944..0000000 --- a/test/sql/quackapi_pg_prepare_cache.test +++ /dev/null @@ -1,99 +0,0 @@ -# name: test/sql/quackapi_pg_prepare_cache.test -# description: PG PQprepare cache must not key on SQL alone (catalog/definition identity) -# group: [sql] - -# -# Requires a live Postgres reachable at QUACKAPI_TEST_PG_DSN (libpq DSN). -# Throwaway setup used in development: -# postgresql://quackapi_pg_cache:qa_pg_cache_test@/quackapi_pg_cache_test?host=/tmp - -require quackapi - -require-env QUACKAPI_TEST_PG_DSN - -# Keep quackapi_request on one worker thread so TLS PGconn + prepare map are shared. -statement ok -SET threads = 1; - -statement ok -SET quackapi_pg_dsn = '${QUACKAPI_TEST_PG_DSN}'; - -# --------------------------------------------------------------------------- -# PQprepare was keyed only by pg_sql. Missing: catalog type OID identity. -# Prepare SELECT …::qa_dpos, DROP+CREATE the domain (new OID, same name), then -# reuse the cached statement name → sticky "cache lookup failed for type". -# Fresh exec (no SQL-only prepare cache) returns the post-redefinition row. -# --------------------------------------------------------------------------- - -statement ok -CREATE OR REPLACE ROUTE qa_pg_redef POST '/qa_pg_redef' AS -DO $$ -BEGIN - DROP DOMAIN IF EXISTS qa_dpos CASCADE; - CREATE DOMAIN qa_dpos AS integer CHECK (VALUE > 0); -END -$$; - -statement ok -CREATE OR REPLACE ROUTE qa_pg_cast GET '/qa_pg_cast/:id' AS - SELECT ($id::INTEGER)::qa_dpos AS v, $id::INTEGER AS id; - -# Boot domain on the TLS libpq connection. -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/qa_pg_redef'); ----- -200 [] - -# First cast prepares with the live domain OID. -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/qa_pg_cast/5'); ----- -200 [{"v":5,"id":5}] - -# A PostgreSQL execution failure is authoritative once the native path has -# dispatched it. It must not silently replay the route against DuckDB. -statement ok -CREATE OR REPLACE ROUTE qa_pg_no_replay GET '/qa_pg_no_replay' AS - SELECT current_setting('quackapi_missing_setting') AS v; - -query I -SELECT status FROM quackapi_request('GET', '/qa_pg_no_replay'); ----- -502 - -# Native libpq I/O and the backend statement both inherit the request deadline. -# The response must terminate as a timeout rather than falling through to the -# DuckDB path after PostgreSQL has accepted the command. -statement ok -CREATE OR REPLACE ROUTE qa_pg_deadline GET '/qa_pg_deadline' AS - SELECT pg_sleep(0.25) AS delayed; - -query I -SELECT status FROM quackapi_request('GET', '/qa_pg_deadline', query_timeout_ms := 100); ----- -504 - -# Wire text must retain its declared SQL type and SQL rewriting must leave -# literals/comments alone. Explicit parameter casts must survive translation. -statement ok -CREATE OR REPLACE ROUTE qa_pg_types GET '/qa_pg_types/:id' AS -SELECT '0012'::TEXT AS code, true AS enabled, false AS disabled, - 'pg.$phantom postgres_query'::TEXT AS literal, - $id::INTEGER AS id, NULL::TEXT AS missing; -- $comment pg.nope - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/qa_pg_types/7'); ----- -200 [{"code":"0012","enabled":true,"disabled":false,"literal":"pg.$phantom postgres_query","id":7,"missing":null}] - -# Redefine domain (new OID, same SQL text for the cast route). -query II -SELECT status, decode(body) FROM quackapi_request('POST', '/qa_pg_redef'); ----- -200 [] - -# Must return the post-redefinition result — not a sticky dead type OID failure. -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/qa_pg_cast/5'); ----- -200 [{"v":5,"id":5}] From e8236c23bc1a15c837a1ad836cd126462922aec1 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:07:12 -0700 Subject: [PATCH 15/93] refactor(stream): remove SSE stream surface --- docs/guide/stream.md | 98 -------- src/include/quackapi_stream.hpp | 28 --- src/quackapi_stream.cpp | 425 -------------------------------- test/sql/quackapi_stream.test | 56 ----- 4 files changed, 607 deletions(-) delete mode 100644 docs/guide/stream.md delete mode 100644 src/include/quackapi_stream.hpp delete mode 100644 src/quackapi_stream.cpp delete mode 100644 test/sql/quackapi_stream.test diff --git a/docs/guide/stream.md b/docs/guide/stream.md deleted file mode 100644 index 919bc42..0000000 --- a/docs/guide/stream.md +++ /dev/null @@ -1,98 +0,0 @@ -# CREATE STREAM — Server-Sent Events (SSE) - -Push a live event stream over HTTP with `text/event-stream`. Each result row becomes one SSE event (`id:` + `data:` JSON). - -All examples run against `build/release/duckdb -unsigned` with `LOAD quackapi;`. - ---- - -## Basic stream - -```sql -CREATE STREAM ticks GET '/ticks' AS -SELECT i AS id, 'tick' AS msg FROM range(3) t(i); -``` - -```sh -curl -N http://127.0.0.1:8000/ticks -``` - -Live output: - -```text -Content-Type: text/event-stream - -id: 0 -data: {"id":0,"msg":"tick"} - -id: 1 -data: {"id":1,"msg":"tick"} - -id: 2 -data: {"id":2,"msg":"tick"} -``` - -Multiple rows (including `UNION ALL`) each become an event: - -```sql -CREATE STREAM once GET '/once' AS -SELECT 1 AS id, 'a' AS msg -UNION ALL -SELECT 2 AS id, 'b' AS msg; -``` - ---- - -## Polling interval - -For long-lived “keep selecting” streams, set an interval: - -```sql -CREATE STREAM live GET '/live' WITH (interval='1s') AS -SELECT now() AS ts; - --- bare number = seconds -CREATE OR REPLACE STREAM live GET '/live' WITH (interval=2) AS -SELECT 1 AS n; - --- milliseconds -CREATE OR REPLACE STREAM live GET '/live' WITH (interval='250ms') AS -SELECT 1 AS n; -``` - -```sql -SELECT name, method, pattern, transport, interval_ms, handler -FROM quackapi_streams(); --- transport is always 'sse' today -``` - ---- - -## WebSocket — not supported (use SSE) - -```sql -CREATE STREAM chat WS '/ws' AS SELECT 1; --- error: WebSocket is not supported (bundled HTTP library has no Upgrade API) -``` - -Use **`CREATE STREAM … GET`** for browser EventSource / SSE clients. See also [coming soon](coming-soon.md). - ---- - -## Notes - -- Streams are **not** listed in `quackapi_routes()` — use `quackapi_streams()`. -- Path must start with `/`. -- Streams do not take `REQUIRE` auth in the current version. -- Ordinary routes win if both match the same path. - -```sql -DROP STREAM ticks; -``` - ---- - -## Next - -- [Policies](policies.md) -- [Static files](static-files.md) diff --git a/src/include/quackapi_stream.hpp b/src/include/quackapi_stream.hpp deleted file mode 100644 index 276876a..0000000 --- a/src/include/quackapi_stream.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/extension/extension_loader.hpp" -#include "duckdb/parser/parser_extension.hpp" - -namespace duckdb { - -//! CREATE [OR REPLACE] STREAM / DROP STREAM syntax. -//! Grammar: -//! CREATE [OR REPLACE] STREAM GET '' -//! [WITH (interval='1s'|1000)] -//! AS -//! DROP STREAM -ParserExtensionParseResult StreamDdlParse(ParserExtensionInfo *, const string &query) { - auto q = QuackapiTrim(query); - auto upper = StringUtil::Upper(q); - - bool or_replace = false; - idx_t pos; - if (StringUtil::StartsWith(upper, "CREATE STREAM ")) { - pos = 14; - } else if (StringUtil::StartsWith(upper, "CREATE OR REPLACE STREAM ")) { - pos = 25; - or_replace = true; - } else if (StringUtil::StartsWith(upper, "DROP STREAM ")) { - auto name = QuackapiTrim(q.substr(12)); - if (name.empty() || name.find(' ') != string::npos) { - return ParserExtensionParseResult("DROP STREAM expects a single stream name"); - } - auto data = make_uniq(); - data->action = "DROP"; - data->stream.name = name; - return ParserExtensionParseResult(std::move(data)); - } else { - return ParserExtensionParseResult(); - } - - auto rest = QuackapiTrim(q.substr(pos)); - // - auto first_space = rest.find(' '); - if (first_space == string::npos) { - return ParserExtensionParseResult("CREATE STREAM GET '' AS - if (!(StringUtil::StartsWith(rest_upper, "AS") && rest.size() > 2 && StringUtil::CharacterIsSpace(rest[2]))) { - return ParserExtensionParseResult("Expected AS ; named params bind like routes - //! 0 = run SELECT once and close the stream. >0 = re-run after each empty/full cycle. - int64_t interval_ms = 0; - QuackapiStreamTransport transport = QuackapiStreamTransport::SSE; -}; - //! Per-database quackapi state: the route registry, auth registry, and running servers. //! Lives in the DatabaseInstance's ObjectCache (non-evictable), so LOAD never //! touches the user's catalog and state dies with the database. @@ -210,14 +172,6 @@ class QuackapiState : public ObjectCacheEntry { //! Shared immutable view for the HTTP hot path (refcount only — no deep copy). shared_ptr> LiveRoutes(); - //! CREATE [OR REPLACE] STREAM. Throws on duplicate name unless or_replace. - void AddStream(const QuackapiStream &stream, bool or_replace); - //! DROP STREAM. Returns false if no such stream. - bool DropStream(const string &name); - vector SnapshotStreams(); - //! Shared immutable view for the HTTP hot path (refcount only — no deep copy). - shared_ptr> LiveStreams(); - //! CREATE [OR REPLACE] GROUP. Throws on duplicate name unless or_replace. void AddGroup(const QuackapiGroup &group, bool or_replace); //! DROP GROUP. Does NOT cascade-drop member routes. Returns false if missing. @@ -250,8 +204,8 @@ class QuackapiState : public ObjectCacheEntry { bool HasServerOnPort(int port); //! Host for the server on port, if any. Returns false if none. bool GetServerHost(int port, string &host_out); - //! (host, port, http_client_active, http_client_reason) for each running server. - vector> ListServers(); + //! (host, port) for each running server. + vector> ListServers(); //! Snapshot one server's request ring; port 0 means the only running server. //! A port is required when more than one server is running. vector SnapshotRequests(int port = 0); @@ -262,31 +216,6 @@ class QuackapiState : public ObjectCacheEntry { //! A port is required when more than one server is running. bool FlushTelemetry(int port = 0); - // --- Thin GraphQL v0 table allowlist (built-in POST /graphql) --- - // Empty allowlist = open mode (all main-schema tables, v0 default). - // Non-empty = only registered tables appear in /graphql and /graphql/schema. - //! Register a main-schema table/view for GraphQL. Throws on duplicate unless or_replace. - void AddGraphqlTable(const string &table_name, bool or_replace); - //! Remove one table from the allowlist. Returns false if not registered. - bool DropGraphqlTable(const string &table_name); - //! Clear allowlist → open catalog mode. - void ClearGraphqlTables(); - //! True when at least one table is registered (allowlist mode). - bool GraphqlAllowlistActive(); - //! True when open mode OR table is on the allowlist. - bool IsGraphqlTableAllowed(const string &table_name); - //! Sorted snapshot of registered table names (empty in open mode). - vector SnapshotGraphqlTables(); - - // --- Named GraphQL mounts (CREATE GRAPHQL ROUTE) --- - void AddGraphqlRoute(const QuackapiGraphqlRoute &route, bool or_replace); - bool DropGraphqlRoute(const string &name); - //! Exact path match (normalized). method upper-case. - bool GetGraphqlRouteByPath(const string &path, const string &method, QuackapiGraphqlRoute &out); - //! True when path is a registered route's schema URL (GET path+"/schema"). - bool GetGraphqlRouteBySchemaPath(const string &path, QuackapiGraphqlRoute &out); - vector SnapshotGraphqlRoutes(); - //! Effective write deadline (seconds) applied to the most recent route //! request. Seeded with serve write_timeout_sec; ApplyRouteIoTimeout //! overwrites it only when it actually extends the socket deadlines. @@ -300,7 +229,6 @@ class QuackapiState : public ObjectCacheEntry { private: void PublishRoutes(); - void PublishStreams(); std::atomic last_effective_write_timeout_sec {0}; struct RateLimitEntry { @@ -315,10 +243,6 @@ class QuackapiState : public ObjectCacheEntry { //! Published under routes_mutex; readers take a shared_ptr copy and drop the lock. shared_ptr> routes_live; - std::mutex streams_mutex; - vector streams; - shared_ptr> streams_live; - std::mutex groups_mutex; vector groups; @@ -326,12 +250,6 @@ class QuackapiState : public ObjectCacheEntry { vector auths; vector api_keys; - //! GraphQL allowlist (case-sensitive catalog names, like duckdb_tables). - std::mutex graphql_mutex; - vector graphql_tables; - //! Named GraphQL path mounts (independent of graphql_tables). - vector graphql_routes; - std::mutex servers_mutex; unordered_map> servers; }; diff --git a/src/quackapi_batteries.cpp b/src/quackapi_batteries.cpp index 164ae0e..d25f223 100644 --- a/src/quackapi_batteries.cpp +++ b/src/quackapi_batteries.cpp @@ -1,13 +1,7 @@ #include "quackapi_server.hpp" -#include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" -#include "duckdb/common/types/uuid.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/main/config.hpp" -#include "duckdb/main/connection.hpp" #include "duckdb/main/database.hpp" -#include "duckdb/main/query_result.hpp" #include "quackapi_state.hpp" @@ -32,381 +26,6 @@ QuackapiLogLevel ParseQuackapiLogLevel(const string &raw) { return QuackapiLogLevel::INFO; } -static const char *LogLevelDuckDBName(QuackapiLogLevel level) { - switch (level) { - case QuackapiLogLevel::SILENT: - case QuackapiLogLevel::ERROR_LEVEL: - return "ERROR"; - case QuackapiLogLevel::WARN: - return "WARNING"; - case QuackapiLogLevel::DEBUG_LEVEL: - return "DEBUG"; - case QuackapiLogLevel::INFO: - default: - return "INFO"; - } -} - -//! Safe serve default when nothing was configured (valsafe HARDENING P1-2). -static constexpr const char *SERVE_DEFAULT_MEMORY_LIMIT = "256MB"; - -static bool IsAtSystemDefaultMemoryLimit(DatabaseInstance &db) { - auto &config = DBConfig::GetConfig(db); - if (!config.file_system) { - return false; - } - auto available = DBConfig::GetSystemAvailableMemory(*config.file_system); - idx_t system_default; - if (available == DBConfigOptions().maximum_memory) { - system_default = available; - } else { - // Match DBConfig::SetDefaultMaxMemory(). - system_default = available * 8 / 10; - } - return config.options.maximum_memory == system_default; -} - -static bool RunSet(Connection &con, const string &sql, string &err_out) { - auto res = con.Query(sql); - if (res->HasError()) { - err_out = res->GetError(); - return false; - } - return true; -} - -string ApplyQuackapiServerDefaults(ClientContext &context, QuackapiServeOptions &opts) { - // Correct-by-default SETs/PRAGMAs for a long-lived HTTP server process. - // Each SET is documented (WHY) and overridable via serve() named params. - // NEVER disables safety (no allow_unsigned_extensions, no disabled checks). - vector applied; - // Build stamp so operators/agents can confirm the loaded extension matches the tree. - applied.push_back(StringUtil::Format("quackapi_request_path=perf (enable_logging=%s access_log=%s) " - "(WHY: per-request Connection reset + uuidv7 ids)", - opts.enable_logging ? "true" : "false", opts.access_log ? "true" : "false")); - Connection con(*context.db); - string err; - - // --- memory_limit --- - // WHY: unbounded RAM is the #1 footgun for multi-tenant HTTP handlers; a - // conservative ceiling prevents a single query from OOMing the host. - // Non-clobber: never overwrite an operator-set DuckDB memory_limit. - { - string limit_to_apply; - if (!opts.memory_limit.empty()) { - limit_to_apply = opts.memory_limit; - } else if (IsAtSystemDefaultMemoryLimit(*context.db)) { - limit_to_apply = SERVE_DEFAULT_MEMORY_LIMIT; - } - if (!limit_to_apply.empty()) { - try { - DBConfig::ParseMemoryLimit(limit_to_apply); - } catch (std::exception &ex) { - throw InvalidInputException("quackapi_serve: invalid memory_limit '%s': %s", limit_to_apply, ex.what()); - } - auto escaped = StringUtil::Replace(limit_to_apply, "'", "''"); - if (RunSet(con, StringUtil::Format("SET memory_limit TO '%s'", escaped), err)) { - applied.push_back( - StringUtil::Format("memory_limit=%s (WHY: RAM guardrail for multi-query HTTP workers; " - "prevents one handler from OOMing the process)", - limit_to_apply)); - } else { - fprintf(stderr, "quackapi: could not SET memory_limit: %s\n", err.c_str()); - } - } else { - applied.push_back("memory_limit= (WHY: non-clobber; left alone)"); - } - } - - // --- preserve_insertion_order --- - // WHY: insertion-order preservation serializes some pipelines; servers almost - // never need row order unless ORDER BY is present — false raises throughput. - { - const char *pio = opts.preserve_insertion_order ? "true" : "false"; - if (RunSet(con, StringUtil::Format("SET preserve_insertion_order = %s", pio), err)) { - applied.push_back(StringUtil::Format("preserve_insertion_order=%s (WHY: %s)", pio, - opts.preserve_insertion_order - ? "operator requested stable scan order" - : "server throughput — allow reordering when no ORDER BY")); - } else { - fprintf(stderr, "quackapi: could not SET preserve_insertion_order: %s\n", err.c_str()); - } - } - - // --- postgres attach OLTP: disable ctid parallel page scan --- - // WHY: with pg_use_ctid_scan=true (DuckDB default), point lookups like - // SELECT … FROM pg.t WHERE id = $id - // profile as POSTGRES_SCAN cumulative_rows_scanned ≈ whole table (~100k+), - // not an index probe. false forces COPY (SELECT … WHERE id=…) so Postgres - // uses the PK. Best-effort: no-op if postgres extension not loaded yet. - if (RunSet(con, "SET pg_use_ctid_scan = false", err)) { - applied.push_back("pg_use_ctid_scan=false (WHY: ATTACH point lookups use PG index via " - "WHERE pushdown, not full ctid range scan)"); - } - - // --- enable_http_metadata_cache --- - // WHY: outbound HTTP (httpfs / curl_httpfs companions) reuses ETag / - // Last-Modified; cuts origin load for repeated remote reads from handlers. - // (enable_object_cache is a DuckDB no-op placeholder — intentionally skipped.) - { - const char *v = opts.enable_http_metadata_cache ? "true" : "false"; - if (RunSet(con, StringUtil::Format("SET enable_http_metadata_cache = %s", v), err)) { - applied.push_back( - StringUtil::Format("enable_http_metadata_cache=%s (WHY: cache HTTP ETag/Last-Modified for " - "outbound companion fetches)", - v)); - } else { - fprintf(stderr, "quackapi: could not SET enable_http_metadata_cache: %s\n", err.c_str()); - } - } - - // --- threads --- - // WHY: default leaves DuckDB at all-cores (correct for a dedicated server). - // Override only when the operator passes threads:= so multi-tenant hosts can - // cap CPU without editing duckdb config files. - if (!opts.threads.empty()) { - auto escaped = StringUtil::Replace(opts.threads, "'", "''"); - // Accept bare int or quoted. - string sql; - bool all_digit = !opts.threads.empty(); - for (char c : opts.threads) { - if (c < '0' || c > '9') { - all_digit = false; - break; - } - } - if (all_digit) { - sql = StringUtil::Format("SET threads = %s", opts.threads); - } else { - sql = StringUtil::Format("SET threads TO '%s'", escaped); - } - if (RunSet(con, sql, err)) { - applied.push_back(StringUtil::Format("threads=%s (WHY: operator-capped worker pool for multi-tenant hosts)", - opts.threads)); - } else { - fprintf(stderr, "quackapi: could not SET threads: %s\n", err.c_str()); - } - } else { - applied.push_back("threads= (WHY: max parallel query work for server)"); - } - - // --- DuckDB built-in logging (OFF by default unless already enabled) --- - // WHY: QueryLog-per-handler-SQL to stdout is a multi-ms tax under load and - // serializes workers. HTTP ops use access_log (structured stderr). Opt in - // with enable_logging:=true when debugging query plans / errors; preserve an - // operator-enabled logger unless enable_logging:=false was explicit. - const auto log_config = context.db->GetLogManager().GetConfig(); - if (opts.enable_logging && opts.log_level != QuackapiLogLevel::SILENT) { - const char *level = LogLevelDuckDBName(opts.log_level); - // Prefer CALL enable_logging (current DuckDB API) — sets storage + level. - auto call = con.Query(StringUtil::Format("CALL enable_logging(level:='%s', storage:='stdout')", level)); - if (call->HasError()) { - // Fall back to SET surface if CALL signature differs. - RunSet(con, "SET enable_logging = true", err); - RunSet(con, StringUtil::Format("SET logging_level = '%s'", level), err); - RunSet(con, "SET logging_storage = 'stdout'", err); - applied.push_back(StringUtil::Format("enable_logging=true logging_level=%s logging_storage=stdout " - "(WHY: built-in query/error capture for server ops; CALL failed: %s)", - level, call->GetError())); - } else { - applied.push_back(StringUtil::Format("CALL enable_logging(level:='%s', storage:='stdout') " - "(WHY: DuckDB built-in logger ON — queries + errors to stdout)", - level)); - } - // Deprecated but still present: keep HTTP client logging on (already true - // by default on v1.5.4 — re-assert for older forks). - if (RunSet(con, "SET enable_http_logging = true", err)) { - applied.push_back("enable_http_logging=true (WHY: outbound HTTP client request log; " - "deprecated DuckDB setting, still effective)"); - } - } else if (log_config.enabled && !opts.enable_logging_explicit) { - applied.push_back("enable_logging= (WHY: non-clobber; left alone)"); - if (log_config.enabled_log_types.find("QueryLog") != log_config.enabled_log_types.end()) { - fprintf(stderr, - "quackapi: WARN DuckDB QueryLog is enabled; per-request logging can reduce HTTP throughput\n"); - } - } else { - // Force OFF when logging was not already enabled, or when the operator - // explicitly requested false; QueryLog serializes every handler SQL. - if (RunSet(con, "SET enable_logging = false", err)) { - applied.push_back("enable_logging=false (WHY: default — QueryLog-per-request kills HTTP RPS; " - "opt in with enable_logging:=true; use access_log for ops)"); - } else { - applied.push_back(StringUtil::Format("enable_logging=", err)); - } - RunSet(con, "SET enable_http_logging = false", err); - } - - // --- Compose quack transport log if the option exists --- - // WHY: when duckdb-quack is loaded, share one log surface with REST. - { - auto &config = DBConfig::GetConfig(*context.db); - Value existing; - if (config.TryGetCurrentSetting("quack_log_level", existing) || - config.TryGetCurrentSetting("quack_logging", existing)) { - // Best-effort: set to a verbose-enough value when present. - if (config.TryGetCurrentSetting("quack_log_level", existing)) { - RunSet(con, "SET quack_log_level = 'info'", err); - applied.push_back("quack_log_level=info (WHY: wire quack transport log when present)"); - } - if (config.TryGetCurrentSetting("quack_logging", existing)) { - RunSet(con, "SET quack_logging = true", err); - applied.push_back("quack_logging=true (WHY: wire quack transport log when present)"); - } - } else { - applied.push_back("quack transport log= (WHY: quack not loaded; no settings to wire)"); - } - } - - // --- Outbound HTTP client: prefer curl_httpfs (pooled libcurl + HTTP/2 + async) --- - // WHY: routes that read remote https (read_json/read_parquet/read_csv/read_text) - // or proxy upstream APIs collapse under concurrency on DuckDB's default - // per-request httplib client. curl_httpfs is a drop-in httpfs client layer. - // CLIENT only — does not replace the inbound httplib SERVER. - // Platform matrix (community description.yml excluded_platforms): unavailable on - // wasm_* and windows_*; available on linux_* and osx_*. - // - // Preference semantics: - // auto — prefer curl_httpfs; fall back to httplib with LOUD reason on - // healthz / quackapi_servers() / stderr (never silent) - // curl — REQUIRE curl_httpfs; fail serve if INSTALL/LOAD fails - // httplib — operator forced stock client (no curl_httpfs install) - { - string pref = StringUtil::Lower(opts.http_client); - StringUtil::Trim(pref); - if (pref.empty()) { - pref = "auto"; - } - if (pref != "auto" && pref != "curl" && pref != "httplib") { - throw InvalidInputException("quackapi_serve: http_client must be 'auto', 'curl', or 'httplib' (got '%s')", - opts.http_client); - } - - if (pref == "httplib") { - // Operator forced stock client — do not INSTALL/LOAD curl_httpfs. - // If curl_httpfs was already LOADed earlier in the process, flip the - // backend back to httplib so this serve matches the knob. - string set_err; - if (RunSet(con, "SET httpfs_client_implementation = 'httplib'", set_err)) { - // ok - } else { - // Setting may not exist until httpfs/curl_httpfs is loaded — fine. - (void)set_err; - } - opts.http_client_active = "httplib"; - opts.http_client_reason = "operator_forced"; - fprintf(stderr, "quackapi.http_client=httplib reason=operator_forced\n"); - applied.push_back("http_client=httplib (WHY: operator forced stock httplib client; " - "no curl_httpfs install)"); - } else { - // auto | curl — probe curl_httpfs. auto falls back; curl fails hard. - string fail_detail; - bool loaded = false; - - // Best-effort: ensure core httpfs first (curl_httpfs is 100% compatible - // with it and usually loads it, but explicit order matches the README). - auto httpfs_load = con.Query("LOAD httpfs"); - if (httpfs_load->HasError()) { - auto httpfs_inst = con.Query("INSTALL httpfs"); - if (!httpfs_inst->HasError()) { - httpfs_load = con.Query("LOAD httpfs"); - } - } - // httpfs failure is non-fatal here — curl_httpfs may still provide the layer. - - auto curl_load = con.Query("LOAD curl_httpfs"); - if (curl_load->HasError()) { - auto curl_inst = con.Query("INSTALL curl_httpfs FROM community"); - if (curl_inst->HasError()) { - fail_detail = curl_inst->GetError(); - } else { - curl_load = con.Query("LOAD curl_httpfs"); - if (curl_load->HasError()) { - fail_detail = curl_load->GetError(); - } else { - loaded = true; - } - } - } else { - loaded = true; - } - - if (loaded) { - // Toggle the shared httpfs client backend. 'curl' selects the - // curl-based implementation (MultiCurl / HTTPFS-Curl depending on - // curl_httpfs version); leave multi_curl default if SET fails. - string set_err; - if (!RunSet(con, "SET httpfs_client_implementation = 'curl'", set_err)) { - // Some builds expose only curl_httpfs_client_implementation. - if (!RunSet(con, "SET curl_httpfs_client_implementation = 'curl'", set_err) && - !RunSet(con, "SET curl_httpfs_client_implementation = 'multi_curl'", set_err)) { - // Still loaded — default after LOAD is already MultiCurl. - (void)set_err; - } - } - opts.http_client_active = "curl"; - opts.http_client_reason.clear(); - fprintf(stderr, "quackapi.http_client=curl\n"); - applied.push_back("http_client=curl (WHY: curl_httpfs — libcurl pool + HTTP/2 + async IO for " - "outbound https reads from handlers; 100% httpfs-compatible)"); - } else if (pref == "curl") { - // Production guarantee: no silent httplib when operator required curl. - const string detail = - fail_detail.empty() ? string("unknown") : StringUtil::Replace(fail_detail, "\n", " "); - fprintf(stderr, "quackapi.http_client=curl FAILED reason=curl_httpfs_unavailable detail=%s\n", - detail.c_str()); - throw InvalidInputException( - "quackapi_serve: http_client='curl' requires curl_httpfs but INSTALL/LOAD failed " - "(platform excluded, offline catalog, or load error). " - "Use http_client:='auto' for graceful fallback, or install curl_httpfs. detail=%s", - detail); - } else { - // auto — graceful but LOUD fallback (healthz + servers + stderr carry reason). - opts.http_client_active = "httplib"; - opts.http_client_reason = "curl_httpfs_unavailable"; - const string detail = - fail_detail.empty() ? string("unknown") : StringUtil::Replace(fail_detail, "\n", " "); - fprintf(stderr, - "quackapi.http_client=httplib reason=curl_httpfs_unavailable " - "WARN=auto_fallback production_should_force_curl_or_ensure_curl_httpfs " - "detail=%s\n", - detail.c_str()); - applied.push_back(StringUtil::Format( - "http_client=httplib reason=curl_httpfs_unavailable (WHY: auto fallback — " - "curl_httpfs not installable/loadable; /healthz + quackapi_servers report reason; " - "production: http_client:='curl' or ensure curl_httpfs; detail=%s)", - detail)); - } - } - } - - // Transport knobs are applied in QuackapiHttpServer ctor (httplib SERVER). - applied.push_back(StringUtil::Format("http keep_alive_max_count=%d keep_alive_timeout_sec=%d " - "(WHY: connection reuse cuts TCP/TLS handshake cost)", - opts.keep_alive_max_count, opts.keep_alive_timeout_sec)); - applied.push_back(StringUtil::Format("http read_timeout_sec=%d write_timeout_sec=%d " - "(WHY: bound stalled clients so workers are not pinned forever)", - opts.read_timeout_sec, opts.write_timeout_sec)); - applied.push_back(StringUtil::Format("http worker_threads=%d (WHY: concurrent request handlers; cap prevents " - "unbounded thread spawn under load)", - opts.worker_threads)); - applied.push_back(StringUtil::Format("http payload_max_length=%llu (WHY: body size DoS guard — 413 above cap)", - (unsigned long long)QUACKAPI_PAYLOAD_MAX_LENGTH)); - const string access_log_destination = - opts.access_log_table.empty() ? (opts.access_log ? "stderr" : "off") : opts.access_log_table; - applied.push_back(StringUtil::Format("access_log=%s log_level=%s (WHY: every request → structured access log " - "for correlation with X-Request-ID)", - access_log_destination, LogLevelDuckDBName(opts.log_level))); - applied.push_back(StringUtil::Format("health_routes=%s (WHY: /health liveness + /healthz readiness out of the box)", - opts.health_routes ? "true" : "false")); - - string summary = StringUtil::Join(applied, "\n"); - if (opts.log_level >= QuackapiLogLevel::INFO) { - fprintf(stderr, "quackapi: server defaults applied:\n%s\n", summary.c_str()); - } - return summary; -} - void ProbeQuackapiRequestIdSource(DatabaseInstance &db, QuackapiServeOptions &opts) { // Request IDs are always core uuidv7 generated in C++ (see NextRequestId). // Never INSTALL/LOAD/SELECT tsid at serve or per request — that path added diff --git a/src/quackapi_ddl.cpp b/src/quackapi_ddl.cpp index ee89bcc..e8b07dd 100644 --- a/src/quackapi_ddl.cpp +++ b/src/quackapi_ddl.cpp @@ -316,7 +316,7 @@ bool ParseRouteTimeoutSec(const string &raw, int32_t &out_sec, string &err) { //! Grammar: //! CREATE [OR REPLACE] ROUTE '' -//! [STATUS ] [REQUIRE ] [FORMAT json|ndjson|csv|parquet|arrow] +//! [STATUS ] [REQUIRE ] [FORMAT json|ndjson|csv] //! [ENVELOPE array|object] [EMPTY STATUS [BODY '']] //! [TIMEOUT |'30s'|'5m'] [SCHEDULE '<6-field cron, seconds first>'] //! [GROUP | IN GROUP ] @@ -552,21 +552,17 @@ ParserExtensionParseResult RouteDdlParse(ParserExtensionInfo *, const string &qu rest = QuackapiTrim(rest.substr(token_end)); continue; } - // [FORMAT json|ndjson|csv|parquet|arrow] + // [FORMAT json|ndjson|csv] if (StringUtil::StartsWith(rest_upper, "FORMAT") && (rest.size() == 6 || StringUtil::CharacterIsSpace(rest[6]))) { rest = QuackapiTrim(rest.substr(6)); auto token_end = NextTokenEnd(rest); if (token_end == 0) { - return ParserExtensionParseResult("FORMAT expects json, ndjson, csv, parquet, or arrow"); + return ParserExtensionParseResult("FORMAT expects json, ndjson, or csv"); } auto fmt = StringUtil::Lower(rest.substr(0, token_end)); - // "arrows" is a friendly alias of the nanoarrow COPY format name. - if (fmt == "arrows") { - fmt = "arrow"; - } - if (fmt != "json" && fmt != "ndjson" && fmt != "csv" && fmt != "parquet" && fmt != "arrow") { - return ParserExtensionParseResult("FORMAT must be json, ndjson, csv, parquet, or arrow"); + if (fmt != "json" && fmt != "ndjson" && fmt != "csv") { + return ParserExtensionParseResult("FORMAT must be json, ndjson, or csv"); } response_format = fmt; rest = QuackapiTrim(rest.substr(token_end)); @@ -1209,7 +1205,6 @@ void ApplyRouteExec(ClientContext &context, TableFunctionInput &data_p, DataChun if (bind_data.route.tags.empty() && !group.tags.empty()) { bind_data.route.tags = group.tags; } - // policy seam: group.policy reserved for future shared policy; unused in v1. } else if (bind_data.route.pattern.empty() || bind_data.route.pattern[0] != '/') { throw InvalidInputException("Route pattern must start with '/'"); } @@ -1258,19 +1253,8 @@ void ApplyRouteExec(ClientContext &context, TableFunctionInput &data_p, DataChun } auto prepared = con.Prepare(bind_data.route.handler_sql); if (prepared->HasError()) { - // Native libpq path (pg_dsn): handler SQL is Postgres dialect and may - // not prepare under DuckDB (domains, DO, INSERT…RETURNING, …). Allow - // CREATE when pg_dsn is configured; request time uses libpq, else 500. - auto err = prepared->GetError(); - auto el = StringUtil::Lower(err); - bool pg_returning = StringUtil::Contains(el, "returning clause not yet supported") && - StringUtil::Contains(el, "postgres"); - Value pg_dsn_setting; - bool has_pg_dsn = context.TryGetCurrentSetting("quackapi_pg_dsn", pg_dsn_setting) && - !pg_dsn_setting.IsNull() && !pg_dsn_setting.GetValue().empty(); - if (!pg_returning && !has_pg_dsn) { - throw InvalidInputException("Invalid handler SQL for route \"%s\": %s", bind_data.route.name, err); - } + throw InvalidInputException("Invalid handler SQL for route \"%s\": %s", bind_data.route.name, + prepared->GetError()); } } state.AddRoute(bind_data.route, bind_data.or_replace); @@ -1377,7 +1361,7 @@ bool ParseQuotedString(const string &s, idx_t start, string &out, idx_t &end) { //! Grammar: //! CREATE [OR REPLACE] [API] GROUP WITH ( -//! prefix='/abs', [auth=|require=], [tags='csv'], [policy=] +//! prefix='/abs', [auth=|require=], [tags='csv'] //! ) //! DROP [API] GROUP //! @@ -1537,12 +1521,8 @@ ParserExtensionParseResult GroupDdlParse(ParserExtensionInfo *, const string &qu group.require_auth = value; } else if (key == "tags") { group.tags = value; - } else if (key == "policy") { - // Seam for future shared policy; stored, unused at request time in v1. - group.policy = value; } else { - return ParserExtensionParseResult("Unknown GROUP option \"" + key + - "\" — expected prefix, auth, tags, policy"); + return ParserExtensionParseResult("Unknown GROUP option \"" + key + "\" — expected prefix, auth, tags"); } } if (!have_prefix) { @@ -1617,7 +1597,6 @@ unique_ptr ApplyGroupBind(ClientContext &, TableFunctionBindInput bind_data->group.prefix = input.inputs[3].GetValue(); bind_data->group.require_auth = input.inputs[4].GetValue(); bind_data->group.tags = input.inputs[5].GetValue(); - bind_data->group.policy = input.inputs[6].GetValue(); BindStatusColumn(return_types, names); return std::move(bind_data); } @@ -1648,7 +1627,7 @@ void ApplyGroupExec(ClientContext &context, TableFunctionInput &data_p, DataChun TableFunction MakeApplyGroupFunction() { return MakeApplyDdlFunction("quackapi_apply_group", {LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, + LogicalType::VARCHAR, LogicalType::VARCHAR}, ApplyGroupExec, ApplyGroupBind); } @@ -1663,7 +1642,6 @@ ParserExtensionPlanResult GroupDdlPlan(ParserExtensionInfo *, ClientContext &, result.parameters.push_back(Value(data.group.prefix)); result.parameters.push_back(Value(data.group.require_auth)); result.parameters.push_back(Value(data.group.tags)); - result.parameters.push_back(Value(data.group.policy)); FinishDdlPlan(result); return result; } diff --git a/src/quackapi_extension.cpp b/src/quackapi_extension.cpp index 4288e2b..fedb1bb 100644 --- a/src/quackapi_extension.cpp +++ b/src/quackapi_extension.cpp @@ -21,13 +21,9 @@ #include "quackapi_auth.hpp" #include "quackapi_ddl.hpp" -#include "quackapi_from_x.hpp" -#include "quackapi_graphql.hpp" #include "quackapi_http_fetch.hpp" #include "quackapi_server.hpp" #include "quackapi_state.hpp" -#include "quackapi_stream.hpp" -#include "quackapi_table_api.hpp" namespace duckdb { @@ -98,16 +94,9 @@ struct ServeBindData : public TableFunctionData { string static_dir; //! Empty (default) = CORS off. Pass '*' or a comma-separated origin list. string cors_origins; - //! Empty = not provided by operator. Named param wins over SET quackapi_memory_limit. - string memory_limit; - //! Batteries defaults — all correct-by-default for servers; overridable. string log_level = "info"; string access_log = "true"; - bool enable_logging = false; bool health_routes = true; - string threads; - bool preserve_insertion_order = false; - bool enable_http_metadata_cache = true; int32_t worker_threads = static_cast(QUACKAPI_DEFAULT_WORKER_THREADS); int32_t keep_alive_max_count = static_cast(QUACKAPI_DEFAULT_KEEP_ALIVE_MAX); int32_t keep_alive_timeout_sec = static_cast(QUACKAPI_DEFAULT_KEEP_ALIVE_TIMEOUT_SEC); @@ -117,10 +106,6 @@ struct ServeBindData : public TableFunctionData { string compression = "auto"; //! Min body size in bytes before compression. Default 1024. idx_t compression_min_bytes = 1024; - //! Outbound HTTP client preference: auto|curl|httplib (default auto). - string http_client = "auto"; - //! Optional libpq DSN for native Postgres execute (bypass ATTACH). - string pg_dsn; //! When true, hold the query open after listen_url until stop / SIGINT / SIGTERM. //! Default false — serve still returns immediately (backward compatible). bool block = false; @@ -236,17 +221,6 @@ static unique_ptr ServeBind(ClientContext &context, TableFunctionB bind_data->cors_origins = setting.GetValue(); } } - // memory_limit named param wins; else fall back to SET quackapi_memory_limit. - // Empty means "not provided" — ApplyQuackapiServerDefaults uses non-clobber logic. - auto mem_entry = input.named_parameters.find("memory_limit"); - if (mem_entry != input.named_parameters.end()) { - bind_data->memory_limit = mem_entry->second.GetValue(); - } else { - Value setting; - if (context.TryGetCurrentSetting("quackapi_memory_limit", setting) && !setting.IsNull()) { - bind_data->memory_limit = setting.GetValue(); - } - } // log_level named param wins; else SET quackapi_log_level. auto log_entry = input.named_parameters.find("log_level"); if (log_entry != input.named_parameters.end()) { @@ -281,32 +255,10 @@ static unique_ptr ServeBind(ClientContext &context, TableFunctionB string table; ParseAccessLogString(bind_data->access_log, enabled, table); } - auto enlog_entry = input.named_parameters.find("enable_logging"); - if (enlog_entry != input.named_parameters.end()) { - bind_data->enable_logging = enlog_entry->second.GetValue(); - bind_data->limits.enable_logging_explicit = true; - } auto health_entry = input.named_parameters.find("health_routes"); if (health_entry != input.named_parameters.end()) { bind_data->health_routes = health_entry->second.GetValue(); } - auto threads_entry = input.named_parameters.find("threads"); - if (threads_entry != input.named_parameters.end()) { - // Accept INTEGER or VARCHAR. - if (threads_entry->second.type().id() == LogicalTypeId::VARCHAR) { - bind_data->threads = threads_entry->second.GetValue(); - } else { - bind_data->threads = threads_entry->second.ToString(); - } - } - auto pio_entry = input.named_parameters.find("preserve_insertion_order"); - if (pio_entry != input.named_parameters.end()) { - bind_data->preserve_insertion_order = pio_entry->second.GetValue(); - } - auto http_meta_entry = input.named_parameters.find("enable_http_metadata_cache"); - if (http_meta_entry != input.named_parameters.end()) { - bind_data->enable_http_metadata_cache = http_meta_entry->second.GetValue(); - } auto wt_entry = input.named_parameters.find("worker_threads"); if (wt_entry != input.named_parameters.end()) { bind_data->worker_threads = wt_entry->second.GetValue(); @@ -343,16 +295,6 @@ static unique_ptr ServeBind(ClientContext &context, TableFunctionB } } // compression_min_bytes named param wins; else SET (default 1024). - // pg_dsn named param wins; else SET quackapi_pg_dsn. - auto pg_dsn_entry = input.named_parameters.find("pg_dsn"); - if (pg_dsn_entry != input.named_parameters.end()) { - bind_data->pg_dsn = pg_dsn_entry->second.GetValue(); - } else { - Value setting; - if (context.TryGetCurrentSetting("quackapi_pg_dsn", setting) && !setting.IsNull()) { - bind_data->pg_dsn = setting.GetValue(); - } - } auto min_entry = input.named_parameters.find("compression_min_bytes"); if (min_entry != input.named_parameters.end()) { auto v = min_entry->second.GetValue(); @@ -370,19 +312,6 @@ static unique_ptr ServeBind(ClientContext &context, TableFunctionB bind_data->compression_min_bytes = static_cast(v); } } - // http_client named param wins; else SET quackapi_http_client (default auto). - auto hc_entry = input.named_parameters.find("http_client"); - if (hc_entry != input.named_parameters.end()) { - bind_data->http_client = hc_entry->second.GetValue(); - } else { - Value setting; - if (context.TryGetCurrentSetting("quackapi_http_client", setting) && !setting.IsNull()) { - auto s = setting.GetValue(); - if (!s.empty()) { - bind_data->http_client = s; - } - } - } auto block_entry = input.named_parameters.find("block"); if (block_entry != input.named_parameters.end()) { bind_data->block = block_entry->second.GetValue(); @@ -434,30 +363,19 @@ static void ServeExec(ClientContext &context, TableFunctionInput &data_p, DataCh // Compose with quack's auth settings when present (no-op if quack unloaded). ComposeQuackAuthSettings(context); - // Batteries-included serve options (all ON / server-optimal by default). QuackapiServeOptions opts = bind_data.limits; opts.static_dir = bind_data.static_dir; opts.cors_origins = bind_data.cors_origins; - opts.memory_limit = bind_data.memory_limit; opts.log_level = ParseQuackapiLogLevel(bind_data.log_level); opts.request_ring = bind_data.request_ring; ParseAccessLogString(bind_data.access_log, opts.access_log, opts.access_log_table); - opts.enable_logging = bind_data.enable_logging; opts.health_routes = bind_data.health_routes; - opts.threads = bind_data.threads; - opts.preserve_insertion_order = bind_data.preserve_insertion_order; - opts.enable_http_metadata_cache = bind_data.enable_http_metadata_cache; opts.worker_threads = bind_data.worker_threads; opts.keep_alive_max_count = bind_data.keep_alive_max_count; opts.keep_alive_timeout_sec = bind_data.keep_alive_timeout_sec; opts.read_timeout_sec = bind_data.read_timeout_sec; opts.write_timeout_sec = bind_data.write_timeout_sec; - opts.http_client = bind_data.http_client; - opts.pg_dsn = bind_data.pg_dsn; - // Apply DuckDB SETs / logging / resource guards (overridable, never unsafe). - // Outbound client: auto prefers curl_httpfs (loud fallback); curl requires it. - ApplyQuackapiServerDefaults(context, opts); // Compose request_id source: community tsid if LOADable, else core uuidv7. ProbeQuackapiRequestIdSource(*context.db, opts); // Auto-register /health + /healthz into the route registry (listed by routes()). @@ -632,8 +550,6 @@ struct RequestBindData : public TableFunctionData { string path; string body; unordered_map req_headers; - //! Optional libpq DSN for this request (named param wins over SET quackapi_pg_dsn). - string pg_dsn; //! Empty (default) keeps in-process requests quiet. Otherwise true, false, or a table name. string access_log; string log_level = "info"; @@ -643,9 +559,8 @@ struct RequestBindData : public TableFunctionData { static unique_ptr RequestBind(ClientContext &context, TableFunctionBindInput &input, vector &return_types, vector &names) { if (input.inputs.size() < 2 || input.inputs.size() > 3) { - throw InvalidInputException( - "quackapi_request(method, path [, body] [, headers := MAP] [, pg_dsn := VARCHAR]) expects 2 or 3 " - "positional args"); + throw InvalidInputException("quackapi_request(method, path [, body] [, headers := MAP]) expects 2 or 3 " + "positional args"); } auto bind_data = make_uniq(); BindResourceLimits(context, input, bind_data->limits); @@ -668,16 +583,6 @@ static unique_ptr RequestBind(ClientContext &context, TableFunctio } } } - // pg_dsn named param wins; else SET quackapi_pg_dsn (empty = DuckDB path). - auto pg_dsn_it = input.named_parameters.find("pg_dsn"); - if (pg_dsn_it != input.named_parameters.end() && !pg_dsn_it->second.IsNull()) { - bind_data->pg_dsn = pg_dsn_it->second.GetValue(); - } else { - Value setting; - if (context.TryGetCurrentSetting("quackapi_pg_dsn", setting) && !setting.IsNull()) { - bind_data->pg_dsn = setting.GetValue(); - } - } auto access_entry = input.named_parameters.find("access_log"); if (access_entry != input.named_parameters.end()) { bind_data->access_log = AccessLogValueString(access_entry->second); @@ -727,7 +632,7 @@ static void RequestExec(ClientContext &context, TableFunctionInput &data_p, Data ParseAccessLogString(bind_data.access_log, opts.access_log, opts.access_log_table); } QuackapiInProcessRequest(*context.db, bind_data.method, bind_data.path, bind_data.body, status, body, content_type, - &bind_data.req_headers, &resp_headers, bind_data.pg_dsn, &opts); + &bind_data.req_headers, &resp_headers, &opts); output.SetValue(0, 0, Value::INTEGER(status)); output.SetValue(1, 0, Value::BLOB(const_data_ptr_cast(body.data()), body.size())); output.SetValue(2, 0, Value(content_type)); @@ -823,7 +728,7 @@ static void RoutesExec(ClientContext &, TableFunctionInput &data_p, DataChunk &o struct ServersBindData : public TableFunctionData {}; struct ServersGlobalState : public GlobalTableFunctionState { - vector> servers; + vector> servers; idx_t offset = 0; }; @@ -835,10 +740,6 @@ static unique_ptr ServersBind(ClientContext &, TableFunctionBindIn names.emplace_back("port"); return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("listen_url"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("http_client"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("http_client_reason"); return make_uniq(); } @@ -855,13 +756,9 @@ static void ServersExec(ClientContext &, TableFunctionInput &data_p, DataChunk & auto &server = state.servers[state.offset]; const auto &host = std::get<0>(server); const auto port = std::get<1>(server); - const auto &http_client = std::get<2>(server); - const auto &http_client_reason = std::get<3>(server); output.SetValue(0, row, Value(host)); output.SetValue(1, row, Value::INTEGER(port)); output.SetValue(2, row, Value(StringUtil::Format("http://%s:%d", host, port))); - output.SetValue(3, row, Value(http_client)); - output.SetValue(4, row, Value(http_client_reason)); row++; state.offset++; } @@ -1141,15 +1038,6 @@ static void LoadInternal(ExtensionLoader &loader) { "CORS allowed origins for quackapi_serve (* or comma-separated list). " "Empty (default) disables CORS. Overridden by cors_origins named parameter.", LogicalType::VARCHAR, Value("")); - // SET quackapi_memory_limit = '4GB' | '512MB' | … - // Empty (default) = leave DuckDB memory_limit alone if already non-default; - // otherwise apply the 256MB serve guard. Serve-time memory_limit := '…' wins. - config.AddExtensionOption("quackapi_memory_limit", - "Memory limit applied by quackapi_serve (e.g. '4GB', '512MB'). " - "Empty (default): do not clobber a non-default DuckDB memory_limit; " - "only apply the 256MB serve default when nothing was configured. " - "Overridden by memory_limit named parameter.", - LogicalType::VARCHAR, Value("")); // SET quackapi_log_level = 'info' | 'debug' | 'warn' | 'error' | 'silent' // Default info — informative access + DuckDB logs, not silent. config.AddExtensionOption("quackapi_log_level", @@ -1185,54 +1073,27 @@ static void LoadInternal(ExtensionLoader &loader) { "Minimum response body size (bytes) before compression. " "Default 1024. Overridden by compression_min_bytes named parameter.", LogicalType::BIGINT, Value::BIGINT(1024)); - // SET quackapi_http_client = 'auto' | 'curl' | 'httplib' - // Default auto: INSTALL+LOAD curl_httpfs; fall back to httplib with loud reason. - // curl: REQUIRE curl_httpfs — fail serve if INSTALL/LOAD fails (no silent fallback). - // httplib: force stock client. Serve-time http_client := '…' wins. - // Inbound server remains httplib regardless. - config.AddExtensionOption("quackapi_http_client", - "Outbound HTTP client for httpfs/route fetches: auto|curl|httplib. " - "Default auto prefers curl_httpfs (pool, HTTP/2, async) and falls back " - "to httplib with http_client_reason on /healthz when unavailable. " - "curl fails serve if curl_httpfs cannot INSTALL/LOAD. Overridden by " - "http_client named parameter. Does not change the inbound HTTP server.", - LogicalType::VARCHAR, Value("auto")); - // SET quackapi_pg_dsn = 'postgresql://…' — native libpq path for serve + request. - // Empty (default) = DuckDB handler path only. Overridden by pg_dsn named parameter. - config.AddExtensionOption("quackapi_pg_dsn", - "Postgres DSN for the native libpq handler path (thread-local PGconn). " - "Empty (default) keeps DuckDB-only execution. Overridden by pg_dsn named " - "parameter on quackapi_serve / quackapi_request.", - LogicalType::VARCHAR, Value("")); - config.AddExtensionOption("quackapi_query_timeout_ms", "Maximum query execution time in milliseconds", LogicalType::BIGINT, Value::BIGINT(30000)); config.AddExtensionOption("quackapi_max_response_bytes", "Maximum uncompressed response bytes", LogicalType::BIGINT, Value::BIGINT(16 * 1024 * 1024)); config.AddExtensionOption("quackapi_max_pending_requests", "Maximum queued HTTP connections", LogicalType::BIGINT, Value::BIGINT(256)); - config.AddExtensionOption("quackapi_graphql_allow_all", "Explicit legacy opt-in to public GraphQL catalog exposure", - LogicalType::BOOLEAN, Value::BOOLEAN(false)); // quackapi_serve() / quackapi_serve(port) with batteries-included options. // All logging / health / server SETs ON by default; every knob overridable. - // Also carries compression + compression_min_bytes + http_client. + // Also carries compression + compression_min_bytes. TableFunctionSet serve_set("quackapi_serve"); TableFunction serve("quackapi_serve", {LogicalType::INTEGER}, ServeExec, ServeBind); serve.named_parameters["host"] = LogicalType::VARCHAR; serve.named_parameters["static_dir"] = LogicalType::VARCHAR; serve.named_parameters["cors_origins"] = LogicalType::VARCHAR; - serve.named_parameters["memory_limit"] = LogicalType::VARCHAR; serve.named_parameters["log_level"] = LogicalType::VARCHAR; serve.named_parameters["slow_request_ms"] = LogicalType::BIGINT; serve.named_parameters["request_ring"] = LogicalType::BIGINT; serve.named_parameters["log_headers"] = LogicalType::VARCHAR; serve.named_parameters["log_query"] = LogicalType::BOOLEAN; serve.named_parameters["access_log"] = LogicalType::ANY; - serve.named_parameters["enable_logging"] = LogicalType::BOOLEAN; serve.named_parameters["health_routes"] = LogicalType::BOOLEAN; - serve.named_parameters["threads"] = LogicalType::VARCHAR; - serve.named_parameters["preserve_insertion_order"] = LogicalType::BOOLEAN; - serve.named_parameters["enable_http_metadata_cache"] = LogicalType::BOOLEAN; serve.named_parameters["worker_threads"] = LogicalType::INTEGER; serve.named_parameters["keep_alive_max_count"] = LogicalType::INTEGER; serve.named_parameters["keep_alive_timeout_sec"] = LogicalType::INTEGER; @@ -1240,8 +1101,6 @@ static void LoadInternal(ExtensionLoader &loader) { serve.named_parameters["write_timeout_sec"] = LogicalType::INTEGER; serve.named_parameters["compression"] = LogicalType::ANY; serve.named_parameters["compression_min_bytes"] = LogicalType::BIGINT; - serve.named_parameters["http_client"] = LogicalType::VARCHAR; - serve.named_parameters["pg_dsn"] = LogicalType::VARCHAR; serve.named_parameters["block"] = LogicalType::BOOLEAN; serve.named_parameters["query_timeout_ms"] = LogicalType::BIGINT; serve.named_parameters["max_response_bytes"] = LogicalType::BIGINT; @@ -1280,7 +1139,6 @@ static void LoadInternal(ExtensionLoader &loader) { request2.named_parameters["slow_request_ms"] = LogicalType::BIGINT; request2.named_parameters["log_headers"] = LogicalType::VARCHAR; request2.named_parameters["log_query"] = LogicalType::BOOLEAN; - request2.named_parameters["pg_dsn"] = LogicalType::VARCHAR; request2.named_parameters["query_timeout_ms"] = LogicalType::BIGINT; request2.named_parameters["max_response_bytes"] = LogicalType::BIGINT; request_set.AddFunction(request2); @@ -1292,7 +1150,6 @@ static void LoadInternal(ExtensionLoader &loader) { request3.named_parameters["slow_request_ms"] = LogicalType::BIGINT; request3.named_parameters["log_headers"] = LogicalType::VARCHAR; request3.named_parameters["log_query"] = LogicalType::BOOLEAN; - request3.named_parameters["pg_dsn"] = LogicalType::VARCHAR; request3.named_parameters["query_timeout_ms"] = LogicalType::BIGINT; request3.named_parameters["max_response_bytes"] = LogicalType::BIGINT; request_set.AddFunction(request3); @@ -1331,9 +1188,6 @@ static void LoadInternal(ExtensionLoader &loader) { // quack_nop_authorization so they can be SET as quack_*_function. RegisterQuackAuthBridgeFunctions(loader); - // CREATE STREAM + quackapi_streams() (SSE push; WS deferred on httplib). - RegisterQuackapiStreamFunctions(loader); - // Outbound client diagnostic — works with Built-In, HTTPFS, MultiCurl, … // Does NOT auto-LOAD curl_httpfs; missing companion must not fail LOAD quackapi. loader.RegisterFunction(ScalarFunction("quackapi_http_util_name", {}, LogicalType::VARCHAR, HttpUtilNameFunction)); @@ -1351,24 +1205,11 @@ static void LoadInternal(ExtensionLoader &loader) { loader.RegisterFunction(GetApplyMiddlewareFunction()); loader.RegisterFunction(GetQuackapiMiddlewaresFunction()); - // quack_from_{fastapi,rails,express,gin}[+_models]: sitting_duck extractors - // that turn a real web-app source tree into route/model IR rows. - RegisterQuackapiFromXFunctions(loader); - - // Thin GraphQL allowlist + named route inspection (CREATE GRAPHQL … DDL). - RegisterQuackapiGraphqlFunctions(loader); - - // CREATE ROUTE / GROUP / AUTH / STREAM / API FOR TABLE / - // GRAPHQL FOR TABLE / GRAPHQL ROUTE — all first-class nouns registered. + // CREATE ROUTE / GROUP / AUTH and middleware DDL. ExtensionCallbackManager::Get(db).Register(RouteDdlParserExtension()); ExtensionCallbackManager::Get(db).Register(GroupDdlParserExtension()); ExtensionCallbackManager::Get(db).Register(AuthDdlParserExtension()); - ExtensionCallbackManager::Get(db).Register(TableApiDdlParserExtension()); ExtensionCallbackManager::Get(db).Register(MiddlewareDdlParserExtension()); - // CREATE STREAM (SSE) - ExtensionCallbackManager::Get(db).Register(StreamDdlParserExtension()); - // CREATE / DROP GRAPHQL FOR TABLE (+ DROP GRAPHQL ALL) + GRAPHQL ROUTE - ExtensionCallbackManager::Get(db).Register(GraphqlDdlParserExtension()); } void QuackapiExtension::Load(ExtensionLoader &loader) { diff --git a/src/quackapi_graphql.cpp b/src/quackapi_graphql.cpp deleted file mode 100644 index 6e8d353..0000000 --- a/src/quackapi_graphql.cpp +++ /dev/null @@ -1,1294 +0,0 @@ -#include "quackapi_graphql.hpp" - -#include - -#include "duckdb/common/exception.hpp" -#include "duckdb/common/string_util.hpp" -#include "duckdb/common/types/value.hpp" -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/main/connection.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/main/prepared_statement.hpp" -#include "duckdb/main/query_result.hpp" -#include "duckdb/parser/parser_extension.hpp" - -#include "quackapi_limits.hpp" -#include "quackapi_state.hpp" -#include "quackapi_util.hpp" - -namespace duckdb { - -namespace { - -string GraphqlError(const string &message) { - return "{\"errors\":[{\"message\":\"" + QuackapiJsonEscape(message) + "\"}]}"; -} - -string QuoteIdent(const string &ident) { - string out = "\""; - for (char c : ident) { - if (c == '"') { - out += "\"\""; - } else { - out += c; - } - } - out += "\""; - return out; -} - -//! GraphQL Name: [_A-Za-z][_0-9A-Za-z]* — character scan, no regex. -bool IsGraphqlNameStart(char c) { - return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_'; -} -bool IsGraphqlNameCont(char c) { - return IsGraphqlNameStart(c) || (c >= '0' && c <= '9'); -} - -void SkipWs(const string &s, idx_t &i) { - while (i < s.size()) { - char c = s[i]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == ',') { - // GraphQL allows commas as insignificant separators. - i++; - continue; - } - // Line comments: # … EOL - if (c == '#') { - while (i < s.size() && s[i] != '\n') { - i++; - } - continue; - } - break; - } -} - -bool MatchChar(const string &s, idx_t &i, char expected) { - SkipWs(s, i); - if (i < s.size() && s[i] == expected) { - i++; - return true; - } - return false; -} - -bool ReadName(const string &s, idx_t &i, string &out) { - SkipWs(s, i); - if (i >= s.size() || !IsGraphqlNameStart(s[i])) { - return false; - } - idx_t start = i; - i++; - while (i < s.size() && IsGraphqlNameCont(s[i])) { - i++; - } - out = s.substr(start, i - start); - return true; -} - -struct GraphqlField { - string name; - vector columns; // empty = invalid for v0 (need selection set) -}; - -struct GraphqlDocument { - vector roots; - string error; -}; - -bool ContainsGraphqlName(const vector &names, const string &name) { - for (auto &existing : names) { - if (existing == name) { - return true; - } - } - return false; -} - -//! Parse only: [query [Name]] { Field { Name+ }+ } -//! Field must have a nested selection set of leaf column names (no deeper nest). -GraphqlDocument ParseThinGraphql(const string &doc) { - GraphqlDocument out; - if (doc.size() > QUACKAPI_GRAPHQL_MAX_DOCUMENT_BYTES) { - out.error = "GraphQL document exceeds the 64 KiB limit"; - return out; - } - idx_t i = 0; - SkipWs(doc, i); - if (i >= doc.size()) { - out.error = "empty GraphQL document"; - return out; - } - - // Optional operation: query [Name] - string first; - idx_t save = i; - if (ReadName(doc, i, first)) { - if (first == "query") { - // Optional operation name — skip one Name if present. - string op_name; - idx_t before = i; - if (ReadName(doc, i, op_name)) { - // If next significant char is '{', op_name was the operation name. - // If we mis-consumed a field name (no op name, bare `{ field {…} }` - // after a mistaken "query" keyword path), we only enter this branch - // when the keyword was literally "query". - SkipWs(doc, i); - if (i < doc.size() && doc[i] == '{') { - // op_name was optional name — fine, leave i at '{' - } else { - // Not a selection set after the name — treat as error. - out.error = "expected selection set '{' after query"; - return out; - } - } else { - i = before; - } - } else if (first == "mutation" || first == "subscription") { - out.error = "mutations and subscriptions are not supported in GraphQL v0"; - return out; - } else { - // Leading name without 'query' keyword — not valid unless we rewind - // and expect '{'. Rewind: document must start with '{'. - i = save; - } - } - - if (!MatchChar(doc, i, '{')) { - out.error = "expected selection set '{'"; - return out; - } - - while (true) { - SkipWs(doc, i); - if (i < doc.size() && doc[i] == '}') { - i++; - break; - } - string field_name; - if (!ReadName(doc, i, field_name)) { - out.error = "expected field name in root selection set"; - return out; - } - if (field_name == "__schema" || field_name == "__type") { - out.error = "introspection fields are not supported in v0; use GET /graphql/schema"; - return out; - } - if (!MatchChar(doc, i, '{')) { - out.error = "v0 requires a column selection set: { " + field_name + " { col1 col2 } }"; - return out; - } - GraphqlField field; - field.name = field_name; - while (true) { - SkipWs(doc, i); - if (i < doc.size() && doc[i] == '}') { - i++; - break; - } - string col; - if (!ReadName(doc, i, col)) { - out.error = "expected column name inside " + field_name; - return out; - } - // No nested selection under columns in v0. - SkipWs(doc, i); - if (i < doc.size() && doc[i] == '{') { - out.error = "nested selections are not supported in GraphQL v0"; - return out; - } - if (i < doc.size() && doc[i] == '(') { - out.error = "field arguments are not supported in GraphQL v0"; - return out; - } - if (ContainsGraphqlName(field.columns, col)) { - out.error = "duplicate column '" + col + "' in " + field_name; - return out; - } - field.columns.push_back(col); - if (field.columns.size() > QUACKAPI_GRAPHQL_MAX_COLUMNS_PER_FIELD) { - out.error = "selection for " + field_name + " exceeds the column limit"; - return out; - } - } - if (field.columns.empty()) { - out.error = "selection set for " + field_name + " must list at least one column"; - return out; - } - for (auto &existing : out.roots) { - if (existing.name == field.name) { - out.error = "duplicate root field '" + field.name + "' is not supported in GraphQL v0"; - return out; - } - } - out.roots.push_back(std::move(field)); - if (out.roots.size() > QUACKAPI_GRAPHQL_MAX_ROOT_FIELDS) { - out.error = "GraphQL document exceeds the root-field limit"; - return out; - } - idx_t total_fields = 0; - for (auto &root : out.roots) { - total_fields += root.columns.size(); - } - if (total_fields > QUACKAPI_GRAPHQL_MAX_TOTAL_FIELDS) { - out.error = "GraphQL document exceeds the total-field limit"; - return out; - } - } - - SkipWs(doc, i); - if (i < doc.size()) { - out.error = "unexpected trailing content after selection set"; - return out; - } - if (out.roots.empty()) { - out.error = "selection set is empty"; - return out; - } - return out; -} - -struct GraphqlCatalogObject { - string database_name; - string schema_name; - string object_name; - vector columns; -}; - -//! Built-in GraphQL is deliberately scoped to the current database's main -//! schema. The catalog identity is kept with every object and reused in the -//! generated SELECT, so an attached catalog or a same-named temporary object -//! cannot be selected after validation. -bool LoadGraphqlCatalog(Connection &con, unordered_map &objects, string &err) { - objects.clear(); - auto res = con.Query( - "WITH objects AS (" - " SELECT database_name, schema_name, table_name AS object_name FROM duckdb_tables() " - " WHERE database_name = current_database() AND schema_name = 'main' AND NOT internal AND NOT temporary " - " UNION ALL " - " SELECT database_name, schema_name, view_name AS object_name FROM duckdb_views() " - " WHERE database_name = current_database() AND schema_name = 'main' AND NOT internal AND NOT temporary" - "), cols AS (" - " SELECT database_name, schema_name, table_name, column_name, column_index FROM duckdb_columns() " - " WHERE database_name = current_database() AND schema_name = 'main' AND NOT internal" - ") " - "SELECT o.database_name, o.schema_name, o.object_name, " - " coalesce(list(c.column_name ORDER BY c.column_index), []) AS columns " - "FROM objects o LEFT JOIN cols c " - " ON c.database_name = o.database_name AND c.schema_name = o.schema_name AND c.table_name = o.object_name " - "GROUP BY ALL ORDER BY o.object_name"); - if (res->HasError()) { - err = res->GetError(); - return false; - } - while (true) { - auto chunk = res->Fetch(); - if (!chunk || chunk->size() == 0) { - break; - } - for (idx_t row = 0; row < chunk->size(); row++) { - GraphqlCatalogObject object; - object.database_name = chunk->GetValue(0, row).ToString(); - object.schema_name = chunk->GetValue(1, row).ToString(); - object.object_name = chunk->GetValue(2, row).ToString(); - auto cols = chunk->GetValue(3, row); - if (!cols.IsNull() && cols.type().id() == LogicalTypeId::LIST) { - for (auto &column : ListValue::GetChildren(cols)) { - if (!column.IsNull()) { - object.columns.push_back(column.ToString()); - } - } - } - objects.emplace(object.object_name, std::move(object)); - } - } - return true; -} - -bool TableExists(Connection &con, const string &table) { - unordered_map objects; - string err; - return LoadGraphqlCatalog(con, objects, err) && objects.find(table) != objects.end(); -} - -bool ColumnExists(Connection &con, const string &table, const string &column) { - unordered_map objects; - string err; - if (!LoadGraphqlCatalog(con, objects, err)) { - return false; - } - auto object = objects.find(table); - return object != objects.end() && ContainsGraphqlName(object->second.columns, column); -} - -string QuoteGraphqlObject(const GraphqlCatalogObject &object) { - return QuoteIdent(object.database_name) + "." + QuoteIdent(object.schema_name) + "." + - QuoteIdent(object.object_name); -} - -bool GraphqlLegacyOpenEnabled(Connection &con) { - auto result = con.Query("SELECT current_setting('quackapi_graphql_allow_all')"); - if (result->HasError()) { - return false; - } - auto chunk = result->Fetch(); - if (!chunk || chunk->size() == 0 || chunk->GetValue(0, 0).IsNull()) { - return false; - } - return StringUtil::Lower(chunk->GetValue(0, 0).ToString()) == "true"; -} - -//! Execute a prevalidated source SELECT as a JSON array. Claims are only bound -//! to named parameters generated by the policy rewriter; GraphQL input never -//! becomes SQL text other than quoted catalog identifiers from the snapshot. -string SelectTableJson(Connection &con, const string &source_sql, const vector &columns, - const GraphqlExecOptions &options, string &err) { - string select_list; - for (idx_t c = 0; c < columns.size(); c++) { - if (c > 0) { - select_list += ", "; - } - select_list += QuoteIdent(columns[c]); - } - // Build json_object('col', "col", …) so types follow DuckDB → JSON. - string json_obj = "json_object("; - for (idx_t c = 0; c < columns.size(); c++) { - if (c > 0) { - json_obj += ", "; - } - json_obj += "'" + StringUtil::Replace(columns[c], "'", "''") + "', " + QuoteIdent(columns[c]); - } - json_obj += ")"; - - string sql = - "SELECT coalesce(json_group_array(" + json_obj + "), '[]'::JSON)::VARCHAR FROM (" + source_sql + ") _gql"; - QuackapiQueryDeadline deadline(con, options.query_timeout_ms); - auto prepared = con.Prepare(sql); - if (prepared->HasError()) { - err = prepared->GetError(); - return {}; - } - case_insensitive_map_t values; - for (auto &entry : prepared->named_param_map) { - auto param = entry.first; - if (!StringUtil::StartsWith(StringUtil::Lower(param), "claims_")) { - err = "unexpected parameter in GraphQL query"; - return {}; - } - string claim_name = param.substr(7); - if (!options.claims) { - values[param] = BoundParameterData(Value()); - continue; - } - auto claim = options.claims->find(claim_name); - values[param] = - claim == options.claims->end() ? BoundParameterData(Value()) : BoundParameterData(Value(claim->second)); - } - auto res = prepared->Execute(values); - if (res->HasError()) { - err = deadline.Expired() ? "query deadline exceeded" : res->GetError(); - return {}; - } - auto chunk = res->Fetch(); - if (!chunk || chunk->size() == 0 || chunk->GetValue(0, 0).IsNull()) { - return "[]"; - } - auto json = chunk->GetValue(0, 0).ToString(); - if (json.size() > options.max_response_bytes) { - err = "GraphQL response exceeds configured byte limit"; - return {}; - } - return json; -} - -//===--------------------------------------------------------------------===// -// CREATE / DROP GRAPHQL FOR TABLE (allowlist for built-in /graphql) -//===--------------------------------------------------------------------===// - -//! Parse a SQL identifier: bare token OR double-quoted with "" escapes. -bool ParseIdent(string &rest, string &out) { - rest = QuackapiTrim(rest); - if (rest.empty()) { - return false; - } - if (rest[0] == '"') { - string result; - idx_t i = 1; - while (i < rest.size()) { - if (rest[i] == '"') { - if (i + 1 < rest.size() && rest[i + 1] == '"') { - result += '"'; - i += 2; - continue; - } - out = result; - rest = QuackapiTrim(rest.substr(i + 1)); - return !out.empty(); - } - result += rest[i]; - i++; - } - return false; // unterminated - } - // Bare identifier: up to whitespace or comma. - idx_t i = 0; - while (i < rest.size() && !StringUtil::CharacterIsSpace(rest[i]) && rest[i] != ',') { - if (rest[i] == '"') { - return false; - } - i++; - } - if (i == 0) { - return false; - } - out = rest.substr(0, i); - rest = QuackapiTrim(rest.substr(i)); - return true; -} - -bool ParseTableList(string &rest, vector &tables, string &err) { - tables.clear(); - while (true) { - string name; - if (!ParseIdent(rest, name)) { - err = "invalid table name — use a bare identifier or \"quoted\"\"ident\""; - return false; - } - if (name.find('\'') != string::npos) { - err = "invalid table name"; - return false; - } - tables.push_back(name); - rest = QuackapiTrim(rest); - if (rest.empty()) { - return true; - } - if (rest[0] == ',') { - rest = QuackapiTrim(rest.substr(1)); - if (rest.empty()) { - err = "trailing comma in table list"; - return false; - } - continue; - } - err = "unexpected trailing content after table list"; - return false; - } -} - -struct GraphqlDdlParseData : public ParserExtensionParseData { - //! "CREATE" | "DROP" | "CLEAR" | "CREATE_ROUTE" | "DROP_ROUTE" - string action; - bool or_replace = false; - vector tables; - //! CREATE/DROP GRAPHQL ROUTE fields - string route_name; - string method = "POST"; - string path; - string require_auth; - idx_t limit = QUACKAPI_GRAPHQL_DEFAULT_LIMIT; - bool has_limit = false; - - unique_ptr Copy() const override { - auto copy = make_uniq(); - copy->action = action; - copy->or_replace = or_replace; - copy->tables = tables; - copy->route_name = route_name; - copy->method = method; - copy->path = path; - copy->require_auth = require_auth; - copy->limit = limit; - copy->has_limit = has_limit; - return std::move(copy); - } - string ToString() const override { - if (action == "CLEAR") { - return "DROP GRAPHQL ALL"; - } - if (action == "DROP_ROUTE") { - return "DROP GRAPHQL ROUTE " + route_name; - } - if (action == "CREATE_ROUTE") { - string s = string(or_replace ? "CREATE OR REPLACE" : "CREATE") + " GRAPHQL ROUTE " + route_name + " " + - method + " '" + path + "' FROM"; - for (idx_t i = 0; i < tables.size(); i++) { - s += (i == 0 ? " " : ", ") + tables[i]; - } - if (!require_auth.empty()) { - s += " REQUIRE " + require_auth; - } - if (has_limit) { - s += " LIMIT " + std::to_string(limit); - } - return s; - } - string s = action + " GRAPHQL FOR TABLE"; - for (idx_t i = 0; i < tables.size(); i++) { - s += (i == 0 ? " " : ", ") + tables[i]; - } - return s; - } -}; - -//! Parse table list that may be followed by REQUIRE / LIMIT keywords. -bool ParseTableListUntilKeywords(string &rest, vector &tables, string &err) { - tables.clear(); - while (true) { - rest = QuackapiTrim(rest); - if (rest.empty()) { - if (tables.empty()) { - err = "expected at least one table name"; - return false; - } - return true; - } - auto upper = StringUtil::Upper(rest); - // Stop before optional clauses. - if ((StringUtil::StartsWith(upper, "REQUIRE") && (rest.size() == 7 || StringUtil::CharacterIsSpace(rest[7]))) || - (StringUtil::StartsWith(upper, "LIMIT") && (rest.size() == 5 || StringUtil::CharacterIsSpace(rest[5])))) { - if (tables.empty()) { - err = "expected at least one table name before " + rest.substr(0, 6); - return false; - } - return true; - } - string name; - if (!ParseIdent(rest, name)) { - err = "invalid table name — use a bare identifier or \"quoted\"\"ident\""; - return false; - } - if (name.find('\'') != string::npos) { - err = "invalid table name"; - return false; - } - tables.push_back(name); - rest = QuackapiTrim(rest); - if (rest.empty()) { - return true; - } - if (rest[0] == ',') { - rest = QuackapiTrim(rest.substr(1)); - if (rest.empty()) { - err = "trailing comma in table list"; - return false; - } - continue; - } - // Next token may be REQUIRE/LIMIT — loop will stop. - upper = StringUtil::Upper(rest); - if ((StringUtil::StartsWith(upper, "REQUIRE") && (rest.size() == 7 || StringUtil::CharacterIsSpace(rest[7]))) || - (StringUtil::StartsWith(upper, "LIMIT") && (rest.size() == 5 || StringUtil::CharacterIsSpace(rest[5])))) { - return true; - } - err = "unexpected trailing content after table list"; - return false; - } -} - -//! Grammar: -//! CREATE [OR REPLACE] GRAPHQL FOR TABLE [,
...] -//! DROP GRAPHQL FOR TABLE
[,
...] -//! DROP GRAPHQL ALL -//! CREATE [OR REPLACE] GRAPHQL ROUTE POST '' FROM -//! [REQUIRE ] [LIMIT ] -//! DROP GRAPHQL ROUTE -ParserExtensionParseResult GraphqlDdlParse(ParserExtensionInfo *, const string &query) { - auto q = QuackapiTrim(query); - auto upper = StringUtil::Upper(q); - - // DROP GRAPHQL ALL — token boundary so "DROP GRAPHQL ALLOWED" is not ours. - if (StringUtil::StartsWith(upper, "DROP GRAPHQL ALL") && - (upper.size() == 16 || (!StringUtil::CharacterIsAlphaNumeric(upper[16]) && upper[16] != '_'))) { - auto tail = QuackapiTrim(q.substr(16)); - if (!tail.empty()) { - return ParserExtensionParseResult("DROP GRAPHQL ALL takes no arguments"); - } - auto data = make_uniq(); - data->action = "CLEAR"; - return ParserExtensionParseResult(std::move(data)); - } - - // DROP GRAPHQL ROUTE - if (StringUtil::StartsWith(upper, "DROP GRAPHQL ROUTE ")) { - auto name = QuackapiTrim(q.substr(19)); - if (name.empty() || name.find(' ') != string::npos) { - return ParserExtensionParseResult("DROP GRAPHQL ROUTE expects a single route name"); - } - auto data = make_uniq(); - data->action = "DROP_ROUTE"; - data->route_name = name; - return ParserExtensionParseResult(std::move(data)); - } - - // CREATE [OR REPLACE] GRAPHQL ROUTE … - bool route_or_replace = false; - idx_t route_pos = 0; - if (StringUtil::StartsWith(upper, "CREATE GRAPHQL ROUTE ")) { - route_pos = 21; - } else if (StringUtil::StartsWith(upper, "CREATE OR REPLACE GRAPHQL ROUTE ")) { - route_pos = 32; - route_or_replace = true; - } - if (route_pos > 0) { - auto rest = QuackapiTrim(q.substr(route_pos)); - // - auto sp = rest.find(' '); - if (sp == string::npos) { - return ParserExtensionParseResult("CREATE GRAPHQL ROUTE POST '' FROM
[, …]"); - } - auto name = rest.substr(0, sp); - rest = QuackapiTrim(rest.substr(sp)); - // POST - sp = rest.find(' '); - if (sp == string::npos) { - return ParserExtensionParseResult("Expected POST '' after GRAPHQL ROUTE name"); - } - auto method = StringUtil::Upper(rest.substr(0, sp)); - if (method != "POST") { - return ParserExtensionParseResult("CREATE GRAPHQL ROUTE only supports POST in v0 (got " + method + ")"); - } - rest = QuackapiTrim(rest.substr(sp)); - // '' - if (rest.empty() || rest[0] != '\'') { - return ParserExtensionParseResult("Expected quoted '' after POST"); - } - auto path_end = rest.find('\'', 1); - if (path_end == string::npos) { - return ParserExtensionParseResult("Unterminated GraphQL route path"); - } - auto path = rest.substr(1, path_end - 1); - if (path.empty() || path[0] != '/') { - return ParserExtensionParseResult("GraphQL route path must be absolute (start with '/')"); - } - rest = QuackapiTrim(rest.substr(path_end + 1)); - // FROM - auto rest_u = StringUtil::Upper(rest); - if (!StringUtil::StartsWith(rest_u, "FROM") || !(rest.size() == 4 || StringUtil::CharacterIsSpace(rest[4]))) { - return ParserExtensionParseResult("Expected FROM
[, …] after path"); - } - rest = QuackapiTrim(rest.substr(4)); - vector tables; - string err; - if (!ParseTableListUntilKeywords(rest, tables, err)) { - return ParserExtensionParseResult("CREATE GRAPHQL ROUTE: " + err); - } - string require_auth; - idx_t limit = QUACKAPI_GRAPHQL_DEFAULT_LIMIT; - bool has_limit = false; - for (int round = 0; round < 4; round++) { - rest = QuackapiTrim(rest); - if (rest.empty()) { - break; - } - rest_u = StringUtil::Upper(rest); - if (StringUtil::StartsWith(rest_u, "REQUIRE") && - (rest.size() == 7 || StringUtil::CharacterIsSpace(rest[7]))) { - rest = QuackapiTrim(rest.substr(7)); - idx_t te = 0; - while (te < rest.size() && !StringUtil::CharacterIsSpace(rest[te])) { - te++; - } - if (te == 0) { - return ParserExtensionParseResult("REQUIRE expects an auth name"); - } - require_auth = rest.substr(0, te); - rest = QuackapiTrim(rest.substr(te)); - continue; - } - if (StringUtil::StartsWith(rest_u, "LIMIT") && - (rest.size() == 5 || StringUtil::CharacterIsSpace(rest[5]))) { - rest = QuackapiTrim(rest.substr(5)); - idx_t te = 0; - while (te < rest.size() && !StringUtil::CharacterIsSpace(rest[te])) { - te++; - } - if (te == 0) { - return ParserExtensionParseResult("LIMIT expects a positive integer"); - } - auto lim_s = rest.substr(0, te); - for (char c : lim_s) { - if (c < '0' || c > '9') { - return ParserExtensionParseResult("LIMIT expects a positive integer"); - } - } - try { - limit = (idx_t)std::stoull(lim_s); - } catch (...) { - return ParserExtensionParseResult("LIMIT expects a positive integer"); - } - if (limit == 0 || limit > 100000) { - return ParserExtensionParseResult("LIMIT must be between 1 and 100000"); - } - has_limit = true; - rest = QuackapiTrim(rest.substr(te)); - continue; - } - return ParserExtensionParseResult("Unexpected trailing content after GRAPHQL ROUTE: " + rest); - } - auto data = make_uniq(); - data->action = "CREATE_ROUTE"; - data->or_replace = route_or_replace; - data->route_name = name; - data->method = method; - data->path = path; - data->tables = std::move(tables); - data->require_auth = require_auth; - data->limit = limit; - data->has_limit = has_limit; - return ParserExtensionParseResult(std::move(data)); - } - - bool or_replace = false; - bool is_drop = false; - idx_t pos = 0; - if (StringUtil::StartsWith(upper, "CREATE GRAPHQL FOR TABLE ")) { - pos = 25; - } else if (StringUtil::StartsWith(upper, "CREATE OR REPLACE GRAPHQL FOR TABLE ")) { - pos = 36; - or_replace = true; - } else if (StringUtil::StartsWith(upper, "DROP GRAPHQL FOR TABLE ")) { - pos = 23; - is_drop = true; - } else { - // Not ours — let the core parser (or another extension) handle it. - return ParserExtensionParseResult(); - } - - auto rest = QuackapiTrim(q.substr(pos)); - if (rest.empty()) { - return ParserExtensionParseResult(is_drop ? "DROP GRAPHQL FOR TABLE expects at least one table name" - : "CREATE GRAPHQL FOR TABLE expects at least one table name"); - } - - vector tables; - string err; - if (!ParseTableList(rest, tables, err)) { - return ParserExtensionParseResult(string(is_drop ? "DROP" : "CREATE") + " GRAPHQL FOR TABLE: " + err); - } - - auto data = make_uniq(); - data->action = is_drop ? "DROP" : "CREATE"; - data->or_replace = or_replace; - data->tables = std::move(tables); - return ParserExtensionParseResult(std::move(data)); -} - -struct ApplyGraphqlBindData : public TableFunctionData { - string action; - bool or_replace = false; - vector tables; - string route_name; - string method = "POST"; - string path; - string require_auth; - idx_t limit = QUACKAPI_GRAPHQL_DEFAULT_LIMIT; - bool finished = false; -}; - -unique_ptr ApplyGraphqlBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - bind_data->action = input.inputs[0].GetValue(); - bind_data->or_replace = input.inputs[1].GetValue(); - // tables as VARCHAR[] (LIST) - if (!input.inputs[2].IsNull() && input.inputs[2].type().id() == LogicalTypeId::LIST) { - for (auto &child : ListValue::GetChildren(input.inputs[2])) { - if (!child.IsNull()) { - bind_data->tables.push_back(child.ToString()); - } - } - } - if (input.inputs.size() > 3 && !input.inputs[3].IsNull()) { - bind_data->route_name = input.inputs[3].GetValue(); - } - if (input.inputs.size() > 4 && !input.inputs[4].IsNull()) { - bind_data->method = input.inputs[4].GetValue(); - } - if (input.inputs.size() > 5 && !input.inputs[5].IsNull()) { - bind_data->path = input.inputs[5].GetValue(); - } - if (input.inputs.size() > 6 && !input.inputs[6].IsNull()) { - bind_data->require_auth = input.inputs[6].GetValue(); - } - if (input.inputs.size() > 7 && !input.inputs[7].IsNull()) { - bind_data->limit = (idx_t)input.inputs[7].GetValue(); - } - BindStatusColumn(return_types, names); - return std::move(bind_data); -} - -void ApplyGraphqlExec(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &bind_data = data_p.bind_data->CastNoConst(); - if (bind_data.finished) { - return; - } - auto &state = QuackapiState::Get(*context.db); - Connection con(*context.db); - - if (bind_data.action == "CLEAR") { - state.ClearGraphqlTables(); - EmitOneShotStatus(output, bind_data.finished, "GraphQL allowlist cleared (catalog closed)"); - return; - } - - if (bind_data.action == "CREATE_ROUTE") { - for (auto &table : bind_data.tables) { - if (!TableExists(con, table)) { - throw InvalidInputException("CREATE GRAPHQL ROUTE: table or view \"%s\" not found in schema main", - table); - } - } - if (!bind_data.require_auth.empty()) { - QuackapiAuth auth; - if (!state.GetAuth(bind_data.require_auth, auth)) { - throw InvalidInputException("CREATE GRAPHQL ROUTE: auth scheme \"%s\" does not exist", - bind_data.require_auth); - } - } - QuackapiGraphqlRoute route; - route.name = bind_data.route_name; - route.method = bind_data.method; - route.path = bind_data.path; - route.tables = bind_data.tables; - route.require_auth = bind_data.require_auth; - route.limit = bind_data.limit == 0 ? QUACKAPI_GRAPHQL_DEFAULT_LIMIT : bind_data.limit; - state.AddGraphqlRoute(route, bind_data.or_replace); - string msg = "GraphQL route " + route.name + ": " + route.method + " " + route.path + " FROM "; - for (idx_t i = 0; i < route.tables.size(); i++) { - if (i > 0) { - msg += ", "; - } - msg += route.tables[i]; - } - if (!route.require_auth.empty()) { - msg += " REQUIRE " + route.require_auth; - } - msg += " LIMIT " + std::to_string(route.limit); - EmitOneShotStatus(output, bind_data.finished, msg); - return; - } - - if (bind_data.action == "DROP_ROUTE") { - if (state.DropGraphqlRoute(bind_data.route_name)) { - EmitOneShotStatus(output, bind_data.finished, "Dropped GraphQL route " + bind_data.route_name); - } else { - throw InvalidInputException("GraphQL route \"%s\" does not exist", bind_data.route_name); - } - return; - } - - if (bind_data.action == "CREATE") { - // Validate the full table list before mutating so a mid-list miss/duplicate - // cannot half-apply earlier names (CREATE GRAPHQL ROUTE already does this). - for (auto &table : bind_data.tables) { - if (!TableExists(con, table)) { - throw InvalidInputException("CREATE GRAPHQL FOR TABLE: table or view \"%s\" not found in schema main", - table); - } - } - if (!bind_data.or_replace) { - auto existing = state.SnapshotGraphqlTables(); - for (auto &table : bind_data.tables) { - for (auto &e : existing) { - if (e == table) { - throw InvalidInputException( - "GraphQL table \"%s\" already registered — use CREATE OR REPLACE GRAPHQL FOR TABLE", table); - } - } - } - } - for (auto &table : bind_data.tables) { - state.AddGraphqlTable(table, bind_data.or_replace); - } - string msg = "GraphQL registered: "; - for (idx_t i = 0; i < bind_data.tables.size(); i++) { - if (i > 0) { - msg += ", "; - } - msg += bind_data.tables[i]; - } - msg += " (allowlist mode on POST /graphql)"; - EmitOneShotStatus(output, bind_data.finished, msg); - return; - } - - if (bind_data.action == "DROP") { - vector dropped; - vector missing; - for (auto &table : bind_data.tables) { - if (state.DropGraphqlTable(table)) { - dropped.push_back(table); - } else { - missing.push_back(table); - } - } - if (dropped.empty()) { - throw InvalidInputException("DROP GRAPHQL FOR TABLE: no matching registered tables"); - } - string msg = "GraphQL unregistered: "; - for (idx_t i = 0; i < dropped.size(); i++) { - if (i > 0) { - msg += ", "; - } - msg += dropped[i]; - } - if (state.SnapshotGraphqlTables().empty()) { - msg += " (allowlist empty → catalog closed)"; - } - if (!missing.empty()) { - msg += "; not registered: "; - for (idx_t i = 0; i < missing.size(); i++) { - if (i > 0) { - msg += ", "; - } - msg += missing[i]; - } - } - EmitOneShotStatus(output, bind_data.finished, msg); - return; - } - - throw InvalidInputException("internal: unknown GraphQL DDL action \"%s\"", bind_data.action); -} - -TableFunction MakeApplyGraphqlFunction() { - return MakeApplyDdlFunction("quackapi_apply_graphql", - {LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::LIST(LogicalType::VARCHAR), - LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, - LogicalType::BIGINT}, - ApplyGraphqlExec, ApplyGraphqlBind); -} - -ParserExtensionPlanResult GraphqlDdlPlan(ParserExtensionInfo *, ClientContext &, - unique_ptr parse_data) { - auto &data = static_cast(*parse_data); - ParserExtensionPlanResult result; - result.function = MakeApplyGraphqlFunction(); - result.parameters.push_back(Value(data.action)); - result.parameters.push_back(Value::BOOLEAN(data.or_replace)); - vector table_vals; - for (auto &t : data.tables) { - table_vals.emplace_back(t); - } - result.parameters.push_back(Value::LIST(LogicalType::VARCHAR, table_vals)); - result.parameters.push_back(Value(data.route_name)); - result.parameters.push_back(Value(data.method)); - result.parameters.push_back(Value(data.path)); - result.parameters.push_back(Value(data.require_auth)); - result.parameters.push_back(Value::BIGINT((int64_t)data.limit)); - FinishDdlPlan(result); - return result; -} - -//===--------------------------------------------------------------------===// -// quackapi_graphql_tables() — inspect allowlist -//===--------------------------------------------------------------------===// - -struct GraphqlTablesBindData : public TableFunctionData {}; - -struct GraphqlTablesGlobalState : public GlobalTableFunctionState { - vector tables; - bool allowlist_active = false; - bool legacy_open = false; - idx_t offset = 0; - bool emitted_mode = false; -}; - -unique_ptr GraphqlTablesBind(ClientContext &, TableFunctionBindInput &, vector &return_types, - vector &names) { - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("mode"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("table_name"); - return make_uniq(); -} - -unique_ptr GraphqlTablesInit(ClientContext &context, TableFunctionInitInput &) { - auto state = make_uniq(); - auto &api = QuackapiState::Get(*context.db); - state->tables = api.SnapshotGraphqlTables(); - Connection con(*context.db); - state->legacy_open = GraphqlLegacyOpenEnabled(con); - state->allowlist_active = !state->legacy_open && !state->tables.empty(); - return std::move(state); -} - -void GraphqlTablesExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - if (!state.allowlist_active) { - // One informational row: explicit legacy open or secure closed mode. - if (state.emitted_mode) { - return; - } - output.SetValue(0, 0, Value(state.legacy_open ? "open" : "disabled")); - output.SetValue(1, 0, Value()); - output.SetCardinality(1); - state.emitted_mode = true; - return; - } - idx_t row = 0; - while (state.offset < state.tables.size() && row < STANDARD_VECTOR_SIZE) { - output.SetValue(0, row, Value("allowlist")); - output.SetValue(1, row, Value(state.tables[state.offset])); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -} // namespace - -bool GraphqlExtractQuery(DatabaseInstance &db, const string &raw_body, string &query_out, string &error_json) { - if (raw_body.empty()) { - error_json = GraphqlError("request body is empty; expected JSON {\"query\":\"...\"}"); - return false; - } - Connection con(db); - auto res = con.Query("SELECT CASE" - " WHEN TRY_CAST(? AS JSON) IS NULL THEN NULL" - " WHEN json_type(?::JSON) != 'OBJECT' THEN NULL" - " ELSE json_extract_string(?::JSON, '$.query')" - " END", - Value(raw_body), Value(raw_body), Value(raw_body)); - if (res->HasError()) { - error_json = GraphqlError("invalid JSON body"); - return false; - } - auto chunk = res->Fetch(); - if (!chunk || chunk->size() == 0 || chunk->GetValue(0, 0).IsNull()) { - error_json = GraphqlError("JSON body must be an object with a non-null \"query\" string"); - return false; - } - query_out = chunk->GetValue(0, 0).ToString(); - if (query_out.empty()) { - error_json = GraphqlError("\"query\" must be a non-empty string"); - return false; - } - return true; -} - -bool TableInList(const vector &tables, const string &name) { - for (auto &t : tables) { - if (t == name) { - return true; - } - } - return false; -} - -string ExecuteGraphqlQuery(DatabaseInstance &db, const string &query, const GraphqlExecOptions &options) { - auto doc = ParseThinGraphql(query); - if (!doc.error.empty()) { - return GraphqlError(doc.error); - } - idx_t limit = options.limit; - if (limit == 0) { - limit = QUACKAPI_GRAPHQL_DEFAULT_LIMIT; - } - limit = std::min(limit, QUACKAPI_GRAPHQL_MAX_TOTAL_ROWS / doc.roots.size()); - - auto &state = QuackapiState::Get(db); - const bool route_mode = options.allowed_tables != nullptr; - Connection con(db); - const bool legacy_open = !route_mode && GraphqlLegacyOpenEnabled(con); - const auto global_tables = state.SnapshotGraphqlTables(); - const bool global_allowlist = !route_mode && !legacy_open; - unordered_map catalog; - string catalog_error; - QuackapiQueryDeadline catalog_deadline(con, options.query_timeout_ms); - if (!LoadGraphqlCatalog(con, catalog, catalog_error)) { - return GraphqlError("schema catalog query failed: " + catalog_error); - } - if (catalog_deadline.Expired()) { - return GraphqlError("schema catalog query deadline exceeded"); - } - string data = "{"; - bool first = true; - for (auto &field : doc.roots) { - if (route_mode) { - if (!TableInList(*options.allowed_tables, field.name)) { - return GraphqlError("table '" + field.name + - "' is not on this GraphQL route — CREATE GRAPHQL ROUTE … FROM " + field.name); - } - } else if (global_allowlist && !TableInList(global_tables, field.name)) { - return GraphqlError("table '" + field.name + "' is not registered for GraphQL — CREATE GRAPHQL FOR TABLE " + - field.name); - } - auto object = catalog.find(field.name); - if (object == catalog.end()) { - return GraphqlError("unknown table '" + field.name + "' (main schema catalog only)"); - } - for (auto &col : field.columns) { - if (!ContainsGraphqlName(object->second.columns, col)) { - return GraphqlError("unknown column '" + col + "' on table '" + field.name + "'"); - } - } - string source_sql = "SELECT "; - for (idx_t c = 0; c < field.columns.size(); c++) { - if (c > 0) { - source_sql += ", "; - } - source_sql += QuoteIdent(field.columns[c]); - } - source_sql += " FROM " + QuoteGraphqlObject(object->second) + " LIMIT " + std::to_string(limit); - string err; - string rows_json = SelectTableJson(con, source_sql, field.columns, options, err); - if (!err.empty()) { - return GraphqlError("query failed for '" + field.name + "': " + err); - } - if (!first) { - data += ","; - } - first = false; - data += "\"" + QuackapiJsonEscape(field.name) + "\":" + rows_json; - if (data.size() > options.max_response_bytes) { - return GraphqlError("GraphQL response exceeds configured byte limit"); - } - } - data += "}"; - return "{\"data\":" + data + "}"; -} - -string GraphqlSchemaNote() { - return "GraphQL v0 catalog-only schema. Full __schema introspection not implemented. " - "CREATE GRAPHQL FOR TABLE registers an allowlist for POST /graphql; empty = closed. " - "SET GLOBAL quackapi_graphql_allow_all = true restores legacy open catalog mode. " - "CREATE GRAPHQL ROUTE mounts named paths with per-route tables."; -} - -string BuildGraphqlSchema(DatabaseInstance &db, const GraphqlExecOptions &options) { - Connection con(db); - auto &state = QuackapiState::Get(db); - const bool route_mode = options.allowed_tables != nullptr; - const bool legacy_open = !route_mode && GraphqlLegacyOpenEnabled(con); - const bool allowlist = !route_mode && !legacy_open; - vector allowed; - if (route_mode) { - allowed = *options.allowed_tables; - } else if (allowlist) { - allowed = state.SnapshotGraphqlTables(); - } - - unordered_map catalog; - string catalog_error; - QuackapiQueryDeadline catalog_deadline(con, options.query_timeout_ms); - if (!LoadGraphqlCatalog(con, catalog, catalog_error)) { - return GraphqlError("schema catalog query failed: " + catalog_error); - } - if (catalog_deadline.Expired()) { - return GraphqlError("schema catalog query deadline exceeded"); - } - - string mode; - if (route_mode) { - mode = options.mode.empty() ? "route" : options.mode; - } else { - mode = legacy_open ? "open" : (allowed.empty() ? "disabled" : "allowlist"); - } - - string body = "{\"mode\":\"" + mode + "\""; - if (route_mode && !options.route_name.empty()) { - body += ",\"route\":\"" + QuackapiJsonEscape(options.route_name) + "\""; - } - body += ",\"tables\":{"; - bool first_table = true; - vector names; - for (auto &entry : catalog) { - names.push_back(entry.first); - } - std::sort(names.begin(), names.end()); - for (auto &table : names) { - if ((route_mode || allowlist) && !TableInList(allowed, table)) { - continue; - } - if (!first_table) { - body += ","; - } - first_table = false; - body += "\"" + QuackapiJsonEscape(table) + "\":["; - for (idx_t c = 0; c < catalog[table].columns.size(); c++) { - if (c > 0) { - body += ","; - } - body += "\"" + QuackapiJsonEscape(catalog[table].columns[c]) + "\""; - } - body += "]"; - if (body.size() > options.max_response_bytes) { - return GraphqlError("GraphQL response exceeds configured byte limit"); - } - } - body += "},\"note\":\"" + QuackapiJsonEscape(GraphqlSchemaNote()) + "\"}"; - return body; -} - -GraphqlDdlParserExtension::GraphqlDdlParserExtension() { - parse_function = GraphqlDdlParse; - plan_function = GraphqlDdlPlan; -} - -//===--------------------------------------------------------------------===// -// quackapi_graphql_routes() — inspect named mounts -//===--------------------------------------------------------------------===// - -struct GraphqlRoutesBindData : public TableFunctionData {}; - -struct GraphqlRoutesGlobalState : public GlobalTableFunctionState { - vector routes; - idx_t offset = 0; -}; - -unique_ptr GraphqlRoutesBind(ClientContext &, TableFunctionBindInput &, vector &return_types, - vector &names) { - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("name"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("method"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("path"); - return_types.emplace_back(LogicalType::LIST(LogicalType::VARCHAR)); - names.emplace_back("tables"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("require_auth"); - return_types.emplace_back(LogicalType::BIGINT); - names.emplace_back("limit"); - return make_uniq(); -} - -unique_ptr GraphqlRoutesInit(ClientContext &context, TableFunctionInitInput &) { - auto state = make_uniq(); - state->routes = QuackapiState::Get(*context.db).SnapshotGraphqlRoutes(); - return std::move(state); -} - -void GraphqlRoutesExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.routes.size() && row < STANDARD_VECTOR_SIZE) { - auto &r = state.routes[state.offset]; - output.SetValue(0, row, Value(r.name)); - output.SetValue(1, row, Value(r.method)); - output.SetValue(2, row, Value(r.path)); - vector table_vals; - for (auto &t : r.tables) { - table_vals.emplace_back(t); - } - output.SetValue(3, row, Value::LIST(LogicalType::VARCHAR, table_vals)); - output.SetValue(4, row, r.require_auth.empty() ? Value() : Value(r.require_auth)); - output.SetValue(5, row, Value::BIGINT((int64_t)r.limit)); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -void RegisterQuackapiGraphqlFunctions(ExtensionLoader &loader) { - TableFunction tf("quackapi_graphql_tables", {}, GraphqlTablesExec, GraphqlTablesBind, GraphqlTablesInit); - loader.RegisterFunction(tf); - TableFunction routes("quackapi_graphql_routes", {}, GraphqlRoutesExec, GraphqlRoutesBind, GraphqlRoutesInit); - loader.RegisterFunction(routes); -} - -} // namespace duckdb diff --git a/src/quackapi_registry.cpp b/src/quackapi_registry.cpp index 0605653..2e1e249 100644 --- a/src/quackapi_registry.cpp +++ b/src/quackapi_registry.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include "duckdb/common/exception.hpp" #include "duckdb/common/helper.hpp" @@ -34,16 +33,10 @@ void QuackapiState::PublishRoutes() { routes_live = make_shared_ptr>(routes); } -void QuackapiState::PublishStreams() { - // Caller holds streams_mutex. - streams_live = make_shared_ptr>(streams); -} - void QuackapiState::AddRoute(const QuackapiRoute &route, bool or_replace) { std::lock_guard lock(routes_mutex); - // Method+pattern must be unique across route names (GraphQL mounts already - // enforce path uniqueness). Otherwise OR REPLACE onto another route's path - // leaves two handlers and last-match / duplicate OpenAPI keys. + // Method+pattern must be unique across route names. Otherwise OR REPLACE + // onto another route's path leaves duplicate handlers and OpenAPI keys. auto collides = [&](const QuackapiRoute &candidate, const string &ignore_name) -> const QuackapiRoute * { for (auto &other : routes) { if (other.name == ignore_name) { @@ -106,66 +99,6 @@ shared_ptr> QuackapiState::LiveRoutes() { return routes_live; } -void QuackapiState::AddStream(const QuackapiStream &stream, bool or_replace) { - std::lock_guard lock(streams_mutex); - auto collides = [&](const QuackapiStream &candidate, const string &ignore_name) -> const QuackapiStream * { - for (auto &other : streams) { - if (other.name == ignore_name) { - continue; - } - if (other.method == candidate.method && other.pattern == candidate.pattern) { - return &other; - } - } - return nullptr; - }; - for (auto it = streams.begin(); it != streams.end(); ++it) { - if (it->name == stream.name) { - if (!or_replace) { - throw InvalidInputException("Stream \"%s\" already exists — use CREATE OR REPLACE STREAM", stream.name); - } - if (auto *hit = collides(stream, stream.name)) { - throw InvalidInputException("Stream path \"%s %s\" already used by stream \"%s\"", stream.method, - stream.pattern, hit->name); - } - *it = stream; - PublishStreams(); - return; - } - } - if (auto *hit = collides(stream, string())) { - throw InvalidInputException("Stream path \"%s %s\" already used by stream \"%s\"", stream.method, - stream.pattern, hit->name); - } - streams.push_back(stream); - PublishStreams(); -} - -bool QuackapiState::DropStream(const string &name) { - std::lock_guard lock(streams_mutex); - for (auto it = streams.begin(); it != streams.end(); ++it) { - if (it->name == name) { - streams.erase(it); - PublishStreams(); - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotStreams() { - auto live = LiveStreams(); - return *live; -} - -shared_ptr> QuackapiState::LiveStreams() { - std::lock_guard lock(streams_mutex); - if (!streams_live) { - PublishStreams(); - } - return streams_live; -} - void QuackapiState::AddGroup(const QuackapiGroup &group, bool or_replace) { std::lock_guard lock(groups_mutex); for (auto it = groups.begin(); it != groups.end(); ++it) { @@ -429,15 +362,12 @@ bool QuackapiState::GetServerHost(int port, string &host_out) { return false; } -vector> QuackapiState::ListServers() { +vector> QuackapiState::ListServers() { std::lock_guard lock(servers_mutex); - vector> result; + vector> result; result.reserve(servers.size()); for (auto &kv : servers) { - const auto &opts = kv.second->Options(); - string client = opts.http_client_active.empty() ? string("httplib") : opts.http_client_active; - string reason = opts.http_client_reason; - result.emplace_back(kv.second->Host(), kv.second->Port(), std::move(client), std::move(reason)); + result.emplace_back(kv.second->Host(), kv.second->Port()); } return result; } @@ -503,175 +433,4 @@ bool QuackapiState::FlushTelemetry(int port) { return false; } -void QuackapiState::AddGraphqlTable(const string &table_name, bool or_replace) { - if (table_name.empty()) { - throw InvalidInputException("CREATE GRAPHQL FOR TABLE expects a non-empty table name"); - } - std::lock_guard lock(graphql_mutex); - for (auto &t : graphql_tables) { - if (t == table_name) { - if (!or_replace) { - throw InvalidInputException( - "GraphQL table \"%s\" already registered — use CREATE OR REPLACE GRAPHQL FOR TABLE", table_name); - } - return; - } - } - graphql_tables.push_back(table_name); -} - -bool QuackapiState::DropGraphqlTable(const string &table_name) { - std::lock_guard lock(graphql_mutex); - for (auto it = graphql_tables.begin(); it != graphql_tables.end(); ++it) { - if (*it == table_name) { - graphql_tables.erase(it); - return true; - } - } - return false; -} - -void QuackapiState::ClearGraphqlTables() { - std::lock_guard lock(graphql_mutex); - graphql_tables.clear(); -} - -bool QuackapiState::GraphqlAllowlistActive() { - std::lock_guard lock(graphql_mutex); - return !graphql_tables.empty(); -} - -bool QuackapiState::IsGraphqlTableAllowed(const string &table_name) { - std::lock_guard lock(graphql_mutex); - if (graphql_tables.empty()) { - return true; // open mode - } - for (auto &t : graphql_tables) { - if (t == table_name) { - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotGraphqlTables() { - std::lock_guard lock(graphql_mutex); - auto out = graphql_tables; - std::sort(out.begin(), out.end()); - return out; -} - -namespace { - -string NormalizeGraphqlMountPath(const string &path) { - string p = path; - while (p.size() > 1 && p.back() == '/') { - p.pop_back(); - } - return p; -} - -bool IsReservedGraphqlPath(const string &path) { - return path == "/graphql" || path == "/graphql/schema"; -} - -} // namespace - -void QuackapiState::AddGraphqlRoute(const QuackapiGraphqlRoute &route_in, bool or_replace) { - if (route_in.name.empty()) { - throw InvalidInputException("CREATE GRAPHQL ROUTE expects a non-empty name"); - } - if (route_in.path.empty() || route_in.path[0] != '/') { - throw InvalidInputException("CREATE GRAPHQL ROUTE path must be absolute (start with '/')"); - } - if (route_in.tables.empty()) { - throw InvalidInputException("CREATE GRAPHQL ROUTE requires FROM
[, …]"); - } - QuackapiGraphqlRoute route = route_in; - route.path = NormalizeGraphqlMountPath(route.path); - route.method = StringUtil::Upper(route.method); - if (route.method != "POST") { - throw InvalidInputException("CREATE GRAPHQL ROUTE only supports POST in v0 (got %s)", route.method); - } - if (IsReservedGraphqlPath(route.path)) { - throw InvalidInputException("CREATE GRAPHQL ROUTE path \"%s\" is reserved for the built-in GraphQL endpoint", - route.path); - } - // Schema sibling must not collide with another mount or reserved paths. - string schema_path = route.path + "/schema"; - if (IsReservedGraphqlPath(schema_path)) { - throw InvalidInputException("CREATE GRAPHQL ROUTE path \"%s\" collides with built-in schema URL", route.path); - } - - std::lock_guard lock(graphql_mutex); - for (auto &existing : graphql_routes) { - if (existing.name == route.name) { - continue; // handled below by name match - } - if (existing.path == route.path) { - throw InvalidInputException("GraphQL route path \"%s\" already used by route \"%s\"", route.path, - existing.name); - } - if (existing.path + "/schema" == route.path || existing.path == schema_path) { - throw InvalidInputException("GraphQL route path \"%s\" collides with schema URL of route \"%s\"", - route.path, existing.name); - } - } - for (auto it = graphql_routes.begin(); it != graphql_routes.end(); ++it) { - if (it->name == route.name) { - if (!or_replace) { - throw InvalidInputException("GraphQL route \"%s\" already exists — use CREATE OR REPLACE GRAPHQL ROUTE", - route.name); - } - *it = route; - return; - } - } - graphql_routes.push_back(route); -} - -bool QuackapiState::DropGraphqlRoute(const string &name) { - std::lock_guard lock(graphql_mutex); - for (auto it = graphql_routes.begin(); it != graphql_routes.end(); ++it) { - if (it->name == name) { - graphql_routes.erase(it); - return true; - } - } - return false; -} - -bool QuackapiState::GetGraphqlRouteByPath(const string &path, const string &method, QuackapiGraphqlRoute &out) { - string norm = NormalizeGraphqlMountPath(path); - string meth = StringUtil::Upper(method); - std::lock_guard lock(graphql_mutex); - for (auto &r : graphql_routes) { - if (r.path == norm && r.method == meth) { - out = r; - return true; - } - } - return false; -} - -bool QuackapiState::GetGraphqlRouteBySchemaPath(const string &path, QuackapiGraphqlRoute &out) { - string norm = NormalizeGraphqlMountPath(path); - std::lock_guard lock(graphql_mutex); - for (auto &r : graphql_routes) { - if (norm == r.path + "/schema") { - out = r; - return true; - } - } - return false; -} - -vector QuackapiState::SnapshotGraphqlRoutes() { - std::lock_guard lock(graphql_mutex); - auto out = graphql_routes; - std::sort(out.begin(), out.end(), - [](const QuackapiGraphqlRoute &a, const QuackapiGraphqlRoute &b) { return a.name < b.name; }); - return out; -} - } // namespace duckdb diff --git a/test/sql/quackapi_batteries.test b/test/sql/quackapi_batteries.test index 71f0ac5..ecced27 100644 --- a/test/sql/quackapi_batteries.test +++ b/test/sql/quackapi_batteries.test @@ -1,39 +1,9 @@ # name: test/sql/quackapi_batteries.test -# description: batteries-included overrides — preserve_insertion_order, health_routes off, http_client force +# description: health route configuration # group: [sql] require quackapi -# log_level extension option defaults to info -query I -SELECT current_setting('quackapi_log_level'); ----- -info - -# http_client extension option defaults to auto (prefer curl_httpfs) -query I -SELECT current_setting('quackapi_http_client'); ----- -auto - -# --------------------------------------------------------------------------- -# Overridable: preserve_insertion_order := true keeps insertion order -# --------------------------------------------------------------------------- -query I -SELECT * FROM quackapi_serve(29202, host := '127.0.0.1', preserve_insertion_order := true); ----- -http://127.0.0.1:29202 - -query I -SELECT current_setting('preserve_insertion_order'); ----- -true - -query I -SELECT * FROM quackapi_stop(29202); ----- -Stopped quackapi server on port 29202 - # --------------------------------------------------------------------------- # health_routes := false → not re-registered after explicit drop # --------------------------------------------------------------------------- @@ -61,22 +31,3 @@ query I SELECT * FROM quackapi_stop(29203); ---- Stopped quackapi server on port 29203 - -# --------------------------------------------------------------------------- -# http_client := 'httplib' forces stock client (always available; no network -# needed for the force path). Active backend + reason on quackapi_servers(). -# --------------------------------------------------------------------------- -query I -SELECT * FROM quackapi_serve(29204, host := '127.0.0.1', http_client := 'httplib'); ----- -http://127.0.0.1:29204 - -query II -SELECT http_client, http_client_reason FROM quackapi_servers() WHERE port = 29204; ----- -httplib operator_forced - -query I -SELECT * FROM quackapi_stop(29204); ----- -Stopped quackapi server on port 29204 diff --git a/test/sql/quackapi_graphql.test b/test/sql/quackapi_graphql.test deleted file mode 100644 index 585dbd8..0000000 --- a/test/sql/quackapi_graphql.test +++ /dev/null @@ -1,325 +0,0 @@ -# name: test/sql/quackapi_graphql.test -# description: Thin GraphQL v0 + closed built-in allowlist + named routes -# group: [sql] - -require quackapi - -require json - -statement ok -CREATE TABLE users AS SELECT 1 AS id, 'ada' AS name; - -statement ok -CREATE TABLE posts AS SELECT 10 AS id, 'hello' AS title; - -statement ok -CREATE TABLE secrets AS SELECT 99 AS id, 'hidden' AS payload; - -# --------------------------------------------------------------------------- -# Built-in mode is closed by default: registration is required before /graphql -# --------------------------------------------------------------------------- -query I -SELECT mode FROM quackapi_graphql_tables(); ----- -disabled - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ users { id name } posts { id title } }"}' -); ----- -200 {"errors":[{"message":"table 'users' is not registered for GraphQL — CREATE GRAPHQL FOR TABLE users"}]} - -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/graphql/schema'); ----- -200 {"mode":"disabled","tables":{},"note":"GraphQL v0 catalog-only schema. Full __schema introspection not implemented. CREATE GRAPHQL FOR TABLE registers an allowlist for POST /graphql; empty = closed. SET GLOBAL quackapi_graphql_allow_all = true restores legacy open catalog mode. CREATE GRAPHQL ROUTE mounts named paths with per-route tables."} - -# Legacy open-catalog behavior needs an explicit database-wide migration opt-in. -statement ok -SET GLOBAL quackapi_graphql_allow_all = true; - -query I -SELECT mode FROM quackapi_graphql_tables(); ----- -open - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ secrets { id } }"}' -); ----- -200 {"data":{"secrets":[{"id":99}]}} - -statement ok -SET GLOBAL quackapi_graphql_allow_all = false; - -# --------------------------------------------------------------------------- -# Allowlist mode: only registered tables -# --------------------------------------------------------------------------- -query I -CREATE GRAPHQL FOR TABLE users, posts; ----- -GraphQL registered: users, posts (allowlist mode on POST /graphql) - -query TT -SELECT mode, table_name FROM quackapi_graphql_tables() ORDER BY table_name; ----- -allowlist posts -allowlist users - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ users { id name } }"}' -); ----- -200 {"data":{"users":[{"id":1,"name":"ada"}]}} - -# secrets is not on the allowlist -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ secrets { id payload } }"}' -); ----- -200 {"errors":[{"message":"table 'secrets' is not registered for GraphQL — CREATE GRAPHQL FOR TABLE secrets"}]} - -# schema only lists allowlisted tables that exist -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/graphql/schema'); ----- -200 {"mode":"allowlist","tables":{"posts":["id","title"],"users":["id","name"]},"note":"GraphQL v0 catalog-only schema. Full __schema introspection not implemented. CREATE GRAPHQL FOR TABLE registers an allowlist for POST /graphql; empty = closed. SET GLOBAL quackapi_graphql_allow_all = true restores legacy open catalog mode. CREATE GRAPHQL ROUTE mounts named paths with per-route tables."} - -# Duplicate without OR REPLACE fails -statement error -CREATE GRAPHQL FOR TABLE users; ----- -already registered - -# OR REPLACE is idempotent -query I -CREATE OR REPLACE GRAPHQL FOR TABLE users; ----- -GraphQL registered: users (allowlist mode on POST /graphql) - -# Unknown catalog table rejected at CREATE -statement error -CREATE GRAPHQL FOR TABLE no_such_table; ----- -not found in schema main - -# DROP one table; allowlist still active while non-empty -query I -DROP GRAPHQL FOR TABLE posts; ----- -GraphQL unregistered: posts - -query TT -SELECT mode, table_name FROM quackapi_graphql_tables(); ----- -allowlist users - -# DROP last table → closed mode again -query I -DROP GRAPHQL FOR TABLE users; ----- -GraphQL unregistered: users (allowlist empty → catalog closed) - -query I -SELECT mode FROM quackapi_graphql_tables(); ----- -disabled - -# secrets remains unavailable after the final registration is dropped -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ secrets { id } }"}' -); ----- -200 {"errors":[{"message":"table 'secrets' is not registered for GraphQL — CREATE GRAPHQL FOR TABLE secrets"}]} - -# DROP GRAPHQL ALL clears even a multi-table allowlist -query I -CREATE GRAPHQL FOR TABLE users, posts, secrets; ----- -GraphQL registered: users, posts, secrets (allowlist mode on POST /graphql) - -query I -DROP GRAPHQL ALL; ----- -GraphQL allowlist cleared (catalog closed) - -query I -SELECT mode FROM quackapi_graphql_tables(); ----- -disabled - -# DROP missing registration fails when none match -statement error -DROP GRAPHQL FOR TABLE users; ----- -no matching registered tables - -# --------------------------------------------------------------------------- -# CREATE GRAPHQL ROUTE — named mounts, independent of global allowlist -# --------------------------------------------------------------------------- -statement ok -CREATE TABLE multi AS SELECT * FROM (VALUES (1),(2),(3)) t(id); - -query I -CREATE GRAPHQL ROUTE public_api POST '/api/gql' FROM users, posts LIMIT 2; ----- -GraphQL route public_api: POST /api/gql FROM users, posts LIMIT 2 - -query IIIII -SELECT name, method, path, require_auth, "limit" FROM quackapi_graphql_routes(); ----- -public_api POST /api/gql NULL 2 - -# Route serves only FROM tables -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ users { id name } }"}' -); ----- -200 {"data":{"users":[{"id":1,"name":"ada"}]}} - -# secrets is not on the named route and remains closed on /graphql -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ secrets { id } }"}' -); ----- -200 {"errors":[{"message":"table 'secrets' is not on this GraphQL route — CREATE GRAPHQL ROUTE … FROM secrets"}]} - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ secrets { id } }"}' -); ----- -200 {"errors":[{"message":"table 'secrets' is not registered for GraphQL — CREATE GRAPHQL FOR TABLE secrets"}]} - -# Global allowlist does not shrink the route's FROM set -query I -CREATE GRAPHQL FOR TABLE users; ----- -GraphQL registered: users (allowlist mode on POST /graphql) - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ posts { id title } }"}' -); ----- -200 {"data":{"posts":[{"id":10,"title":"hello"}]}} - -query I -DROP GRAPHQL ALL; ----- -GraphQL allowlist cleared (catalog closed) - -# Schema for the route -query II -SELECT status, decode(body) FROM quackapi_request('GET', '/api/gql/schema'); ----- -200 {"mode":"route","route":"public_api","tables":{"posts":["id","title"],"users":["id","name"]},"note":"GraphQL v0 catalog-only schema. Full __schema introspection not implemented. CREATE GRAPHQL FOR TABLE registers an allowlist for POST /graphql; empty = closed. SET GLOBAL quackapi_graphql_allow_all = true restores legacy open catalog mode. CREATE GRAPHQL ROUTE mounts named paths with per-route tables."} - -# LIMIT honored on multi-row table (route with LIMIT 1) -query I -CREATE OR REPLACE GRAPHQL ROUTE public_api POST '/api/gql' FROM multi LIMIT 1; ----- -GraphQL route public_api: POST /api/gql FROM multi LIMIT 1 - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ multi { id } }"}' -); ----- -200 {"data":{"multi":[{"id":1}]}} - -# Reserved path rejected -statement error -CREATE GRAPHQL ROUTE bad POST '/graphql' FROM users; ----- -reserved - -# REQUIRE auth -query I -CREATE AUTH gql_site AS API_KEY; ----- -Auth gql_site: API_KEY - -query I -SELECT * FROM quackapi_add_api_key('gql_site', 'gql-secret-key', 'bob'); ----- -bob - -query I -CREATE OR REPLACE GRAPHQL ROUTE public_api POST '/api/gql' FROM users REQUIRE gql_site; ----- -GraphQL route public_api: POST /api/gql FROM users REQUIRE gql_site LIMIT 100 - -query I -SELECT status FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ users { id } }"}' -); ----- -401 - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ users { id } }"}', - headers := MAP {'X-API-Key': 'gql-secret-key'} -); ----- -200 {"data":{"users":[{"id":1}]}} - -# DROP route → 404 on path; built-in remains closed until explicitly registered -query I -DROP GRAPHQL ROUTE public_api; ----- -Dropped GraphQL route public_api - -query I -SELECT status FROM quackapi_request( - 'POST', '/api/gql', - '{"query":"{ users { id } }"}' -); ----- -404 - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ users { id } }"}' -); ----- -200 {"errors":[{"message":"table 'users' is not registered for GraphQL — CREATE GRAPHQL FOR TABLE users"}]} - -query I -CREATE GRAPHQL FOR TABLE users; ----- -GraphQL registered: users (allowlist mode on POST /graphql) - -query II -SELECT status, decode(body) FROM quackapi_request( - 'POST', '/graphql', - '{"query":"{ users { id } }"}' -); ----- -200 {"data":{"users":[{"id":1}]}} - -query I -SELECT count(*) FROM quackapi_graphql_routes(); ----- -0 diff --git a/test/sql/quackapi_lifecycle.test b/test/sql/quackapi_lifecycle.test index d196d94..e097ecd 100644 --- a/test/sql/quackapi_lifecycle.test +++ b/test/sql/quackapi_lifecycle.test @@ -118,56 +118,6 @@ DROP ROUTE keeper; statement ok DROP ROUTE mover; -# --------------------------------------------------------------------------- -# Bug: CREATE GRAPHQL FOR TABLE a, b half-applied when a later name failed -# (missing table or duplicate) — earlier tables stayed on the allowlist. -# --------------------------------------------------------------------------- -statement ok -CREATE TABLE users AS SELECT 1 AS id; - -statement ok -CREATE TABLE posts AS SELECT 1 AS id; - -statement error -CREATE GRAPHQL FOR TABLE users, no_such_table; ----- -not found - -query I -SELECT count(*) FROM quackapi_graphql_tables() WHERE table_name = 'users'; ----- -0 - -statement ok -CREATE GRAPHQL FOR TABLE users; - -statement error -CREATE GRAPHQL FOR TABLE posts, users; ----- -already registered - -query I -SELECT count(*) FROM quackapi_graphql_tables() WHERE table_name = 'posts'; ----- -0 - -statement ok -DROP GRAPHQL FOR TABLE users; - -# --------------------------------------------------------------------------- -# Bug: CREATE API FOR TABLE skipped handler validation — missing table still -# registered list/get routes that 500'd on first request. -# --------------------------------------------------------------------------- -statement error -CREATE API FOR TABLE no_such_table; ----- -Invalid handler SQL - -query I -SELECT count(*) FROM quackapi_routes() WHERE name IN ('no_such_table_list', 'no_such_table_get'); ----- -0 - # --------------------------------------------------------------------------- # Bug: serve(health_routes=true) → stop → serve(health_routes=false) left # __quackapi_health* in the registry (RegisterQuackapiHealthRoutes no-op). diff --git a/test/sql/quackapi_logging_defaults.test b/test/sql/quackapi_logging_defaults.test deleted file mode 100644 index 1ca1864..0000000 --- a/test/sql/quackapi_logging_defaults.test +++ /dev/null @@ -1,62 +0,0 @@ -# name: test/sql/quackapi_logging_defaults.test -# description: quackapi_serve preserves prior DuckDB logging unless explicitly disabled -# group: [sql] - -require quackapi - -# An operator-enabled logger survives the serve defaults. -statement ok -CALL enable_logging(storage := 'memory'); - -query I -SELECT * FROM quackapi_serve(29501, host := '127.0.0.1'); ----- -http://127.0.0.1:29501 - -query I -SELECT current_setting('enable_logging'); ----- -1 - -query I -SELECT * FROM quackapi_stop(29501); ----- -Stopped quackapi server on port 29501 - -# Without prior logging, the serve default remains off. -statement ok -SET enable_logging = false; - -query I -SELECT * FROM quackapi_serve(29502, host := '127.0.0.1'); ----- -http://127.0.0.1:29502 - -query I -SELECT current_setting('enable_logging'); ----- -0 - -query I -SELECT * FROM quackapi_stop(29502); ----- -Stopped quackapi server on port 29502 - -# An explicit false still overrides an operator-enabled logger. -statement ok -CALL enable_logging(storage := 'memory'); - -query I -SELECT * FROM quackapi_serve(29503, host := '127.0.0.1', enable_logging := false); ----- -http://127.0.0.1:29503 - -query I -SELECT current_setting('enable_logging'); ----- -0 - -query I -SELECT * FROM quackapi_stop(29503); ----- -Stopped quackapi server on port 29503 diff --git a/test/sql/quackapi_memory_limit.test b/test/sql/quackapi_memory_limit.test deleted file mode 100644 index 4ac38b2..0000000 --- a/test/sql/quackapi_memory_limit.test +++ /dev/null @@ -1,54 +0,0 @@ -# name: test/sql/quackapi_memory_limit.test -# description: memory_limit knob — named param and SET quackapi_memory_limit reach DuckDB via serve -# group: [sql] - -require quackapi - -# Default: empty string (non-clobber / safe-default logic) -query I -SELECT current_setting('quackapi_memory_limit'); ----- -(empty) - -# --------------------------------------------------------------------------- -# Named param wins and applies into DuckDB memory_limit -# --------------------------------------------------------------------------- -query I -SELECT * FROM quackapi_serve(29102, host := '127.0.0.1', memory_limit := '4GB'); ----- -http://127.0.0.1:29102 - -query I -SELECT current_setting('memory_limit'); ----- -3.7 GiB - -query I -SELECT * FROM quackapi_stop(29102); ----- -Stopped quackapi server on port 29102 - -# --------------------------------------------------------------------------- -# Extension setting applies when named param omitted -# --------------------------------------------------------------------------- -statement ok -SET quackapi_memory_limit = '512MB'; - -query I -SELECT * FROM quackapi_serve(29103, host := '127.0.0.1'); ----- -http://127.0.0.1:29103 - -query I -SELECT current_setting('memory_limit'); ----- -488.2 MiB - -query I -SELECT * FROM quackapi_stop(29103); ----- -Stopped quackapi server on port 29103 - -# Cleanup setting -statement ok -SET quackapi_memory_limit = ''; diff --git a/test/sql/quackapi_routes.test b/test/sql/quackapi_routes.test index 22fd5e4..c1ceaa9 100644 --- a/test/sql/quackapi_routes.test +++ b/test/sql/quackapi_routes.test @@ -178,7 +178,7 @@ Dropped route spaced statement error CREATE ROUTE bad_fmt GET '/x' FORMAT protobuf AS SELECT 1; ---- -FORMAT must be json, ndjson, csv, parquet, or arrow +FORMAT must be json, ndjson, or csv # Handler SQL is validated at CREATE time (not deferred to first request) statement error diff --git a/test/ultra/routes.sql b/test/ultra/routes.sql index 4cc32d9..4c65985 100644 --- a/test/ultra/routes.sql +++ b/test/ultra/routes.sql @@ -93,9 +93,6 @@ CREATE ROUTE matrix_small GET '/matrix/small' CREATE ROUTE matrix_ndjson GET '/matrix/ndjson' FORMAT ndjson AS SELECT id, name, price, tags FROM matrix_items ORDER BY id; -CREATE STREAM matrix_stream GET '/matrix/stream' - AS SELECT i AS id FROM range(3) t(i); - CREATE ROUTE matrix_csv GET '/matrix/csv' FORMAT csv AS SELECT id, name FROM matrix_items WHERE id <= 2 ORDER BY id; diff --git a/test/ultra/serve_quack.sh b/test/ultra/serve_quack.sh index 4b80277..bc3e71a 100755 --- a/test/ultra/serve_quack.sh +++ b/test/ultra/serve_quack.sh @@ -35,7 +35,7 @@ exec 3>"$FIFO" printf "LOAD '%s';\n" "${EXT//\'/\'\'}" >&3 cat "${ROOT}/test/ultra/routes.sql" >&3 -printf "SELECT * FROM quackapi_serve(%s, host := '%s', health_routes := false, access_log := false, enable_logging := false, compression := 'auto', cors_origins := 'http://example.test');\n" "$PORT" "$HOST" >&3 +printf "SELECT * FROM quackapi_serve(%s, host := '%s', health_routes := false, access_log := false, compression := 'auto', cors_origins := 'http://example.test');\n" "$PORT" "$HOST" >&3 ready=0 for _ in $(seq 1 120); do From a74c60a8d18e5a82f8d46d79fbe5dded9a71eb28 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:20:27 -0700 Subject: [PATCH 19/93] fix(encoding): bind json functions in transaction scope --- src/quackapi_response_encoding.cpp | 67 ++++++++++++------------- test/sql/quackapi_access_log.test | 2 + test/sql/quackapi_bug_state.test | 2 + test/sql/quackapi_cache_key.test | 2 + test/sql/quackapi_format.test | 2 + test/sql/quackapi_lifecycle.test | 2 + test/sql/quackapi_middleware.test | 2 + test/sql/quackapi_rate_limit.test | 2 + test/sql/quackapi_request.test | 2 + test/sql/quackapi_request_id.test | 2 + test/sql/quackapi_resource_limits.test | 2 + test/sql/quackapi_route_errors.test | 2 + test/sql/quackapi_schedule_cronjob.test | 2 + test/sql/quackapi_telemetry_record.test | 2 + test/sql/quackapi_telemetry_redact.test | 2 + 15 files changed, 59 insertions(+), 36 deletions(-) diff --git a/src/quackapi_response_encoding.cpp b/src/quackapi_response_encoding.cpp index a033d13..fe14e76 100644 --- a/src/quackapi_response_encoding.cpp +++ b/src/quackapi_response_encoding.cpp @@ -103,10 +103,8 @@ class QuackapiJsonChunkEncoder::State { } // json_object supplies DuckDB's own recursive serializer for every column // type (including MAP, STRUCT, LIST, and JSON), rather than Value::ToString. - auto load = con.Query("LOAD json"); - if (load->HasError()) { - throw InvalidInputException("JSON encoder: could not LOAD json: %s", load->GetError()); - } + // The JSON extension is a declared runtime dependency; don't LOAD it while + // a route result is live. vector> args; for (auto col : data_cols) { if (col >= names.size()) { @@ -115,13 +113,15 @@ class QuackapiJsonChunkEncoder::State { args.push_back(make_uniq(Value(names[col]))); args.push_back(make_uniq(types[col], UnsafeNumericCast(col))); } - FunctionBinder binder(*con.context); - ErrorData error; - expression = binder.BindScalarFunction(DEFAULT_SCHEMA, "json_object", std::move(args), error); - if (!expression) { - throw InvalidInputException("JSON encoder: could not bind json_object: %s", error.Message()); - } - executor = make_uniq(*con.context, *expression); + con.context->RunFunctionInTransaction([&]() { + FunctionBinder binder(*con.context); + ErrorData error; + expression = binder.BindScalarFunction(DEFAULT_SCHEMA, "json_object", std::move(args), error); + if (!expression) { + throw InvalidInputException("JSON encoder: could not bind json_object: %s", error.Message()); + } + executor = make_uniq(*con.context, *expression); + }); if (mode == JsonRowMode::ARRAY) { writer.Append("["); } @@ -217,38 +217,33 @@ class QuackapiCsvChunkEncoder::State { if (names.size() != types.size()) { throw InvalidInputException("CSV encoder: result name/type count mismatch"); } - bool needs_json = false; for (auto col : data_cols) { if (col >= names.size()) { throw InvalidInputException("CSV encoder: data column index is out of bounds"); } - needs_json = needs_json || IsNestedCsvCell(types[col]); - } - if (needs_json) { - auto load = con.Query("LOAD json"); - if (load->HasError()) { - throw InvalidInputException("CSV encoder: could not LOAD json: %s", load->GetError()); - } } - for (auto col : data_cols) { - unique_ptr cell = - make_uniq(types[col], UnsafeNumericCast(col)); - if (IsNestedCsvCell(types[col])) { - vector> args; - args.push_back(std::move(cell)); - FunctionBinder binder(*con.context); - ErrorData error; - cell = binder.BindScalarFunction(DEFAULT_SCHEMA, "to_json", std::move(args), error); - if (!cell) { - throw InvalidInputException("CSV encoder: could not bind to_json: %s", error.Message()); + con.context->RunFunctionInTransaction([&]() { + for (auto col : data_cols) { + unique_ptr cell = + make_uniq(types[col], UnsafeNumericCast(col)); + if (IsNestedCsvCell(types[col])) { + vector> args; + args.push_back(std::move(cell)); + FunctionBinder binder(*con.context); + ErrorData error; + cell = binder.BindScalarFunction(DEFAULT_SCHEMA, "to_json", std::move(args), error); + if (!cell) { + throw InvalidInputException("CSV encoder: could not bind to_json: %s", error.Message()); + } } + casts.push_back( + BoundCastExpression::AddCastToType(*con.context, std::move(cell), LogicalType::VARCHAR)); + output_types.push_back(LogicalType::VARCHAR); } - casts.push_back(BoundCastExpression::AddCastToType(*con.context, std::move(cell), LogicalType::VARCHAR)); - output_types.push_back(LogicalType::VARCHAR); - } - if (!casts.empty()) { - executor = make_uniq(*con.context, casts); - } + if (!casts.empty()) { + executor = make_uniq(*con.context, casts); + } + }); writer.options.null_str = {""}; writer.options.dialect_options.header = CSVOption(true, true); writer.writer_options.newline_writing_mode = CSVNewLineMode::WRITE_AFTER; diff --git a/test/sql/quackapi_access_log.test b/test/sql/quackapi_access_log.test index d45a101..ebb53f4 100644 --- a/test/sql/quackapi_access_log.test +++ b/test/sql/quackapi_access_log.test @@ -4,6 +4,8 @@ require quackapi +require json + statement ok CREATE TABLE access_log ( request_id VARCHAR, diff --git a/test/sql/quackapi_bug_state.test b/test/sql/quackapi_bug_state.test index a0a7aa0..733f8b3 100644 --- a/test/sql/quackapi_bug_state.test +++ b/test/sql/quackapi_bug_state.test @@ -16,6 +16,8 @@ require quackapi +require json + # --------------------------------------------------------------------------- # 1) Thread-local static JSON body cache (ThreadRequestCache::static_json_bodies) # Zero-param handlers must not freeze the first serialized body forever. diff --git a/test/sql/quackapi_cache_key.test b/test/sql/quackapi_cache_key.test index a878bc6..6bf6bcb 100644 --- a/test/sql/quackapi_cache_key.test +++ b/test/sql/quackapi_cache_key.test @@ -4,6 +4,8 @@ require quackapi +require json + # Keep quackapi_request on one worker thread so TLS state is shared across calls. statement ok SET threads = 1; diff --git a/test/sql/quackapi_format.test b/test/sql/quackapi_format.test index 8d97463..d6456a6 100644 --- a/test/sql/quackapi_format.test +++ b/test/sql/quackapi_format.test @@ -4,6 +4,8 @@ require quackapi +require json + statement ok CREATE OR REPLACE ROUTE items GET '/items' AS SELECT 1 AS id, 'alice' AS name UNION ALL SELECT 2, 'bob'; diff --git a/test/sql/quackapi_lifecycle.test b/test/sql/quackapi_lifecycle.test index e097ecd..04f9676 100644 --- a/test/sql/quackapi_lifecycle.test +++ b/test/sql/quackapi_lifecycle.test @@ -4,6 +4,8 @@ require quackapi +require json + # --------------------------------------------------------------------------- # Bug: CREATE OR REPLACE AUTH API_KEY → JWT leaves hashed keys that resurrect # when replaced back to API_KEY (DropAuth clears keys; AddAuth did not). diff --git a/test/sql/quackapi_middleware.test b/test/sql/quackapi_middleware.test index 38ca5d4..73a4ccf 100644 --- a/test/sql/quackapi_middleware.test +++ b/test/sql/quackapi_middleware.test @@ -4,6 +4,8 @@ require quackapi +require json + # --------------------------------------------------------------------------- # Definitions are introspectable and OR REPLACE preserves creation order. # --------------------------------------------------------------------------- diff --git a/test/sql/quackapi_rate_limit.test b/test/sql/quackapi_rate_limit.test index 9feeae4..93428d7 100644 --- a/test/sql/quackapi_rate_limit.test +++ b/test/sql/quackapi_rate_limit.test @@ -4,6 +4,8 @@ require quackapi +require json + # --- DDL registration --- statement ok CREATE ROUTE rl1 GET '/rl1' RATE LIMIT 5 PER 10 AS SELECT 1 AS x; diff --git a/test/sql/quackapi_request.test b/test/sql/quackapi_request.test index e1fefe8..dcff317 100644 --- a/test/sql/quackapi_request.test +++ b/test/sql/quackapi_request.test @@ -4,6 +4,8 @@ require quackapi +require json + # --------------------------------------------------------------------------- # CREATE routes (registry only — no quackapi_serve / no port) # --------------------------------------------------------------------------- diff --git a/test/sql/quackapi_request_id.test b/test/sql/quackapi_request_id.test index 64a4817..2aff6c3 100644 --- a/test/sql/quackapi_request_id.test +++ b/test/sql/quackapi_request_id.test @@ -4,6 +4,8 @@ require quackapi +require json + statement ok CREATE OR REPLACE ROUTE echo_rid GET '/echo-rid' AS SELECT $request_id::VARCHAR AS id; diff --git a/test/sql/quackapi_resource_limits.test b/test/sql/quackapi_resource_limits.test index e181f72..7926807 100644 --- a/test/sql/quackapi_resource_limits.test +++ b/test/sql/quackapi_resource_limits.test @@ -4,6 +4,8 @@ require quackapi +require json + statement ok CREATE ROUTE resource_large GET '/resource-large' AS SELECT repeat('x', 4096) AS text; diff --git a/test/sql/quackapi_route_errors.test b/test/sql/quackapi_route_errors.test index 89e8f5b..dd07801 100644 --- a/test/sql/quackapi_route_errors.test +++ b/test/sql/quackapi_route_errors.test @@ -4,6 +4,8 @@ require quackapi +require json + query I CREATE OR REPLACE ROUTE sql POST '/sql' AS SELECT * FROM query($sql); ---- diff --git a/test/sql/quackapi_schedule_cronjob.test b/test/sql/quackapi_schedule_cronjob.test index 1d0a7b2..9626c24 100644 --- a/test/sql/quackapi_schedule_cronjob.test +++ b/test/sql/quackapi_schedule_cronjob.test @@ -4,6 +4,8 @@ require quackapi +require json + require cronjob statement ok diff --git a/test/sql/quackapi_telemetry_record.test b/test/sql/quackapi_telemetry_record.test index ae698d1..32638c5 100644 --- a/test/sql/quackapi_telemetry_record.test +++ b/test/sql/quackapi_telemetry_record.test @@ -4,6 +4,8 @@ require quackapi +require json + statement ok CREATE TABLE telemetry_log ( request_id VARCHAR, diff --git a/test/sql/quackapi_telemetry_redact.test b/test/sql/quackapi_telemetry_redact.test index d5435e2..f684071 100644 --- a/test/sql/quackapi_telemetry_redact.test +++ b/test/sql/quackapi_telemetry_redact.test @@ -4,6 +4,8 @@ require quackapi +require json + statement ok CREATE TABLE redaction_log ( path VARCHAR, From 89f55562ac4fd571c96c7a0add85f53be890db82 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:24:48 -0700 Subject: [PATCH 20/93] Remove fetch API and isolate POST result contract --- CMakeLists.txt | 2 +- src/include/quackapi_http_fetch.hpp | 95 --- src/include/quackapi_post_executor.hpp | 2 +- src/include/quackapi_post_provider.hpp | 2 +- src/include/quackapi_post_result.hpp | 36 + src/include/quackapi_state.hpp | 12 - src/include/quackapi_thread_join.hpp | 19 - src/quackapi_extension.cpp | 44 +- src/quackapi_http_fetch.cpp | 910 ------------------------ src/quackapi_post.cpp | 1 + src/quackapi_post_result.cpp | 52 ++ src/quackapi_registry.cpp | 7 +- test/cpp/quackapi_thread_join_test.cpp | 29 - test/sql/quackapi_access_log.test | 24 +- test/sql/quackapi_batteries.test | 33 - test/sql/quackapi_http_pool.test | 136 ---- test/sql/quackapi_lifecycle.test | 26 - test/sql/quackapi_requests_ring.test | 34 +- test/sql/quackapi_route_timeout.test | 97 +-- test/sql/quackapi_telemetry_record.test | 4 +- test/sql/quackapi_telemetry_status.test | 13 +- 21 files changed, 130 insertions(+), 1448 deletions(-) delete mode 100644 src/include/quackapi_http_fetch.hpp create mode 100644 src/include/quackapi_post_result.hpp delete mode 100644 src/include/quackapi_thread_join.hpp delete mode 100644 src/quackapi_http_fetch.cpp create mode 100644 src/quackapi_post_result.cpp delete mode 100644 test/cpp/quackapi_thread_join_test.cpp delete mode 100644 test/sql/quackapi_batteries.test delete mode 100644 test/sql/quackapi_http_pool.test diff --git a/CMakeLists.txt b/CMakeLists.txt index fa8cef0..94dbd8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ set(EXTENSION_SOURCES src/quackapi_batteries.cpp src/quackapi_ddl.cpp src/quackapi_auth.cpp - src/quackapi_http_fetch.cpp + src/quackapi_post_result.cpp src/quackapi_post.cpp src/quackapi_post_executor.cpp src/quackapi_post_provider.cpp diff --git a/src/include/quackapi_http_fetch.hpp b/src/include/quackapi_http_fetch.hpp deleted file mode 100644 index a371ce9..0000000 --- a/src/include/quackapi_http_fetch.hpp +++ /dev/null @@ -1,95 +0,0 @@ -//===----------------------------------------------------------------------===// -// quackapi_http_fetch.hpp -// -// Outbound HTTP for quackapi. Two paths, one rule: the TCP connection MUST -// survive across requests. -// -// 1. http:// → the VENDORED httplib client, checked out of a per-host -// free-list pool (QuackapiHttpPool). Keep-alive + TCP_NODELAY. -// Full method support, no companion extension required. -// 2. https:// → DuckDB's HTTPUtil, so `LOAD curl_httpfs` / `LOAD httpfs` -// transparently supplies TLS. Pooled the same way, via -// HTTPUtil::Request(request, client) which reuses the client -// we hand it instead of building a fresh one. -// -// Why the pool exists: HTTPUtil::Request(request) — the single-argument form — -// declares a local `unique_ptr client;` and lets it die at the end -// of the call (duckdb/src/main/http/http_util.cpp:128). Every request therefore -// pays a fresh DNS + TCP + (TLS) handshake. The community `http_client` -// extension has the same shape, which is why a route calling http_post() once -// per request measured ~40ms against an upstream whose own floor was 13ms, -// flat across worker_threads — while ten calls inside ONE query amortised to -// 9.2ms. Connections were only ever reused WITHIN a query execution. A route -// handler makes exactly one call per request: the worst case. -// -// Never shell out to the curl binary. Never link libcurl into quackapi. -//===----------------------------------------------------------------------===// -#pragma once - -#include "duckdb/common/http_util.hpp" -#include "duckdb/common/string.hpp" -#include "duckdb/common/types.hpp" -#include "duckdb/common/unordered_map.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/main/extension/extension_loader.hpp" - -namespace duckdb { - -//! Result of an in-database outbound HTTP call. -struct QuackapiHttpFetchResult { - HTTPStatusCode status = HTTPStatusCode::INVALID; - string body; - string reason; - bool success = false; - //! Empty unless the transport failed before a response (DNS, connect, …). - string request_error; - HTTPHeaders headers; - //! True when this call reused a pooled connection rather than dialling. - bool reused_connection = false; - - bool Ok() const { - return success && request_error.empty(); - } -}; - -//! Snapshot of the outbound connection pool, exposed as quackapi_http_pool(). -struct QuackapiHttpPoolStats { - string host; - idx_t idle = 0; - idx_t dialed = 0; - idx_t reused = 0; -}; - -struct QuackapiHttpFetch { - //! Active util name: "MultiCurl", "HTTPFS-Curl", "HTTPFS", "Built-In", … - static string ActiveHttpUtilName(DatabaseInstance &db); - - //! GET url. headers are optional extra request headers (e.g. Authorization). - //! stall_ms > 0 (plain http:// only): after sending the request, wait that - //! many milliseconds before reading the response. Test seam for server - //! write-timeout / per-route timeout_sec — a compute-bound handler cannot - //! trip socket IO deadlines. - static QuackapiHttpFetchResult Get(DatabaseInstance &db, const string &url, - const unordered_map &extra_headers = {}, int32_t stall_ms = 0); - - //! POST url with a raw body and Content-Type. - //! http:// needs nothing loaded; https:// requires httpfs / curl_httpfs. - static QuackapiHttpFetchResult Post(DatabaseInstance &db, const string &url, const string &body, - const string &content_type = "application/x-www-form-urlencoded", - const unordered_map &extra_headers = {}); - - //! Per-host pool counters, for tests and quackapi_http_pool(). - static vector PoolStats(); - //! Drop every idle connection (test isolation; not needed in production). - static void ResetPool(); -}; - -//! Shared SQL conversion helpers for the fetch and POST scalar functions. -LogicalType QuackapiHttpFetchResultType(); -Value QuackapiHttpFetchResultValue(const QuackapiHttpFetchResult &result); -unordered_map QuackapiHttpHeadersFromValue(const Value &value); - -//! quackapi_fetch / quackapi_post / quackapi_http_pool / quackapi_parallel_fetch. -void RegisterQuackapiHttpFetchFunctions(ExtensionLoader &loader); - -} // namespace duckdb diff --git a/src/include/quackapi_post_executor.hpp b/src/include/quackapi_post_executor.hpp index 201381e..20b08e9 100644 --- a/src/include/quackapi_post_executor.hpp +++ b/src/include/quackapi_post_executor.hpp @@ -5,7 +5,7 @@ //===----------------------------------------------------------------------===// #pragma once -#include "quackapi_http_fetch.hpp" +#include "quackapi_post_result.hpp" #include "duckdb/storage/object_cache.hpp" #include diff --git a/src/include/quackapi_post_provider.hpp b/src/include/quackapi_post_provider.hpp index a5a347f..7ea5fb1 100644 --- a/src/include/quackapi_post_provider.hpp +++ b/src/include/quackapi_post_provider.hpp @@ -6,7 +6,7 @@ //===----------------------------------------------------------------------===// #pragma once -#include "quackapi_http_fetch.hpp" +#include "quackapi_post_result.hpp" namespace duckdb { diff --git a/src/include/quackapi_post_result.hpp b/src/include/quackapi_post_result.hpp new file mode 100644 index 0000000..74ae80b --- /dev/null +++ b/src/include/quackapi_post_result.hpp @@ -0,0 +1,36 @@ +//===----------------------------------------------------------------------===// +// quackapi_post_result.hpp +// +// Shared response type and SQL conversion helpers for quackapi_post. +//===----------------------------------------------------------------------===// +#pragma once + +#include "duckdb/common/http_util.hpp" +#include "duckdb/common/string.hpp" +#include "duckdb/common/types.hpp" +#include "duckdb/common/unordered_map.hpp" + +namespace duckdb { + +//! Result of an outbound POST call. +struct QuackapiHttpFetchResult { + HTTPStatusCode status = HTTPStatusCode::INVALID; + string body; + string reason; + bool success = false; + //! Empty unless the transport failed before a response (DNS, connect, …). + string request_error; + HTTPHeaders headers; + //! True when this call reused a provider connection. + bool reused_connection = false; + + bool Ok() const { + return success && request_error.empty(); + } +}; + +LogicalType QuackapiHttpFetchResultType(); +Value QuackapiHttpFetchResultValue(const QuackapiHttpFetchResult &result); +unordered_map QuackapiHttpHeadersFromValue(const Value &value); + +} // namespace duckdb diff --git a/src/include/quackapi_state.hpp b/src/include/quackapi_state.hpp index 7c790e8..7071360 100644 --- a/src/include/quackapi_state.hpp +++ b/src/include/quackapi_state.hpp @@ -216,21 +216,9 @@ class QuackapiState : public ObjectCacheEntry { //! A port is required when more than one server is running. bool FlushTelemetry(int port = 0); - //! Effective write deadline (seconds) applied to the most recent route - //! request. Seeded with serve write_timeout_sec; ApplyRouteIoTimeout - //! overwrites it only when it actually extends the socket deadlines. - //! Test/introspection seam — not a registry field. - void SetLastEffectiveWriteTimeoutSec(int32_t sec) { - last_effective_write_timeout_sec.store(sec, std::memory_order_relaxed); - } - int32_t GetLastEffectiveWriteTimeoutSec() const { - return last_effective_write_timeout_sec.load(std::memory_order_relaxed); - } - private: void PublishRoutes(); - std::atomic last_effective_write_timeout_sec {0}; struct RateLimitEntry { int count = 0; std::chrono::steady_clock::time_point expires; diff --git a/src/include/quackapi_thread_join.hpp b/src/include/quackapi_thread_join.hpp deleted file mode 100644 index c47d233..0000000 --- a/src/include/quackapi_thread_join.hpp +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -namespace duckdb { - -// Join before captured state is destroyed, including when a later thread fails -// to start. Destroying an already-joinable std::thread would terminate the host. -template -struct JoinOutboundThreads { - THREADS &threads; - ~JoinOutboundThreads() { - for (auto &thread : threads) { - if (thread.joinable()) { - thread.join(); - } - } - } -}; - -} // namespace duckdb diff --git a/src/quackapi_extension.cpp b/src/quackapi_extension.cpp index fedb1bb..7505bad 100644 --- a/src/quackapi_extension.cpp +++ b/src/quackapi_extension.cpp @@ -16,12 +16,13 @@ #include "duckdb/main/connection.hpp" #include "duckdb/main/database.hpp" #include "duckdb/main/extension/extension_loader.hpp" +#include "duckdb/main/extension_helper.hpp" #include "duckdb/main/extension_callback_manager.hpp" #include "duckdb/main/settings.hpp" #include "quackapi_auth.hpp" #include "quackapi_ddl.hpp" -#include "quackapi_http_fetch.hpp" +#include "quackapi_post.hpp" #include "quackapi_server.hpp" #include "quackapi_state.hpp" @@ -1002,29 +1003,6 @@ static void TelemetryFlushExec(ClientContext &context, TableFunctionInput &data_ bind_data.finished = true; } -//===--------------------------------------------------------------------===// -// quackapi_http_util_name() — active outbound HTTPUtil (no curl_httpfs dep) -//===--------------------------------------------------------------------===// -// When curl_httpfs is LOADed this is typically "MultiCurl". Same underlying -// DBConfig::GetHTTPUtil().GetName() that curl_httpfs_http_util_name() reads. - -static void HttpUtilNameFunction(DataChunk &, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - result.Reference(Value(QuackapiHttpFetch::ActiveHttpUtilName(db))); -} - -//===--------------------------------------------------------------------===// -// quackapi_last_write_timeout_sec() — effective write deadline of last request -//===--------------------------------------------------------------------===// -// Reports what HandleRequest / ApplyRouteIoTimeout actually applied — serve -// write_timeout_sec when no route override ran, or the route timeout_sec when -// ApplyRouteIoTimeout extended the socket deadlines. Not a registry lookup. - -static void LastWriteTimeoutSecFunction(DataChunk &, ExpressionState &state, Vector &result) { - auto &qa = QuackapiState::Get(*state.GetContext().db); - result.Reference(Value::INTEGER(qa.GetLastEffectiveWriteTimeoutSec())); -} - //===--------------------------------------------------------------------===// // Load //===--------------------------------------------------------------------===// @@ -1033,6 +1011,10 @@ static void LoadInternal(ExtensionLoader &loader) { // SET quackapi_cors_origins = '*' | 'https://app.example,https://admin.example' // Default empty = CORS off. Serve-time cors_origins := '…' overrides this SET. auto &db = loader.GetDatabaseInstance(); + if (!ExtensionHelper::TryAutoLoadAvailableExtension(db, "json")) { + throw IOException("QuackAPI requires the DuckDB JSON extension to be available and loadable before loading " + "quackapi"); + } auto &config = DBConfig::GetConfig(db); config.AddExtensionOption("quackapi_cors_origins", "CORS allowed origins for quackapi_serve (* or comma-separated list). " @@ -1188,19 +1170,7 @@ static void LoadInternal(ExtensionLoader &loader) { // quack_nop_authorization so they can be SET as quack_*_function. RegisterQuackAuthBridgeFunctions(loader); - // Outbound client diagnostic — works with Built-In, HTTPFS, MultiCurl, … - // Does NOT auto-LOAD curl_httpfs; missing companion must not fail LOAD quackapi. - loader.RegisterFunction(ScalarFunction("quackapi_http_util_name", {}, LogicalType::VARCHAR, HttpUtilNameFunction)); - - // Effective write deadline applied to the most recent route request - // (ApplyRouteIoTimeout / serve write_timeout_sec). Test/introspection seam. - loader.RegisterFunction( - ScalarFunction("quackapi_last_write_timeout_sec", {}, LogicalType::INTEGER, LastWriteTimeoutSecFunction)); - - // Outbound HTTP with a connection pool that survives ACROSS requests: - // quackapi_fetch / quackapi_post / quackapi_http_pool. See the header of - // quackapi_http_fetch.hpp for why the un-pooled path costs ~27ms a call. - RegisterQuackapiHttpFetchFunctions(loader); + RegisterQuackapiPostFunctions(loader); loader.RegisterFunction(GetApplyMiddlewareFunction()); loader.RegisterFunction(GetQuackapiMiddlewaresFunction()); diff --git a/src/quackapi_http_fetch.cpp b/src/quackapi_http_fetch.cpp deleted file mode 100644 index 6e6e53f..0000000 --- a/src/quackapi_http_fetch.cpp +++ /dev/null @@ -1,910 +0,0 @@ -#include "quackapi_http_fetch.hpp" -#include "quackapi_post.hpp" -#include "quackapi_thread_join.hpp" -#include "quackapi_limits.hpp" - -#include -#include -#include -#include - -#include "duckdb/common/exception.hpp" -#include "duckdb/common/exception/http_exception.hpp" -#include "duckdb/common/string_util.hpp" -#include "duckdb/function/scalar_function.hpp" -#include "duckdb/function/table_function.hpp" -#include "duckdb/main/config.hpp" -#include "duckdb/main/client_context.hpp" - -#include "httplib.hpp" - -namespace duckdb { - -namespace { - -constexpr idx_t MAX_IDLE_PER_HOST = 64; -// Concurrency cap when one chunk carries many independent calls. Kept at or below -// MAX_IDLE_PER_HOST so a fan-out cannot dial more sockets than the pool will keep, -// which would turn every burst into a churn of one-shot connections. -constexpr idx_t MAX_OUTBOUND_FANOUT = 32; -// Read/write: LLM upstreams are slow; the caller cancels. Connect stays short so -// a dead peer (or CI loopback after stop) fails fast instead of parking a worker. -constexpr time_t OUTBOUND_TIMEOUT_SECONDS = 600; -constexpr time_t OUTBOUND_CONNECT_TIMEOUT_SECONDS = 5; - -void InsertExtraHeaders(HTTPHeaders &headers, const unordered_map &extra) { - for (auto &kv : extra) { - headers.Insert(kv.first, kv.second); - } -} - -QuackapiHttpFetchResult FromResponse(unique_ptr response) { - QuackapiHttpFetchResult out; - if (!response) { - out.request_error = "null HTTP response"; - return out; - } - out.status = response->status; - out.body = std::move(response->body); - out.reason = std::move(response->reason); - out.success = response->Success(); - out.request_error = response->GetRequestError(); - out.headers = std::move(response->headers); - return out; -} - -//===--------------------------------------------------------------------===// -// Connection pool -//===--------------------------------------------------------------------===// -// Free-list per proto_host_port. Acquire pops (or dials); Release pushes back. -// A pool rather than one shared client per host on purpose: httplib::Client -// serialises requests on its socket, so a single shared client would trade the -// handshake cost for a concurrency ceiling of 1. N idle clients give both reuse -// and parallelism — the same shape as httpx's pool on the FastAPI side. - -struct PooledPlain { - unique_ptr client; -}; - -struct PooledUtil { - unique_ptr client; -}; - -struct HostPool { - vector plain_idle; - vector util_idle; - idx_t dialed = 0; - idx_t reused = 0; -}; - -struct ConnectionPool { - static ConnectionPool &Get() { - static ConnectionPool instance; - return instance; - } - - std::mutex lock; - unordered_map hosts; -}; - -//! Split "http://host:port/a/b?c" into origin + path. Mirrors HTTPUtil::DecomposeURL. -void SplitURL(const string &url, string &origin, string &path) { - HTTPUtil::DecomposeURL(url, path, origin); - if (path.empty()) { - path = "/"; - } -} - -bool IsPlainHTTP(const string &url) { - return StringUtil::StartsWith(StringUtil::Lower(url), "http://"); -} - -//! Split "http://host:port" (from SplitURL origin) into host + port. -void SplitOriginHostPort(const string &origin, string &host, int &port) { - host.clear(); - port = 80; - string rest = origin; - if (StringUtil::StartsWith(StringUtil::Lower(rest), "http://")) { - rest = rest.substr(7); - } else if (StringUtil::StartsWith(StringUtil::Lower(rest), "https://")) { - rest = rest.substr(8); - port = 443; - } - // Strip any leftover path (should not be present on an origin). - auto slash = rest.find('/'); - if (slash != string::npos) { - rest = rest.substr(0, slash); - } - auto colon = rest.rfind(':'); - if (colon != string::npos && rest.find(']') == string::npos) { - // hostname:port (skip IPv6 bracket form for this test seam) - host = rest.substr(0, colon); - port = std::stoi(rest.substr(colon + 1)); - } else if (colon != string::npos && rest.front() == '[') { - // [ipv6]:port - auto br = rest.rfind(']'); - if (br != string::npos && br + 1 < rest.size() && rest[br + 1] == ':') { - host = rest.substr(1, br - 1); - port = std::stoi(rest.substr(br + 2)); - } else { - host = rest; - } - } else { - host = rest; - } -} - -//! Parse a raw HTTP/1.x response into status + body. Tolerates truncation. -void ParseRawHttpResponse(const string &raw, QuackapiHttpFetchResult &out) { - auto header_end = raw.find("\r\n\r\n"); - string header_block; - if (header_end == string::npos) { - header_block = raw; - out.body.clear(); - } else { - header_block = raw.substr(0, header_end); - out.body = raw.substr(header_end + 4); - } - auto line_end = header_block.find("\r\n"); - string status_line = line_end == string::npos ? header_block : header_block.substr(0, line_end); - // "HTTP/1.1 200 OK" - auto sp1 = status_line.find(' '); - if (sp1 != string::npos) { - auto sp2 = status_line.find(' ', sp1 + 1); - string code = sp2 == string::npos ? status_line.substr(sp1 + 1) : status_line.substr(sp1 + 1, sp2 - sp1 - 1); - try { - int status = std::stoi(code); - out.status = HTTPUtil::ToStatusCode(status); - out.success = status >= 200 && status < 400; - if (sp2 != string::npos) { - out.reason = status_line.substr(sp2 + 1); - } - } catch (...) { - out.request_error = "failed to parse HTTP status line"; - } - } else if (!raw.empty()) { - out.request_error = "malformed HTTP response"; - } - if (line_end != string::npos) { - idx_t pos = line_end + 2; - while (pos < header_block.size()) { - auto next = header_block.find("\r\n", pos); - string line = next == string::npos ? header_block.substr(pos) : header_block.substr(pos, next - pos); - auto colon = line.find(':'); - if (colon != string::npos) { - string key = line.substr(0, colon); - string val = line.substr(colon + 1); - StringUtil::Trim(key); - StringUtil::Trim(val); - out.headers.Insert(key, val); - } - if (next == string::npos) { - break; - } - pos = next + 2; - } - } -} - -//! Plain HTTP GET that sends the request, waits stall_ms, then reads. Matches -//! a raw socket that stops reading — the only client shape that trips httplib -//! server write_timeout / ApplyRouteIoTimeout. -QuackapiHttpFetchResult PlainGetWithStall(const string &url, const unordered_map &extra_headers, - int32_t stall_ms) { - QuackapiHttpFetchResult out; - string origin, path; - SplitURL(url, origin, path); - string host; - int port = 80; - SplitOriginHostPort(origin, host, port); - if (host.empty()) { - out.request_error = "stall_ms: could not parse host from URL"; - return out; - } - - duckdb_httplib::Error conn_error = duckdb_httplib::Error::Success; - auto sock = duckdb_httplib::detail::create_client_socket( - host, "", port, AF_UNSPEC, /*tcp_nodelay=*/true, /*ipv6_v6only=*/false, duckdb_httplib::default_socket_options, - OUTBOUND_CONNECT_TIMEOUT_SECONDS, 0, OUTBOUND_TIMEOUT_SECONDS, 0, OUTBOUND_TIMEOUT_SECONDS, 0, /*intf=*/"", - conn_error); - if (sock == INVALID_SOCKET) { - out.request_error = "stall_ms connect: " + duckdb_httplib::to_string(conn_error); - return out; - } - - string req = "GET " + path + " HTTP/1.1\r\nHost: " + host; - if (port != 80) { - req += ":" + std::to_string(port); - } - req += "\r\nConnection: close\r\n"; - for (auto &kv : extra_headers) { - req += kv.first + ": " + kv.second + "\r\n"; - } - req += "\r\n"; - - const char *send_ptr = req.data(); - size_t send_left = req.size(); - while (send_left > 0) { - auto n = duckdb_httplib::detail::send_socket(sock, send_ptr, send_left, CPPHTTPLIB_SEND_FLAGS); - if (n <= 0) { - duckdb_httplib::detail::close_socket(sock); - out.request_error = "stall_ms: send failed"; - return out; - } - send_ptr += n; - send_left -= static_cast(n); - } - - if (stall_ms > 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(stall_ms)); - } - - string raw; - char buf[16384]; - while (true) { - auto n = duckdb_httplib::detail::read_socket(sock, buf, sizeof(buf), CPPHTTPLIB_RECV_FLAGS); - if (n <= 0) { - break; - } - raw.append(buf, static_cast(n)); - } - duckdb_httplib::detail::close_socket(sock); - - if (raw.empty()) { - out.request_error = "stall_ms: empty response (peer closed during stall)"; - return out; - } - ParseRawHttpResponse(raw, out); - return out; -} - -unique_ptr DialPlain(const string &origin) { - auto client = make_uniq(origin); - client->set_keep_alive(true); - // Same Nagle/delayed-ACK deadlock the server side hit (quackapi_server.cpp:917), - // just on the other end of the socket: without this a small POST body waits on - // the peer's delayed ACK before the request is even complete. - client->set_tcp_nodelay(true); - client->set_follow_location(true); - client->set_decompress(true); - client->set_read_timeout(OUTBOUND_TIMEOUT_SECONDS, 0); - client->set_write_timeout(OUTBOUND_TIMEOUT_SECONDS, 0); - client->set_connection_timeout(OUTBOUND_CONNECT_TIMEOUT_SECONDS, 0); - return client; -} - -duckdb_httplib::Headers ToHttplibHeaders(const unordered_map &extra) { - duckdb_httplib::Headers headers; - for (auto &kv : extra) { - headers.emplace(kv.first, kv.second); - } - return headers; -} - -QuackapiHttpFetchResult FromHttplibResult(const duckdb_httplib::Result &res) { - QuackapiHttpFetchResult out; - if (res.error() != duckdb_httplib::Error::Success) { - out.request_error = duckdb_httplib::to_string(res.error()); - return out; - } - auto &response = res.value(); - out.status = HTTPUtil::ToStatusCode(response.status); - out.body = response.body; - out.reason = response.reason; - out.success = response.status >= 200 && response.status < 400; - for (auto &entry : response.headers) { - out.headers.Insert(entry.first, entry.second); - } - return out; -} - -//! Run `call` against a pooled plain-HTTP client for `origin`, then return the -//! client to the pool. The client is only recycled when the transport stayed -//! healthy — a broken socket must not be handed to the next request. -template -QuackapiHttpFetchResult WithPlainClient(const string &origin, CALL &&call) { - unique_ptr client; - bool reused = false; - { - auto &pool = ConnectionPool::Get(); - std::lock_guard guard(pool.lock); - auto &host = pool.hosts[origin]; - if (!host.plain_idle.empty()) { - client = std::move(host.plain_idle.back().client); - host.plain_idle.pop_back(); - host.reused++; - reused = true; - } else { - host.dialed++; - } - } - if (!client) { - client = DialPlain(origin); - } - // Reset on every checkout: a previous request's short deadline must not leak. - const auto timeout_ms = QuackapiRemainingTimeoutMillis(OUTBOUND_TIMEOUT_SECONDS * 1000); - client->set_max_timeout(static_cast(timeout_ms)); - client->set_read_timeout(static_cast(timeout_ms / 1000), static_cast((timeout_ms % 1000) * 1000)); - client->set_write_timeout(static_cast(timeout_ms / 1000), static_cast((timeout_ms % 1000) * 1000)); - const auto connect_ms = std::min(timeout_ms, OUTBOUND_CONNECT_TIMEOUT_SECONDS * 1000); - client->set_connection_timeout(static_cast(connect_ms / 1000), - static_cast((connect_ms % 1000) * 1000)); - - QuackapiHttpFetchResult result; - bool healthy = false; - try { - auto res = call(*client); - healthy = res.error() == duckdb_httplib::Error::Success; - result = FromHttplibResult(res); - } catch (...) { - // Drop the client on the floor; a half-written socket is not reusable. - throw; - } - result.reused_connection = reused; - - if (healthy) { - auto &pool = ConnectionPool::Get(); - std::lock_guard guard(pool.lock); - auto &host = pool.hosts[origin]; - if (host.plain_idle.size() < MAX_IDLE_PER_HOST) { - host.plain_idle.push_back(PooledPlain {std::move(client)}); - } - } - return result; -} - -//! Same checkout/return dance for the HTTPUtil path (https, or a loaded -//! curl_httpfs the operator wants used). The two-argument HTTPUtil::Request -//! reuses the client we pass instead of constructing a throwaway one. -template -QuackapiHttpFetchResult WithUtilClient(DatabaseInstance &db, const string &url, BUILD &&build) { - auto &http_util = HTTPUtil::Get(db); - string origin, path; - SplitURL(url, origin, path); - const auto key = http_util.GetName() + "|" + origin; - - unique_ptr client; - bool reused = false; - { - auto &pool = ConnectionPool::Get(); - std::lock_guard guard(pool.lock); - auto &host = pool.hosts[key]; - if (!host.util_idle.empty()) { - client = std::move(host.util_idle.back().client); - host.util_idle.pop_back(); - host.reused++; - reused = true; - } else { - host.dialed++; - } - } - - auto params = http_util.InitializeParameters(db, url); - params->keep_alive = true; - const auto timeout_ms = QuackapiRemainingTimeoutMillis(OUTBOUND_TIMEOUT_SECONDS * 1000); - params->timeout = timeout_ms / 1000; - params->timeout_usec = (timeout_ms % 1000) * 1000; - auto result = build(http_util, *params, client); - result.reused_connection = reused; - - if (client && result.request_error.empty()) { - auto &pool = ConnectionPool::Get(); - std::lock_guard guard(pool.lock); - auto &host = pool.hosts[key]; - if (host.util_idle.size() < MAX_IDLE_PER_HOST) { - host.util_idle.push_back(PooledUtil {std::move(client)}); - } - } - return result; -} - -} // namespace - -//===--------------------------------------------------------------------===// -// Public API -//===--------------------------------------------------------------------===// - -string QuackapiHttpFetch::ActiveHttpUtilName(DatabaseInstance &db) { - return HTTPUtil::Get(db).GetName(); -} - -QuackapiHttpFetchResult QuackapiHttpFetch::Get(DatabaseInstance &db, const string &url, - const unordered_map &extra_headers, int32_t stall_ms) { - if (stall_ms < 0) { - throw InvalidInputException("quackapi_fetch stall_ms must be >= 0"); - } - if (stall_ms > 0) { - if (!IsPlainHTTP(url)) { - throw InvalidInputException("quackapi_fetch stall_ms requires an http:// URL (plain TCP stall seam)"); - } - return PlainGetWithStall(url, extra_headers, stall_ms); - } - if (IsPlainHTTP(url)) { - string origin, path; - SplitURL(url, origin, path); - auto headers = ToHttplibHeaders(extra_headers); - return WithPlainClient(origin, [&](duckdb_httplib::Client &client) { return client.Get(path, headers); }); - } - - return WithUtilClient(db, url, [&](HTTPUtil &http_util, HTTPParams ¶ms, unique_ptr &client) { - HTTPHeaders headers(db); - InsertExtraHeaders(headers, extra_headers); - GetRequestInfo request(url, headers, params, /*response_handler=*/nullptr, - /*content_handler=*/nullptr); - request.try_request = true; - return FromResponse(http_util.Request(request, client)); - }); -} - -QuackapiHttpFetchResult QuackapiHttpFetch::Post(DatabaseInstance &db, const string &url, const string &body, - const string &content_type, - const unordered_map &extra_headers) { - if (IsPlainHTTP(url)) { - // The vendored httplib implements POST, so plain HTTP needs no companion - // extension at all — unlike the HTTPUtil path below. - string origin, path; - SplitURL(url, origin, path); - auto headers = ToHttplibHeaders(extra_headers); - const auto &ct = content_type.empty() ? string("application/octet-stream") : content_type; - return WithPlainClient(origin, - [&](duckdb_httplib::Client &client) { return client.Post(path, headers, body, ct); }); - } - - // Built-In HTTPLibClient does not implement POST (http_util.cpp). Surface a - // clear error pointing operators at curl_httpfs / httpfs rather than a raw - // NotImplementedException deep in the client. - auto &util = HTTPUtil::Get(db); - const auto util_name = util.GetName(); - if (util_name == "Built-In") { - throw InvalidConfigurationException( - "quackapi outbound POST over https requires an HTTP client with full method support. " - "LOAD curl_httpfs (recommended) or LOAD httpfs, then retry. Active HTTPUtil is '%s'.", - util_name); - } - - return WithUtilClient(db, url, [&](HTTPUtil &http_util, HTTPParams ¶ms, unique_ptr &client) { - HTTPHeaders headers(db); - if (!content_type.empty()) { - headers.Insert("Content-Type", content_type); - } - InsertExtraHeaders(headers, extra_headers); - PostRequestInfo request(url, headers, params, const_data_ptr_cast(body.data()), body.size()); - request.try_request = true; - return FromResponse(http_util.Request(request, client)); - }); -} - -vector QuackapiHttpFetch::PoolStats() { - vector out; - auto &pool = ConnectionPool::Get(); - std::lock_guard guard(pool.lock); - for (auto &entry : pool.hosts) { - QuackapiHttpPoolStats stats; - stats.host = entry.first; - stats.idle = entry.second.plain_idle.size() + entry.second.util_idle.size(); - stats.dialed = entry.second.dialed; - stats.reused = entry.second.reused; - out.push_back(std::move(stats)); - } - return out; -} - -void QuackapiHttpFetch::ResetPool() { - // Drop idle keep-alive sockets *before* httplib Server join. Stop paths - // detach ~QuackapiHttpServer; if pooled clients still hold the peer open, - // worker join can block until read timeout (600s) and CI process exit hangs - // with no log upload (Windows runner "lost communication"). - auto &pool = ConnectionPool::Get(); - std::lock_guard guard(pool.lock); - for (auto &entry : pool.hosts) { - for (auto &plain : entry.second.plain_idle) { - if (plain.client) { - plain.client->stop(); - } - } - // HTTPUtil clients: unique_ptr reset is enough (no keep-alive join path - // into our inbound Server). - entry.second.plain_idle.clear(); - entry.second.util_idle.clear(); - } - pool.hosts.clear(); -} - -//===--------------------------------------------------------------------===// -// SQL surface -//===--------------------------------------------------------------------===// - -namespace { - -LogicalType FetchResultType() { - child_list_t children; - children.emplace_back("status", LogicalType::INTEGER); - children.emplace_back("reason", LogicalType::VARCHAR); - children.emplace_back("body", LogicalType::VARCHAR); - children.emplace_back("headers", LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); - children.emplace_back("error", LogicalType::VARCHAR); - children.emplace_back("reused_connection", LogicalType::BOOLEAN); - return LogicalType::STRUCT(std::move(children)); -} - -Value ToValue(const QuackapiHttpFetchResult &result) { - vector header_keys; - vector header_values; - for (auto &entry : result.headers) { - header_keys.emplace_back(entry.first); - header_values.emplace_back(entry.second); - } - child_list_t children; - children.emplace_back("status", Value::INTEGER(static_cast(result.status))); - children.emplace_back("reason", Value(result.reason)); - children.emplace_back("body", Value(result.body)); - children.emplace_back("headers", Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(header_keys), - std::move(header_values))); - children.emplace_back("error", - result.request_error.empty() ? Value(LogicalType::VARCHAR) : Value(result.request_error)); - children.emplace_back("reused_connection", Value::BOOLEAN(result.reused_connection)); - return Value::STRUCT(std::move(children)); -} - -//! MAP(VARCHAR, VARCHAR) argument → header map. NULL / missing = no extras. -unordered_map HeadersFromValue(const Value &value) { - unordered_map out; - if (value.IsNull()) { - return out; - } - auto &entries = MapValue::GetChildren(value); - for (auto &entry : entries) { - auto &kv = StructValue::GetChildren(entry); - if (kv.size() != 2 || kv[0].IsNull()) { - continue; - } - out[kv[0].ToString()] = kv[1].IsNull() ? string() : kv[1].ToString(); - } - return out; -} - -int32_t StallMsFromArgs(DataChunk &args, idx_t row, idx_t col) { - if (args.ColumnCount() <= col) { - return 0; - } - auto v = args.data[col].GetValue(row); - if (v.IsNull()) { - return 0; - } - return v.GetValue(); -} - -//! Issue one independent outbound call per row of a chunk, CONCURRENTLY. -//! -//! Why this exists: these are scalar functions, so a chunk can carry up to -//! STANDARD_VECTOR_SIZE independent requests. Run serially, N calls against an -//! upstream with latency L cost N*L while the machine idles and the connection -//! pool sits unused. The pool already tolerates concurrent checkout (every path -//! takes ConnectionPool::lock), so fanning out is the whole fix. -//! -//! `perform(i)` must be safe to call from any thread. Exceptions are captured per -//! row rather than allowed to escape: WithPlainClient deliberately rethrows on a -//! half-written socket, and an exception crossing a std::thread boundary would -//! call std::terminate and take the server down with it. Argument validation that -//! must still raise to the caller therefore happens BEFORE the fan-out. -//! -//! The single-request case — what a route handler doing one upstream call per -//! request looks like — stays fully inline. No thread is created for it. -template -void FanOutRequests(idx_t count, const vector &skip, vector &out, PERFORM &&perform) { - const auto deadline = QuackapiCurrentDeadline(); - auto run_one = [&](idx_t i) { - QuackapiSetCurrentDeadline(deadline); - try { - out[i] = perform(i); - } catch (std::exception &e) { - out[i].request_error = e.what(); - } catch (...) { - out[i].request_error = "unknown error during outbound HTTP request"; - } - }; - - idx_t active = 0; - for (idx_t i = 0; i < count; i++) { - if (!skip[i]) { - active++; - } - } - if (active == 0) { - return; - } - if (active == 1) { - for (idx_t i = 0; i < count; i++) { - if (!skip[i]) { - run_one(i); - } - } - return; - } - - const idx_t n_threads = active < MAX_OUTBOUND_FANOUT ? active : MAX_OUTBOUND_FANOUT; - std::atomic cursor {0}; - vector threads; - threads.reserve(n_threads); - JoinOutboundThreads> join_threads {threads}; - for (idx_t t = 0; t < n_threads; t++) { - threads.emplace_back([&]() { - for (;;) { - const idx_t i = cursor.fetch_add(1); - if (i >= count) { - return; - } - if (!skip[i]) { - run_one(i); - } - } - }); - } -} - -void FetchScalar(DataChunk &args, ExpressionState &state, Vector &result) { - auto &db = *state.GetContext().db; - const auto count = args.size(); - result.SetVectorType(VectorType::FLAT_VECTOR); - - // Extract every argument up front, single-threaded: DataChunk/Vector reads are - // not safe to interleave with the fan-out below. stall_ms is validated here too, - // so its InvalidInputException still reaches the caller instead of being captured - // into a per-row error by the fan-out. - vector skip(count, false); - vector urls(count); - vector> headers(count); - vector stalls(count, 0); - for (idx_t i = 0; i < count; i++) { - auto url = args.data[0].GetValue(i); - if (url.IsNull()) { - skip[i] = true; - continue; - } - urls[i] = url.ToString(); - if (args.ColumnCount() == 2) { - // Overload is either (url, headers MAP) or (url, stall_ms INTEGER). - if (args.data[1].GetType().id() == LogicalTypeId::INTEGER || - args.data[1].GetType().id() == LogicalTypeId::BIGINT) { - stalls[i] = StallMsFromArgs(args, i, 1); - } else { - headers[i] = HeadersFromValue(args.data[1].GetValue(i)); - } - } else if (args.ColumnCount() >= 3) { - headers[i] = HeadersFromValue(args.data[1].GetValue(i)); - stalls[i] = StallMsFromArgs(args, i, 2); - } - if (stalls[i] < 0) { - throw InvalidInputException("quackapi_fetch stall_ms must be >= 0"); - } - if (stalls[i] > 0 && !IsPlainHTTP(urls[i])) { - throw InvalidInputException("quackapi_fetch stall_ms requires an http:// URL (plain TCP stall seam)"); - } - } - - vector out(count); - FanOutRequests(count, skip, out, - [&](idx_t i) { return QuackapiHttpFetch::Get(db, urls[i], headers[i], stalls[i]); }); - - for (idx_t i = 0; i < count; i++) { - result.SetValue(i, skip[i] ? Value(FetchResultType()) : ToValue(out[i])); - } -} - -struct PoolGlobalState : public GlobalTableFunctionState { - vector rows; - idx_t offset = 0; -}; - -struct FetchTableBindData : public TableFunctionData { - string url; - unordered_map headers; - int32_t stall_ms = 0; - bool finished = false; -}; - -unique_ptr PoolBind(ClientContext &, TableFunctionBindInput &, vector &return_types, - vector &names) { - names = {"host", "idle", "dialed", "reused"}; - return_types = {LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT, LogicalType::BIGINT}; - return nullptr; -} - -unique_ptr PoolInit(ClientContext &, TableFunctionInitInput &) { - auto state = make_uniq(); - state->rows = QuackapiHttpFetch::PoolStats(); - return std::move(state); -} - -void PoolExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.rows.size() && row < STANDARD_VECTOR_SIZE) { - auto &entry = state.rows[state.offset]; - output.SetValue(0, row, Value(entry.host)); - output.SetValue(1, row, Value::BIGINT(NumericCast(entry.idle))); - output.SetValue(2, row, Value::BIGINT(NumericCast(entry.dialed))); - output.SetValue(3, row, Value::BIGINT(NumericCast(entry.reused))); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -//! Concurrent GETs — the seam SQLLogic needs for claim-race / fan-out tests. -//! `FROM range(n)` around scalar quackapi_fetch constant-folds to one request; -//! this table function fires n real threads and returns one row per attempt. -struct ParallelFetchBindData : public TableFunctionData { - string url; - int32_t n = 1; -}; - -struct ParallelFetchRow { - int32_t idx = 0; - int32_t status = 0; - string body; - string error; -}; - -struct ParallelFetchGlobalState : public GlobalTableFunctionState { - vector rows; - idx_t offset = 0; -}; - -unique_ptr ParallelFetchBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind = make_uniq(); - bind->url = input.inputs[0].GetValue(); - bind->n = input.inputs[1].GetValue(); - if (bind->n < 1) { - throw InvalidInputException("quackapi_parallel_fetch: n must be >= 1"); - } - if (bind->n > 256) { - throw InvalidInputException("quackapi_parallel_fetch: n max is 256"); - } - names = {"idx", "status", "body", "error"}; - return_types = {LogicalType::INTEGER, LogicalType::INTEGER, LogicalType::VARCHAR, LogicalType::VARCHAR}; - return std::move(bind); -} - -unique_ptr ParallelFetchInit(ClientContext &context, TableFunctionInitInput &input) { - auto &bind = input.bind_data->Cast(); - auto state = make_uniq(); - state->rows.resize(NumericCast(bind.n)); - const auto deadline = QuackapiCurrentDeadline(); - auto &db = *context.db; - vector threads; - threads.reserve(NumericCast(bind.n)); - JoinOutboundThreads> join_threads {threads}; - for (int32_t i = 0; i < bind.n; i++) { - threads.emplace_back([&db, &bind, &state, i, deadline]() { - QuackapiSetCurrentDeadline(deadline); - auto &row = state->rows[NumericCast(i)]; - row.idx = i; - try { - auto res = QuackapiHttpFetch::Get(db, bind.url); - row.status = static_cast(res.status); - row.body = std::move(res.body); - row.error = std::move(res.request_error); - } catch (std::exception &ex) { - row.error = ex.what(); - } catch (...) { - row.error = "unknown error during outbound HTTP request"; - } - }); - } - for (auto &t : threads) { - t.join(); - } - return std::move(state); -} - -void ParallelFetchExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.rows.size() && row < STANDARD_VECTOR_SIZE) { - auto &r = state.rows[state.offset]; - output.SetValue(0, row, Value::INTEGER(r.idx)); - output.SetValue(1, row, Value::INTEGER(r.status)); - output.SetValue(2, row, Value(r.body)); - output.SetValue(3, row, r.error.empty() ? Value(LogicalType::VARCHAR) : Value(r.error)); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -} // namespace - -LogicalType QuackapiHttpFetchResultType() { - return FetchResultType(); -} - -Value QuackapiHttpFetchResultValue(const QuackapiHttpFetchResult &result) { - return ToValue(result); -} - -unordered_map QuackapiHttpHeadersFromValue(const Value &value) { - return HeadersFromValue(value); -} - -void RegisterQuackapiHttpFetchFunctions(ExtensionLoader &loader) { - const auto header_type = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); - const auto result_type = FetchResultType(); - - // VOLATILE on every overload, deliberately. Without it DuckDB constant-folds - // a call whose arguments are all literals — `FROM range(1000)` around a fetch - // with a fixed URL executes ONE request and copies the answer 1000 times. - // That is wrong for a function with side effects and it silently invalidates - // any benchmark built on it. - ScalarFunctionSet fetch_set("quackapi_fetch"); - ScalarFunction fetch1("quackapi_fetch", {LogicalType::VARCHAR}, result_type, FetchScalar); - fetch1.stability = FunctionStability::VOLATILE; - ScalarFunction fetch2("quackapi_fetch", {LogicalType::VARCHAR, header_type}, result_type, FetchScalar); - fetch2.stability = FunctionStability::VOLATILE; - // stall_ms: after send, wait N ms before reading (socket write-timeout test seam). - ScalarFunction fetch_stall("quackapi_fetch", {LogicalType::VARCHAR, LogicalType::INTEGER}, result_type, - FetchScalar); - fetch_stall.stability = FunctionStability::VOLATILE; - ScalarFunction fetch_hdr_stall("quackapi_fetch", {LogicalType::VARCHAR, header_type, LogicalType::INTEGER}, - result_type, FetchScalar); - fetch_hdr_stall.stability = FunctionStability::VOLATILE; - fetch_set.AddFunction(fetch1); - fetch_set.AddFunction(fetch2); - fetch_set.AddFunction(fetch_stall); - fetch_set.AddFunction(fetch_hdr_stall); - loader.RegisterFunction(fetch_set); - - // Table form exposes named stall_ms := N (scalars have no named_parameters). - TableFunction fetch_tf( - "quackapi_fetch", {LogicalType::VARCHAR}, - [](ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &bind = data_p.bind_data->CastNoConst(); - if (bind.finished) { - return; - } - auto result = QuackapiHttpFetch::Get(*context.db, bind.url, bind.headers, bind.stall_ms); - auto value = ToValue(result); - auto &children = StructValue::GetChildren(value); - for (idx_t c = 0; c < children.size(); c++) { - output.SetValue(c, 0, children[c]); - } - output.SetCardinality(1); - bind.finished = true; - }, - [](ClientContext &, TableFunctionBindInput &input, vector &return_types, - vector &names) -> unique_ptr { - auto bind = make_uniq(); - if (input.inputs.empty() || input.inputs[0].IsNull()) { - throw InvalidInputException("quackapi_fetch(url, stall_ms := N): url must be non-NULL"); - } - bind->url = input.inputs[0].ToString(); - auto headers_it = input.named_parameters.find("headers"); - if (headers_it != input.named_parameters.end() && !headers_it->second.IsNull()) { - bind->headers = HeadersFromValue(headers_it->second); - } - auto stall_it = input.named_parameters.find("stall_ms"); - if (stall_it != input.named_parameters.end() && !stall_it->second.IsNull()) { - bind->stall_ms = stall_it->second.GetValue(); - } - auto result_type = FetchResultType(); - for (auto &child : StructType::GetChildTypes(result_type)) { - names.push_back(child.first); - return_types.push_back(child.second); - } - return std::move(bind); - }); - fetch_tf.named_parameters["headers"] = header_type; - fetch_tf.named_parameters["stall_ms"] = LogicalType::INTEGER; - loader.RegisterFunction(fetch_tf); - - RegisterQuackapiPostFunctions(loader); - - loader.RegisterFunction(TableFunction("quackapi_http_pool", {}, PoolExec, PoolBind, PoolInit)); - - TableFunction parallel_fetch("quackapi_parallel_fetch", {LogicalType::VARCHAR, LogicalType::INTEGER}, - ParallelFetchExec, ParallelFetchBind, ParallelFetchInit); - loader.RegisterFunction(parallel_fetch); -} - -} // namespace duckdb diff --git a/src/quackapi_post.cpp b/src/quackapi_post.cpp index 3d296c3..afa6961 100644 --- a/src/quackapi_post.cpp +++ b/src/quackapi_post.cpp @@ -1,6 +1,7 @@ #include "quackapi_post.hpp" #include "quackapi_post_executor.hpp" #include "quackapi_post_provider.hpp" +#include "quackapi_post_result.hpp" #include "duckdb/function/scalar_function.hpp" #include "duckdb/main/config.hpp" diff --git a/src/quackapi_post_result.cpp b/src/quackapi_post_result.cpp new file mode 100644 index 0000000..addab22 --- /dev/null +++ b/src/quackapi_post_result.cpp @@ -0,0 +1,52 @@ +#include "quackapi_post_result.hpp" + +#include "duckdb/common/types/value.hpp" + +namespace duckdb { + +LogicalType QuackapiHttpFetchResultType() { + child_list_t children; + children.emplace_back("status", LogicalType::INTEGER); + children.emplace_back("reason", LogicalType::VARCHAR); + children.emplace_back("body", LogicalType::VARCHAR); + children.emplace_back("headers", LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)); + children.emplace_back("error", LogicalType::VARCHAR); + children.emplace_back("reused_connection", LogicalType::BOOLEAN); + return LogicalType::STRUCT(std::move(children)); +} + +Value QuackapiHttpFetchResultValue(const QuackapiHttpFetchResult &result) { + vector header_keys; + vector header_values; + for (auto &entry : result.headers) { + header_keys.emplace_back(entry.first); + header_values.emplace_back(entry.second); + } + child_list_t children; + children.emplace_back("status", Value::INTEGER(static_cast(result.status))); + children.emplace_back("reason", Value(result.reason)); + children.emplace_back("body", Value(result.body)); + children.emplace_back("headers", Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(header_keys), + std::move(header_values))); + children.emplace_back("error", + result.request_error.empty() ? Value(LogicalType::VARCHAR) : Value(result.request_error)); + children.emplace_back("reused_connection", Value::BOOLEAN(result.reused_connection)); + return Value::STRUCT(std::move(children)); +} + +unordered_map QuackapiHttpHeadersFromValue(const Value &value) { + unordered_map out; + if (value.IsNull()) { + return out; + } + for (auto &entry : MapValue::GetChildren(value)) { + auto &kv = StructValue::GetChildren(entry); + if (kv.size() != 2 || kv[0].IsNull()) { + continue; + } + out[kv[0].ToString()] = kv[1].IsNull() ? string() : kv[1].ToString(); + } + return out; +} + +} // namespace duckdb diff --git a/src/quackapi_registry.cpp b/src/quackapi_registry.cpp index 2e1e249..1b8e724 100644 --- a/src/quackapi_registry.cpp +++ b/src/quackapi_registry.cpp @@ -11,7 +11,6 @@ #include "duckdb/main/database.hpp" #include "quackapi_auth.hpp" -#include "quackapi_http_fetch.hpp" #include "quackapi_server.hpp" namespace duckdb { @@ -298,9 +297,6 @@ bool QuackapiState::StopServer(int port) { if (!to_destroy) { return false; } - // Close pooled outbound keep-alives first so Server worker join is not - // blocked by idle clients still holding the listen-side connection open. - QuackapiHttpFetch::ResetPool(); to_destroy->StopAccepting(); // Brief delay so a self-stop from inside a route can finish its response // before ~Server joins the thread pool. @@ -323,8 +319,7 @@ void QuackapiState::StopAllServers() { } servers.clear(); } - // Same as StopServer: drain free-list before joining httplib workers. - QuackapiHttpFetch::ResetPool(); + // Stop accepting before joining listener and request-handler threads. for (auto &srv : to_destroy) { if (srv) { srv->StopAccepting(); diff --git a/test/cpp/quackapi_thread_join_test.cpp b/test/cpp/quackapi_thread_join_test.cpp deleted file mode 100644 index d508257..0000000 --- a/test/cpp/quackapi_thread_join_test.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// Standalone regression: compile with -std=c++17 -pthread -Isrc/include. -// Simulate a failure after one worker starts without exhausting host threads. -#include "quackapi_thread_join.hpp" - -#include -#include -#include -#include -#include - -int main() { - std::atomic completed {0}; - try { - std::vector threads; - duckdb::JoinOutboundThreads> join_threads {threads}; - threads.emplace_back([&completed]() { completed++; }); - throw std::runtime_error("simulated later thread-start failure"); - } catch (const std::runtime_error &) { - assert(completed == 1); - } - { - std::vector threads; - duckdb::JoinOutboundThreads> join_threads {threads}; - threads.emplace_back([&completed]() { completed++; }); - threads.emplace_back([&completed]() { completed++; }); - threads[0].join(); // Already-joined workers must also be safe. - } - assert(completed == 3); -} diff --git a/test/sql/quackapi_access_log.test b/test/sql/quackapi_access_log.test index ebb53f4..6bccec0 100644 --- a/test/sql/quackapi_access_log.test +++ b/test/sql/quackapi_access_log.test @@ -23,6 +23,7 @@ CREATE TABLE access_log ( statement ok CREATE ROUTE access_log_route GET '/access-log' AS SELECT 'ok' AS result; +CREATE ROUTE access_log_post POST '/access-log' AS SELECT 'ok' AS result; query I SELECT status @@ -82,7 +83,7 @@ SELECT * FROM quackapi_serve(29461, host := '127.0.0.1', access_log := 'access_l http://127.0.0.1:29461 query I -SELECT (quackapi_fetch('http://127.0.0.1:29461/access-log')).status; +SELECT (quackapi_post('http://127.0.0.1:29461/access-log', '{}')).status; ---- 200 @@ -100,7 +101,7 @@ DROP TABLE access_log_retry; # The request still succeeds while the writer is backing off and the missing # table means its access row cannot be appended. query I -SELECT (quackapi_fetch('http://127.0.0.1:29461/access-log')).status; +SELECT (quackapi_post('http://127.0.0.1:29461/access-log', '{}')).status; ---- 200 @@ -125,7 +126,7 @@ statement ok SELECT sleep_ms(1200); query I -SELECT (quackapi_fetch('http://127.0.0.1:29461/access-log')).status; +SELECT (quackapi_post('http://127.0.0.1:29461/access-log', '{}')).status; ---- 200 @@ -142,23 +143,6 @@ SELECT * FROM quackapi_stop(29461); ---- Stopped quackapi server on port 29461 -# The existing readiness surface reports table-writer overflow state. -query I -SELECT * FROM quackapi_serve(29231, host := '127.0.0.1', access_log := 'access_log'); ----- -http://127.0.0.1:29231 - -query I -SELECT contains(body, '"access_log_overflow_count":0') -FROM quackapi_fetch('http://127.0.0.1:29231/healthz'); ----- -true - -query I -SELECT * FROM quackapi_stop(29231); ----- -Stopped quackapi server on port 29231 - # A missing operator table must not fail the in-process request. query I SELECT status diff --git a/test/sql/quackapi_batteries.test b/test/sql/quackapi_batteries.test deleted file mode 100644 index ecced27..0000000 --- a/test/sql/quackapi_batteries.test +++ /dev/null @@ -1,33 +0,0 @@ -# name: test/sql/quackapi_batteries.test -# description: health route configuration -# group: [sql] - -require quackapi - -# --------------------------------------------------------------------------- -# health_routes := false → not re-registered after explicit drop -# --------------------------------------------------------------------------- -query I -DROP ROUTE __quackapi_health; ----- -Dropped route __quackapi_health - -query I -DROP ROUTE __quackapi_healthz; ----- -Dropped route __quackapi_healthz - -query I -SELECT * FROM quackapi_serve(29203, host := '127.0.0.1', health_routes := false); ----- -http://127.0.0.1:29203 - -query I -SELECT count(*) FROM quackapi_routes() WHERE name LIKE '__quackapi_health%'; ----- -0 - -query I -SELECT * FROM quackapi_stop(29203); ----- -Stopped quackapi server on port 29203 diff --git a/test/sql/quackapi_http_pool.test b/test/sql/quackapi_http_pool.test deleted file mode 100644 index 8737739..0000000 --- a/test/sql/quackapi_http_pool.test +++ /dev/null @@ -1,136 +0,0 @@ -# name: test/sql/quackapi_http_pool.test -# description: Outbound quackapi_fetch / quackapi_post reuse pooled connections -# group: [sql] - -require quackapi - -require json - -# Serve a tiny echo API and then call it from the SAME process. No external -# network: the pool is exercised against quackapi's own server. -statement ok -CREATE ROUTE ping GET '/ping' AS SELECT 'pong' AS msg; - -statement ok -CREATE ROUTE echo POST '/echo' AS SELECT $body::VARCHAR AS got; - -query I -SELECT * FROM quackapi_serve(29301, host := '127.0.0.1', access_log := false); ----- -http://127.0.0.1:29301 - -# --- GET --------------------------------------------------------------------- -query I -SELECT (quackapi_fetch('http://127.0.0.1:29301/ping')).status; ----- -200 - -query I -SELECT (quackapi_fetch('http://127.0.0.1:29301/ping')).body; ----- -[{"msg":"pong"}] - -# Extra headers travel as a MAP argument. -query I -SELECT (quackapi_fetch('http://127.0.0.1:29301/ping', MAP{'X-Test': 'yes'})).status; ----- -200 - -# --- POST -------------------------------------------------------------------- -# Plain http:// needs NO companion extension — the vendored httplib implements -# POST, unlike DuckDB's built-in HTTPUtil client. -query I -SELECT (quackapi_post('http://127.0.0.1:29301/echo', to_json({a: 1})::VARCHAR)).status; ----- -200 - -# The body arrives intact — $body binds the raw payload we sent. -query I -SELECT (quackapi_post('http://127.0.0.1:29301/echo', to_json({a: 1})::VARCHAR)).body; ----- -[{"got":"{\"a\":1}"}] - -# --- pooling ----------------------------------------------------------------- -# The point of the whole exercise: the second call must NOT dial again. -query I -SELECT (quackapi_fetch('http://127.0.0.1:29301/ping')).reused_connection; ----- -true - -# One origin, one dial, many reuses. -query I -SELECT count(*) FROM quackapi_http_pool() WHERE host = 'http://127.0.0.1:29301'; ----- -1 - -query I -SELECT dialed = 1 AND reused >= 4 AND idle >= 1 -FROM quackapi_http_pool() WHERE host = 'http://127.0.0.1:29301'; ----- -true - -# --- VOLATILE ---------------------------------------------------------------- -# Without FunctionStability::VOLATILE, DuckDB constant-folds a literal-argument -# call and range(N) would issue ONE request while reporting N rows. Assert the -# server actually saw every one of them by watching TOTAL requests (dialed + -# reused) climb. -# -# Total, not `reused` alone: a chunk carrying N independent calls fans them out -# concurrently, so the burst DIALS up to the fan-out width instead of reusing one -# socket N times sequentially. Which counter moves is an artefact of the execution -# model; the invariant this test exists to defend is that N rows issue N requests. -statement ok -CREATE OR REPLACE TEMP TABLE before AS - SELECT dialed + reused AS n FROM quackapi_http_pool() WHERE host = 'http://127.0.0.1:29301'; - -statement ok -SELECT (quackapi_fetch('http://127.0.0.1:29301/ping')).status AS s FROM range(10) t(i); - -query I -SELECT (SELECT dialed + reused FROM quackapi_http_pool() WHERE host = 'http://127.0.0.1:29301') - - (SELECT n FROM before) >= 10; ----- -true - -# --- fan-out: one chunk, many concurrent calls -------------------------------- -# A chunk carrying N independent calls issues them concurrently rather than one -# blocking round trip at a time. Asserted on RESULTS, not elapsed time: a -# "finishes within X ms" check would be measuring the machine under CI load, not -# this code. What must hold is that concurrency loses nothing — every row gets -# its own correct answer, none are dropped, none error, and the total request -# count still matches the row count. -statement ok -CREATE OR REPLACE TEMP TABLE fanout_before AS - SELECT dialed + reused AS n FROM quackapi_http_pool() WHERE host = 'http://127.0.0.1:29301'; - -query III -SELECT count(*) AS n, - count(CASE WHEN (r).status = 200 THEN 1 END) AS ok, - count(CASE WHEN (r).error IS NOT NULL THEN 1 END) AS errs -FROM (SELECT quackapi_fetch('http://127.0.0.1:29301/ping') AS r FROM range(64) t(i)); ----- -64 64 0 - -# Every one of the 64 rows became a real request (no folding, no coalescing). -query I -SELECT (SELECT dialed + reused FROM quackapi_http_pool() WHERE host = 'http://127.0.0.1:29301') - - (SELECT n FROM fanout_before) = 64; ----- -true - -# --- transport errors are values, not exceptions ------------------------------ -query I -SELECT (quackapi_fetch('http://127.0.0.1:29399/nope')).error IS NOT NULL; ----- -true - -# NULL url → NULL struct, no request. -query I -SELECT (quackapi_fetch(NULL::VARCHAR)).status IS NULL; ----- -true - -# stop closes the listener and (via StopServer) drains the free-list so idle -# keep-alive clients cannot pin httplib workers across process teardown. -statement ok -SELECT * FROM quackapi_stop(29301); diff --git a/test/sql/quackapi_lifecycle.test b/test/sql/quackapi_lifecycle.test index 04f9676..c342be9 100644 --- a/test/sql/quackapi_lifecycle.test +++ b/test/sql/quackapi_lifecycle.test @@ -120,32 +120,6 @@ DROP ROUTE keeper; statement ok DROP ROUTE mover; -# --------------------------------------------------------------------------- -# Bug: serve(health_routes=true) → stop → serve(health_routes=false) left -# __quackapi_health* in the registry (RegisterQuackapiHealthRoutes no-op). -# --------------------------------------------------------------------------- -statement ok -SELECT * FROM quackapi_serve(29451, host := '127.0.0.1'); - -query I -SELECT count(*) FROM quackapi_routes() WHERE starts_with(name, '__quackapi_health'); ----- -2 - -statement ok -SELECT * FROM quackapi_stop(29451); - -statement ok -SELECT * FROM quackapi_serve(29451, host := '127.0.0.1', health_routes := false); - -query I -SELECT count(*) FROM quackapi_routes() WHERE starts_with(name, '__quackapi_health'); ----- -0 - -statement ok -SELECT * FROM quackapi_stop(29451); - # OpenAPI after DROP ROUTE: path must vanish (live snapshot — regression lock). statement ok CREATE ROUTE oa GET '/oa' AS SELECT 1 AS n; diff --git a/test/sql/quackapi_requests_ring.test b/test/sql/quackapi_requests_ring.test index 23136e0..f70c907 100644 --- a/test/sql/quackapi_requests_ring.test +++ b/test/sql/quackapi_requests_ring.test @@ -11,13 +11,13 @@ SELECT count(*) FROM quackapi_requests(); 0 statement ok -CREATE OR REPLACE ROUTE ring_first GET '/ring/first' AS SELECT 'first' AS result; +CREATE OR REPLACE ROUTE ring_first POST '/ring/first' AS SELECT 'first' AS result; statement ok -CREATE OR REPLACE ROUTE ring_second GET '/ring/second' AS SELECT 'second' AS result; +CREATE OR REPLACE ROUTE ring_second POST '/ring/second' AS SELECT 'second' AS result; statement ok -CREATE OR REPLACE ROUTE ring_fail GET '/ring/fail' AS SELECT * FROM query($sql); +CREATE OR REPLACE ROUTE ring_fail POST '/ring/fail' AS SELECT * FROM query($sql); # Access-log output is disabled, but the ring still records every request. query I @@ -26,17 +26,17 @@ SELECT * FROM quackapi_serve(29561, host := '127.0.0.1', access_log := false); http://127.0.0.1:29561 query I -SELECT quackapi_fetch('http://127.0.0.1:29561/ring/first').status; +SELECT quackapi_post('http://127.0.0.1:29561/ring/first', '{}').status; ---- 200 query I -SELECT quackapi_fetch('http://127.0.0.1:29561/ring/missing').status; +SELECT quackapi_post('http://127.0.0.1:29561/ring/missing', '{}').status; ---- 404 query I -SELECT quackapi_fetch('http://127.0.0.1:29561/ring/second').status; +SELECT quackapi_post('http://127.0.0.1:29561/ring/second', '{}').status; ---- 200 @@ -60,8 +60,8 @@ SELECT * FROM quackapi_serve(29566, host := '127.0.0.1', access_log := false, http://127.0.0.1:29566 query I -SELECT (quackapi_fetch( - 'http://127.0.0.1:29566/ring/first?token=abc&q=x', +SELECT (quackapi_post( + 'http://127.0.0.1:29566/ring/first?token=abc&q=x', '{}', 'application/json', MAP {'Authorization': 'Bearer do-not-store', 'x-tenant-id': 'tenant-42'} )).status; ---- @@ -87,7 +87,7 @@ SELECT * FROM quackapi_serve(29567, host := '127.0.0.1', access_log := false); http://127.0.0.1:29567 query I -SELECT status FROM quackapi_fetch('http://127.0.0.1:29567/ring/second'); +SELECT status FROM quackapi_post('http://127.0.0.1:29567/ring/second', '{}'); ---- 200 @@ -108,13 +108,13 @@ SELECT * FROM quackapi_serve(29562, host := '127.0.0.1', access_log := false, re http://127.0.0.1:29562 statement ok -SELECT quackapi_fetch('http://127.0.0.1:29562/ring/two-one').status; +SELECT quackapi_post('http://127.0.0.1:29562/ring/two-one', '{}').status; statement ok -SELECT quackapi_fetch('http://127.0.0.1:29562/ring/two-two').status; +SELECT quackapi_post('http://127.0.0.1:29562/ring/two-two', '{}').status; statement ok -SELECT quackapi_fetch('http://127.0.0.1:29562/ring/two-three').status; +SELECT quackapi_post('http://127.0.0.1:29562/ring/two-three', '{}').status; query I SELECT count(*) FROM quackapi_requests(29562); @@ -138,10 +138,10 @@ SELECT * FROM quackapi_serve(29563, host := '127.0.0.1', access_log := false, re http://127.0.0.1:29563 statement ok -SELECT quackapi_fetch('http://127.0.0.1:29563/ring/zero-one').status; +SELECT quackapi_post('http://127.0.0.1:29563/ring/zero-one', '{}').status; statement ok -SELECT quackapi_fetch('http://127.0.0.1:29563/ring/zero-two').status; +SELECT quackapi_post('http://127.0.0.1:29563/ring/zero-two', '{}').status; query I SELECT count(*) FROM quackapi_requests(29563); @@ -163,10 +163,10 @@ SELECT * FROM quackapi_serve(29565, host := '127.0.0.1', access_log := false); http://127.0.0.1:29565 statement ok -SELECT quackapi_fetch('http://127.0.0.1:29565/ring/setting-one').status; +SELECT quackapi_post('http://127.0.0.1:29565/ring/setting-one', '{}').status; statement ok -SELECT quackapi_fetch('http://127.0.0.1:29565/ring/setting-two').status; +SELECT quackapi_post('http://127.0.0.1:29565/ring/setting-two', '{}').status; query I SELECT count(*) FROM quackapi_requests(29565); @@ -188,7 +188,7 @@ SELECT * FROM quackapi_serve(29564, host := '127.0.0.1', access_log := false); http://127.0.0.1:29564 query I -SELECT quackapi_fetch('http://127.0.0.1:29564/ring/fail?sql=SELECT%20nope%20FROM%20nowhere').status; +SELECT quackapi_post('http://127.0.0.1:29564/ring/fail?sql=SELECT%20nope%20FROM%20nowhere', '{}').status; ---- 500 diff --git a/test/sql/quackapi_route_timeout.test b/test/sql/quackapi_route_timeout.test index 3eb44cc..14ea771 100644 --- a/test/sql/quackapi_route_timeout.test +++ b/test/sql/quackapi_route_timeout.test @@ -1,12 +1,12 @@ # name: test/sql/quackapi_route_timeout.test -# description: Per-route TIMEOUT / WITH (timeout_sec) extends httplib IO deadlines +# description: Per-route TIMEOUT / WITH (timeout_sec) metadata # group: [sql] require quackapi require json -# --- DDL: TIMEOUT clause + WITH (timeout_sec) land on the registry ------------ +# TIMEOUT clause and WITH (timeout_sec) are registered as route metadata. query I CREATE ROUTE slow_clause GET '/slow-clause' TIMEOUT 180 AS SELECT 1 AS n; ---- @@ -39,7 +39,6 @@ SELECT timeout_sec FROM quackapi_routes() WHERE name = 'slow_dur'; ---- 180 -# Default remains 0 (serve-level 30s IO timeouts apply). query I CREATE ROUTE fast GET '/fast' AS SELECT 'ok' AS msg; ---- @@ -59,95 +58,3 @@ statement error CREATE ROUTE bad2 GET '/bad2' WITH (timeout_sec := 180, response := 'json') AS SELECT 1; ---- Unknown WITH option - -# --- Effective write deadline (all platforms) -------------------------------- -# Prove ApplyRouteIoTimeout ran (or did not) via quackapi_last_write_timeout_sec, -# which records what the apply path actually did — not the registry timeout_sec. - -statement ok -CREATE OR REPLACE ROUTE probe_default GET '/probe-default' AS SELECT 1 AS n; - -statement ok -CREATE OR REPLACE ROUTE probe_optin GET '/probe-optin' - WITH (timeout_sec := 30) - AS SELECT 1 AS n; - -query I -SELECT * FROM quackapi_serve(29350, host := '127.0.0.1', access_log := false, - read_timeout_sec := 2, write_timeout_sec := 2, - compression := false); ----- -http://127.0.0.1:29350 - -# No route timeout_sec → serve-level write_timeout_sec remains effective. -query I -SELECT status FROM quackapi_fetch('http://127.0.0.1:29350/probe-default'); ----- -200 - -query I -SELECT quackapi_last_write_timeout_sec(); ----- -2 - -# Route WITH timeout_sec := 30 → ApplyRouteIoTimeout applied 30. -query I -SELECT status FROM quackapi_fetch('http://127.0.0.1:29350/probe-optin'); ----- -200 - -query I -SELECT quackapi_last_write_timeout_sec(); ----- -30 - -query I -SELECT * FROM quackapi_stop(29350); ----- -Stopped quackapi server on port 29350 - -# --- Live HTTP stall: control/treatment (POSIX only) ------------------------- -# Windows receive-window autotuning absorbs the body, so client stall cannot create write backpressure. -require notwindows - -# ApplyRouteIoTimeout raises httplib socket write deadlines. A compute-bound -# handler never stalls the socket, so sleep_ms cannot prove the feature. -# Observable difference: large body + client that sends then waits before -# reading (quackapi_fetch stall_ms). Serve write_timeout_sec short; stall longer. -# Control (no opt-in) truncates/fails; treatment (timeout_sec) completes. - -statement ok -CREATE OR REPLACE ROUTE big_default GET '/big-default' AS - SELECT repeat('x', 10000000) AS payload; - -statement ok -CREATE OR REPLACE ROUTE big_optin GET '/big-optin' - WITH (timeout_sec := 30) - AS SELECT repeat('x', 10000000) AS payload; - -query I -SELECT * FROM quackapi_serve(29350, host := '127.0.0.1', access_log := false, - read_timeout_sec := 2, write_timeout_sec := 2, - compression := false); ----- -http://127.0.0.1:29350 - -# Expected complete JSON body length: [{"payload":"<10M x>"}] = 10_000_014 -# Control: stall 6s > write_timeout 2s → truncated or failed (body shorter). -query I -SELECT length(body) < 10000014 AS truncated_or_short -FROM quackapi_fetch('http://127.0.0.1:29350/big-default', stall_ms := 6000); ----- -true - -# Treatment: per-route timeout_sec := 30 covers the stall → full body. -query I -SELECT length(body) >= 10000014 AS complete -FROM quackapi_fetch('http://127.0.0.1:29350/big-optin', stall_ms := 6000); ----- -true - -query I -SELECT * FROM quackapi_stop(29350); ----- -Stopped quackapi server on port 29350 diff --git a/test/sql/quackapi_telemetry_record.test b/test/sql/quackapi_telemetry_record.test index 32638c5..b22440f 100644 --- a/test/sql/quackapi_telemetry_record.test +++ b/test/sql/quackapi_telemetry_record.test @@ -61,7 +61,7 @@ CREATE OR REPLACE ROUTE telemetry_fail POST '/telemetry/fail' AS SELECT * FROM query($sql); statement ok -CREATE OR REPLACE ROUTE telemetry_ok GET '/telemetry/ok' AS +CREATE OR REPLACE ROUTE telemetry_ok POST '/telemetry/ok' AS SELECT 'ok' AS result; # Valid W3C trace context is retained while the server creates a fresh span. @@ -153,7 +153,7 @@ SELECT * FROM quackapi_serve(29520, host := '127.0.0.1', access_log := 'warn_log http://127.0.0.1:29520 query I -SELECT (quackapi_fetch('http://127.0.0.1:29520/telemetry/ok')).status; +SELECT (quackapi_post('http://127.0.0.1:29520/telemetry/ok', '{}')).status; ---- 200 diff --git a/test/sql/quackapi_telemetry_status.test b/test/sql/quackapi_telemetry_status.test index 1239afe..b7362f7 100644 --- a/test/sql/quackapi_telemetry_status.test +++ b/test/sql/quackapi_telemetry_status.test @@ -4,6 +4,9 @@ require quackapi +statement ok +CREATE ROUTE telemetry_probe POST '/telemetry/probe' AS SELECT 'ok' AS result; + statement ok CREATE TABLE telemetry_status_log ( request_id VARCHAR, @@ -19,7 +22,7 @@ SELECT * FROM quackapi_serve(29580, host := '127.0.0.1', access_log := 'telemetr http://127.0.0.1:29580 query I -SELECT quackapi_fetch('http://127.0.0.1:29580/health').status; +SELECT quackapi_post('http://127.0.0.1:29580/telemetry/probe', '{}').status; ---- 200 @@ -39,7 +42,7 @@ query III SELECT method, path, status FROM telemetry_status_log; ---- -GET /health 200 +POST /telemetry/probe 200 query I SELECT * FROM quackapi_stop(29580); @@ -56,12 +59,6 @@ SELECT sink FROM quackapi_telemetry_status(); ---- stderr -query I -SELECT contains(body, '"telemetry"') AND contains(body, '"access_log_overflow_count"') -FROM quackapi_fetch('http://127.0.0.1:29581/healthz'); ----- -true - query I SELECT * FROM quackapi_stop(29581); ---- From 7e4c7921e95f63f4f9d4f00b475535dc601514b0 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:23:56 -0700 Subject: [PATCH 21/93] refactor(binding): parse request JSON natively --- src/include/quackapi_request_binding.hpp | 4 +- src/quackapi_request_binding.cpp | 173 +++++------------------ src/quackapi_server.cpp | 123 +++------------- test/sql/quackapi_route_errors.test | 4 +- 4 files changed, 56 insertions(+), 248 deletions(-) diff --git a/src/include/quackapi_request_binding.hpp b/src/include/quackapi_request_binding.hpp index 8d7588d..9b1a354 100644 --- a/src/include/quackapi_request_binding.hpp +++ b/src/include/quackapi_request_binding.hpp @@ -20,12 +20,14 @@ struct JsonBodyField { } }; +enum class JsonBodyParseResult : uint8_t { SUCCESS, INVALID_JSON, NOT_OBJECT }; + bool IsJsonMediaType(const string &media); bool IsFormUrlEncodedMediaType(const string &media); bool IsMultipartMediaType(const string &media); bool IsBodyMethod(const string &method); void ParseFormUrlEncoded(const string &body, case_insensitive_map_t &out); -bool TryExtractFlatJsonObject(const string &raw_body, case_insensitive_map_t &fields); +JsonBodyParseResult ParseJsonBodyFields(const string &raw_body, case_insensitive_map_t &fields); void ParseQueryStringIntoParams(const string &query, duckdb_httplib::Params ¶ms); } // namespace duckdb diff --git a/src/quackapi_request_binding.cpp b/src/quackapi_request_binding.cpp index 1646da6..239c333 100644 --- a/src/quackapi_request_binding.cpp +++ b/src/quackapi_request_binding.cpp @@ -1,6 +1,9 @@ #include "quackapi_request_binding.hpp" +#include + #include "duckdb/common/string_util.hpp" +#include "yyjson.hpp" namespace duckdb { bool IsJsonMediaType(const string &media) { @@ -44,149 +47,43 @@ void ParseFormUrlEncoded(const string &body, case_insensitive_map_t &out i = amp + 1; } } -//! Fast path: flat JSON object of string/number/bool/null scalars (no DuckDB round-trip). -//! Returns true on success. false → caller may fall back to SQL extract or error. -bool TryExtractFlatJsonObject(const string &raw_body, case_insensitive_map_t &fields) { - const char *s = raw_body.c_str(); - const char *end = s + raw_body.size(); - auto skip_ws = [&]() { - while (s < end && (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')) { - s++; - } - }; - auto is_digit = [](char c) { - return c >= '0' && c <= '9'; - }; - // This path deliberately accepts only unescaped JSON strings. Escapes require - // full JSON decoding (in particular, UTF-16 surrogate handling), so leave them - // to DuckDB instead of attempting a partial implementation here. - auto parse_plain_string = [&](string &out) { - if (s >= end || *s != '"') { - return false; - } - s++; - while (s < end && *s != '"') { - const auto ch = static_cast(*s); - if (ch < 0x20 || *s == '\\') { - return false; - } - out.push_back(*s++); - } - if (s >= end) { - return false; - } - s++; - return true; - }; - auto parse_number = [&]() { - const char *start = s; - if (s < end && *s == '-') { - s++; - } - if (s >= end) { - return false; - } - if (*s == '0') { - s++; - if (s < end && is_digit(*s)) { - return false; - } - } else if (*s >= '1' && *s <= '9') { - do { - s++; - } while (s < end && is_digit(*s)); - } else { - return false; - } - if (s < end && *s == '.') { - s++; - if (s >= end || !is_digit(*s)) { - return false; - } - do { - s++; - } while (s < end && is_digit(*s)); - } - if (s < end && (*s == 'e' || *s == 'E')) { - s++; - if (s < end && (*s == '+' || *s == '-')) { - s++; - } - if (s >= end || !is_digit(*s)) { - return false; - } - do { - s++; - } while (s < end && is_digit(*s)); - } - return s > start; - }; - skip_ws(); - if (s >= end || *s != '{') { - return false; - } - s++; +JsonBodyParseResult ParseJsonBodyFields(const string &raw_body, case_insensitive_map_t &fields) { fields.clear(); - skip_ws(); - if (s < end && *s == '}') { - s++; - skip_ws(); - return s == end; + if (raw_body.empty()) { + return JsonBodyParseResult::INVALID_JSON; } - while (s < end) { - skip_ws(); - string key; - if (!parse_plain_string(key)) { - return false; - } - skip_ws(); - if (s >= end || *s != ':') { - return false; - } - s++; - skip_ws(); - if (s >= end) { - return false; - } - string val; - bool explicit_null = false; - if (*s == '"') { - if (!parse_plain_string(val)) { - return false; - } - } else if (*s == 'n' && s + 4 <= end && string(s, 4) == "null") { - explicit_null = true; - s += 4; - } else if (*s == 't' && s + 4 <= end && string(s, 4) == "true") { - val = "true"; - s += 4; - } else if (*s == 'f' && s + 5 <= end && string(s, 5) == "false") { - val = "false"; - s += 5; - } else if (*s == '-' || (*s >= '0' && *s <= '9')) { - const char *start = s; - if (!parse_number()) { - return false; - } - val.assign(start, s - start); + auto document = + duckdb_yyjson::yyjson_read(raw_body.data(), raw_body.size(), duckdb_yyjson::YYJSON_READ_BIGNUM_AS_RAW); + if (!document) { + return JsonBodyParseResult::INVALID_JSON; + } + unique_ptr doc(document, + duckdb_yyjson::yyjson_doc_free); + auto root = duckdb_yyjson::yyjson_doc_get_root(doc.get()); + if (!root || !duckdb_yyjson::yyjson_is_obj(root)) { + return JsonBodyParseResult::NOT_OBJECT; + } + auto iter = duckdb_yyjson::yyjson_obj_iter_with(root); + duckdb_yyjson::yyjson_val *key = nullptr; + while ((key = duckdb_yyjson::yyjson_obj_iter_next(&iter))) { + auto value = duckdb_yyjson::yyjson_obj_iter_get_val(key); + string name(duckdb_yyjson::yyjson_get_str(key), duckdb_yyjson::yyjson_get_len(key)); + if (duckdb_yyjson::yyjson_is_null(value)) { + fields[name] = {string(), true}; + } else if (duckdb_yyjson::yyjson_is_str(value)) { + fields[name] = {string(duckdb_yyjson::yyjson_get_str(value), duckdb_yyjson::yyjson_get_len(value)), false}; } else { - // nested object/array — not flat; fall back - return false; - } - fields[key] = {val, explicit_null}; - skip_ws(); - if (s < end && *s == ',') { - s++; - continue; - } - if (s < end && *s == '}') { - s++; - skip_ws(); - return s == end; + size_t length = 0; + auto serialized = duckdb_yyjson::yyjson_val_write(value, duckdb_yyjson::YYJSON_WRITE_NOFLAG, &length); + if (!serialized) { + fields.clear(); + return JsonBodyParseResult::INVALID_JSON; + } + fields[name] = {string(serialized, length), false}; + std::free(serialized); } - return false; } - return false; + return JsonBodyParseResult::SUCCESS; } //! Parse application/x-www-form-urlencoded query into httplib Params. void ParseQueryStringIntoParams(const string &query, duckdb_httplib::Params ¶ms) { diff --git a/src/quackapi_server.cpp b/src/quackapi_server.cpp index 0075e26..03818e7 100644 --- a/src/quackapi_server.cpp +++ b/src/quackapi_server.cpp @@ -99,9 +99,9 @@ struct QuackapiHttplibServer : duckdb_httplib::Server { //! SO_* is restored to serve defaults in QuackapiHttplibServer after //! process_request returns (keep-alive safe). Do not restore in a HandleRequest //! destructor — that runs before httplib's write_response. -//! Records the applied write deadline on qa_state only after this path runs -//! (early return leaves the serve-level seed from HandleRequest unchanged). -void ApplyRouteIoTimeout(time_t timeout_sec, QuackapiState &qa_state) { +//! Route timeout introspection was removed; the socket deadlines remain local +//! to the active request. +void ApplyRouteIoTimeout(time_t timeout_sec) { if (timeout_sec <= 0) { return; } @@ -114,7 +114,6 @@ void ApplyRouteIoTimeout(time_t timeout_sec, QuackapiState &qa_state) { duckdb_httplib::detail::set_socket_opt_time(quackapi_tls_sock, SOL_SOCKET, SO_RCVTIMEO, timeout_sec, 0); duckdb_httplib::detail::set_socket_opt_time(quackapi_tls_sock, SOL_SOCKET, SO_SNDTIMEO, timeout_sec, 0); } - qa_state.SetLastEffectiveWriteTimeoutSec(static_cast(timeout_sec)); } //! Content-Encoding choice after Accept-Encoding negotiation. @@ -348,9 +347,6 @@ string ValidationErrorJsonSchema(Connection &con, const string &msg, const strin return ValidationErrorJson("body", "_schema", msg, type); } -//! Preserve whether a JSON member was an actual JSON null. A string value of -//! "null" remains a string and must not silently become SQL NULL. - //! Media type from Content-Type (strip parameters; lowercased). string ContentTypeMedia(const case_insensitive_map_t &headers) { auto it = headers.find("Content-Type"); @@ -366,88 +362,6 @@ string ContentTypeMedia(const case_insensitive_map_t &headers) { return StringUtil::Lower(ct); } -//! Extract top-level JSON object fields as string values for binding. -//! On invalid JSON sets err_json and returns false. Arrays/non-objects → model_attributes_type. -bool ExtractJsonBodyFields(Connection &con, const string &raw_body, case_insensitive_map_t &fields, - string &err_json) { - fields.clear(); - if (raw_body.empty()) { - err_json = ValidationErrorJsonBody("JSON decode error", "json_invalid"); - return false; - } - // Hot path: flat object without spinning DuckDB (POST /write etc.). - if (TryExtractFlatJsonObject(raw_body, fields)) { - return true; - } - // json_each lives in the JSON extension and is not available when automatic - // extension loading is disabled. Loading a bundled extension is idempotent. - auto json_load = con.Query("LOAD json"); - if (json_load->HasError()) { - err_json = ValidationErrorJsonBody("JSON extension unavailable", "value_error"); - return false; - } - // Validate JSON parse first (TRY_CAST → NULL on failure). - auto check = con.Query("SELECT TRY_CAST(? AS JSON) IS NOT NULL", Value(raw_body)); - if (check->HasError()) { - err_json = ValidationErrorJsonBody("JSON decode error", "json_invalid"); - return false; - } - auto check_chunk = check->Fetch(); - if (!check_chunk || check_chunk->size() == 0 || !check_chunk->GetValue(0, 0).GetValue()) { - err_json = ValidationErrorJsonBody("JSON decode error", "json_invalid"); - return false; - } - // Must be an object to extract named fields (FastAPI body model). - auto type_res = con.Query("SELECT json_type(?::JSON)", Value(raw_body)); - if (type_res->HasError()) { - err_json = ValidationErrorJsonBody("JSON decode error", "json_invalid"); - return false; - } - auto type_chunk = type_res->Fetch(); - string jtype = type_chunk && type_chunk->size() > 0 && !type_chunk->GetValue(0, 0).IsNull() - ? type_chunk->GetValue(0, 0).GetValue() - : string(); - if (jtype != "OBJECT") { - err_json = ValidationErrorJsonBody("Input should be a valid dictionary or object to extract fields from", - "model_attributes_type"); - return false; - } - // json_each supplies the literal key, so keys containing dots or other JSONPath - // metacharacters cannot be reinterpreted as a path expression. - auto fields_res = con.Query("SELECT key, " - " CASE type " - " WHEN 'VARCHAR' THEN json_extract_string(value, '$') " - " WHEN 'NULL' THEN NULL " - " ELSE CAST(value AS VARCHAR) " - " END AS val " - "FROM json_each(?::JSON)", - Value(raw_body)); - if (fields_res->HasError()) { - err_json = ValidationErrorJsonBody("JSON decode error", "json_invalid"); - return false; - } - while (true) { - auto chunk = fields_res->Fetch(); - if (!chunk || chunk->size() == 0) { - break; - } - for (idx_t row = 0; row < chunk->size(); row++) { - auto key_v = chunk->GetValue(0, row); - if (key_v.IsNull()) { - continue; - } - string key = key_v.GetValue(); - auto val_v = chunk->GetValue(1, row); - if (val_v.IsNull()) { - fields[key] = {string(), true}; - continue; - } - fields[key] = {val_v.GetValue(), false}; - } - } - return true; -} - //! Validate body against BODY SCHEMA using community json_schema extension. //! json_schema_validate returns true on pass and THROWS on fail — wrap with try(). bool ValidateBodySchema(Connection &con, const string &schema, const string &raw_body, string &err_json) { @@ -497,16 +411,6 @@ bool ValidateBodySchema(Connection &con, const string &schema, const string &raw bool TransformTypedBody(Connection &con, const string &body_type, const string &raw_body, Value &out, string &err_json) { auto result = con.Query("SELECT try(json_transform_strict(?::JSON, ?::JSON))", Value(raw_body), Value(body_type)); - // json_transform is supplied by DuckDB's bundled json extension. Test and - // embedded deployments may turn automatic extension loading off, so load it - // explicitly only after DuckDB identifies this precise missing dependency. - if (result->HasError() && result->GetError().find("json extension") != string::npos) { - auto load = con.Query("LOAD json"); - if (!load->HasError()) { - result = - con.Query("SELECT try(json_transform_strict(?::JSON, ?::JSON))", Value(raw_body), Value(body_type)); - } - } if (!result->HasError()) { auto chunk = result->Fetch(); if (chunk && chunk->size() > 0 && !chunk->GetValue(0, 0).IsNull()) { @@ -2409,11 +2313,8 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd // Per-route IO timeout (CREATE ROUTE … TIMEOUT / WITH (timeout_sec := N)). // Extends httplib SocketStream select deadlines + SO_RCVTIMEO/SO_SNDTIMEO for // this connection through handler execute and response write. - // Seed introspection with serve write_timeout_sec; ApplyRouteIoTimeout - // overwrites only when it actually applies a route override. - qa_state.SetLastEffectiveWriteTimeoutSec(options.write_timeout_sec); if (match.route.timeout_sec > 0) { - ApplyRouteIoTimeout(static_cast(match.route.timeout_sec), qa_state); + ApplyRouteIoTimeout(static_cast(match.route.timeout_sec)); } // ---- AUTH ENFORCEMENT (before prepare/execute) ---- @@ -2637,10 +2538,18 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd // A native BODY TYPE bound only through $body can be an object or // array. It does not need the flat-field extractor; the native // transform below validates the complete payload directly. - if (needs_json_field_extraction && !ExtractJsonBodyFields(con, req.body, body_fields, err_json)) { - SetJson(res, 422, err_json); - finish(); - return; + if (needs_json_field_extraction) { + auto parse_result = ParseJsonBodyFields(req.body, body_fields); + if (parse_result != JsonBodyParseResult::SUCCESS) { + err_json = parse_result == JsonBodyParseResult::INVALID_JSON + ? ValidationErrorJsonBody("JSON decode error", "json_invalid") + : ValidationErrorJsonBody( + "Input should be a valid dictionary or object to extract fields from", + "model_attributes_type"); + SetJson(res, 422, err_json); + finish(); + return; + } } if (!match.route.body_schema.empty()) { if (!ValidateBodySchema(con, match.route.body_schema, req.body, err_json)) { diff --git a/test/sql/quackapi_route_errors.test b/test/sql/quackapi_route_errors.test index dd07801..6d5d471 100644 --- a/test/sql/quackapi_route_errors.test +++ b/test/sql/quackapi_route_errors.test @@ -31,12 +31,12 @@ query I CREATE TABLE quackapi_sql_empty (x INTEGER); ---- -# Dynamic SQL is validated before delegation so parser errors cannot become 200 [] +# SQL errors raised by an ordinary registered route remain server errors. query II SELECT status, contains(decode(body), 'SELEC') FROM quackapi_request('POST', '/sql', '{"sql":"SELEC 1"}'); ---- -422 true +500 true # Valid SELECT response shape is unchanged. query II From 907200936b99eb86d1ef31d5d55a50dfd6042011 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:25:09 -0700 Subject: [PATCH 22/93] fix(encoding): count only response body bytes --- src/include/quackapi_response_encoding.hpp | 3 - src/quackapi_response_encoding.cpp | 69 ---------------------- src/quackapi_server.cpp | 35 ++++------- 3 files changed, 11 insertions(+), 96 deletions(-) diff --git a/src/include/quackapi_response_encoding.hpp b/src/include/quackapi_response_encoding.hpp index 1f0c231..8fef771 100644 --- a/src/include/quackapi_response_encoding.hpp +++ b/src/include/quackapi_response_encoding.hpp @@ -1,7 +1,6 @@ #pragma once #include "duckdb/common/types.hpp" -#include "duckdb/common/types/value.hpp" #include "httplib.hpp" #include @@ -23,7 +22,6 @@ class QuackapiResponseWriter { explicit QuackapiResponseWriter(idx_t max_bytes); bool Append(const char *data, idx_t length); bool Append(const string &data); - bool SetMaxBytes(idx_t max_bytes); idx_t BytesWritten() const; bool ExceededLimit() const; const string &Data() const; @@ -76,7 +74,6 @@ class QuackapiCsvChunkEncoder { unique_ptr state; }; -string ValueToJson(const Value &value); bool IsSpecialResponseColumn(const string &name); void SetJson(duckdb_httplib::Response &res, int status, const string &body); void SetInternalError(duckdb_httplib::Response &res, const string &server_side_detail); diff --git a/src/quackapi_response_encoding.cpp b/src/quackapi_response_encoding.cpp index fe14e76..b805706 100644 --- a/src/quackapi_response_encoding.cpp +++ b/src/quackapi_response_encoding.cpp @@ -1,6 +1,5 @@ #include "quackapi_response_encoding.hpp" -#include #include #include @@ -73,14 +72,6 @@ bool QuackapiResponseWriter::Append(const string &append_data) { return Append(append_data.data(), append_data.size()); } -bool QuackapiResponseWriter::SetMaxBytes(idx_t max_bytes) { - this->max_bytes = max_bytes; - if (bytes_written > max_bytes) { - exceeded_limit = true; - } - return !exceeded_limit; -} - idx_t QuackapiResponseWriter::BytesWritten() const { return bytes_written; } @@ -321,66 +312,6 @@ void QuackapiCsvChunkEncoder::Finish() { state->Finish(); } -//! Render a DuckDB Value as JSON. The database typed the column — the JSON -//! representation follows the type, not string-formatting heuristics. -string ValueToJson(const Value &value) { - if (value.IsNull()) { - return "null"; - } - auto &type = value.type(); - switch (type.id()) { - case LogicalTypeId::BOOLEAN: - return value.GetValue() ? "true" : "false"; - case LogicalTypeId::TINYINT: - case LogicalTypeId::SMALLINT: - case LogicalTypeId::INTEGER: - case LogicalTypeId::BIGINT: - case LogicalTypeId::HUGEINT: - case LogicalTypeId::UTINYINT: - case LogicalTypeId::USMALLINT: - case LogicalTypeId::UINTEGER: - case LogicalTypeId::UBIGINT: - case LogicalTypeId::DECIMAL: - return value.ToString(); - case LogicalTypeId::FLOAT: - case LogicalTypeId::DOUBLE: { - auto d = value.GetValue(); - if (std::isnan(d) || std::isinf(d)) { - // JSON has no NaN/Infinity — mirror FastAPI/ujson and emit null - return "null"; - } - return value.ToString(); - } - case LogicalTypeId::LIST: { - auto &children = ListValue::GetChildren(value); - string result = "["; - for (idx_t i = 0; i < children.size(); i++) { - if (i > 0) { - result += ","; - } - result += ValueToJson(children[i]); - } - result += "]"; - return result; - } - case LogicalTypeId::STRUCT: { - auto &children = StructValue::GetChildren(value); - auto &child_types = StructType::GetChildTypes(type); - string result = "{"; - for (idx_t i = 0; i < children.size(); i++) { - if (i > 0) { - result += ","; - } - result += "\"" + QuackapiJsonEscape(child_types[i].first) + "\":" + ValueToJson(children[i]); - } - result += "}"; - return result; - } - default: - return "\"" + QuackapiJsonEscape(value.ToString()) + "\""; - } -} - //! True if column name is a special response control column (stripped from body). bool IsSpecialResponseColumn(const string &name) { auto lower = StringUtil::Lower(name); diff --git a/src/quackapi_server.cpp b/src/quackapi_server.cpp index 03818e7..2da18c9 100644 --- a/src/quackapi_server.cpp +++ b/src/quackapi_server.cpp @@ -3005,7 +3005,6 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd string location_value; vector set_cookie_values; - idx_t header_value_bytes = 0; idx_t rows_fetched = 0; bool limit_exceeded = false; while (true) { @@ -3018,38 +3017,20 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd // Keep response-control columns outside the body while reading them // from every chunk, matching the old first-Location/all-Cookies rules. - for (idx_t row = 0; row < chunk->size() && !limit_exceeded; row++) { + for (idx_t row = 0; row < chunk->size(); row++) { if (location_col < chunk->ColumnCount()) { auto value = chunk->GetValue(location_col, row); - if (!value.IsNull()) { - auto text = value.ToString(); - if (text.size() > max_response_bytes - header_value_bytes) { - limit_exceeded = true; - } else { - header_value_bytes += text.size(); - if (location_value.empty() && !text.empty()) { - location_value = text; - } - } + if (location_value.empty() && !value.IsNull()) { + location_value = value.ToString(); } } if (set_cookie_col < chunk->ColumnCount()) { auto value = chunk->GetValue(set_cookie_col, row); if (!value.IsNull()) { - auto text = value.ToString(); - if (text.size() > max_response_bytes - header_value_bytes) { - limit_exceeded = true; - } else { - header_value_bytes += text.size(); - set_cookie_values.push_back(std::move(text)); - } + set_cookie_values.push_back(value.ToString()); } } } - if (limit_exceeded || !response_writer.SetMaxBytes(max_response_bytes - header_value_bytes)) { - limit_exceeded = true; - break; - } if (mode == ResponseMode::HTML || mode == ResponseMode::TEXT) { auto col = data_cols[0]; @@ -3100,7 +3081,13 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd // 0-row EMPTY STATUS: override success STATUS (any FORMAT / html/text). if (rows_fetched == 0 && match.route.empty_status != 0) { - SetJson(res, match.route.empty_status, EmptyResultBody(match.route)); + auto empty_body = EmptyResultBody(match.route); + if (empty_body.size() > max_response_bytes) { + SetJson(res, 507, "{\"detail\":\"Response exceeds configured byte limit\"}"); + finish(); + return; + } + SetJson(res, match.route.empty_status, empty_body); finish(); return; } From 82f293c96e290fe81b3cd287ed6ed14841999a6d Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:28:51 -0700 Subject: [PATCH 23/93] Load JSON dependencies without implicit installs --- src/quackapi_ddl.cpp | 23 +++++++---------------- src/quackapi_extension.cpp | 8 +++++++- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/quackapi_ddl.cpp b/src/quackapi_ddl.cpp index e8b07dd..01ba491 100644 --- a/src/quackapi_ddl.cpp +++ b/src/quackapi_ddl.cpp @@ -1218,28 +1218,19 @@ void ApplyRouteExec(ClientContext &context, TableFunctionInput &data_p, DataChun { Connection con(*context.db); if (!bind_data.route.body_schema.empty()) { - // BODY SCHEMA depends on the community json_schema extension. Resolve it - // while creating the route, outside any request deadline, so the first - // request cannot unexpectedly pay the installation/network cost. + // BODY SCHEMA depends on the community json_schema extension. Load only + // an extension the caller has already made available; never install it + // from route DDL. auto load = con.Query("LOAD json_schema"); if (load->HasError()) { - auto install = con.Query("INSTALL json_schema FROM community"); - if (install->HasError()) { - throw InvalidInputException("BODY SCHEMA requires the json_schema extension: %s", - install->GetError()); - } - load = con.Query("LOAD json_schema"); - } - if (load->HasError()) { - throw InvalidInputException("BODY SCHEMA requires the json_schema extension: %s", load->GetError()); + throw InvalidInputException("BODY SCHEMA requires the json_schema extension to be available and " + "loadable; load json_schema before creating this route: %s", + load->GetError()); } // Catch malformed schema text at CREATE time rather than on the first // request. TRY_CAST returns NULL for invalid JSON without throwing. - auto json_load = con.Query("LOAD json"); - if (json_load->HasError()) { - throw InvalidInputException("BODY SCHEMA requires the json extension: %s", json_load->GetError()); - } + // The core json extension is loaded when QuackAPI loads. auto schema_json = con.Query("SELECT TRY_CAST(? AS JSON) IS NOT NULL", Value(bind_data.route.body_schema)); if (schema_json->HasError()) { diff --git a/src/quackapi_extension.cpp b/src/quackapi_extension.cpp index 7505bad..f281a7c 100644 --- a/src/quackapi_extension.cpp +++ b/src/quackapi_extension.cpp @@ -1011,7 +1011,13 @@ static void LoadInternal(ExtensionLoader &loader) { // SET quackapi_cors_origins = '*' | 'https://app.example,https://admin.example' // Default empty = CORS off. Serve-time cors_origins := '…' overrides this SET. auto &db = loader.GetDatabaseInstance(); - if (!ExtensionHelper::TryAutoLoadAvailableExtension(db, "json")) { + // Load a statically linked DuckDB JSON extension through its generated + // loader first, then accept an already available loadable extension. Neither + // path installs a dependency as a side effect of LOAD quackapi. + DuckDB duckdb(db); + const auto json_load = ExtensionHelper::LoadExtension(duckdb, "json"); + if (json_load != ExtensionLoadResult::LOADED_EXTENSION && + !ExtensionHelper::TryAutoLoadAvailableExtension(db, "json")) { throw IOException("QuackAPI requires the DuckDB JSON extension to be available and loadable before loading " "quackapi"); } From 960b3dff41ac3aa17dd7b108eeeabcb01928a7cc Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:27:07 -0700 Subject: [PATCH 24/93] Reuse reviewed outbound POST clients safely --- src/quackapi_post_provider.cpp | 94 ++++++++++++++-------------------- 1 file changed, 39 insertions(+), 55 deletions(-) diff --git a/src/quackapi_post_provider.cpp b/src/quackapi_post_provider.cpp index de2d604..775a18d 100644 --- a/src/quackapi_post_provider.cpp +++ b/src/quackapi_post_provider.cpp @@ -6,54 +6,26 @@ #include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" -#include - namespace duckdb { namespace { constexpr int64_t OUTBOUND_TIMEOUT_FALLBACK_MS = 600 * 1000; -//! Connection settings that must match before a worker may reuse its provider -//! client. Request headers and timeouts stay request-local. +//! Identity of a provider client that may be reinitialized by the next request. struct ClientConfig { string provider_name; - string params_type; string origin; - string http_proxy; - idx_t http_proxy_port = 0; - string http_proxy_username; - string http_proxy_password; - bool keep_alive = true; - bool follow_location = true; - bool override_verify_ssl = false; - bool verify_ssl = true; - - bool Matches(const HTTPUtil &util, const HTTPParams ¶ms, const string &candidate_origin) const { - // The common fields below cannot fingerprint provider-specific parameter - // subclasses. Until a provider exposes a configuration comparison API, - // only reuse the exact base parameter type; replace provider-specialized - // clients for every request rather than risk carrying stale settings. - return typeid(params) == typeid(HTTPParams) && provider_name == util.GetName() && - params_type == typeid(params).name() && origin == candidate_origin && http_proxy == params.http_proxy && - (http_proxy.empty() || http_proxy_port == params.http_proxy_port) && - http_proxy_username == params.http_proxy_username && http_proxy_password == params.http_proxy_password && - keep_alive == params.keep_alive && follow_location == params.follow_location && - override_verify_ssl == params.override_verify_ssl && verify_ssl == params.verify_ssl; + + bool Matches(const HTTPUtil &util, const string &candidate_origin) const { + const auto name = util.GetName(); + const bool reviewed_provider = name == "HTTPFS-Curl" || name == "MultiCurl"; + return reviewed_provider && provider_name == name && origin == candidate_origin; } - void Set(const HTTPUtil &util, const HTTPParams ¶ms, const string &candidate_origin) { + void Set(const HTTPUtil &util, const string &candidate_origin) { provider_name = util.GetName(); - params_type = typeid(params).name(); origin = candidate_origin; - http_proxy = params.http_proxy; - http_proxy_port = http_proxy.empty() ? 0 : params.http_proxy_port; - http_proxy_username = params.http_proxy_username; - http_proxy_password = params.http_proxy_password; - keep_alive = params.keep_alive; - follow_location = params.follow_location; - override_verify_ssl = params.override_verify_ssl; - verify_ssl = params.verify_ssl; } }; @@ -136,25 +108,8 @@ QuackapiHttpFetchResult QuackapiPostProvider::Post(DatabaseInstance &db, Quackap auto params = http_util.InitializeParameters(db, request.url); params->keep_alive = true; params->retries = 0; // POST is not safe to replay after an ambiguous transport failure. - const auto timeout_ms = QuackapiRemainingTimeoutMillis(OUTBOUND_TIMEOUT_FALLBACK_MS); - params->timeout = static_cast(timeout_ms / 1000); - params->timeout_usec = static_cast((timeout_ms % 1000) * 1000); - const bool reused = state.client && state.client_util == &http_util && state.has_client_config && - state.client_config.Matches(http_util, *params, origin); - if (!reused) { - DiscardClient(state); - state.client = http_util.InitializeClient(*params, origin); - if (!state.client) { - throw InvalidConfigurationException(UnsupportedProviderMessage(provider_name)); - } - state.client_util = &http_util; - state.client_config.Set(http_util, *params, origin); - state.has_client_config = true; - } else { - // Timeouts are request-specific. Reapply them to a worker-reused client. - state.client->Initialize(*params); - } + state.client_config.Matches(http_util, origin); HTTPHeaders headers(db); if (!request.content_type.empty()) { @@ -167,8 +122,35 @@ QuackapiHttpFetchResult QuackapiPostProvider::Post(DatabaseInstance &db, Quackap PostRequestInfo info(request.url, headers, *params, const_data_ptr_cast(request.body.data()), request.body.size()); info.try_request = true; - // The caller can be interrupted while this worker is being scheduled or - // while its provider client is prepared. Never send after that signal. + // Prepare the provider client with the latest remaining deadline. Parameter + // initialization and client setup happen before this point and can consume + // part of the caller's budget. + ThrowIfCancelled(request); + const auto timeout_ms = QuackapiRemainingTimeoutMillis(OUTBOUND_TIMEOUT_FALLBACK_MS); + params->timeout = static_cast(timeout_ms / 1000); + params->timeout_usec = static_cast((timeout_ms % 1000) * 1000); + if (!reused) { + DiscardClient(state); + state.client = http_util.InitializeClient(*params, origin); + if (!state.client) { + throw InvalidConfigurationException(UnsupportedProviderMessage(provider_name)); + } + state.client_util = &http_util; + state.client_config.Set(http_util, origin); + state.has_client_config = true; + } + + // Client setup itself may use part of the remaining budget. Refresh timeout + // parameters and initialize with the current configuration before sending. + const auto send_timeout_ms = QuackapiRemainingTimeoutMillis(OUTBOUND_TIMEOUT_FALLBACK_MS); + params->timeout = static_cast(send_timeout_ms / 1000); + params->timeout_usec = static_cast((send_timeout_ms % 1000) * 1000); + // The reviewed providers fully apply current HTTPParams on Initialize. This + // includes changed proxy/TLS/auth settings; no short-lived params reference + // is retained. For fresh clients this also replaces the initial timeout. + state.client->Initialize(*params); + + // Never send after setup consumes the remaining deadline. ThrowIfCancelled(request); QuackapiHttpFetchResult result; try { @@ -177,6 +159,8 @@ QuackapiHttpFetchResult QuackapiPostProvider::Post(DatabaseInstance &db, Quackap DiscardClient(state); throw InvalidConfigurationException(UnsupportedProviderMessage(provider_name)); } + // This reports HTTPClient object reuse only; the provider does not expose + // whether its underlying TCP socket was reused. result.reused_connection = reused; if (!result.request_error.empty()) { DiscardClient(state); From 26cc79e2d4e9ce7918e09cd18b4c4bb09b0a2f8d Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:30:16 -0700 Subject: [PATCH 25/93] Propagate route deadlines into POST workers --- src/include/quackapi_limits.hpp | 4 +++ src/quackapi_limits.cpp | 58 ++++++++++++++++++++++++++++++++- src/quackapi_post_executor.cpp | 2 +- 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/include/quackapi_limits.hpp b/src/include/quackapi_limits.hpp index c6d2a17..1a07b26 100644 --- a/src/include/quackapi_limits.hpp +++ b/src/include/quackapi_limits.hpp @@ -6,6 +6,7 @@ namespace duckdb { class Connection; +class ClientContext; struct QuackapiDeadlineState; //! Shared watchdog, not a new OS thread for every request. The guard must die @@ -25,6 +26,9 @@ class QuackapiQueryDeadline { //! Outbound clients inherit the request's remaining execution budget. std::chrono::steady_clock::time_point QuackapiCurrentDeadline(); +//! Get the active deadline for a ClientContext, including work scheduled on a +//! DuckDB task thread where thread-local state is not inherited. +std::chrono::steady_clock::time_point QuackapiCurrentDeadline(ClientContext &context); void QuackapiSetCurrentDeadline(std::chrono::steady_clock::time_point deadline); int64_t QuackapiRemainingTimeoutMillis(int64_t fallback_ms); } // namespace duckdb diff --git a/src/quackapi_limits.cpp b/src/quackapi_limits.cpp index 8a2d812..94fc260 100644 --- a/src/quackapi_limits.cpp +++ b/src/quackapi_limits.cpp @@ -1,11 +1,13 @@ #include "quackapi_limits.hpp" #include "duckdb/main/connection.hpp" +#include "duckdb/main/client_context.hpp" #include #include #include #include #include #include +#include namespace duckdb { static thread_local std::chrono::steady_clock::time_point current_deadline = @@ -18,6 +20,38 @@ struct QuackapiDeadlineState { std::atomic expired {false}; }; +static std::mutex deadline_registry_mutex; +static std::unordered_map>> deadline_registry; + +static void RegisterDeadline(ClientContext *context, const std::shared_ptr &state) { + if (!context) { + return; + } + std::lock_guard guard(deadline_registry_mutex); + auto &stack = deadline_registry[context]; + stack.erase(std::remove_if(stack.begin(), stack.end(), [](const auto &entry) { return entry.expired(); }), + stack.end()); + stack.push_back(state); +} + +static void UnregisterDeadline(ClientContext *context, const std::shared_ptr &state) { + if (!context) { + return; + } + std::lock_guard guard(deadline_registry_mutex); + auto found = deadline_registry.find(context); + if (found == deadline_registry.end()) { + return; + } + auto &stack = found->second; + stack.erase(std::remove_if(stack.begin(), stack.end(), + [&](const auto &entry) { return entry.expired() || entry.lock() == state; }), + stack.end()); + if (stack.empty()) { + deadline_registry.erase(found); + } +} + class DeadlineWatchdog { public: DeadlineWatchdog() : worker([this]() { Run(); }) { @@ -77,11 +111,16 @@ static DeadlineWatchdog &Watchdog() { QuackapiQueryDeadline::QuackapiQueryDeadline(Connection &connection, int64_t timeout_ms) : previous(current_deadline) { state = std::make_shared(); state->connection = &connection; - state->deadline = std::min(previous, std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms)); + const auto context_deadline = connection.context ? QuackapiCurrentDeadline(*connection.context) + : std::chrono::steady_clock::time_point::max(); + state->deadline = std::min(std::min(previous, context_deadline), + std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms)); current_deadline = state->deadline; + RegisterDeadline(connection.context.get(), state); Watchdog().Add(state); } QuackapiQueryDeadline::~QuackapiQueryDeadline() { + UnregisterDeadline(connection.context.get(), state); { std::lock_guard guard(state->mutex); state->connection = nullptr; @@ -94,6 +133,23 @@ bool QuackapiQueryDeadline::Expired() const { std::chrono::steady_clock::time_point QuackapiCurrentDeadline() { return current_deadline; } +std::chrono::steady_clock::time_point QuackapiCurrentDeadline(ClientContext &context) { + std::lock_guard guard(deadline_registry_mutex); + auto found = deadline_registry.find(&context); + if (found == deadline_registry.end()) { + return std::chrono::steady_clock::time_point::max(); + } + auto &stack = found->second; + while (!stack.empty()) { + auto state = stack.back().lock(); + if (state) { + return state->deadline; + } + stack.pop_back(); + } + deadline_registry.erase(found); + return std::chrono::steady_clock::time_point::max(); +} void QuackapiSetCurrentDeadline(std::chrono::steady_clock::time_point deadline) { current_deadline = deadline; } diff --git a/src/quackapi_post_executor.cpp b/src/quackapi_post_executor.cpp index ff8ed88..c011088 100644 --- a/src/quackapi_post_executor.cpp +++ b/src/quackapi_post_executor.cpp @@ -109,7 +109,7 @@ class QuackapiPostExecutor::Impl { vector Execute(ClientContext &context, vector requests, const vector &skip) { - const auto deadline = QuackapiCurrentDeadline(); + const auto deadline = QuackapiCurrentDeadline(context); auto batch = make_shared_ptr(); batch->results.resize(requests.size()); batch->cancellation_flags.resize(requests.size()); From a56fdcad6daa18b8ad1a3a7fce21473725e4c2d8 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:31:46 -0700 Subject: [PATCH 26/93] fix: defer JSON requirement until serving --- src/quackapi_extension.cpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/quackapi_extension.cpp b/src/quackapi_extension.cpp index f281a7c..14967d1 100644 --- a/src/quackapi_extension.cpp +++ b/src/quackapi_extension.cpp @@ -16,7 +16,6 @@ #include "duckdb/main/connection.hpp" #include "duckdb/main/database.hpp" #include "duckdb/main/extension/extension_loader.hpp" -#include "duckdb/main/extension_helper.hpp" #include "duckdb/main/extension_callback_manager.hpp" #include "duckdb/main/settings.hpp" @@ -1011,16 +1010,6 @@ static void LoadInternal(ExtensionLoader &loader) { // SET quackapi_cors_origins = '*' | 'https://app.example,https://admin.example' // Default empty = CORS off. Serve-time cors_origins := '…' overrides this SET. auto &db = loader.GetDatabaseInstance(); - // Load a statically linked DuckDB JSON extension through its generated - // loader first, then accept an already available loadable extension. Neither - // path installs a dependency as a side effect of LOAD quackapi. - DuckDB duckdb(db); - const auto json_load = ExtensionHelper::LoadExtension(duckdb, "json"); - if (json_load != ExtensionLoadResult::LOADED_EXTENSION && - !ExtensionHelper::TryAutoLoadAvailableExtension(db, "json")) { - throw IOException("QuackAPI requires the DuckDB JSON extension to be available and loadable before loading " - "quackapi"); - } auto &config = DBConfig::GetConfig(db); config.AddExtensionOption("quackapi_cors_origins", "CORS allowed origins for quackapi_serve (* or comma-separated list). " From d424a3d54efab37c1b86371d29df14315611e68b Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:31:54 -0700 Subject: [PATCH 27/93] fix: avoid installing JSON schema from route DDL --- src/quackapi_ddl.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/quackapi_ddl.cpp b/src/quackapi_ddl.cpp index 01ba491..f0866a5 100644 --- a/src/quackapi_ddl.cpp +++ b/src/quackapi_ddl.cpp @@ -4,6 +4,7 @@ #include "duckdb/main/client_context.hpp" #include "duckdb/main/connection.hpp" #include "duckdb/main/database.hpp" +#include "duckdb/main/extension_helper.hpp" #include "duckdb/main/prepared_statement.hpp" #include "duckdb/parser/parser_extension.hpp" @@ -1218,19 +1219,21 @@ void ApplyRouteExec(ClientContext &context, TableFunctionInput &data_p, DataChun { Connection con(*context.db); if (!bind_data.route.body_schema.empty()) { - // BODY SCHEMA depends on the community json_schema extension. Load only - // an extension the caller has already made available; never install it - // from route DDL. - auto load = con.Query("LOAD json_schema"); - if (load->HasError()) { + // Load dependencies only when already available locally; route DDL must + // never install an extension as a side effect. + if (!ExtensionHelper::TryAutoLoadAvailableExtension(*context.db, "json")) { + throw InvalidInputException( + "BODY SCHEMA requires the DuckDB JSON extension; load json before creating " + "this route"); + } + if (!ExtensionHelper::TryAutoLoadAvailableExtension(*context.db, "json_schema")) { throw InvalidInputException("BODY SCHEMA requires the json_schema extension to be available and " - "loadable; load json_schema before creating this route: %s", - load->GetError()); + "loadable; make it available locally before creating this route"); } // Catch malformed schema text at CREATE time rather than on the first // request. TRY_CAST returns NULL for invalid JSON without throwing. - // The core json extension is loaded when QuackAPI loads. + // The core json extension was loaded above without an install fallback. auto schema_json = con.Query("SELECT TRY_CAST(? AS JSON) IS NOT NULL", Value(bind_data.route.body_schema)); if (schema_json->HasError()) { From da31c39cfb3e8cfc44db99bd1410a8c17e809213 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:32:33 -0700 Subject: [PATCH 28/93] refactor(logging): use DuckDB native access logs --- src/include/quackapi_logging.hpp | 5 +- src/include/quackapi_server.hpp | 88 +--- src/include/quackapi_state.hpp | 9 - src/quackapi_extension.cpp | 349 ++-------------- src/quackapi_logging.cpp | 85 ++-- src/quackapi_registry.cpp | 61 --- src/quackapi_server.cpp | 516 +----------------------- test/sql/quackapi_access_log.test | 162 -------- test/sql/quackapi_requests_ring.test | 209 ---------- test/sql/quackapi_telemetry_record.test | 171 -------- test/sql/quackapi_telemetry_redact.test | 78 ---- test/sql/quackapi_telemetry_status.test | 65 --- 12 files changed, 115 insertions(+), 1683 deletions(-) delete mode 100644 test/sql/quackapi_access_log.test delete mode 100644 test/sql/quackapi_requests_ring.test delete mode 100644 test/sql/quackapi_telemetry_record.test delete mode 100644 test/sql/quackapi_telemetry_redact.test delete mode 100644 test/sql/quackapi_telemetry_status.test diff --git a/src/include/quackapi_logging.hpp b/src/include/quackapi_logging.hpp index 040f17b..e08b1f4 100644 --- a/src/include/quackapi_logging.hpp +++ b/src/include/quackapi_logging.hpp @@ -1,9 +1,12 @@ #pragma once +#include "duckdb/logging/logging.hpp" + namespace duckdb { +class DatabaseInstance; struct QuackapiRequestRecord; -void EmitAccessLogStderr(const QuackapiRequestRecord &entry); +void EmitQuackapiAccessLog(DatabaseInstance &db, const QuackapiRequestRecord &entry, LogLevel level); } // namespace duckdb diff --git a/src/include/quackapi_server.hpp b/src/include/quackapi_server.hpp index 9f6ad14..0866fe7 100644 --- a/src/include/quackapi_server.hpp +++ b/src/include/quackapi_server.hpp @@ -2,7 +2,6 @@ #include #include -#include #include #include @@ -21,7 +20,6 @@ namespace duckdb { class ClientContext; class DatabaseInstance; -class Value; //! Max request body accepted by quackapi (8 MiB). Larger bodies get 413. static constexpr size_t QUACKAPI_PAYLOAD_MAX_LENGTH = 8ull * 1024ull * 1024ull; @@ -60,18 +58,12 @@ struct QuackapiServeOptions { QuackapiLogLevel log_level = QuackapiLogLevel::INFO; //! Requests at or above this duration are included at WARN level. int64_t slow_request_ms = 1000; - //! Emit one structured access-log line per request (stderr, JSON). Default true. + //! Emit one structured request event through DuckDB's native logger. Default true. bool access_log = true; - //! When non-empty, append access-log rows to this operator-created table instead - //! of stderr. The writer falls back to stderr if the table cannot be used. - string access_log_table; - //! Header names permitted in the access-log map; empty avoids retaining headers. + //! Header names permitted in the structured access log; empty omits headers. string log_headers; //! Preserve the raw query string in the access log after key-based redaction. bool log_query = false; - //! Number of completed TCP requests retained for quackapi_requests(). - //! Zero disables the ring and its allocation. - int64_t request_ring = 10000; // --- Batteries: health routes (ON by default) --- //! Auto-register GET /health + GET /healthz. Default true. @@ -105,7 +97,7 @@ struct QuackapiServeOptions { idx_t compression_min_bytes = 1024; }; -//! One completed request, shared by the stderr and table access-log sinks. +//! One completed request, emitted through DuckDB's native logger. struct QuackapiRequestRecord { string request_id; int64_t received_at_micros; @@ -135,70 +127,6 @@ struct QuackapiRequestRecord { string query; }; -struct QuackapiTelemetryStatus { - string sink; - string target; - idx_t queued = 0; - idx_t exported_total = 0; - idx_t dropped_total = 0; - string last_error; - int64_t last_export_age_ms = -1; -}; - -class QuackapiTelemetrySink { -public: - virtual ~QuackapiTelemetrySink() = default; - - virtual void Enqueue(const QuackapiRequestRecord &record) = 0; - virtual void FlushPending() = 0; - virtual void Flush() = 0; - virtual QuackapiTelemetryStatus Status() const = 0; -}; - -//! Convert the opt-in request fields to the access-log MAP/VARCHAR values. -Value QuackapiRequestHeadersValue(const QuackapiRequestRecord &entry); -Value QuackapiRequestQueryValue(const QuackapiRequestRecord &entry); - -//! Fixed storage keeps request recording bounded while the mutex makes snapshots -//! safe without holding the request path behind a reader's work. -class QuackapiRequestRing { -public: - explicit QuackapiRequestRing(size_t capacity) : records(capacity) { - } - - void Push(const QuackapiRequestRecord &record) { - std::lock_guard lock(mutex); - if (records.empty()) { - return; - } - records[head] = record; - head = (head + 1) % records.size(); - if (count < records.size()) { - count++; - } - } - - std::vector Snapshot() const { - std::lock_guard lock(mutex); - std::vector result; - result.reserve(count); - if (records.empty()) { - return result; - } - const auto first = count == records.size() ? head : 0; - for (size_t i = 0; i < count; i++) { - result.push_back(records[(first + i) % records.size()]); - } - return result; - } - -private: - std::vector records; - mutable std::mutex mutex; - size_t head = 0; - size_t count = 0; -}; - //! Parse log_level named param / setting. Accepts silent|error|warn|info|debug //! (case-insensitive). Unknown → INFO. QuackapiLogLevel ParseQuackapiLogLevel(const string &raw); @@ -247,14 +175,6 @@ class QuackapiHttpServer { const QuackapiServeOptions &Options() const { return options; } - //! Copy the retained records oldest-first while holding only the ring lock. - std::vector SnapshotRequests() const; - //! Keep per-sink counters queryable without exposing sink ownership to the registry. - std::vector SnapshotTelemetryStatus() const; - //! Flush queued telemetry without stopping the server. - void FlushTelemetryPending(); - //! Compatibility counter used by /healthz. - idx_t AccessLogOverflowCount() const; //! True while the TCP listener thread is alive (false after StopAccepting). bool IsRunning() const { return is_running.load(); @@ -283,8 +203,6 @@ class QuackapiHttpServer { unique_ptr server; std::vector listen_threads; std::atomic is_running {false}; - unique_ptr request_ring; - std::vector> telemetry_sinks; }; //! In-process HTTP-shape invoke (no TCP). Builds a Request, runs Dispatch, returns diff --git a/src/include/quackapi_state.hpp b/src/include/quackapi_state.hpp index 7071360..f2c2909 100644 --- a/src/include/quackapi_state.hpp +++ b/src/include/quackapi_state.hpp @@ -206,15 +206,6 @@ class QuackapiState : public ObjectCacheEntry { bool GetServerHost(int port, string &host_out); //! (host, port) for each running server. vector> ListServers(); - //! Snapshot one server's request ring; port 0 means the only running server. - //! A port is required when more than one server is running. - vector SnapshotRequests(int port = 0); - //! Snapshot telemetry sinks; port 0 means the only running server. - //! A port is required when more than one server is running. - vector SnapshotTelemetryStatus(int port = 0); - //! Flush queued telemetry for one running server. Port 0 means the only server. - //! A port is required when more than one server is running. - bool FlushTelemetry(int port = 0); private: void PublishRoutes(); diff --git a/src/quackapi_extension.cpp b/src/quackapi_extension.cpp index 14967d1..7ec5987 100644 --- a/src/quackapi_extension.cpp +++ b/src/quackapi_extension.cpp @@ -95,7 +95,7 @@ struct ServeBindData : public TableFunctionData { //! Empty (default) = CORS off. Pass '*' or a comma-separated origin list. string cors_origins; string log_level = "info"; - string access_log = "true"; + bool access_log = true; bool health_routes = true; int32_t worker_threads = static_cast(QUACKAPI_DEFAULT_WORKER_THREADS); int32_t keep_alive_max_count = static_cast(QUACKAPI_DEFAULT_KEEP_ALIVE_MAX); @@ -112,33 +112,26 @@ struct ServeBindData : public TableFunctionData { //! First Exec emitted listen_url; second Exec blocks when block=true. bool started = false; bool finished = false; - int64_t request_ring = 10000; }; -static void ParseAccessLogString(const string &raw, bool &enabled, string &table) { - auto value = StringUtil::Lower(raw); - if (value == "true") { - enabled = true; - table.clear(); - } else if (value == "false") { - enabled = false; - table.clear(); - } else if (!raw.empty()) { - enabled = true; - table = raw; - } else { - throw InvalidInputException("quackapi_serve: access_log must be true, false, or a table name"); - } -} - -static string AccessLogValueString(const Value &value) { +static bool AccessLogValue(const Value &value) { if (value.IsNull()) { - throw InvalidInputException("quackapi_serve: access_log must not be NULL"); + throw InvalidInputException("access_log must not be NULL"); + } + if (value.type().id() == LogicalTypeId::BOOLEAN) { + return value.GetValue(); } if (value.type().id() == LogicalTypeId::VARCHAR) { - return value.GetValue(); + auto raw = StringUtil::Lower(value.GetValue()); + StringUtil::Trim(raw); + if (raw == "true") { + return true; + } + if (raw == "false") { + return false; + } } - return value.ToString(); + throw InvalidInputException("access_log must be true or false; table sinks are no longer supported"); } static void BindResourceLimits(ClientContext &context, TableFunctionBindInput &input, QuackapiServeOptions &opts) { @@ -234,26 +227,9 @@ static unique_ptr ServeBind(ClientContext &context, TableFunctionB } } } - // request_ring named param wins; else SET quackapi_request_ring. - auto ring_entry = input.named_parameters.find("request_ring"); - if (ring_entry != input.named_parameters.end()) { - bind_data->request_ring = ring_entry->second.GetValue(); - } else { - Value setting; - if (context.TryGetCurrentSetting("quackapi_request_ring", setting) && !setting.IsNull()) { - bind_data->request_ring = setting.GetValue(); - } - } - // The ring is allocated up front at serve time, so bound it to keep a typo from exhausting memory. - if (bind_data->request_ring < 0 || bind_data->request_ring > 1000000) { - throw InvalidInputException("quackapi_serve: request_ring must be between 0 and 1000000"); - } auto access_entry = input.named_parameters.find("access_log"); if (access_entry != input.named_parameters.end()) { - bind_data->access_log = AccessLogValueString(access_entry->second); - bool enabled; - string table; - ParseAccessLogString(bind_data->access_log, enabled, table); + bind_data->access_log = AccessLogValue(access_entry->second); } auto health_entry = input.named_parameters.find("health_routes"); if (health_entry != input.named_parameters.end()) { @@ -367,8 +343,7 @@ static void ServeExec(ClientContext &context, TableFunctionInput &data_p, DataCh opts.static_dir = bind_data.static_dir; opts.cors_origins = bind_data.cors_origins; opts.log_level = ParseQuackapiLogLevel(bind_data.log_level); - opts.request_ring = bind_data.request_ring; - ParseAccessLogString(bind_data.access_log, opts.access_log, opts.access_log_table); + opts.access_log = bind_data.access_log; opts.health_routes = bind_data.health_routes; opts.worker_threads = bind_data.worker_threads; opts.keep_alive_max_count = bind_data.keep_alive_max_count; @@ -550,8 +525,8 @@ struct RequestBindData : public TableFunctionData { string path; string body; unordered_map req_headers; - //! Empty (default) keeps in-process requests quiet. Otherwise true, false, or a table name. - string access_log; + //! In-process requests are quiet by default; access_log := true opts into native logging. + bool access_log = false; string log_level = "info"; bool finished = false; }; @@ -585,10 +560,7 @@ static unique_ptr RequestBind(ClientContext &context, TableFunctio } auto access_entry = input.named_parameters.find("access_log"); if (access_entry != input.named_parameters.end()) { - bind_data->access_log = AccessLogValueString(access_entry->second); - bool enabled; - string table; - ParseAccessLogString(bind_data->access_log, enabled, table); + bind_data->access_log = AccessLogValue(access_entry->second); } // log_level named param wins; otherwise use SET quackapi_log_level. auto log_entry = input.named_parameters.find("log_level"); @@ -627,10 +599,7 @@ static void RequestExec(ClientContext &context, TableFunctionInput &data_p, Data unordered_map resp_headers; QuackapiServeOptions opts = bind_data.limits; opts.log_level = ParseQuackapiLogLevel(bind_data.log_level); - opts.access_log = false; - if (!bind_data.access_log.empty()) { - ParseAccessLogString(bind_data.access_log, opts.access_log, opts.access_log_table); - } + opts.access_log = bind_data.access_log; QuackapiInProcessRequest(*context.db, bind_data.method, bind_data.path, bind_data.body, status, body, content_type, &bind_data.req_headers, &resp_headers, &opts); output.SetValue(0, 0, Value::INTEGER(status)); @@ -765,243 +734,6 @@ static void ServersExec(ClientContext &, TableFunctionInput &data_p, DataChunk & output.SetCardinality(row); } -//===--------------------------------------------------------------------===// -// quackapi_requests([port]) — recent TCP requests from the in-memory ring -//===--------------------------------------------------------------------===// - -struct RequestsBindData : public TableFunctionData { - int32_t port = 0; -}; - -struct RequestsGlobalState : public GlobalTableFunctionState { - vector records; - idx_t offset = 0; -}; - -static Value RequestsValue(const QuackapiRequestRecord &record, idx_t column) { - switch (column) { - case 0: - return Value(record.request_id); - case 1: - return Value::TIMESTAMP(timestamp_t(record.received_at_micros)); - case 2: - return Value(record.method); - case 3: - return Value(record.path); - case 4: - return record.route_name.empty() ? Value() : Value(record.route_name); - case 5: - return record.route_path.empty() ? Value() : Value(record.route_path); - case 6: - return Value::INTEGER(record.status); - case 7: - return Value::DOUBLE(record.duration_ms); - case 8: - return record.sql_prepare_ms < 0 ? Value() : Value::DOUBLE(record.sql_prepare_ms); - case 9: - return record.sql_execute_ms < 0 ? Value() : Value::DOUBLE(record.sql_execute_ms); - case 10: - return record.rows_out < 0 ? Value() : Value::BIGINT(record.rows_out); - case 11: - return Value::BIGINT(record.bytes_in); - case 12: - return Value::BIGINT(record.bytes_out); - case 13: - return Value(record.client_ip); - case 14: - return Value(record.user_agent); - case 15: - return record.http_version.empty() ? Value() : Value(record.http_version); - case 16: - return record.error_type.empty() ? Value() : Value(record.error_type); - case 17: - return record.error_message.empty() ? Value() : Value(record.error_message); - case 18: - return record.trace_id.empty() ? Value() : Value(record.trace_id); - case 19: - return record.span_id.empty() ? Value() : Value(record.span_id); - case 20: - return record.parent_span_id.empty() ? Value() : Value(record.parent_span_id); - case 21: - return Value::BOOLEAN(record.sampled); - case 22: - return QuackapiRequestHeadersValue(record); - case 23: - return QuackapiRequestQueryValue(record); - default: - throw InternalException("unknown quackapi request column"); - } -} - -static unique_ptr RequestsBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - const bool has_port = !input.inputs.empty() && !input.inputs[0].IsNull(); - if (has_port) { - bind_data->port = input.inputs[0].GetValue(); - } - if (has_port && (bind_data->port < 1 || bind_data->port > 65535)) { - throw InvalidInputException("quackapi_requests: port must be between 1 and 65535"); - } - const LogicalType types[] = {LogicalType::VARCHAR, - LogicalType::TIMESTAMP, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::INTEGER, - LogicalType::DOUBLE, - LogicalType::DOUBLE, - LogicalType::DOUBLE, - LogicalType::BIGINT, - LogicalType::BIGINT, - LogicalType::BIGINT, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::VARCHAR, - LogicalType::BOOLEAN, - LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR), - LogicalType::VARCHAR}; - const char *const columns[] = {"request_id", "received_at", "method", "path", "route_name", - "route_path", "status", "duration_ms", "sql_prepare_ms", "sql_execute_ms", - "rows_out", "bytes_in", "bytes_out", "client_ip", "user_agent", - "http_version", "error_type", "error_message", "trace_id", "span_id", - "parent_span_id", "sampled", "headers", "query"}; - for (idx_t i = 0; i < 24; i++) { - return_types.emplace_back(types[i]); - names.emplace_back(columns[i]); - } - return std::move(bind_data); -} - -static unique_ptr RequestsInit(ClientContext &context, TableFunctionInitInput &input) { - auto state = make_uniq(); - auto &bind_data = input.bind_data->Cast(); - state->records = QuackapiState::Get(*context.db).SnapshotRequests(bind_data.port); - return std::move(state); -} - -static void RequestsExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.records.size() && row < STANDARD_VECTOR_SIZE) { - auto &record = state.records[state.offset]; - for (idx_t column = 0; column < 24; column++) { - output.SetValue(column, row, RequestsValue(record, column)); - } - row++; - state.offset++; - } - output.SetCardinality(row); -} - -//===--------------------------------------------------------------------===// -// Keep sink health queryable without making the writer implementation public. -//===--------------------------------------------------------------------===// - -struct TelemetryStatusBindData : public TableFunctionData { - int32_t port = 0; -}; - -struct TelemetryStatusGlobalState : public GlobalTableFunctionState { - vector statuses; - idx_t offset = 0; -}; - -static unique_ptr TelemetryStatusBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - const bool has_port = !input.inputs.empty() && !input.inputs[0].IsNull(); - if (has_port) { - bind_data->port = input.inputs[0].GetValue(); - } - if (has_port && (bind_data->port < 1 || bind_data->port > 65535)) { - throw InvalidInputException("quackapi_telemetry_status: port must be between 1 and 65535"); - } - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("sink"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("target"); - return_types.emplace_back(LogicalType::BIGINT); - names.emplace_back("queued"); - return_types.emplace_back(LogicalType::BIGINT); - names.emplace_back("exported_total"); - return_types.emplace_back(LogicalType::BIGINT); - names.emplace_back("dropped_total"); - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("last_error"); - return_types.emplace_back(LogicalType::BIGINT); - names.emplace_back("last_export_age_ms"); - return std::move(bind_data); -} - -static unique_ptr TelemetryStatusInit(ClientContext &context, TableFunctionInitInput &input) { - auto state = make_uniq(); - auto &bind_data = input.bind_data->Cast(); - state->statuses = QuackapiState::Get(*context.db).SnapshotTelemetryStatus(bind_data.port); - return std::move(state); -} - -static void TelemetryStatusExec(ClientContext &, TableFunctionInput &data_p, DataChunk &output) { - auto &state = data_p.global_state->Cast(); - idx_t row = 0; - while (state.offset < state.statuses.size() && row < STANDARD_VECTOR_SIZE) { - auto &status = state.statuses[state.offset]; - output.SetValue(0, row, Value(status.sink)); - output.SetValue(1, row, Value(status.target)); - output.SetValue(2, row, Value::BIGINT(static_cast(status.queued))); - output.SetValue(3, row, Value::BIGINT(static_cast(status.exported_total))); - output.SetValue(4, row, Value::BIGINT(static_cast(status.dropped_total))); - output.SetValue(5, row, Value(status.last_error)); - output.SetValue(6, row, Value::BIGINT(status.last_export_age_ms)); - row++; - state.offset++; - } - output.SetCardinality(row); -} - -struct TelemetryFlushBindData : public TableFunctionData { - int32_t port = 0; - bool finished = false; -}; - -static unique_ptr TelemetryFlushBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { - auto bind_data = make_uniq(); - const bool has_port = !input.inputs.empty() && !input.inputs[0].IsNull(); - if (has_port) { - bind_data->port = input.inputs[0].GetValue(); - } - if (has_port && (bind_data->port < 1 || bind_data->port > 65535)) { - throw InvalidInputException("quackapi_telemetry_flush: port must be between 1 and 65535"); - } - return_types.emplace_back(LogicalType::VARCHAR); - names.emplace_back("status"); - return std::move(bind_data); -} - -static void TelemetryFlushExec(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { - auto &bind_data = data_p.bind_data->CastNoConst(); - if (bind_data.finished) { - return; - } - if (!QuackapiState::Get(*context.db).FlushTelemetry(bind_data.port)) { - output.SetValue(0, 0, Value("No quackapi server to flush")); - } else if (bind_data.port == 0) { - output.SetValue(0, 0, Value("Flushed telemetry for all quackapi servers")); - } else { - output.SetValue(0, 0, - Value(StringUtil::Format("Flushed telemetry for quackapi server on port %d", bind_data.port))); - } - output.SetCardinality(1); - bind_data.finished = true; -} - //===--------------------------------------------------------------------===// // Load //===--------------------------------------------------------------------===// @@ -1025,19 +757,15 @@ static void LoadInternal(ExtensionLoader &loader) { "Requests at or above this duration are logged at warn level. " "Default 1000ms. Overridden by slow_request_ms named parameter.", LogicalType::BIGINT, Value::BIGINT(1000)); - config.AddExtensionOption("quackapi_request_ring", - "Number of recent TCP requests retained by quackapi_requests(). " - "Default 10000; zero disables the in-memory ring. Overridden by request_ring named " - "parameter.", - LogicalType::BIGINT, Value::BIGINT(10000)); config.AddExtensionOption("quackapi_log_headers", - "Comma-separated request-header allowlist for access-log maps; empty (default) " - "avoids retaining headers. Overridden by log_headers named parameter.", + "Comma-separated request-header allowlist for native access logs; empty (default) " + "omits headers. Overridden by log_headers named parameter.", LogicalType::VARCHAR, Value("")); - config.AddExtensionOption("quackapi_log_query", - "Preserve a redacted raw query string in access logs; default false. Overridden by " - "log_query named parameter.", - LogicalType::BOOLEAN, Value::BOOLEAN(false)); + config.AddExtensionOption( + "quackapi_log_query", + "Preserve a redacted raw query string in native access logs; default false. Overridden by " + "log_query named parameter.", + LogicalType::BOOLEAN, Value::BOOLEAN(false)); // SET quackapi_compression = 'auto'|'gzip'|'zstd'|'off'. // Default auto. Legacy true/false values map to auto/off. config.AddExtensionOption("quackapi_compression", @@ -1066,7 +794,6 @@ static void LoadInternal(ExtensionLoader &loader) { serve.named_parameters["cors_origins"] = LogicalType::VARCHAR; serve.named_parameters["log_level"] = LogicalType::VARCHAR; serve.named_parameters["slow_request_ms"] = LogicalType::BIGINT; - serve.named_parameters["request_ring"] = LogicalType::BIGINT; serve.named_parameters["log_headers"] = LogicalType::VARCHAR; serve.named_parameters["log_query"] = LogicalType::BOOLEAN; serve.named_parameters["access_log"] = LogicalType::ANY; @@ -1134,26 +861,6 @@ static void LoadInternal(ExtensionLoader &loader) { loader.RegisterFunction(TableFunction("quackapi_routes", {}, RoutesExec, RoutesBind, RoutesInit)); loader.RegisterFunction(TableFunction("quackapi_servers", {}, ServersExec, ServersBind, ServersInit)); - TableFunctionSet requests_set("quackapi_requests"); - TableFunction requests1("quackapi_requests", {LogicalType::INTEGER}, RequestsExec, RequestsBind, RequestsInit); - requests_set.AddFunction(requests1); - requests1.arguments.clear(); - requests_set.AddFunction(requests1); - loader.RegisterFunction(requests_set); - TableFunctionSet telemetry_status_set("quackapi_telemetry_status"); - TableFunction telemetry_status1("quackapi_telemetry_status", {LogicalType::INTEGER}, TelemetryStatusExec, - TelemetryStatusBind, TelemetryStatusInit); - telemetry_status_set.AddFunction(telemetry_status1); - telemetry_status1.arguments.clear(); - telemetry_status_set.AddFunction(telemetry_status1); - loader.RegisterFunction(telemetry_status_set); - TableFunctionSet telemetry_flush_set("quackapi_telemetry_flush"); - TableFunction telemetry_flush1("quackapi_telemetry_flush", {LogicalType::INTEGER}, TelemetryFlushExec, - TelemetryFlushBind); - telemetry_flush_set.AddFunction(telemetry_flush1); - telemetry_flush1.arguments.clear(); - telemetry_flush_set.AddFunction(telemetry_flush1); - loader.RegisterFunction(telemetry_flush_set); loader.RegisterFunction(GetQuackapiGroupsFunction()); // Auth inspection + API key management (secrets/hashes never exposed). diff --git a/src/quackapi_logging.cpp b/src/quackapi_logging.cpp index af1d71f..60861c8 100644 --- a/src/quackapi_logging.cpp +++ b/src/quackapi_logging.cpp @@ -1,38 +1,75 @@ #include "quackapi_logging.hpp" -#include +#include +#include "duckdb/common/string_util.hpp" +#include "duckdb/logging/log_type.hpp" +#include "duckdb/logging/logger.hpp" #include "quackapi_server.hpp" #include "quackapi_util.hpp" namespace duckdb { -string JsonEscape(const string &value) { - return QuackapiJsonEscape(value); +namespace { + +string JsonString(const string &value) { + return "\"" + QuackapiJsonEscape(value) + "\""; +} + +string JsonOptional(const string &value) { + return value.empty() ? "null" : JsonString(value); } -void EmitAccessLogStderr(const QuackapiRequestRecord &entry) { - string path_esc; - path_esc.reserve(entry.path.size() + 8); - for (unsigned char c : entry.path) { - if (c == '"' || c == '\\') { - path_esc += '\\'; - path_esc += static_cast(c); - } else if (c < 0x20) { - char buf[8]; - snprintf(buf, sizeof(buf), "\\u%04x", c); - path_esc += buf; - } else { - path_esc += static_cast(c); +string JsonOptionalDouble(double value) { + return value < 0 ? "null" : StringUtil::Format("%.3f", value); +} + +string JsonOptionalInteger(int64_t value) { + return value < 0 ? "null" : std::to_string(value); +} + +string BuildAccessLogJson(const QuackapiRequestRecord &entry) { + string result = StringUtil::Format( + "{\"type\":\"access\",\"request_id\":%s,\"received_at_micros\":%lld,\"method\":%s,\"path\":%s," + "\"route_name\":%s,\"route_path\":%s,\"status\":%d,\"duration_ms\":%.3f," + "\"sql_prepare_ms\":%s,\"sql_execute_ms\":%s,\"rows_out\":%s,\"bytes_in\":%lld,\"bytes_out\":%lld," + "\"client_ip\":%s,\"user_agent\":%s,\"http_version\":%s,\"error_type\":%s,\"error_message\":%s," + "\"trace_id\":%s,\"span_id\":%s,\"parent_span_id\":%s,\"sampled\":%s,\"headers\":", + JsonString(entry.request_id), static_cast(entry.received_at_micros), JsonString(entry.method), + JsonString(entry.path), JsonOptional(entry.route_name), JsonOptional(entry.route_path), entry.status, + entry.duration_ms, JsonOptionalDouble(entry.sql_prepare_ms), JsonOptionalDouble(entry.sql_execute_ms), + JsonOptionalInteger(entry.rows_out), static_cast(entry.bytes_in), + static_cast(entry.bytes_out), JsonString(entry.client_ip), JsonString(entry.user_agent), + JsonOptional(entry.http_version), JsonOptional(entry.error_type), JsonOptional(entry.error_message), + JsonOptional(entry.trace_id), JsonOptional(entry.span_id), JsonOptional(entry.parent_span_id), + entry.sampled ? "true" : "false"); + if (!entry.headers_logged) { + result += "null"; + } else { + result += "{"; + bool first = true; + for (auto &header : entry.headers) { + if (!first) { + result += ","; + } + first = false; + result += JsonString(header.first) + ":" + JsonString(header.second); } + result += "}"; + } + result += ",\"query\":"; + result += entry.query_logged ? JsonString(entry.query) : "null"; + result += "}"; + return result; +} + +} // namespace + +void EmitQuackapiAccessLog(DatabaseInstance &db, const QuackapiRequestRecord &entry, LogLevel level) { + auto &logger = Logger::Get(db); + if (!logger.ShouldLog(DefaultLogType::NAME, level)) { + return; } - fprintf(stderr, - "{\"type\":\"access\",\"method\":\"%s\",\"path\":\"%s\",\"status\":%d," - "\"latency_ms\":%.3f,\"request_id\":\"%s\",\"bytes\":%llu,\"client_ip\":\"%s\"," - "\"user_agent\":\"%s\",\"route_path\":\"%s\",\"trace_id\":\"%s\"}\n", - JsonEscape(entry.method).c_str(), path_esc.c_str(), entry.status, entry.duration_ms, - JsonEscape(entry.request_id).c_str(), (unsigned long long)entry.bytes_out, - JsonEscape(entry.client_ip).c_str(), JsonEscape(entry.user_agent).c_str(), - JsonEscape(entry.route_path).c_str(), JsonEscape(entry.trace_id).c_str()); + logger.WriteLog(DefaultLogType::NAME, level, BuildAccessLogJson(entry)); } } // namespace duckdb diff --git a/src/quackapi_registry.cpp b/src/quackapi_registry.cpp index 1b8e724..bea4519 100644 --- a/src/quackapi_registry.cpp +++ b/src/quackapi_registry.cpp @@ -367,65 +367,4 @@ vector> QuackapiState::ListServers() { return result; } -vector QuackapiState::SnapshotRequests(int port) { - std::lock_guard lock(servers_mutex); - if (port == 0) { - if (servers.empty()) { - return {}; - } - if (servers.size() > 1) { - throw InvalidInputException("quackapi_requests(): port is required when multiple servers are running"); - } - return servers.begin()->second->SnapshotRequests(); - } - for (auto &kv : servers) { - if (kv.second->Port() == port) { - return kv.second->SnapshotRequests(); - } - } - return {}; -} - -vector QuackapiState::SnapshotTelemetryStatus(int port) { - std::lock_guard lock(servers_mutex); - if (port == 0) { - if (servers.empty()) { - return {}; - } - if (servers.size() > 1) { - throw InvalidInputException( - "quackapi_telemetry_status(): port is required when multiple servers are running"); - } - return servers.begin()->second->SnapshotTelemetryStatus(); - } - for (auto &kv : servers) { - if (kv.second->Port() == port) { - return kv.second->SnapshotTelemetryStatus(); - } - } - return {}; -} - -bool QuackapiState::FlushTelemetry(int port) { - std::lock_guard lock(servers_mutex); - if (port == 0) { - if (servers.empty()) { - return false; - } - if (servers.size() > 1) { - throw InvalidInputException( - "quackapi_telemetry_flush(): port is required when multiple servers are running"); - } - servers.begin()->second->FlushTelemetryPending(); - return true; - } - for (auto &kv : servers) { - if (kv.second->Port() == port) { - kv.second->FlushTelemetryPending(); - return true; - } - } - return false; -} - } // namespace duckdb diff --git a/src/quackapi_server.cpp b/src/quackapi_server.cpp index 2da18c9..e189463 100644 --- a/src/quackapi_server.cpp +++ b/src/quackapi_server.cpp @@ -4,19 +4,16 @@ #include "quackapi_request_binding.hpp" #include "quackapi_response_encoding.hpp" -#include #include #include #include #include #include -#include #include #include #include "duckdb/common/case_insensitive_map.hpp" #include "duckdb/common/exception.hpp" -#include "duckdb/common/limits.hpp" #include "duckdb/common/file_open_flags.hpp" #include "duckdb/common/file_system.hpp" #include "duckdb/common/helper.hpp" @@ -27,10 +24,10 @@ #include "duckdb/common/types/uuid.hpp" #include "duckdb/common/types/value.hpp" #include "duckdb/common/types/timestamp.hpp" -#include "duckdb/main/appender.hpp" #include "duckdb/main/client_context.hpp" #include "duckdb/main/connection.hpp" #include "duckdb/main/database.hpp" +#include "duckdb/main/extension_helper.hpp" #include "duckdb/main/prepared_statement.hpp" #include "duckdb/main/query_result.hpp" #include "duckdb/parser/keyword_helper.hpp" @@ -891,436 +888,9 @@ bool MethodListContains(const vector &list, const string &method) { } // namespace -enum class QuackapiAccessLogColumn : uint8_t { - REQUEST_ID, - RECEIVED_AT, - METHOD, - PATH, - ROUTE_NAME, - ROUTE_PATH, - STATUS, - DURATION_MS, - SQL_PREPARE_MS, - SQL_EXECUTE_MS, - ROWS_OUT, - BYTES_IN, - BYTES_OUT, - CLIENT_IP, - USER_AGENT, - HTTP_VERSION, - ERROR_TYPE, - ERROR_MESSAGE, - TRACE_ID, - SPAN_ID, - PARENT_SPAN_ID, - SAMPLED, - HEADERS, - QUERY, -}; - -static constexpr const char *QUACKAPI_ACCESS_LOG_COLUMNS[] = { - "request_id", "received_at", "method", "path", "route_name", "route_path", - "status", "duration_ms", "sql_prepare_ms", "sql_execute_ms", "rows_out", "bytes_in", - "bytes_out", "client_ip", "user_agent", "http_version", "error_type", "error_message", - "trace_id", "span_id", "parent_span_id", "sampled", "headers", "query"}; - -static std::mutex quackapi_access_log_flush_mutex; - -Value QuackapiRequestHeadersValue(const QuackapiRequestRecord &entry) { - if (!entry.headers_logged) { - return Value(); - } - vector keys; - vector values; - keys.reserve(entry.headers.size()); - values.reserve(entry.headers.size()); - for (auto &header : entry.headers) { - keys.emplace_back(header.first); - values.emplace_back(header.second); - } - return Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(keys), std::move(values)); -} - -Value QuackapiRequestQueryValue(const QuackapiRequestRecord &entry) { - return entry.query_logged ? Value(entry.query) : Value(); -} - -static Value AccessLogValue(const QuackapiRequestRecord &entry, QuackapiAccessLogColumn column) { - switch (column) { - case QuackapiAccessLogColumn::REQUEST_ID: - return Value(entry.request_id); - case QuackapiAccessLogColumn::RECEIVED_AT: - return Value::TIMESTAMP(timestamp_t(entry.received_at_micros)); - case QuackapiAccessLogColumn::METHOD: - return Value(entry.method); - case QuackapiAccessLogColumn::PATH: - return Value(entry.path); - case QuackapiAccessLogColumn::ROUTE_NAME: - return entry.route_name.empty() ? Value() : Value(entry.route_name); - case QuackapiAccessLogColumn::ROUTE_PATH: - return entry.route_path.empty() ? Value() : Value(entry.route_path); - case QuackapiAccessLogColumn::STATUS: - return Value::INTEGER(entry.status); - case QuackapiAccessLogColumn::DURATION_MS: - return Value::DOUBLE(entry.duration_ms); - case QuackapiAccessLogColumn::SQL_PREPARE_MS: - return entry.sql_prepare_ms < 0 ? Value() : Value::DOUBLE(entry.sql_prepare_ms); - case QuackapiAccessLogColumn::SQL_EXECUTE_MS: - return entry.sql_execute_ms < 0 ? Value() : Value::DOUBLE(entry.sql_execute_ms); - case QuackapiAccessLogColumn::ROWS_OUT: - return entry.rows_out < 0 ? Value() : Value::BIGINT(entry.rows_out); - case QuackapiAccessLogColumn::BYTES_IN: - return Value::BIGINT(entry.bytes_in); - case QuackapiAccessLogColumn::BYTES_OUT: - return Value::BIGINT(entry.bytes_out); - case QuackapiAccessLogColumn::CLIENT_IP: - return Value(entry.client_ip); - case QuackapiAccessLogColumn::USER_AGENT: - return Value(entry.user_agent); - case QuackapiAccessLogColumn::HTTP_VERSION: - return entry.http_version.empty() ? Value() : Value(entry.http_version); - case QuackapiAccessLogColumn::ERROR_TYPE: - return entry.error_type.empty() ? Value() : Value(entry.error_type); - case QuackapiAccessLogColumn::ERROR_MESSAGE: - return entry.error_message.empty() ? Value() : Value(entry.error_message); - case QuackapiAccessLogColumn::TRACE_ID: - return entry.trace_id.empty() ? Value() : Value(entry.trace_id); - case QuackapiAccessLogColumn::SPAN_ID: - return entry.span_id.empty() ? Value() : Value(entry.span_id); - case QuackapiAccessLogColumn::PARENT_SPAN_ID: - return entry.parent_span_id.empty() ? Value() : Value(entry.parent_span_id); - case QuackapiAccessLogColumn::SAMPLED: - return Value::BOOLEAN(entry.sampled); - case QuackapiAccessLogColumn::HEADERS: - return QuackapiRequestHeadersValue(entry); - case QuackapiAccessLogColumn::QUERY: - return QuackapiRequestQueryValue(entry); - } - throw InternalException("unknown quackapi access-log column"); -} - -static vector SplitAccessLogTable(const string &table) { - vector parts = StringUtil::Split(table, '.'); - for (auto &part : parts) { - StringUtil::Trim(part); - } - if (parts.empty() || parts.size() > 3) { - throw InvalidInputException("access_log table must be a table, schema.table, or database.schema.table"); - } - for (auto &part : parts) { - if (part.empty()) { - throw InvalidInputException("access_log table name contains an empty identifier"); - } - } - return parts; -} - -static unique_ptr AccessLogTableInfo(Connection &con, const vector &parts) { - if (parts.size() == 1) { - return con.TableInfo(parts[0]); - } - if (parts.size() == 2) { - return con.TableInfo(parts[0], parts[1]); - } - return con.TableInfo(parts[0], parts[1], parts[2]); -} - -static unique_ptr AccessLogAppender(Connection &con, const vector &parts) { - if (parts.size() == 1) { - return make_uniq(con, parts[0]); - } - if (parts.size() == 2) { - return make_uniq(con, parts[0], parts[1]); - } - return make_uniq(con, parts[0], parts[1], parts[2]); -} - static constexpr const char *QUACKAPI_STATIC_LOG_MARKER = "X-Quackapi-Static-Access-Log"; static constexpr const char *QUACKAPI_STATIC_LOG_STARTED = "X-Quackapi-Static-Started"; static constexpr const char *QUACKAPI_STATIC_LOG_RECEIVED = "X-Quackapi-Static-Received"; -static constexpr idx_t QUACKAPI_ACCESS_LOG_QUEUE_CAPACITY = 10000; - -static int64_t TelemetryExportAgeMs(const std::chrono::steady_clock::time_point &last_export, bool has_export) { - if (!has_export) { - return -1; - } - auto age = std::chrono::duration_cast(std::chrono::steady_clock::now() - last_export); - return age.count() < 0 ? 0 : age.count(); -} - -class QuackapiStderrTelemetrySink final : public QuackapiTelemetrySink { -public: - void Enqueue(const QuackapiRequestRecord &entry) override { - // Keep the existing line writer synchronous so stderr retains its current ordering. - EmitAccessLogStderr(entry); - // Atomics, not a mutex: this runs on every request and must not serialize workers. - exported_total.fetch_add(1, std::memory_order_relaxed); - last_export_ticks.store(std::chrono::steady_clock::now().time_since_epoch().count(), std::memory_order_relaxed); - } - - void FlushPending() override { - } - - void Flush() override { - } - - QuackapiTelemetryStatus Status() const override { - QuackapiTelemetryStatus result; - result.sink = "stderr"; - result.exported_total = exported_total.load(std::memory_order_relaxed); - auto ticks = last_export_ticks.load(std::memory_order_relaxed); - result.last_export_age_ms = TelemetryExportAgeMs( - std::chrono::steady_clock::time_point(std::chrono::steady_clock::duration(ticks)), ticks != NO_EXPORT); - return result; - } - -private: - static constexpr int64_t NO_EXPORT = NumericLimits::Minimum(); - std::atomic exported_total {0}; - std::atomic last_export_ticks {NO_EXPORT}; -}; - -class QuackapiTableTelemetrySink final : public QuackapiTelemetrySink { -public: - QuackapiTableTelemetrySink(DatabaseInstance &db, string table_p) - : db_ptr(db.shared_from_this()), table(std::move(table_p)) { - worker = std::thread(&QuackapiTableTelemetrySink::Run, this); - } - - ~QuackapiTableTelemetrySink() override { - Flush(); - } - - void Enqueue(const QuackapiRequestRecord &entry) override { - bool fallback; - bool overflow = false; - bool warn_overflow = false; - idx_t overflow_count_snapshot = 0; - { - std::lock_guard lock(mutex); - fallback = failed && std::chrono::steady_clock::now() < retry_at; - if (!fallback) { - if (pending.size() >= QUACKAPI_ACCESS_LOG_QUEUE_CAPACITY) { - overflow = true; - overflow_count_snapshot = ++overflow_count; - if (!overflow_warning_emitted) { - overflow_warning_emitted = true; - warn_overflow = true; - } - } else { - pending.push_back(entry); - if (pending.size() >= 100) { - condition.notify_one(); - } - } - } - } - if (overflow) { - if (warn_overflow) { - fprintf(stderr, - "quackapi: access_log table \"%s\" queue full; falling back to stderr " - "(overflow_count=%llu)\n", - table.c_str(), (unsigned long long)overflow_count_snapshot); - } - EmitAccessLogStderr(entry); - } else if (fallback) { - EmitAccessLogStderr(entry); - } - } - - void FlushPending() override { - std::unique_lock lock(mutex); - if (!worker.joinable() || stopping || (pending.empty() && !flushing)) { - return; - } - flush_requested = true; - condition.notify_one(); - condition.wait(lock, [&] { return !flush_requested && !flushing; }); - } - - void Flush() override { - Shutdown(); - } - - QuackapiTelemetryStatus Status() const override { - std::lock_guard lock(mutex); - QuackapiTelemetryStatus result; - result.sink = "table"; - result.target = table; - result.queued = pending.size(); - result.exported_total = exported_total; - result.dropped_total = overflow_count; - result.last_error = last_error; - result.last_export_age_ms = TelemetryExportAgeMs(last_export, has_export); - return result; - } - -private: - void Shutdown() { - if (!worker.joinable()) { - return; - } - { - std::lock_guard lock(mutex); - stopping = true; - } - condition.notify_one(); - worker.join(); - } - - void FlushBatch(const vector &batch) { - if (table_parts.empty()) { - table_parts = SplitAccessLogTable(table); - } - std::lock_guard write_lock(quackapi_access_log_flush_mutex); - auto db = db_ptr.lock(); - if (!db) { - throw InvalidInputException("database is shutting down"); - } - Connection con(*db); - auto info = AccessLogTableInfo(con, table_parts); - if (!info) { - throw CatalogException("access_log table \"%s\" does not exist", table); - } - auto appender = AccessLogAppender(con, table_parts); - appender->ClearColumns(); - - vector selected; - for (idx_t i = 0; i < sizeof(QUACKAPI_ACCESS_LOG_COLUMNS) / sizeof(QUACKAPI_ACCESS_LOG_COLUMNS[0]); i++) { - for (auto &column : info->columns) { - if (!column.Generated() && StringUtil::CIEquals(column.Name(), QUACKAPI_ACCESS_LOG_COLUMNS[i])) { - appender->AddColumn(column.Name()); - selected.push_back(static_cast(i)); - break; - } - } - } - if (selected.empty()) { - throw InvalidInputException("access_log table has no recognized request columns"); - } - for (auto &entry : batch) { - appender->BeginRow(); - for (auto column : selected) { - appender->Append(AccessLogValue(entry, column)); - } - appender->EndRow(); - } - appender->Flush(); - } - - void MarkExported(idx_t count) { - std::lock_guard lock(mutex); - exported_total += count; - last_export = std::chrono::steady_clock::now(); - has_export = true; - } - - void MarkFailed(const string &reason) { - std::lock_guard lock(mutex); - last_error = reason; - if (!failed) { - failed = true; - fprintf(stderr, - "quackapi: access_log table \"%s\" failed; falling back to stderr " - "(overflow_count=%llu): %s\n", - table.c_str(), (unsigned long long)overflow_count, reason.c_str()); - } - retry_at = std::chrono::steady_clock::now() + std::chrono::seconds(retry_delay_seconds); - retry_delay_seconds = std::min(retry_delay_seconds * 2, 60); - } - - void MarkRecovered() { - std::lock_guard lock(mutex); - if (!failed) { - return; - } - failed = false; - retry_delay_seconds = 1; - fprintf(stderr, "quackapi: access_log table \"%s\" recovered; resuming table logging\n", table.c_str()); - } - - void Run() { - try { - table_parts = SplitAccessLogTable(table); - } catch (std::exception &ex) { - MarkFailed(ex.what()); - } - for (;;) { - vector batch; - { - std::unique_lock lock(mutex); - condition.wait_for(lock, std::chrono::seconds(1), - [&] { return stopping || flush_requested || pending.size() >= 100; }); - batch.assign(std::make_move_iterator(pending.begin()), std::make_move_iterator(pending.end())); - pending.clear(); - if (batch.empty() && flush_requested) { - flush_requested = false; - condition.notify_all(); - } - if (batch.empty() && stopping) { - break; - } - if (!batch.empty()) { - flushing = true; - } - } - if (batch.empty()) { - continue; - } - bool use_stderr; - { - std::lock_guard lock(mutex); - use_stderr = failed && std::chrono::steady_clock::now() < retry_at; - } - if (!use_stderr) { - try { - FlushBatch(batch); - MarkExported(batch.size()); - MarkRecovered(); - } catch (std::exception &ex) { - MarkFailed(ex.what()); - use_stderr = true; - } catch (...) { - MarkFailed("unknown append error"); - use_stderr = true; - } - } - if (use_stderr) { - for (auto &entry : batch) { - EmitAccessLogStderr(entry); - } - } - { - std::lock_guard lock(mutex); - flushing = false; - if (flush_requested && pending.empty()) { - flush_requested = false; - } - condition.notify_all(); - } - } - } - - weak_ptr db_ptr; - string table; - vector table_parts; - mutable std::mutex mutex; - std::condition_variable condition; - vector pending; - std::thread worker; - bool stopping = false; - bool flush_requested = false; - bool flushing = false; - bool failed = false; - std::chrono::steady_clock::time_point retry_at; - int64_t retry_delay_seconds = 1; - idx_t overflow_count = 0; - idx_t exported_total = 0; - string last_error; - std::chrono::steady_clock::time_point last_export; - bool has_export = false; - bool overflow_warning_emitted = false; -}; string SanitizeClientRequestId(const string &s); static void MakeTraceContext(const string &traceparent, string &trace_id, string &span_id, string &parent_span_id, @@ -1338,12 +908,10 @@ QuackapiHttpServer::QuackapiHttpServer(DatabaseInstance &db, const string &host_ : db_ptr(db.shared_from_this()), host(host_p), port(port_p), cors_origins(opts.cors_origins), options(opts), started_at(std::chrono::steady_clock::now()), compression(opts.compression), compression_min_bytes(opts.compression_min_bytes) { - if (options.access_log) { - if (options.access_log_table.empty()) { - telemetry_sinks.push_back(make_uniq()); - } else { - telemetry_sinks.push_back(make_uniq(db, options.access_log_table)); - } + // JSON is a runtime dependency for typed request bodies and validation. Load + // an available copy once the database is initialized; never install it here. + if (!db.ExtensionIsLoaded("json") && !ExtensionHelper::TryAutoLoadAvailableExtension(db, "json")) { + throw IOException("QuackAPI requires the DuckDB JSON extension; load json before serving requests"); } // In-process only (quackapi_request): no TCP server object, no bind. if (!bind_and_listen) { @@ -1352,9 +920,6 @@ QuackapiHttpServer::QuackapiHttpServer(DatabaseInstance &db, const string &host_ } server = make_uniq(); - if (options.request_ring > 0) { - request_ring = make_uniq(static_cast(options.request_ring)); - } // Static files (FastAPI StaticFiles equivalent). httplib checks file // requests before route handlers, so API routes always win over files. @@ -1760,12 +1325,10 @@ static QuackapiRequestRecord MakeRequestRecord(const duckdb_httplib::Request &re } void QuackapiHttpServer::EmitAccessLog(const QuackapiRequestRecord &entry) { - if (request_ring) { - request_ring->Push(entry); - } if (!options.access_log) { return; } + LogLevel level; bool emit = false; switch (options.log_level) { case QuackapiLogLevel::DEBUG_LEVEL: @@ -1784,53 +1347,19 @@ void QuackapiHttpServer::EmitAccessLog(const QuackapiRequestRecord &entry) { if (!emit) { return; } - for (auto &sink : telemetry_sinks) { - sink->Enqueue(entry); - } -} - -std::vector QuackapiHttpServer::SnapshotRequests() const { - return request_ring ? request_ring->Snapshot() : std::vector(); -} - -std::vector QuackapiHttpServer::SnapshotTelemetryStatus() const { - std::vector result; - result.reserve(telemetry_sinks.size()); - for (auto &sink : telemetry_sinks) { - result.push_back(sink->Status()); - } - return result; -} - -void QuackapiHttpServer::FlushTelemetryPending() { - for (auto &sink : telemetry_sinks) { - sink->FlushPending(); - } -} - -idx_t QuackapiHttpServer::AccessLogOverflowCount() const { - idx_t result = 0; - for (auto &status : SnapshotTelemetryStatus()) { - result += status.dropped_total; + if (entry.status >= 500) { + level = LogLevel::LOG_ERROR; + } else if (entry.status >= 400 || entry.duration_ms >= static_cast(options.slow_request_ms)) { + level = LogLevel::LOG_WARNING; + } else if (options.log_level == QuackapiLogLevel::DEBUG_LEVEL) { + level = LogLevel::LOG_DEBUG; + } else { + level = LogLevel::LOG_INFO; } - return result; -} - -static string TelemetryHealthJson(const std::vector &statuses) { - string result = "["; - for (idx_t i = 0; i < statuses.size(); i++) { - if (i > 0) { - result += ","; - } - const auto &status = statuses[i]; - result += StringUtil::Format("{\"sink\":\"%s\",\"queued\":%llu,\"exported_total\":%llu," - "\"dropped_total\":%llu,\"last_error\":\"%s\"}", - QuackapiJsonEscape(status.sink), (unsigned long long)status.queued, - (unsigned long long)status.exported_total, - (unsigned long long)status.dropped_total, QuackapiJsonEscape(status.last_error)); + auto db = db_ptr.lock(); + if (db) { + EmitQuackapiAccessLog(*db, entry, level); } - result += "]"; - return result; } void QuackapiHttpServer::ApplyCorsHeaders(const duckdb_httplib::Request &req, duckdb_httplib::Response &res) { @@ -2113,18 +1642,14 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd } auto uptime_sec = std::chrono::duration_cast(std::chrono::steady_clock::now() - started_at).count(); - auto telemetry_status = SnapshotTelemetryStatus(); - auto access_log_overflow_count = AccessLogOverflowCount(); if (ready) { SetJson( res, 200, StringUtil::Format( "{\"status\":\"ok\",\"version\":\"%s\",\"uptime_sec\":%lld," - "\"request_id_source\":\"%s\",\"access_log_overflow_count\":%llu," - "\"telemetry\":%s}", + "\"request_id_source\":\"%s\"}", QuackapiJsonEscape(version), (long long)uptime_sec, - QuackapiJsonEscape(options.request_id_source.empty() ? "uuidv7" : options.request_id_source), - (unsigned long long)access_log_overflow_count, TelemetryHealthJson(telemetry_status))); + QuackapiJsonEscape(options.request_id_source.empty() ? "uuidv7" : options.request_id_source))); } else { SetJson(res, 503, "{\"status\":\"not_ready\",\"detail\":\"database handle check failed\"}"); } @@ -3176,9 +2701,6 @@ void QuackapiHttpServer::Close() { QuackapiHttpServer::~QuackapiHttpServer() { try { Close(); - for (auto &sink : telemetry_sinks) { - sink->Flush(); - } } catch (std::exception &) { } } diff --git a/test/sql/quackapi_access_log.test b/test/sql/quackapi_access_log.test deleted file mode 100644 index 6bccec0..0000000 --- a/test/sql/quackapi_access_log.test +++ /dev/null @@ -1,162 +0,0 @@ -# name: test/sql/quackapi_access_log.test -# description: Table-backed access logging for serve and the in-process TestClient -# group: [sql] - -require quackapi - -require json - -statement ok -CREATE TABLE access_log ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - route_name VARCHAR, - status INTEGER, - duration_ms DOUBLE, - bytes_out BIGINT, - client_ip VARCHAR, - user_agent VARCHAR, - operator_note VARCHAR DEFAULT 'kept' -); - -statement ok -CREATE ROUTE access_log_route GET '/access-log' AS SELECT 'ok' AS result; -CREATE ROUTE access_log_post POST '/access-log' AS SELECT 'ok' AS result; - -query I -SELECT status -FROM quackapi_request('GET', '/access-log', access_log := 'access_log'); ----- -200 - -query I -SELECT count(*) FROM access_log; ----- -1 - -query III -SELECT route_name, status, operator_note -FROM access_log -WHERE path = '/access-log'; ----- -access_log_route 200 kept - -query I -SELECT status -FROM quackapi_request('GET', '/missing', access_log := 'access_log'); ----- -404 - -query II -SELECT route_name IS NULL, status -FROM access_log -WHERE path = '/missing'; ----- -true 404 - -query I -SELECT count(*) FROM access_log; ----- -2 - -# A transiently unavailable table falls back to stderr, then resumes table -# logging after the capped retry backoff when the table is recreated. -statement ok -CREATE TABLE access_log_retry ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - route_name VARCHAR, - status INTEGER, - duration_ms DOUBLE, - bytes_out BIGINT, - client_ip VARCHAR, - user_agent VARCHAR -); - -query I -SELECT * FROM quackapi_serve(29461, host := '127.0.0.1', access_log := 'access_log_retry'); ----- -http://127.0.0.1:29461 - -query I -SELECT (quackapi_post('http://127.0.0.1:29461/access-log', '{}')).status; ----- -200 - -statement ok -SELECT sleep_ms(1100); - -query I -SELECT count(*) FROM access_log_retry; ----- -1 - -statement ok -DROP TABLE access_log_retry; - -# The request still succeeds while the writer is backing off and the missing -# table means its access row cannot be appended. -query I -SELECT (quackapi_post('http://127.0.0.1:29461/access-log', '{}')).status; ----- -200 - -statement ok -SELECT sleep_ms(1200); - -statement ok -CREATE TABLE access_log_retry ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - route_name VARCHAR, - status INTEGER, - duration_ms DOUBLE, - bytes_out BIGINT, - client_ip VARCHAR, - user_agent VARCHAR -); - -statement ok -SELECT sleep_ms(1200); - -query I -SELECT (quackapi_post('http://127.0.0.1:29461/access-log', '{}')).status; ----- -200 - -statement ok -SELECT sleep_ms(1100); - -query I -SELECT count(*) FROM access_log_retry; ----- -1 - -query I -SELECT * FROM quackapi_stop(29461); ----- -Stopped quackapi server on port 29461 - -# A missing operator table must not fail the in-process request. -query I -SELECT status -FROM quackapi_request('GET', '/access-log', access_log := 'missing_access_log'); ----- -200 - -query I -SELECT status -FROM quackapi_request('GET', '/access-log', access_log := false); ----- -200 - -query I -SELECT count(*) FROM access_log; ----- -2 diff --git a/test/sql/quackapi_requests_ring.test b/test/sql/quackapi_requests_ring.test deleted file mode 100644 index f70c907..0000000 --- a/test/sql/quackapi_requests_ring.test +++ /dev/null @@ -1,209 +0,0 @@ -# name: test/sql/quackapi_requests_ring.test -# description: Recent TCP requests retained by quackapi_requests() -# group: [sql] - -require quackapi - -# No listener means the built-in ring surface is an empty relation. -query I -SELECT count(*) FROM quackapi_requests(); ----- -0 - -statement ok -CREATE OR REPLACE ROUTE ring_first POST '/ring/first' AS SELECT 'first' AS result; - -statement ok -CREATE OR REPLACE ROUTE ring_second POST '/ring/second' AS SELECT 'second' AS result; - -statement ok -CREATE OR REPLACE ROUTE ring_fail POST '/ring/fail' AS SELECT * FROM query($sql); - -# Access-log output is disabled, but the ring still records every request. -query I -SELECT * FROM quackapi_serve(29561, host := '127.0.0.1', access_log := false); ----- -http://127.0.0.1:29561 - -query I -SELECT quackapi_post('http://127.0.0.1:29561/ring/first', '{}').status; ----- -200 - -query I -SELECT quackapi_post('http://127.0.0.1:29561/ring/missing', '{}').status; ----- -404 - -query I -SELECT quackapi_post('http://127.0.0.1:29561/ring/second', '{}').status; ----- -200 - -query II -SELECT status, path FROM quackapi_requests() ORDER BY received_at; ----- -200 /ring/first -404 /ring/missing -200 /ring/second - -query I -SELECT * FROM quackapi_stop(29561); ----- -Stopped quackapi server on port 29561 - -# Opt-in request headers and query are retained in the ring after redaction. -query I -SELECT * FROM quackapi_serve(29566, host := '127.0.0.1', access_log := false, - log_headers := 'authorization,x-tenant-id', log_query := true); ----- -http://127.0.0.1:29566 - -query I -SELECT (quackapi_post( - 'http://127.0.0.1:29566/ring/first?token=abc&q=x', '{}', 'application/json', - MAP {'Authorization': 'Bearer do-not-store', 'x-tenant-id': 'tenant-42'} -)).status; ----- -200 - -# The outbound client may reorder query parameters, so assert on content, not order. -query IIIII -SELECT headers['authorization'], headers['x-tenant-id'], - contains(query, 'token=[REDACTED]'), contains(query, 'q=x'), contains(query, 'abc') -FROM quackapi_requests(29566); ----- -[REDACTED] tenant-42 true true false - -query I -SELECT * FROM quackapi_stop(29566); ----- -Stopped quackapi server on port 29566 - -# Defaults retain neither optional request field. -query I -SELECT * FROM quackapi_serve(29567, host := '127.0.0.1', access_log := false); ----- -http://127.0.0.1:29567 - -query I -SELECT status FROM quackapi_post('http://127.0.0.1:29567/ring/second', '{}'); ----- -200 - -query II -SELECT headers IS NULL, query IS NULL FROM quackapi_requests(29567); ----- -true true - -query I -SELECT * FROM quackapi_stop(29567); ----- -Stopped quackapi server on port 29567 - -# The ring keeps only the newest two records, in oldest-first order. -query I -SELECT * FROM quackapi_serve(29562, host := '127.0.0.1', access_log := false, request_ring := 2); ----- -http://127.0.0.1:29562 - -statement ok -SELECT quackapi_post('http://127.0.0.1:29562/ring/two-one', '{}').status; - -statement ok -SELECT quackapi_post('http://127.0.0.1:29562/ring/two-two', '{}').status; - -statement ok -SELECT quackapi_post('http://127.0.0.1:29562/ring/two-three', '{}').status; - -query I -SELECT count(*) FROM quackapi_requests(29562); ----- -2 - -query I -SELECT string_agg(path, ',' ORDER BY received_at) FROM quackapi_requests(29562); ----- -/ring/two-two,/ring/two-three - -query I -SELECT * FROM quackapi_stop(29562); ----- -Stopped quackapi server on port 29562 - -# A zero-sized ring does not retain requests. -query I -SELECT * FROM quackapi_serve(29563, host := '127.0.0.1', access_log := false, request_ring := 0); ----- -http://127.0.0.1:29563 - -statement ok -SELECT quackapi_post('http://127.0.0.1:29563/ring/zero-one', '{}').status; - -statement ok -SELECT quackapi_post('http://127.0.0.1:29563/ring/zero-two', '{}').status; - -query I -SELECT count(*) FROM quackapi_requests(29563); ----- -0 - -query I -SELECT * FROM quackapi_stop(29563); ----- -Stopped quackapi server on port 29563 - -# The SET option applies when the named parameter is omitted. -statement ok -SET quackapi_request_ring = 1; - -query I -SELECT * FROM quackapi_serve(29565, host := '127.0.0.1', access_log := false); ----- -http://127.0.0.1:29565 - -statement ok -SELECT quackapi_post('http://127.0.0.1:29565/ring/setting-one', '{}').status; - -statement ok -SELECT quackapi_post('http://127.0.0.1:29565/ring/setting-two', '{}').status; - -query I -SELECT count(*) FROM quackapi_requests(29565); ----- -1 - -query I -SELECT * FROM quackapi_stop(29565); ----- -Stopped quackapi server on port 29565 - -statement ok -RESET quackapi_request_ring; - -# Failed requests remain queryable for SQL error filtering. -query I -SELECT * FROM quackapi_serve(29564, host := '127.0.0.1', access_log := false); ----- -http://127.0.0.1:29564 - -query I -SELECT quackapi_post('http://127.0.0.1:29564/ring/fail?sql=SELECT%20nope%20FROM%20nowhere', '{}').status; ----- -500 - -query II -SELECT status, path FROM quackapi_requests(29564) WHERE status >= 500; ----- -500 /ring/fail - -query I -SELECT * FROM quackapi_stop(29564); ----- -Stopped quackapi server on port 29564 - -# The ring is allocated at serve time, so oversized values are rejected. -statement error -SELECT * FROM quackapi_serve(29569, host := '127.0.0.1', request_ring := 1000001); ----- -request_ring must be between 0 and 1000000 diff --git a/test/sql/quackapi_telemetry_record.test b/test/sql/quackapi_telemetry_record.test deleted file mode 100644 index b22440f..0000000 --- a/test/sql/quackapi_telemetry_record.test +++ /dev/null @@ -1,171 +0,0 @@ -# name: test/sql/quackapi_telemetry_record.test -# description: Rich request records, inbound W3C trace context, and log levels -# group: [sql] - -require quackapi - -require json - -statement ok -CREATE TABLE telemetry_log ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - route_name VARCHAR, - route_path VARCHAR, - status INTEGER, - duration_ms DOUBLE, - sql_prepare_ms DOUBLE, - sql_execute_ms DOUBLE, - rows_out BIGINT, - bytes_in BIGINT, - bytes_out BIGINT, - client_ip VARCHAR, - user_agent VARCHAR, - http_version VARCHAR, - error_type VARCHAR, - error_message VARCHAR, - trace_id VARCHAR, - span_id VARCHAR, - parent_span_id VARCHAR, - sampled BOOLEAN -); - -statement ok -CREATE TABLE old_log ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - route_name VARCHAR, - status INTEGER, - duration_ms DOUBLE, - bytes_out BIGINT, - client_ip VARCHAR, - user_agent VARCHAR -); - -statement ok -CREATE TABLE warn_log ( - request_id VARCHAR, - status INTEGER -); - -statement ok -CREATE OR REPLACE ROUTE telemetry_item GET '/telemetry/items/{id}' AS - SELECT $id::INTEGER AS id, $trace_id::VARCHAR AS trace_id, $span_id::VARCHAR AS span_id; - -statement ok -CREATE OR REPLACE ROUTE telemetry_fail POST '/telemetry/fail' AS - SELECT * FROM query($sql); - -statement ok -CREATE OR REPLACE ROUTE telemetry_ok POST '/telemetry/ok' AS - SELECT 'ok' AS result; - -# Valid W3C trace context is retained while the server creates a fresh span. -query I -SELECT status -FROM quackapi_request( - 'GET', '/telemetry/items/7', - headers := MAP {'traceparent': '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01'}, - access_log := 'telemetry_log' -); ----- -200 - -query III -SELECT trace_id, parent_span_id, length(span_id) -FROM telemetry_log -WHERE path = '/telemetry/items/7'; ----- -4bf92f3577b34da6a3ce929d0e0e4736 00f067aa0ba902b7 16 - -# Malformed and all-zero parents start fresh traces. -query I -SELECT status FROM quackapi_request( - 'GET', '/telemetry/items/8', - headers := MAP {'traceparent': 'not-a-traceparent'}, - access_log := 'telemetry_log' -); ----- -200 - -query I -SELECT status FROM quackapi_request( - 'GET', '/telemetry/items/9', - headers := MAP {'traceparent': '00-00000000000000000000000000000000-0000000000000000-01'}, - access_log := 'telemetry_log' -); ----- -200 - -query I -SELECT count(*) = 2 - AND bool_and(regexp_matches(trace_id, '^[0-9a-f]{32}$')) - AND bool_and(parent_span_id IS NULL) - AND bool_and(trace_id <> '00000000000000000000000000000000') -FROM telemetry_log -WHERE path IN ('/telemetry/items/8', '/telemetry/items/9'); ----- -true - -# Handler SQL can bind the server span context. -query I -SELECT decode(body) LIKE '%"trace_id":"%' -FROM quackapi_request('GET', '/telemetry/items/10'); ----- -true - -# A failing route records an error type and keeps its 5xx message. -query I -SELECT status -FROM quackapi_request( - 'POST', '/telemetry/fail', '{"sql":"SELECT nope FROM nowhere"}', access_log := 'telemetry_log' -); ----- -500 - -query I -SELECT error_type IS NOT NULL AND error_message IS NOT NULL -FROM telemetry_log -WHERE path = '/telemetry/fail'; ----- -true - -# New record fields are opt-in: the original ten-column table still appends. -query I -SELECT status -FROM quackapi_request('GET', '/telemetry/ok', access_log := 'old_log'); ----- -200 - -query I -SELECT count(*) FROM old_log WHERE path = '/telemetry/ok'; ----- -1 - -# WARN excludes a successful request from the table sink. -query I -SELECT * FROM quackapi_serve(29520, host := '127.0.0.1', access_log := 'warn_log', log_level := 'warn'); ----- -http://127.0.0.1:29520 - -query I -SELECT (quackapi_post('http://127.0.0.1:29520/telemetry/ok', '{}')).status; ----- -200 - -statement ok -SELECT sleep_ms(1100); - -query I -SELECT count(*) FROM warn_log; ----- -0 - -query I -SELECT * FROM quackapi_stop(29520); ----- -Stopped quackapi server on port 29520 diff --git a/test/sql/quackapi_telemetry_redact.test b/test/sql/quackapi_telemetry_redact.test deleted file mode 100644 index f684071..0000000 --- a/test/sql/quackapi_telemetry_redact.test +++ /dev/null @@ -1,78 +0,0 @@ -# name: test/sql/quackapi_telemetry_redact.test -# description: Opt-in access-log header and query redaction -# group: [sql] - -require quackapi - -require json - -statement ok -CREATE TABLE redaction_log ( - path VARCHAR, - status INTEGER, - headers MAP(VARCHAR, VARCHAR), - query VARCHAR -); - -statement ok -CREATE OR REPLACE ROUTE redaction_route GET '/redaction' AS - SELECT 'ok' AS result; - -query I -SELECT status -FROM quackapi_request( - 'GET', - '/redaction', - headers := MAP { - 'Authorization': 'Bearer do-not-store', - 'X-Tenant-ID': 'tenant-42', - 'X-Not-Logged': 'omit-me' - }, - access_log := 'redaction_log', - log_headers := ' authorization, X-TENANT-ID ' -); ----- -200 - -query III -SELECT headers['authorization'], headers['x-tenant-id'], headers['x-not-logged'] IS NULL -FROM redaction_log -WHERE path = '/redaction'; ----- -[REDACTED] tenant-42 true - -query I -SELECT status -FROM quackapi_request( - 'GET', - '/redaction?token=abc&q=x', - access_log := 'redaction_log', - log_query := true -); ----- -200 - -query I -SELECT query -FROM redaction_log -WHERE query IS NOT NULL; ----- -token=[REDACTED]&q=x - -query I -SELECT status -FROM quackapi_request( - 'GET', - '/redaction?token=abc&q=x', - headers := MAP {'X-Tenant-ID': 'tenant-42'}, - access_log := 'redaction_log' -); ----- -200 - -query II -SELECT query IS NULL, headers IS NULL -FROM redaction_log -WHERE query IS NULL AND headers IS NULL; ----- -true true diff --git a/test/sql/quackapi_telemetry_status.test b/test/sql/quackapi_telemetry_status.test deleted file mode 100644 index b7362f7..0000000 --- a/test/sql/quackapi_telemetry_status.test +++ /dev/null @@ -1,65 +0,0 @@ -# name: test/sql/quackapi_telemetry_status.test -# description: Per-sink telemetry status and health compatibility fields -# group: [sql] - -require quackapi - -statement ok -CREATE ROUTE telemetry_probe POST '/telemetry/probe' AS SELECT 'ok' AS result; - -statement ok -CREATE TABLE telemetry_status_log ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - status INTEGER -); - -query I -SELECT * FROM quackapi_serve(29580, host := '127.0.0.1', access_log := 'telemetry_status_log'); ----- -http://127.0.0.1:29580 - -query I -SELECT quackapi_post('http://127.0.0.1:29580/telemetry/probe', '{}').status; ----- -200 - -query I -SELECT * FROM quackapi_telemetry_flush(29580); ----- -Flushed telemetry for quackapi server on port 29580 - -query II -SELECT sink, exported_total -FROM quackapi_telemetry_status() -ORDER BY sink; ----- -table 1 - -query III -SELECT method, path, status -FROM telemetry_status_log; ----- -POST /telemetry/probe 200 - -query I -SELECT * FROM quackapi_stop(29580); ----- -Stopped quackapi server on port 29580 - -query I -SELECT * FROM quackapi_serve(29581, host := '127.0.0.1'); ----- -http://127.0.0.1:29581 - -query I -SELECT sink FROM quackapi_telemetry_status(); ----- -stderr - -query I -SELECT * FROM quackapi_stop(29581); ----- -Stopped quackapi server on port 29581 From 923e5ee0b7f1b541bce79983db308284c6fda79b Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:33:53 -0700 Subject: [PATCH 29/93] Fix deadline propagation C++11 compatibility --- src/include/quackapi_limits.hpp | 1 + src/quackapi_limits.cpp | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/include/quackapi_limits.hpp b/src/include/quackapi_limits.hpp index 1a07b26..6e6c3e3 100644 --- a/src/include/quackapi_limits.hpp +++ b/src/include/quackapi_limits.hpp @@ -21,6 +21,7 @@ class QuackapiQueryDeadline { private: std::shared_ptr state; + ClientContext *context; std::chrono::steady_clock::time_point previous; }; diff --git a/src/quackapi_limits.cpp b/src/quackapi_limits.cpp index 94fc260..a84998a 100644 --- a/src/quackapi_limits.cpp +++ b/src/quackapi_limits.cpp @@ -29,7 +29,8 @@ static void RegisterDeadline(ClientContext *context, const std::shared_ptr guard(deadline_registry_mutex); auto &stack = deadline_registry[context]; - stack.erase(std::remove_if(stack.begin(), stack.end(), [](const auto &entry) { return entry.expired(); }), + stack.erase(std::remove_if(stack.begin(), stack.end(), + [](const std::weak_ptr &entry) { return entry.expired(); }), stack.end()); stack.push_back(state); } @@ -45,7 +46,9 @@ static void UnregisterDeadline(ClientContext *context, const std::shared_ptrsecond; stack.erase(std::remove_if(stack.begin(), stack.end(), - [&](const auto &entry) { return entry.expired() || entry.lock() == state; }), + [&](const std::weak_ptr &entry) { + return entry.expired() || entry.lock() == state; + }), stack.end()); if (stack.empty()) { deadline_registry.erase(found); @@ -108,7 +111,8 @@ static DeadlineWatchdog &Watchdog() { return watchdog; } -QuackapiQueryDeadline::QuackapiQueryDeadline(Connection &connection, int64_t timeout_ms) : previous(current_deadline) { +QuackapiQueryDeadline::QuackapiQueryDeadline(Connection &connection, int64_t timeout_ms) + : context(connection.context.get()), previous(current_deadline) { state = std::make_shared(); state->connection = &connection; const auto context_deadline = connection.context ? QuackapiCurrentDeadline(*connection.context) @@ -116,11 +120,11 @@ QuackapiQueryDeadline::QuackapiQueryDeadline(Connection &connection, int64_t tim state->deadline = std::min(std::min(previous, context_deadline), std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms)); current_deadline = state->deadline; - RegisterDeadline(connection.context.get(), state); + RegisterDeadline(context, state); Watchdog().Add(state); } QuackapiQueryDeadline::~QuackapiQueryDeadline() { - UnregisterDeadline(connection.context.get(), state); + UnregisterDeadline(context, state); { std::lock_guard guard(state->mutex); state->connection = nullptr; From d896e6086dc6b861202344ca1d755bf48f230bd8 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:34:04 -0700 Subject: [PATCH 30/93] perf(encoding): move completed response buffers --- src/include/quackapi_response_encoding.hpp | 1 + src/quackapi_response_encoding.cpp | 8 ++++++-- src/quackapi_server.cpp | 13 +++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/include/quackapi_response_encoding.hpp b/src/include/quackapi_response_encoding.hpp index 8fef771..80ba57a 100644 --- a/src/include/quackapi_response_encoding.hpp +++ b/src/include/quackapi_response_encoding.hpp @@ -25,6 +25,7 @@ class QuackapiResponseWriter { idx_t BytesWritten() const; bool ExceededLimit() const; const string &Data() const; + string TakeData(); private: idx_t max_bytes; diff --git a/src/quackapi_response_encoding.cpp b/src/quackapi_response_encoding.cpp index b805706..85b0df7 100644 --- a/src/quackapi_response_encoding.cpp +++ b/src/quackapi_response_encoding.cpp @@ -84,6 +84,10 @@ const string &QuackapiResponseWriter::Data() const { return exceeded_limit ? EmptyString() : data; } +string QuackapiResponseWriter::TakeData() { + return std::move(data); +} + class QuackapiJsonChunkEncoder::State { public: State(Connection &con, const vector &names, const vector &types, @@ -139,8 +143,8 @@ class QuackapiJsonChunkEncoder::State { if (!format.validity.RowIsValid(index)) { throw InternalException("JSON encoder: json_object returned NULL"); } - auto row = data[index].GetString(); - if (!writer.Append(row.data(), row.size())) { + auto row = data[index]; + if (!writer.Append(row.GetData(), row.GetSize())) { return; } if (mode == JsonRowMode::NDJSON && !writer.Append("\n")) { diff --git a/src/quackapi_server.cpp b/src/quackapi_server.cpp index e189463..39ffd26 100644 --- a/src/quackapi_server.cpp +++ b/src/quackapi_server.cpp @@ -1951,8 +1951,8 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd for (auto &kv : match.path_params) { provided[kv.first] = {"path", kv.second}; } - // Server-stamped request id — bindable as $request_id in handler SQL - // (and libpq path). Wins over any client query/path of the same name. + // Server-stamped request id — bindable as $request_id in handler SQL. + // Wins over any client query/path of the same name. provided["request_id"] = {"server", request_id}; provided["trace_id"] = {"server", trace_id}; provided["span_id"] = {"server", span_id}; @@ -2647,20 +2647,21 @@ void QuackapiHttpServer::HandleRequest(const duckdb_httplib::Request &req, duckd if (mode == ResponseMode::HTML || mode == ResponseMode::TEXT) { res.status = match.route.status; - res.set_content(response_writer.Data(), + res.set_content(response_writer.TakeData(), mode == ResponseMode::HTML ? "text/html; charset=utf-8" : "text/plain; charset=utf-8"); } else if (body_format == BodyFormat::CSV) { res.status = match.route.status; - res.set_content(response_writer.Data(), "text/csv; charset=utf-8"); + res.set_content(response_writer.TakeData(), "text/csv; charset=utf-8"); } else if (body_format == BodyFormat::NDJSON) { res.status = match.route.status; - res.set_content(response_writer.Data(), "application/x-ndjson"); + res.set_content(response_writer.TakeData(), "application/x-ndjson"); } else if (object_envelope && rows_fetched > 1) { SetJson(res, 500, "{\"detail\":\"ENVELOPE object requires exactly one row, got " + std::to_string(rows_fetched) + "\"}"); } else { - SetJson(res, match.route.status, response_writer.Data()); + res.status = match.route.status; + res.set_content(response_writer.TakeData(), "application/json"); } // Keep body so Content-Length is correct; httplib omits the body for HEAD. } catch (std::exception &ex) { From 7ab047d7e13a295e7cc24801c34a705c7e92dcce Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:36:00 -0700 Subject: [PATCH 31/93] test: load JSON before starting QuackAPI --- test/sql/quackapi_compression.test | 1 + test/sql/quackapi_serve_block.test | 1 + 2 files changed, 2 insertions(+) diff --git a/test/sql/quackapi_compression.test b/test/sql/quackapi_compression.test index fc0070f..9355465 100644 --- a/test/sql/quackapi_compression.test +++ b/test/sql/quackapi_compression.test @@ -2,6 +2,7 @@ # description: response compression mode and threshold configuration # group: [sql] +require json require quackapi query I diff --git a/test/sql/quackapi_serve_block.test b/test/sql/quackapi_serve_block.test index 94290bb..6d7b39e 100644 --- a/test/sql/quackapi_serve_block.test +++ b/test/sql/quackapi_serve_block.test @@ -2,6 +2,7 @@ # description: quackapi_serve(block := …) + quackapi_wait readiness # group: [sql] +require json require quackapi # --------------------------------------------------------------------------- From 9ee2967e5cb6d5647ecde7b679dbf74d951d1850 Mon Sep 17 00:00:00 2001 From: Alok Subbarao Date: Wed, 14 Oct 2026 12:36:12 -0700 Subject: [PATCH 32/93] docs: document current QuackAPI surface and migration paths --- .../workflows/MainDistributionPipeline.yml | 47 +- README.md | 597 +++--------------- description.yml | 153 ++--- docs/FASTAPI_FEATURE_DEMAND.md | 3 + docs/FASTAPI_PARITY.md | 3 + docs/FASTAPI_TRANSPORT_DEMAND.md | 4 + docs/FASTAPI_ULTRA_MATRIX.md | 3 + docs/FEATURE_STATUS.md | 4 + docs/MIDDLEWARE.md | 14 +- docs/MIGRATION.md | 75 +++ docs/RESOURCE_LIMITS.md | 99 ++- docs/REVIEW_2026_09_07.md | 3 + docs/SUPERIORITY_ORDER.md | 3 + docs/community-page.md | 367 ++--------- docs/curl_httpfs.md | 186 ++---- docs/fastapi-parity.md | 3 + docs/gemini-ext-catalog-map.md | 3 + docs/guide/auth.md | 6 +- docs/guide/coming-soon.md | 24 +- docs/guide/extension-composition.md | 295 ++------- docs/guide/headers-cookies-redirects.md | 44 +- docs/guide/request-bodies.md | 2 +- docs/index.md | 262 ++------ docs/reference/ddl.md | 398 +++--------- docs/reference/functions.md | 492 +++------------ packaging/description.yml | 153 ++--- 26 files changed, 652 insertions(+), 2591 deletions(-) create mode 100644 docs/MIGRATION.md diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 051d188..424cdcd 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -46,32 +46,15 @@ jobs: extension_name: quackapi format_checks: "format" - # Own job (not the reusable distribution workflow): needs `services:` for - # Postgres so require-env QUACKAPI_TEST_PG_DSN tests actually run. Linux - # amd64 only — does not publish artifacts or expand the release matrix. - linux-amd64-pg-sqllogic: - name: Linux amd64 SQLLogic with Postgres + # Focused Linux validation for the SQLLogic suite and real HTTP contracts. + # The released extension uses DuckDB SQL only; no external database service + # is required. + linux-amd64-sqllogic: + name: Linux amd64 SQLLogic runs-on: ubuntu-24.04 env: GEN: ninja DUCKDB_GIT_VERSION: v1.5.6 - ENABLE_EXTENSION_AUTOLOADING: "1" - ENABLE_EXTENSION_AUTOINSTALL: "1" - QUACKAPI_TEST_PG_DSN: postgresql://quackapi_pg_cache:qa_pg_cache_test@127.0.0.1:5432/quackapi_pg_cache_test - services: - postgres: - image: postgres:16 - env: - POSTGRES_USER: quackapi_pg_cache - POSTGRES_PASSWORD: qa_pg_cache_test - POSTGRES_DB: quackapi_pg_cache_test - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U quackapi_pg_cache -d quackapi_pg_cache_test" - --health-interval 5s - --health-timeout 5s - --health-retries 10 steps: - name: Checkout uses: actions/checkout@v4 @@ -92,20 +75,14 @@ jobs: - name: Install build deps run: | sudo apt-get update - sudo apt-get install -y ninja-build ccache libpq-dev postgresql-client + sudo apt-get install -y ninja-build ccache - name: Setup Ccache uses: hendrikmuhs/ccache-action@main continue-on-error: true with: - key: ccache-quackapi-pg-sqllogic-linux-amd64-v1.5.6 - restore-keys: ccache-quackapi-pg-sqllogic-linux-amd64- - - - name: Wait for Postgres - run: | - until pg_isready -h 127.0.0.1 -p 5432 -U quackapi_pg_cache -d quackapi_pg_cache_test; do - sleep 1 - done + key: ccache-quackapi-sqllogic-linux-amd64-v1.5.6 + restore-keys: ccache-quackapi-sqllogic-linux-amd64- - name: Build extension run: make release -j"$(nproc)" @@ -113,14 +90,6 @@ jobs: - name: Run quackapi SQLLogic run: ./build/release/test/unittest "test/sql/quackapi*" - - name: Verify outbound worker cleanup - run: | - c++ -std=c++17 -pthread -Wall -Wextra -Werror -Isrc/include test/cpp/quackapi_thread_join_test.cpp -o /tmp/quackapi-thread-join-test - /tmp/quackapi-thread-join-test - - - name: Verify build without libpq - run: c++ -std=c++17 -fsyntax-only -isystem duckdb/src/include -Isrc/include src/quackapi_pg.cpp - - name: Verify request database lifetime run: | c++ -std=c++17 -pthread -Wall -Wextra -Werror -isystem duckdb/src/include test/cpp/quackapi_lifetime_test.cpp -Lbuild/release/src -lduckdb -Wl,-rpath,"$PWD/build/release/src" -o /tmp/quackapi-lifetime-test diff --git a/README.md b/README.md index e3fbc03..7d11c59 100644 --- a/README.md +++ b/README.md @@ -1,559 +1,124 @@ # quackapi -**A FastAPI-class web framework that lives inside DuckDB.** - -Routes are DDL. Handlers are SQL. Request validation is the database type system. -One process is the database, the HTTP server, and — when you load companion -extensions — the PDF engine, HTML renderer, crawler, and fake-data generator. - -``` -browser ──► DuckDB [ quackapi · pdf · tera · fakeit · webbed · … ] -``` - -There is no app server, no ORM, and no schema re-declared in a second language. -The query already types the response. - -> **Thesis:** the backend *is* the database. -> Prior art: PostgREST, Datasette, Hasura, Oracle APEX. The novel part is one -> **embedded OLAP** process as DB + HTTP framework + (optional) PDF/renderer, -> driven by `CREATE ROUTE` DDL. -> -> **Inspiration:** quackapi was also inspired by the Query.Farm [`httpserver`](https://github.com/Query-farm/httpserver) DuckDB community extension. - ---- - -## Five-line quickstart - -Requires **DuckDB v1.5.6** (linux, macOS, windows_amd64). +QuackAPI serves HTTP routes whose handlers are DuckDB SQL. Request values bind +as prepared parameters, DuckDB types validate them, and query results become +HTTP responses. ```sql INSTALL quackapi FROM community; +LOAD json; LOAD quackapi; -CREATE ROUTE hello GET '/hello' AS SELECT 'world' AS msg; -CREATE ROUTE item GET '/items/:id' AS SELECT $id::INTEGER AS id; +CREATE ROUTE hello GET '/hello' AS SELECT 'world' AS message; +CREATE ROUTE item GET '/items/:id' AS SELECT $id::INTEGER AS id; SELECT * FROM quackapi_serve(8000); --- returns immediately; pass block := true to hold until stop / SIGINT / SIGTERM - --- Readiness (replaces sleep/lsof/curl loops): --- SELECT ready, listen_url FROM quackapi_wait(8000, 5000); ``` ```sh curl http://127.0.0.1:8000/hello -# [{"msg":"world"}] +# [{"message":"world"}] curl http://127.0.0.1:8000/items/42 # [{"id":42}] - -curl http://127.0.0.1:8000/items/abc -# {"detail":[{"loc":["path","id"],"msg":"…","type":"type_error"}]} -# HTTP 422 — FastAPI-shaped validation error -``` - -> **Parser note:** `CREATE ROUTE` is registered when you `LOAD quackapi`. Feed -> `LOAD` first in a sequential session (interactive shell or stdin), then the -> DDL. Putting `LOAD` and `CREATE ROUTE` in one `duckdb -c "…"` string can parse -> the route before the extension is loaded. - ---- - -## CREATE ROUTE — the model - -```sql -CREATE [OR REPLACE] ROUTE '' - [STATUS ] - [REQUIRE ] - [SCHEDULE '<6-field cron expression, seconds first>'] - [GROUP | IN GROUP ] - [BODY SCHEMA ''] - [BODY TYPE ''] - [PARAM [] [HEADER|COOKIE|QUERY [wire-name]] - [DEFAULT ] [GE|GT|LE|LT ] [MIN_LENGTH|MAX_LENGTH ] …] - AS ; - - - Methods: GET, POST, PUT, DELETE, PATCH, HEAD. - - Path segments `:id` / `{id}` and query/body fields bind to `$id`. - - Cast/constraint failures return **422** with FastAPI-shaped - `{"detail":[{"loc":[...],"msg":...,"type":...}]}`. - - Default JSON response is an **array of row objects** (SQL result-set - semantics). A single column named `html` or `text` returns raw HTML/text; - `location` / `set_cookie` set response headers. - - `quackapi_serve([port], host := …, static_dir := …, cors_origins := …, memory_limit := …)` - uses DuckDB's bundled httplib (background listener + workers). - - Built-in OpenAPI: `GET /openapi.json`, `GET /docs`, `GET /redoc`. - - ## Other DDL / functions - - - **Auth:** `CREATE AUTH … AS API_KEY | JWT (SECRET … [, ALGORITHM HS256])`, - `REQUIRE` on routes, `quackapi_add_api_key`, `quackapi_auths`, claims as - `$claims_*`. JWT verify is self-contained (DuckDB-bundled mbedtls + - `json_*` claims) — does not depend on community `crypto` (that extension - is hash/HMAC only, not JWT parse/exp/nbf/claims). - - **Groups:** `CREATE GROUP … WITH (prefix=…, auth=…, tags=…)` — path prefix - + auth inheritance (`quackapi_groups()`). - - **Table API:** `CREATE API FOR TABLE t [AT '/path'] [KEY 'id']` → GET list - + GET by key only. - - **Queue:** `CREATE QUEUE`, `quackapi_enqueue/dequeue/ack/nack`, table - `quackapi_jobs` (broker-less; compose `cronjob` for workers). - - **SSE:** `CREATE STREAM … GET '/path' AS ; - -DROP ROUTE ; -``` - -| Element | Detail | -|---------|--------| -| Method | `GET` `POST` `PUT` `DELETE` `PATCH` `HEAD` | -| Pattern | Absolute path starting with `/` (or relative when using `GROUP`) | -| Path params | `:id` or `{id}` → `$id` | -| Query / body / form / multipart | Same `$name` binding; `$body` for raw payload when declared | -| STATUS | Default 200 | -| REQUIRE | Named scheme from `CREATE AUTH` | -| PARAM | Optional types, defaults, numeric/string constraints, header/cookie sources | -| BODY SCHEMA | Optional JSON Schema string for request body validation | -| Handler | Prepared at CREATE; invalid SQL fails CREATE (not first request) | - -Routes update **live** while the server is running. - -### Response modes - -| Handler shape | HTTP body | -|---------------|-----------| -| Multiple columns / ordinary names | `application/json` — **array of row objects** | -| Single column `html` | `text/html; charset=utf-8` | -| Single column `text` | `text/plain; charset=utf-8` | -| Column `location` | sets `Location` (pair with `STATUS 3xx`) | -| Column `set_cookie` / `set-cookie` | sets `Set-Cookie` (stripped from JSON) | - -JSON types follow DuckDB column types (numbers, booleans, nulls, nested lists/structs). - -### Error shapes (verified) - -| Situation | Status | -|-----------|--------| -| Type / constraint / missing required param | **422** `{"detail":[{loc,msg,type},…]}` | -| Unknown path | **404** | -| Wrong method | **405** + `Allow` | -| Auth failure | **401** | -| Body too large (> 8 MiB) | **413** | -| Handler internal error | **500** `{"detail":"Internal Server Error"}` (SQL text not leaked) | - ---- - -## Examples - -### JSON route - -```sql -CREATE ROUTE get_doc GET '/api/documents/:id' AS -SELECT * FROM documents WHERE id = $id::INTEGER; -``` - -### HTML route - -```sql -CREATE ROUTE home GET '/' AS -SELECT '

quackapi

' AS html; - --- With community tera: --- SELECT tera_render(template, ctx) AS html FROM …; -``` - -### Static directory + CORS - -```sql -SET quackapi_cors_origins = 'https://app.example'; -SELECT * FROM quackapi_serve( - 8000, - host := '0.0.0.0', - static_dir := './static', - cors_origins := '*' -- named param overrides SET -); -``` - -### Typed params + 422 - -```sql -CREATE ROUTE search GET '/search' - PARAM limit INTEGER DEFAULT 10 LE 100 - AS -SELECT id, name FROM users -WHERE name ILIKE $q::VARCHAR || '%' -LIMIT $limit::INTEGER; -``` - -`limit=101` → 422 `less_than_equal`; `limit=abc` → 422 `type_error` with -`loc=["query","limit"]`. - -### POST mutation - -```sql -CREATE ROUTE decide POST '/api/suggestions/:id/decision' STATUS 201 - PARAM status VARCHAR - PARAM actor VARCHAR DEFAULT 'reviewer' - AS -INSERT INTO decisions BY NAME -SELECT $id::INTEGER AS suggestion_id, $status AS status, $actor AS actor -RETURNING *; -``` - -```sh -curl -X POST http://127.0.0.1:8000/api/suggestions/1/decision \ - -H 'Content-Type: application/json' \ - -d '{"status":"accepted"}' -``` -### Auth - -```sql -CREATE AUTH site AS API_KEY; -- default header X-API-Key -SELECT * FROM quackapi_add_api_key('site', 'super-secret', 'alice'); - -CREATE ROUTE me GET '/me' REQUIRE site AS -SELECT $claims_sub AS user; -``` - -```sql -CREATE AUTH jwt_auth AS JWT ( SECRET 'test-secret', ALGORITHM HS256 ); -``` - -### Serve / inspect / stop - -```sql +CREATE ROUTE hello GET '/hello' AS SELECT 'world' AS message; SELECT * FROM quackapi_serve(8000); -SELECT * FROM quackapi_routes(); -SELECT * FROM quackapi_servers(); -SELECT * FROM quackapi_stop(8000); ``` -Built-in (not in `quackapi_routes()`): `/openapi.json`, `/docs`, `/redoc`. - ---- - -## Function table - -Surfaces registered by the extension (source: `src/quackapi_extension.cpp` and -related modules). Auto-detection on the community site will list overloads after -`LOAD`; this table is the human guide. - -### HTTP lifecycle - -| Name | Kind | Purpose | -|------|------|---------| -| `quackapi_serve([port], host, static_dir, cors_origins, memory_limit, log_level, access_log, enable_logging, health_routes, threads, preserve_insertion_order, enable_http_metadata_cache, worker_threads, keep_alive_max_count, keep_alive_timeout_sec, read_timeout_sec, write_timeout_sec, compression, compression_min_bytes, http_client, pg_dsn, block, query_timeout_ms, max_response_bytes, max_pending_requests)` | table | Start listener (default `127.0.0.1:8000`) | -| `quackapi_wait(port [, timeout_ms], host := …)` | table | TCP readiness probe → `ready`, `listen_url` | -| `quackapi_stop([port])` | table | Stop one or all servers | -| `quackapi_routes()` | table | Registry: name, method, pattern, status, handler, require_auth, group_name, tags, format, envelope, empty_status, timeout_sec | -| `quackapi_servers()` | table | host, port, listen_url, http_client, http_client_reason | -| `quackapi_cors_origins` | setting | CORS allow-list (`*` or CSV); empty = off | -| `quackapi_compression` / `quackapi_compression_min_bytes` | settings | Response coding (`auto`, `gzip`, `zstd`, `off`) and 1024-byte default threshold | - -### Auth - -| Name | Kind | Purpose | -|------|------|---------| -| `CREATE` / `DROP AUTH` | DDL | API_KEY or JWT HS256 schemes | -| `quackapi_add_api_key(auth, key, subject)` | table | Store SHA-256 hash of key | -| `quackapi_auths()` | table | name, kind, header (no secrets) | -| `quackapi_verify_auth(scheme, auth_string)` | scalar | Policy engine probe | -| `quackapi_authentication` / `quackapi_authorization` | scalar | Bridges for core quack RPC settings | - -### Groups - -| Name | Kind | Purpose | -|------|------|---------| -| `CREATE` / `DROP GROUP` | DDL | Prefix + default auth + OpenAPI tags | -| `quackapi_groups()` | table | name, prefix, require_auth, tags, members | - -### Table API - -| Name | Kind | Purpose | -|------|------|---------| -| `CREATE API FOR TABLE …` | DDL | Registers GET list + GET by key only | - -### Queue - -| Name | Kind | Purpose | -|------|------|---------| -| `CREATE` / `DROP QUEUE` | DDL | Register queue options | -| `quackapi_enqueue` | scalar | Enqueue JSON/text payload → job_id | -| `quackapi_dequeue` | table | Claim up to n jobs | -| `quackapi_ack` / `quackapi_nack` | scalar | Complete / retry / dead-letter | -| `quackapi_queues()` | table | depth, in_flight, dead, options | -| `quackapi_jobs` | table (catalog) | Durable job rows | - -### Streams & policies - -| Name | Kind | Purpose | -|------|------|---------| -| `CREATE` / `DROP STREAM` | DDL | SSE (`GET` only); WS rejected | -| `quackapi_streams()` | table | Inspect streams | -| `CREATE ROW ACCESS POLICY` / `CREATE MASKING POLICY` | DDL | Claims-oriented policies | -| `ALTER TABLE …` policy bind | DDL | Attach/detach | -| `quackapi_policies()` | table | Inspect policies | - -### Other - -| Name | Kind | Purpose | -|------|------|---------| -| `quackapi_fetch(url[, headers])` | scalar | Pooled outbound GET → `STRUCT(status, reason, body, headers, error, reused_connection)` | -| `quackapi_post(url, body[, ct[, headers]])` | scalar | Pooled outbound POST, same result shape | -| `quackapi_http_pool()` | table | Outbound pool per origin: `host, idle, dialed, reused` | -| `quackapi_http_util_name()` | scalar | Active outbound HTTPUtil name | -| `quackapi_request(method, path [, body], …)` | table | In-process request → status, BLOB body, content type, headers | -| `quackapi_middlewares()` | table | Inspect SQL middleware definitions | - -Internal apply helpers (`quackapi_apply_route`, `quackapi_apply_auth`, …) exist -for the planner and are not part of the public app API. - ---- - -## Platforms & signed-build feasibility - -| Item | Value | -|------|--------| -| Target DuckDB | **v1.5.6** (CI + submodule pin) | -| Language / build | C++11 / **cmake** | -| Extra toolchains | **None** (no vcpkg, no Rust, no Python at build) | -| Linked deps | DuckDB **bundled httplib** + **mbedtls** only | -| Outbound HTTPS | Core `HTTPUtil`; optional `LOAD curl_httpfs` upgrades process-wide client | - -| DuckDB arch | Community CI intent | -|-------------|---------------------| -| `linux_amd64`, `linux_arm64` | Build & sign | -| `osx_amd64`, `osx_arm64` | Build & sign | -| `wasm_mvp`, `wasm_eh`, `wasm_threads` | **Excluded** — no server sockets | -| `windows_amd64` | Build & sign | -| `windows_amd64_mingw`, `windows_amd64_rtools`, `windows_arm64` | **Excluded** — not enabled by the repository descriptor | - -Repo workflow: `duckdb/extension-ci-tools` `@v1.5.6`, -`duckdb_version: v1.5.6`, same `exclude_archs` as `description.yml`. - -**Feasibility:** high for the non-excluded platforms — pure CMake extension -template shape, no exotic system libraries. Blocker before first signed publish: -clean release tag with **no merge-conflict markers** in `src/`, green -`make release` + `test/http/run_all.sh` on at least one host arch. +Route handlers use prepared parameters. Authentication controls access to a +route; it does not add database-wide row filtering or rewrite arbitrary SQL by +tenant. Write tenant predicates into handler SQL or use explicitly scoped +views. ---- +## Outbound POST -## Honest limits +`quackapi_post(url, body[, content_type[, headers]])` performs volatile HTTP +and HTTPS POSTs through DuckDB's HTTPUtil provider. `curl_httpfs` is +recommended and `httpfs` is also supported. Load a provider explicitly; the +extension does not install one. Results preserve input row alignment and +contain status, reason, body, headers, transport error, and connection-reuse +metadata. `quackapi_post_concurrency` bounds simultaneous requests for a +database instance. -1. **Single-writer OLTP** — DuckDB file locking; suitable for small concurrent - reviewer sets, not high-write multi-tenant SaaS. -2. **Instance-scoped registry** — routes/auth/groups/streams/queue *options* - die with the process; re-run DDL on boot. Job **rows** in `quackapi_jobs` - persist. -3. **Serve memory guard** — `quackapi_serve` applies a 256MB default only when - no operator or serve memory limit is configured; use `memory_limit := '4GB'` - or `SET quackapi_memory_limit` for large PDF/HTML work. -4. **Body size** — 8 MiB max payload. -5. **JWT** — HS256 only. -6. **Table API** — read-only GET scaffold. -7. **No WebSocket** on the HTTP sidecar (SSE via `CREATE STREAM` only). -8. **Unsigned until accepted** — community `INSTALL` path goes live after the - community-extensions PR and signing pipeline succeed. +For GET requests, use DuckDB's HTTP reader functions. QuackAPI has no public +fetch or pool-diagnostics API. ---- +## Scope -## Related links +The HTTP response formats are JSON, NDJSON, and CSV. QuackAPI does not provide +the former experimental queue, row-access or masking policy, SSE, GraphQL, +generated table/source-import, native PostgreSQL, Arrow, or Parquet surfaces. +`radio` is a separate message bus; it does not provide the removed queue's +lease, acknowledgement, or retry semantics. Arrow Flight and ADBC are separate +client/protocol choices and are not QuackAPI runtime dependencies. DuckDB's +ordinary SQL transaction and database-file concurrency behavior still applies. -- Source: [github.com/asubbarao/quackapi](https://github.com/asubbarao/quackapi) -- FastAPI parity scorecard: [`docs/FASTAPI_PARITY.md`](FASTAPI_PARITY.md) -- Queue semantics: [`docs/QUEUE.md`](QUEUE.md) -- Outbound HTTP: [`docs/curl_httpfs.md`](curl_httpfs.md) -- Submission YAML: [`description.yml`](../description.yml) +Build target: DuckDB v1.5.6. The extension declares DuckDB JSON as a build +dependency; load JSON before serving routes that use it. `BODY SCHEMA` also +requires `json_schema` to be available locally. Neither is installed by +QuackAPI as a side effect. diff --git a/docs/curl_httpfs.md b/docs/curl_httpfs.md index 781dd59..c057164 100644 --- a/docs/curl_httpfs.md +++ b/docs/curl_httpfs.md @@ -1,160 +1,64 @@ -# Outbound HTTP and curl_httpfs +# Outbound HTTP providers -quackapi’s **inbound** server uses DuckDB’s bundled httplib. That is intentional -and unchanged by this document. +`quackapi_post` sends outbound HTTP and HTTPS requests through DuckDB's +`HTTPUtil` provider API. Load a supported provider explicitly before the first +call. QuackAPI does not install or select a provider automatically. -**Outbound** HTTP (anything a route handler fetches via `read_text` / -`read_json` / `read_parquet` / `read_csv` over `https://`, and C++ token-exchange -via `QuackapiHttpFetch`) goes through DuckDB’s shared `HTTPUtil` layer — never -the `curl` CLI. - -## Batteries default: `curl_httpfs` - -On every `quackapi_serve()`, batteries **prefer** the community -[curl_httpfs](https://github.com/dentiny/duckdb-curl-filesystem) extension as -the outbound HTTP client: - -- libcurl **connection pooling** -- **HTTP/2** -- **async** network IO -- **100% compatible** with httpfs (`SET httpfs_client_implementation=…`) - -Flow (same pattern as auto-LOAD companions): - -1. Best-effort `LOAD` / `INSTALL` **httpfs** -2. `INSTALL curl_httpfs FROM community` + `LOAD curl_httpfs` (if not already loaded) -3. `SET httpfs_client_implementation = 'curl'` -4. Log `quackapi.http_client=curl` and record `http_client=curl` on `/healthz` - and `quackapi_servers()` - -A web-framework server that fetches remote data needs a production-grade pooled -client; without it, throughput collapses under concurrency on DuckDB’s default -per-request httplib client. - -### Platform coverage - -Community `description.yml` for curl_httpfs excludes: - -| Excluded | Available (ships binaries) | -|---|---| -| `wasm_mvp`, `wasm_eh`, `wasm_threads` | `linux_amd64`, `linux_arm64`, … | -| `windows_amd64`, `windows_amd64_mingw`, `windows_amd64_rtools` | `osx_amd64`, `osx_arm64` | - -### Preference semantics (`http_client`) - -| Surface | Values | Default | -|---|---|---| -| `quackapi_serve(…, http_client := '…')` | `auto` \| `curl` \| `httplib` | `auto` | -| `SET quackapi_http_client = '…'` | same | `auto` | - -Named param wins over the SET. - -| Value | Behavior | -|---|---| -| **`auto`** | Prefer curl_httpfs. If INSTALL/LOAD fails, **fall back to httplib** and report **loudly**: stderr line, `/healthz` `http_client` + `http_client_reason`, `quackapi_servers()`. | -| **`curl`** | **Require** curl_httpfs. If INSTALL/LOAD fails, **serve fails** (no silent fallback). Production guarantee. | -| **`httplib`** | Skip curl_httpfs install; force stock client. Logs `reason=operator_forced`. | - -```sql --- Default (prefer curl_httpfs; loud fallback if unavailable) -SELECT * FROM quackapi_serve(8000); - --- Production: require curl_httpfs or refuse to serve -SELECT * FROM quackapi_serve(8000, http_client := 'curl'); - --- Explicit stock client (dev / unsupported platform) -SELECT * FROM quackapi_serve(8000, http_client := 'httplib'); - -SET quackapi_http_client = 'curl'; -SELECT * FROM quackapi_serve(8001); -``` - -### auto fallback (loud, never silent) - -When `http_client` is `auto` and curl_httpfs cannot install/load: - -```text -quackapi.http_client=httplib reason=curl_httpfs_unavailable WARN=auto_fallback … -``` - -`/healthz` then reports: - -```json -{"status":"ok", …, "http_client":"httplib", "http_client_reason":"curl_httpfs_unavailable"} -``` - -`quackapi_servers()` has the same `http_client` / `http_client_reason` columns. - -### Production recommendation - -On **linux** and **osx** (platforms where community curl_httpfs ships), production -should either: - -1. **`http_client := 'curl'`** (or `SET quackapi_http_client = 'curl'`) so serve - fails closed if the pooled client is missing, **or** -2. Keep `auto` but **alert on** `/healthz` when - `http_client_reason == "curl_httpfs_unavailable"`. - -Do not ship concurrent outbound https under silent httplib fallback. - -### Confirm the active client - -```sql --- After serve -SELECT host, port, listen_url, http_client, http_client_reason -FROM quackapi_servers(); - --- Readiness JSON includes the same fields --- GET /healthz → {"status":"ok", …, "http_client":"curl", "http_client_reason":""} - --- Active HTTPUtil name (MultiCurl / HTTPFS-Curl after curl_httpfs, Built-In otherwise) -SELECT quackapi_http_util_name(); -``` - -### Manual load (optional) - -Batteries already do this on serve. Manual load is still fine for non-serve -sessions: +`curl_httpfs` is recommended. `httpfs` is also supported. The `curl_httpfs` +provider can provide connection reuse and HTTP/2 according to its own build +and configuration; the exact behavior depends on the provider version and +platform. ```sql -INSTALL curl_httpfs FROM community; LOAD curl_httpfs; -SET httpfs_client_implementation = 'curl'; -- or leave default MultiCurl LOAD quackapi; -``` -### Transparent acceleration for SQL handlers +SELECT (quackapi_post( + 'https://api.example.test/items', + '{"name":"new item"}', + 'application/json', + MAP {'Authorization': 'Bearer …'} +)).status; +``` -Route handlers are ordinary SELECTs. If they call httpfs surfaces, the active -client speeds them up with no quackapi change: +To use `httpfs`, load it instead: ```sql -CREATE ROUTE proxy GET '/proxy' AS -SELECT content FROM read_text('https://example.com/data.json'); +LOAD httpfs; +LOAD quackapi; ``` -See `examples/proxy_curl_httpfs.sql` if present. +If neither supported HTTPUtil provider is loaded, `quackapi_post` returns a +configuration error for each non-NULL input row. Each input row retains its +position and produces one aligned result. Requests are not retried after +transport errors. -### C++ outbound (OAuth/OIDC) +Sub-second request deadlines use DuckDB's `HTTPParams::timeout_usec` field. +The `curl_httpfs` MultiCurl implementation still needs the provider fix in +[duckdb-curl-filesystem PR #75](https://github.com/dentiny/duckdb-curl-filesystem/pull/75) +to honor that fractional remainder. The `httpfs` HTTPFS-Curl path also has a +separate compatibility change pending. Treat sub-second deadline enforcement +as pending until the corresponding provider fix is merged and present in the +loaded provider. -Use `QuackapiHttpFetch` (`src/quackapi_http_fetch.{hpp,cpp}`). It only depends -on core `HTTPUtil` headers. When curl_httpfs is loaded (including via -batteries), token-exchange POSTs use the curl client automatically. When only -the built-in util is active, POST fails fast with a message to load curl_httpfs. +## GET requests -## What not to do +For GET requests inside DuckDB SQL, use DuckDB's reader functions such as +`read_text`, `read_json`, and `read_csv` after loading the extension that +provides the reader. QuackAPI has no public `quackapi_fetch` function. -| Anti-pattern | Why | -|---|---| -| Hard-link libcurl / curl_httpfs into quackapi | Separate extension; composition is `SetHTTPUtil` | -| `system("curl ...")` / subprocess | No secrets integration, no pool, unsafe | -| Replace the **inbound** httplib server with curl_httpfs | curl_httpfs is the **client** layer only | -| Rely on silent httplib when you need production pool/HTTP2 | Use `http_client := 'curl'` or alert on `http_client_reason` | +```sql +LOAD httpfs; -## Load order +SELECT content +FROM read_text('https://example.com/data.txt'); +``` -1. `LOAD quackapi` then `quackapi_serve` (batteries prefer curl_httpfs for you), **or** -2. `LOAD curl_httpfs` then `LOAD quackapi` then serve +QuackAPI's inbound server is separate from outbound provider selection. It +continues to use DuckDB's bundled HTTP server implementation. The provider +does not replace the server and does not add TLS termination for inbound +connections. -Either order works for SQL handlers as long as the curl client is active before -the request that performs outbound I/O. +See the [POST function reference](reference/functions.md#outbound-post) and +[migration guide](MIGRATION.md) for the retained SQL contract and removed +transport APIs. diff --git a/docs/fastapi-parity.md b/docs/fastapi-parity.md index e730b32..949ef5f 100644 --- a/docs/fastapi-parity.md +++ b/docs/fastapi-parity.md @@ -1,5 +1,8 @@ # FastAPI → quackapi parity +> Historical scorecard. Its feature list predates the current API surface; see +> the [current migration guide](MIGRATION.md). + **Headline:** the versioned HTTP conformance harness scores **89 / 89 (100%)** against FastAPI-shaped expectations. | | | diff --git a/docs/gemini-ext-catalog-map.md b/docs/gemini-ext-catalog-map.md index 618ee29..2dbe736 100644 --- a/docs/gemini-ext-catalog-map.md +++ b/docs/gemini-ext-catalog-map.md @@ -1,5 +1,8 @@ # Gemini brainstorm × `ext_catalog_clean` — composition map +> Historical catalog research. Its QuackAPI feature claims predate the API +> cleanup; see the [current migration guide](MIGRATION.md). + **Source:** live quack `:9494` `ext_catalog_clean` (2026-07-29). **Method:** `ILIKE` on `extension_name` / `content`; samples only from `query IS NOT NULL` or README blocks — no invented signatures. **Thesis:** compose first. Prefer a **recipe** (LOAD + `CREATE ROUTE` / `CREATE STREAM` / `CREATE QUEUE`) over new C++. Thin C++ only when the wire format or registry lives inside quackapi. diff --git a/docs/guide/auth.md b/docs/guide/auth.md index 505edd1..1a167fb 100644 --- a/docs/guide/auth.md +++ b/docs/guide/auth.md @@ -118,7 +118,9 @@ CREATE ROUTE me GET '/me' REQUIRE site AS SELECT $claims_sub AS user; ``` -Claims power [row access & masking policies](policies.md). +Claims are available to the authenticated handler as SQL parameters. Route +authentication does not filter rows or rewrite arbitrary SQL by tenant; add +tenant predicates to the handler or an explicitly scoped view. --- @@ -164,4 +166,4 @@ SELECT (quackapi_verify_auth('site', 'wrong')).ok; -- false ## Next - [CREATE GROUP](groups.md) — shared auth on a prefix -- [Policies](policies.md) — claims-keyed row filters and masks +- [Migration guide](../MIGRATION.md) — removed policy APIs and the current tenant-filter boundary diff --git a/docs/guide/coming-soon.md b/docs/guide/coming-soon.md index 342f6a5..c5014ff 100644 --- a/docs/guide/coming-soon.md +++ b/docs/guide/coming-soon.md @@ -1,14 +1,16 @@ -# Coming / in progress +# Scope boundaries -These features are **not** documented as done. They are actively designed or building. Do not rely on them in production recipes yet. +This page records areas outside the current QuackAPI HTTP API. These are +not advertised as implemented or as scheduled replacements. -| Feature | Intent | Status | -|---------|--------|--------| -| **WebSocket routes** | Browser RFC6455 duplex | **Blocked** on transport (HTTP library has no Upgrade API). Use [CREATE STREAM (SSE)](stream.md) instead | -| **OIDC / OAuth2 browser SSO** | `CREATE AUTH … OIDC` | Designed — JWT/API_KEY only today | -| **Signed cookie sessions + CSRF** | Browser session cookies | Designed | -| **Static URL prefix** | Mount `static_dir` under `/assets` | Partial — root `static_dir` only | +| Surface | Current boundary | +|---|---| +| Browser WebSocket upgrade | QuackAPI does not provide an HTTP WebSocket endpoint. `radio` is a separate extension and does not add WebSocket support to QuackAPI. | +| SSE and streaming responses | QuackAPI returns bounded JSON, NDJSON, or CSV responses; it does not provide the removed `CREATE STREAM`/SSE surface. | +| Row filtering and column masking | Route auth protects endpoints but does not rewrite arbitrary SQL or enforce tenant row policies. Put tenant filters in handler SQL or scoped views. | +| Durable job queue | No built-in queue with lease, acknowledgement, visibility-timeout, and retry guarantees remains. A `radio` message bus does not supply those job semantics. | +| GraphQL, generated table APIs, and source import | Define explicit SQL handlers with `CREATE ROUTE`; these former experimental APIs are not included. | +| Arrow Flight | A separate protocol and deployment choice; it is not hosted by QuackAPI or a runtime dependency. | -Authoritative ledger: [FEATURE_STATUS.md](../FEATURE_STATUS.md) sections 2.5 and 4. - -When a row moves to “BUILT & MERGED,” this page will shrink and a full guide will land under `docs/guide/`. +See the [migration guide](../MIGRATION.md) for removed SQL names and current +alternatives. diff --git a/docs/guide/extension-composition.md b/docs/guide/extension-composition.md index 2e3c9e9..6a68bfc 100644 --- a/docs/guide/extension-composition.md +++ b/docs/guide/extension-composition.md @@ -1,273 +1,72 @@ -# Extension composition — handlers are SQL +# Compose SQL handlers with DuckDB extensions -**The whole product idea:** `CREATE ROUTE` / `CREATE STREAM` handlers are ordinary SQL. -Whatever you can `LOAD` and call from a `SELECT`, a request can call. - -No C++ rewrite per companion. No second process for “the PDF service.” Same DuckDB session, -same address space, same type system. +QuackAPI routes run ordinary SQL in DuckDB. A handler can use tables, views, +scalar functions, and table functions already available in that database. +Loading a companion extension is an explicit deployment choice. ```text -browser ──► quackapi (SQL handlers + thin GraphQL) - │ - ├── local tables / views - ├── CREATE ROUTE / API FOR TABLE (REST → your SQL) - ├── POST /graphql | GRAPHQL ROUTE (GQL → SELECT …) - ├── curl_httpfs / httpfs (read_text, read_json, …) - ├── http_client (http_get, http_post, …) - ├── sitting_duck (AST / quack_from_*; optional GQL parse recipe) - ├── quack (quack_query / ATTACH) - └── pdf / tera / … (community companions) -``` - -**Dual surface:** GraphQL + ordinary HTTP on the same session — both resolve to SQL ([graphql-v0.md](graphql-v0.md)). Companions run *inside* handlers; they are not a second API product. - -Full DDL and function tables stay in the [README](../../README.md) and -[function reference](../reference/functions.md). This page is **recipes only**. - ---- - -## Recipe 1 — Outbound gateway - -Join remote HTTP with local tables inside one route. Prefer DuckDB’s shared -HTTP stack so `quackapi_serve` batteries (`curl_httpfs` when available) speed -you up for free — see [curl_httpfs](../curl_httpfs.md). - -### GET proxy via `read_text` / httpfs - -Runnable shape in-tree: [`examples/proxy_curl_httpfs.sql`](../../examples/proxy_curl_httpfs.sql). - -```sql -LOAD curl_httpfs; -- optional; batteries also try this on serve -LOAD quackapi; - -CREATE OR REPLACE ROUTE proxy GET '/proxy/:url' AS -SELECT - $url AS requested_url, - content AS body, - length(content) AS bytes -FROM read_text($url); - -CREATE OR REPLACE ROUTE http_util GET '/http_util' AS -SELECT quackapi_http_util_name() AS http_util; - -SELECT * FROM quackapi_serve(18080); -``` - -```sh -# percent-encode the remote URL into :url -curl -sS "http://127.0.0.1:18080/http_util" -# [{"http_util":"MultiCurl"}] # when curl_httpfs is active -``` - -There is **no** public SQL `quackapi_fetch` / `quackapi_post`. C++ `QuackapiHttpFetch` -is an internal HTTPUtil helper (OAuth/OIDC wave). Route authors use **SQL readers** -or a community HTTP extension. - -### POST / partner call via community `http_client` - -When you need explicit POST + headers (not just GET of a URL): - -```sql -LOAD http_client; -LOAD quackapi; - -CREATE OR REPLACE ROUTE proxy_score POST '/score' AS -SELECT http_post( - 'https://api.partner.com/v1/score', - map { - 'Authorization': 'Bearer ' || getenv('PARTNER_TOKEN'), - 'Content-Type': 'application/json' - }, - $body::JSON -) AS partner_response; -``` - -(Shape from the [handler bridge notes](../../bridges/from_x/docs/handler_bridge.md). -Confirm `http_post` arity against the `http_client` extension you load.) - -Join local state the same way you always would: - -```sql -CREATE ROUTE enrich GET '/orders/:id' AS -SELECT o.*, r.content AS remote_blob -FROM orders o -CROSS JOIN read_text(o.partner_url) r -WHERE o.id = $id::INTEGER; -``` - ---- - -## Recipe 2 — SSE live (honest transport) - -Push rows as Server-Sent Events. Full guide: [CREATE STREAM](stream.md). - -```sql -LOAD quackapi; - -CREATE STREAM ticks GET '/ticks' AS -SELECT i AS id, 'tick' AS msg FROM range(3) t(i); - --- long-lived poll -CREATE OR REPLACE STREAM live GET '/live' WITH (interval='1s') AS -SELECT now() AS ts; - -SELECT * FROM quackapi_serve(8000); -``` - -```sh -curl -N http://127.0.0.1:8000/ticks -# Content-Type: text/event-stream -# id: 0 -# data: {"id":0,"msg":"tick"} -``` - -**Honest limits (do not plan around these):** - -| Want | Reality | -|------|---------| -| Browser `EventSource` / SSE | **Yes** — `CREATE STREAM … GET` | -| WebSocket Upgrade | **No** — bundled httplib has no Upgrade API; `CREATE STREAM … WS` errors | -| Streams in `quackapi_routes()` | **No** — use `quackapi_streams()` (`transport` is always `'sse'` today) | -| `REQUIRE` auth on streams | Not in the current version | - -A bus extension (e.g. community pub/sub) can feed the `SELECT` behind the stream -when LOADed; quackapi only owns the SSE wire format. - ---- - -## Recipe 3 — `sitting_duck` / `quack_from_x` - -Point at an existing app tree; get **route + model IR rows**. Native table functions -ship in quackapi and **auto-INSTALL/LOAD `sitting_duck` FROM community** on first use. - -```sql -LOAD quackapi; - --- Routes: method, path, handler_name, file, … -SELECT method, path, handler_name -FROM quack_from_fastapi('bridges/from_x/fixtures/fastapi_mini') -ORDER BY method, path; --- GET /articles/{slug} get_article --- POST /login login - --- Models → field IR (then BODY SCHEMA by hand or generator) -SELECT model_name, field_name, field_type, is_required -FROM quack_from_fastapi_models('bridges/from_x/fixtures/fastapi_mini'); +HTTP client → QuackAPI route → prepared DuckDB SQL → response + │ + ├── tables and views + └── explicitly loaded extensions ``` -Siblings: `quack_from_rails`, `quack_from_express`, `quack_from_gin` (+ `_models`). +## Read upstream data -**What is automatic vs not:** - -| Automatic | Escape hatch | -|-----------|----------------| -| Decorators / DSL → route IR | Imperative handler bodies (never transpiled) | -| Pydantic / validates / tags → field IR | Rewrite body as SQL, macro, or [queue](queue.md) worker | -| Review IR → write `CREATE ROUTE` | Keep a thin external service and call it (recipe 1) | - -More narrative, corpus numbers, and migration tips: [Coming from FastAPI](../from-fastapi.md). -Bridge layout and tests: [`bridges/from_x/`](../../bridges/from_x/README.md). - ---- - -## Recipe 4 — Quack mesh - -**Composition, not a second server inside quackapi.** When the community **`quack`** -extension is LOADed, a handler can fan out to another DuckDB process over the -quack remote protocol — same pattern as any other TVF. +Use DuckDB's HTTP readers inside a route after loading a reader provider such +as `httpfs` or `curl_httpfs`: ```sql -LOAD quack; -- separate extension; not bundled into quackapi +LOAD json; +LOAD httpfs; LOAD quackapi; --- Illustrative shape — use the quack version you actually install. --- Common forms in the wild: --- quack_query('quack:host:9494', 'SELECT …', token => '…') --- ATTACH 'quack:host:9494' AS edge (DISABLE_SSL true); - -CREATE ROUTE edge_health GET '/mesh/health' AS -SELECT * -FROM quack_query( - 'quack:localhost:9494', - 'SELECT 1 AS ok', - token := 'your-token' -); - -CREATE ROUTE edge_table GET '/mesh/items' AS -SELECT * -FROM quack_query( - 'quack:localhost:9494', - 'SELECT * FROM items LIMIT 100', - token := 'your-token' -); -``` - -Auth bridge scalars on the **quackapi** side for RPC-style checks (used by -quack-compatible setups; no extra process): - -```sql -SELECT (quackapi_verify_auth('site', 'k-secret')).ok; -SELECT quackapi_authentication('sess', 'token', 'token'); -SELECT quackapi_authorization('sess', 'SELECT 1'); +CREATE ROUTE remote_status GET '/remote-status' AS +SELECT * FROM read_json('https://api.example.test/status'); ``` -quackapi still only listens with `quackapi_serve`. Mesh nodes are whatever quack -servers you already run. - ---- +QuackAPI does not install HTTP providers when a server starts. Reader behavior +and supported schemes come from the loaded DuckDB extension. -## Recipe 5 — PDF companion (one process) +## Send an outbound POST -README showcase: Closure-style PDF review is **one DuckDB process** — routes + -community `pdf` + optional `tera` + tables. Function names come from the **`pdf`** -extension you load (not from quackapi). +`quackapi_post` uses DuckDB's HTTPUtil provider and requires a POST-capable +provider to be explicitly loaded. `curl_httpfs` is recommended; `httpfs` is +also supported. ```sql -INSTALL pdf FROM community; -LOAD pdf; +LOAD json; +LOAD curl_httpfs; LOAD quackapi; --- Typical community surfaces (names as used in product docs): --- read_pdf_words, pdf_redact, … --- Confirm against the pdf extension README for your DuckDB version. - -CREATE ROUTE doc_words GET '/docs/:id/words' AS -SELECT * -FROM read_pdf_words(d.path) -JOIN documents d ON d.id = $id::INTEGER; - --- HTML page if tera is loaded: --- LOAD tera; --- CREATE ROUTE case_page GET '/cases/:id' AS --- SELECT tera_render(template, ctx) AS html --- FROM app_templates, … WHERE name = 'case.html'; - -SELECT * FROM quackapi_serve(8000, memory_limit := '4GB'); -- headroom for PDF work +CREATE ROUTE submit POST '/submit' AS +SELECT (quackapi_post( + 'https://api.example.test/submit', + json_object('id', $id::INTEGER)::VARCHAR, + 'application/json' +)).status AS upstream_status; ``` -Calling the “PDF service” is a function call in the same address space — not an RPC. - ---- - -## Non-goals (explicit) - -These are **out of scope** for quackapi composition stories and this guide: +`quackapi_post` keeps one result row per input row, preserves its NULL and +volatile-function behavior, and reports status, response body, headers, and +transport errors. Configure `quackapi_post_concurrency` to bound simultaneous +requests. See [Outbound POST providers](../curl_httpfs.md). -| Non-goal | Why | -|----------|-----| -| GraphQL | Not a quackapi transport; stay REST + SQL | -| Asio / custom async server rewrite | Inbound remains httplib | -| Browser WebSocket Upgrade | Use SSE (`CREATE STREAM … GET`) | -| `WITH HISTORY` / time-travel as a product feature | Ordinary DuckDB time-travel if *you* enable it; not a quackapi API | +## Keep route security explicit -Also not invented here: public `quackapi_fetch` / `quackapi_post` SQL TVFs. +`CREATE AUTH` and `REQUIRE` decide whether a caller may invoke a route. They do +not apply row filters to arbitrary SQL. Put tenant constraints in the handler +query or a scoped view and use DuckDB privileges for database access. There is +no built-in row-access or masking-policy rewrite. ---- +## Protocol boundaries -## Next +QuackAPI currently serves ordinary bounded HTTP route responses in JSON, +NDJSON, and CSV. The extension does not host GraphQL, SSE, WebSocket, Arrow +Flight, or PostgreSQL protocols. DuckDB Flight/ADBC clients and the `radio` +message-bus extension can be composed separately, but they do not add those +protocols to the QuackAPI listener. In particular, `radio` does not implement +the durable queue lease, acknowledgement, and retry behavior of the removed +QuackAPI queue prototype. -1. [Routes & params](routes-and-params.md) — muscle memory for `$name::TYPE` -2. [SSE streams](stream.md) — full `CREATE STREAM` grammar -3. [Outbound curl_httpfs](../curl_httpfs.md) — client selection + `/healthz` -4. [from-fastapi](../from-fastapi.md) — migration narrative -5. [Function reference](../reference/functions.md) — every `quackapi_*` +See the [migration guide](../MIGRATION.md) for removed function and DDL names. diff --git a/docs/guide/headers-cookies-redirects.md b/docs/guide/headers-cookies-redirects.md index fa54a4e..5ede8f1 100644 --- a/docs/guide/headers-cookies-redirects.md +++ b/docs/guide/headers-cookies-redirects.md @@ -32,44 +32,12 @@ curl -i http://127.0.0.1:8000/echo-rid -H 'X-Request-ID: client-rid-deadbeef' # [{"id":"client-rid-deadbeef"}] ``` -Structured access log lines (when `access_log:=true`, the default) include -`request_id` on stderr as JSON (`"type":"access",…`). - -### Table-backed access logs - -`access_log` also accepts `false` to disable logging or a table name to append -one row per request to an operator-created table. The table writer recognizes -these columns by name: `request_id`, `received_at`, `method`, `path`, -`route_name`, `status`, `duration_ms`, `bytes_out`, `client_ip`, and -`user_agent`. Other columns are left to their defaults. `route_name` is NULL -for a 404 and `static` for a response served by `static_dir`. - -```sql -CREATE TABLE http_access_log ( - request_id VARCHAR, - received_at TIMESTAMP, - method VARCHAR, - path VARCHAR, - route_name VARCHAR, - status INTEGER, - duration_ms DOUBLE, - bytes_out BIGINT, - client_ip VARCHAR, - user_agent VARCHAR, - operator_note VARCHAR DEFAULT 'keep' -); - -SELECT * FROM quackapi_serve(8000, access_log := 'http_access_log'); -``` - -Appends are batched at 100 rows or one second. The pending queue is capped at -10,000 entries; overflow is written to stderr and counted in `/healthz` as -`access_log_overflow_count`. A missing or broken table never changes the -response: quackapi emits one warning for the failure episode, falls back to -the stderr JSON log, and retries with backoff from one second to 60 seconds. A -successful retry emits a recovery line and resumes table logging. The in-process -test client accepts the same option, for example -`quackapi_request('GET', '/health', access_log := 'http_access_log')`. +Request observations use DuckDB's native logging and can be filtered through +DuckDB's log level. Access logging is enabled by default and can be disabled +with `access_log := false`. Header and query logging are opt-in: configure +`log_headers` with an explicit header allowlist and enable `log_query` only +when needed. Sensitive values are redacted before logging. QuackAPI does not +provide a request-ring table or a table-backed logging sink. --- diff --git a/docs/guide/request-bodies.md b/docs/guide/request-bodies.md index b7f4013..d21c3ed 100644 --- a/docs/guide/request-bodies.md +++ b/docs/guide/request-bodies.md @@ -138,7 +138,7 @@ SELECT $title::VARCHAR AS title, ## BODY SCHEMA (JSON Schema validation) -Add `BODY SCHEMA ''` to validate the JSON object **before** the handler runs. This uses DuckDB’s `json_schema` machinery under the hood. The `json_schema` extension is loaded when the route is created (and installed from community if needed), so request handling never performs an installation. +Add `BODY SCHEMA ''` to validate the JSON object **before** the handler runs. This uses DuckDB’s `json_schema` extension. Make both `json` and `json_schema` available locally before creating the route; QuackAPI never installs either extension as a side effect. ```sql CREATE ROUTE create_user POST '/users' STATUS 201 diff --git a/docs/index.md b/docs/index.md index f97fbbb..6268e2e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,226 +1,36 @@ -# quackapi - -**A FastAPI-class web framework that lives inside DuckDB.** - -You write routes. The database is the type system, the request validator, and the response encoder. One process serves HTTP and runs SQL. - -``` -browser ──► DuckDB [ quackapi ] -``` - -If you already know “routes,” you know enough to start. - ---- - -## Five-line hello world - -Build (once): - -```sh -git clone --recurse-submodules https://github.com/asubbarao/quackapi -cd quackapi -GEN=ninja make release -./build/release/duckdb -unsigned -``` - -In the DuckDB shell: - -```sql -LOAD quackapi; - -CREATE ROUTE hello GET '/hello' AS SELECT 'world' AS msg; -CREATE ROUTE item GET '/items/:id' AS SELECT $id::INTEGER AS id; - -SELECT * FROM quackapi_serve(8000); -``` - -In another terminal (outputs below are from a live run against `build/release/duckdb -unsigned`): - -```sh -curl http://127.0.0.1:8000/hello -# [{"msg":"world"}] - -curl http://127.0.0.1:8000/items/42 -# [{"id":42}] - -curl http://127.0.0.1:8000/items/abc -# {"detail":[{"loc":["path","id"],"msg":"Input should be a valid integer","type":"type_error"}]} -# HTTP 422 -``` - -That is the whole product idea: - -1. **`CREATE ROUTE`** registers an HTTP endpoint whose handler is SQL. -2. **`$id::INTEGER`** (and friends) type-check request values. -3. Failures return **FastAPI-shaped 422** JSON: `{ "detail": [ { "loc", "msg", "type" } ] }`. -4. Success returns a **JSON array of row objects** by default — your `SELECT` list *is* the response model. - -> **Parser tip:** run `LOAD quackapi;` first in a sequential session (interactive shell or FIFO stdin). Putting `LOAD` and `CREATE ROUTE` in one `duckdb -c "…"` string can parse the route before the extension is loaded. - ---- - -## You write routes. Everything else is included. - -| You need | You write | Built-in | -|----------|-----------|----------| -| HTTP methods | `GET` / `POST` / `PUT` / `PATCH` / `DELETE` / `HEAD` | 405 + `Allow`, auto-HEAD for GET | -| Path / query params | `$name::TYPE` or `PARAM name TYPE …` | 422 validation | -| JSON / form / multipart bodies | same `$name` namespace | body binder | -| Auth | `CREATE AUTH` + `REQUIRE` | API key + JWT HS256 | -| Instant list/get CRUD | `CREATE API FOR TABLE` | expands to two GET routes | -| Versioned prefix + shared auth | `CREATE GROUP` | APIRouter-style | -| Background jobs | `CREATE QUEUE` + enqueue/dequeue/ack | durable `quackapi_jobs` table | -| Live push | `CREATE STREAM … GET` | Server-Sent Events | -| Row security | `CREATE ROW ACCESS POLICY` | claims-keyed filters | -| Column redaction | `CREATE MASKING POLICY` | claims-keyed masks | -| Static files | `static_dir := '…'` on serve | unrouted GETs | -| OpenAPI | open the browser | `/openapi.json`, `/docs`, `/redoc` | -| Thin GraphQL (v0) + dual surface | REST + GraphQL both → SQL; `FOR TABLE` + `GRAPHQL ROUTE` | [graphql-v0.md](guide/graphql-v0.md) | - -Start the server with: - -```sql -SELECT * FROM quackapi_serve(8000); --- → http://127.0.0.1:8000 -``` - -Stop with: - -```sql -SELECT * FROM quackapi_stop(); --- or SELECT * FROM quackapi_stop(8000); -``` - ---- - -## Mental model (one page) - -### Request → bind → SQL → response - -1. HTTP request hits a registered pattern (`/items/:id`). -2. Path, query, header, cookie, and body values land in the same **`$name`** parameter namespace. -3. Handler SQL runs with those parameters prepared. -4. Result rows become JSON (or `html` / `text` / redirect / Set-Cookie — see [headers, cookies, redirects](guide/headers-cookies-redirects.md)). - -### Response envelope - -| Handler shape | Response | -|---------------|----------| -| Multiple columns (default) | `application/json` **array of objects** | -| `ENVELOPE object` (FORMAT json) | bare **JSON object** (exactly one row); 0 rows → `null` unless `EMPTY STATUS` | -| `EMPTY STATUS [BODY '…']` | 0-row result → that status (default body `{"detail":"Not Found"}`) | -| Single column named `html` | `text/html; charset=utf-8` raw body | -| Single column named `text` | `text/plain; charset=utf-8` raw body | -| Column `location` (+ `STATUS 3xx`) | redirect (`Location` header) | -| Column `set_cookie` / `set-cookie` | `Set-Cookie` header (stripped from JSON) | - -Example of typed JSON (bool / number / null preserved): - -```sql -CREATE ROUTE page_json GET '/json' AS -SELECT 'world' AS msg, 42 AS n, true AS ok, NULL::INTEGER AS missing; -``` - -```sh -curl http://127.0.0.1:8000/json -# [{"msg":"world","n":42,"ok":true,"missing":null}] -``` - -Single-row object (opt-in; array remains the default): - -```sql -CREATE ROUTE health GET '/api/health' ENVELOPE object AS -SELECT 'ok' AS status; --- → {"status":"ok"} -``` - -### Live registry - -Routes created **after** serve is already running are live immediately. Inspect them: - -```sql -SELECT name, method, pattern, status, require_auth -FROM quackapi_routes(); -``` - -Handler SQL is validated at `CREATE` time. Broken SQL fails at create — not on first request. - ---- - -## Guided tour (read top to bottom) - -### Task guides - -1. [Routes, typed params, and 422](guide/routes-and-params.md) -2. [Request bodies (JSON, form, multipart, BODY SCHEMA)](guide/request-bodies.md) -3. [Headers, cookies, redirects, status codes, content types](guide/headers-cookies-redirects.md) -4. [Auth (API key + JWT) and REQUIRE](guide/auth.md) -5. [CREATE API FOR TABLE (instant CRUD reads)](guide/table-api.md) -6. [CREATE GROUP (prefix + shared auth = versioning)](guide/groups.md) -7. [CREATE QUEUE (background jobs + worker)](guide/queue.md) -8. [CREATE STREAM (SSE)](guide/stream.md) -9. [Extension composition (fetch, SSE, from_x, mesh, PDF)](guide/extension-composition.md) -10. [Row access & masking policies](guide/policies.md) -11. [Static files (`static_dir`)](guide/static-files.md) -12. [OpenAPI, Swagger UI, ReDoc](guide/openapi.md) -13. [Thin GraphQL v0 (catalog tables)](guide/graphql-v0.md) -14. [Response compression](guide/compression.md) -15. [SQL middleware](MIDDLEWARE.md) - -### Reference - -- [DDL grammar (every CREATE noun)](reference/ddl.md) -- [SQL functions (`quackapi_*`)](reference/functions.md) - -### FastAPI readers - -- [FastAPI parity map (89/89)](fastapi-parity.md) -- [Coming from FastAPI (`quack_from_X`)](from-fastapi.md) - -### Feature ledger - -Authoritative built-vs-not list: [FEATURE_STATUS.md](FEATURE_STATUS.md). - ---- - -## Install & load - -### From source (today) - -```sh -GEN=ninja make release -./build/release/duckdb -unsigned -``` - -```sql -LOAD quackapi; --- or absolute path: --- LOAD '/path/to/build/release/extension/quackapi/quackapi.duckdb_extension'; -``` - -### Community - -```sql -INSTALL quackapi FROM community; -LOAD quackapi; -``` - -**Target DuckDB:** v1.5.6. Platforms: Linux/macOS amd64+arm64 and Windows -amd64. wasm, Windows MinGW/rtools, and Windows arm64 are excluded. - ---- - -## Honest limits (so you plan correctly) - -- **Single-writer DuckDB** — fine for many concurrent readers and light writes; not a pooled OLTP app tier. -- **JWT** — HS256 only today (no RS256 / OIDC browser flow yet). -- **`CREATE API FOR TABLE`** — read routes only (list + get by key). -- **WebSocket** — not supported on the HTTP transport; use [SSE streams](guide/stream.md). -- **Response gzip / zstd** — [compression guide](guide/compression.md). -- Registry (routes, auth, groups, queues, streams) lives on the **database instance**. Re-run DDL after reopen. Queue **jobs** (`quackapi_jobs`) are normal tables and survive restart. - ---- - -## Next step - -Open [Routes, typed params, and 422](guide/routes-and-params.md) and run every `curl` against your own `quackapi_serve` process. +# QuackAPI documentation + +QuackAPI registers HTTP routes in DuckDB and executes each handler as SQL. +Start with the [README quickstart](../README.md), then use these guides for the +retained API. + +## Getting started + +- [Routes and typed parameters](guide/routes-and-params.md) +- [Request bodies](guide/request-bodies.md) +- [Authentication](guide/auth.md) +- [Groups](guide/groups.md) +- [Static files](guide/static-files.md) +- [Compression](guide/compression.md) +- [OpenAPI](guide/openapi.md) + +## Reference + +- [Function reference](reference/functions.md) +- [DDL reference](reference/ddl.md) +- [Middleware](MIDDLEWARE.md) +- [Request budgets and limits](RESOURCE_LIMITS.md) +- [Outbound POST providers](curl_httpfs.md) +- [Removed APIs and migration](MIGRATION.md) + +## Scope and operating limits + +The current HTTP response formats are JSON, NDJSON, and CSV. Route +authentication does not enforce row-level policies on arbitrary SQL. DuckDB's +ordinary SQL and database-file concurrency rules apply to requests; the HTTP +server is not a multi-writer OLTP tier. Arrow Flight remains a separate +protocol. + +Historical feature research in this repository predates the API cleanup. Use +the [migration guide](MIGRATION.md) and current references above for supported +behavior. diff --git a/docs/reference/ddl.md b/docs/reference/ddl.md index 931402c..db2184d 100644 --- a/docs/reference/ddl.md +++ b/docs/reference/ddl.md @@ -1,367 +1,125 @@ -# DDL reference — every CREATE noun +# DDL reference -Grammar below is taken from the parser extensions in `src/`. Examples were run against `build/release/duckdb -unsigned` with `LOAD quackapi;`. +Load QuackAPI before executing its DDL so DuckDB has registered the parser +extensions. Load DuckDB's JSON extension before QuackAPI when the database will +serve JSON routes or use JSON body validation: -Drop forms exist for each noun unless noted. - ---- +```sql +LOAD json; +LOAD quackapi; +``` -## 1. CREATE ROUTE +## `CREATE ROUTE` ```sql -CREATE [OR REPLACE] ROUTE '' - [STATUS ] - [REQUIRE ] - [RATE LIMIT PER [BY ip|token|key]] - [FORMAT json|ndjson|csv|parquet|arrow] - [ENVELOPE array|object] - [EMPTY STATUS [BODY '']] - [TIMEOUT |'30s'|'5m'|'1h'] - [GROUP | IN GROUP ] +CREATE [OR REPLACE] ROUTE '' + [STATUS ] + [FORMAT json | ndjson | csv] + [EMPTY STATUS ] + [REQUIRE ] + [GROUP ] + [SCHEDULE ''] [BODY SCHEMA ''] - [BODY TYPE ''] - [PARAM [] [HEADER|COOKIE|QUERY [wire-name]] - [DEFAULT ] [GE|GT|LE|LT ] [MIN_LENGTH|MAX_LENGTH ] …] - [WITH (timeout_sec [=|:=] |'30s')] - AS ; + [BODY TYPE ] + [PARAM ] +AS ; DROP ROUTE ; ``` -| Piece | Rules | -|-------|--------| -| **METHOD** | `GET`, `POST`, `PUT`, `DELETE`, `PATCH`, `HEAD` | -| **pattern** | Quoted. Must start with `/` unless the route is in a GROUP (relative join allowed). Captures: `:id` or `{id}` → `$id` | -| **STATUS** | Integer 100–599. Default 200 | -| **REQUIRE** | Auth scheme name (checked at request time) | -| **RATE LIMIT** | Fixed-window limit; `BY ip` is the default, and `BY token` / `BY key` use the authenticated subject or credential digest. Exceeding the limit returns `429` with `Retry-After`. | -| **FORMAT** | Response body for row data: `json` (default, array of objects), `ndjson` (`application/x-ndjson`), `csv` (`text/csv`), `parquet` (`application/vnd.apache.parquet` file bytes, magic `PAR1`), `arrow` (Arrow IPC stream via community `nanoarrow` `FORMAT ARROWS`, `application/vnd.apache.arrow.stream`, magic `0xFFFFFFFF`). Explicit `ndjson`/`csv`/`parquet`/`arrow` win over `Accept`; default/`json` allows Accept negotiation (`application/x-ndjson`, `application/jsonl`, `text/csv`, `application/vnd.apache.parquet`, `application/parquet`, `application/vnd.apache.arrow.stream`, `application/vnd.apache.arrow.file`). Column modes `html`/`text` still win. | -| **ENVELOPE** | JSON shape for `FORMAT json`: `array` (default, `[{…}]`) or `object` (exactly one row → `{…}`; 0 rows → `null` unless `EMPTY STATUS`; >1 rows → 500). Only valid with `FORMAT json`. OpenAPI advertises `type: object` when set. | -| **EMPTY STATUS** | When the handler returns **0 rows**, respond with this HTTP status instead of success `STATUS` + `[]`/`null`. Optional `BODY ''` (default `{"detail":"Not Found"}`). Works with array or object envelope. Listed in OpenAPI responses. | -| **TIMEOUT** / **WITH (timeout_sec)** | Per-request httplib socket read/write deadline in seconds (also `'30s'` / `'5m'` / `'1h'`, max 24h). Default omit/`0` = serve defaults (`quackapi_serve` `read_timeout_sec`/`write_timeout_sec`, usually 30). When set, the matched request extends `SocketStream` select timeouts and `SO_RCVTIMEO`/`SO_SNDTIMEO` on that connection through handler + response write. Synonyms: `TIMEOUT 180` or `WITH (timeout_sec := 180)`. | -| **GROUP / IN GROUP** | Join group prefix + inherit auth/tags | -| **BODY SCHEMA** | Quoted JSON Schema string; the `json_schema` extension is loaded when the route is created (and may be installed from community); may appear before or after PARAM | -| **BODY TYPE** | Quoted DuckDB `json_transform` structure; native typed body validation; may appear before or after PARAM | -| **PARAM** | Zero or more. Types: INTEGER/INT, BIGINT, VARCHAR/TEXT/STRING, BOOLEAN/BOOL, DOUBLE, FLOAT/REAL, HUGEINT, UBIGINT, UINTEGER | -| **AS** | Any SQL returning a result; validated at CREATE time | - -**Example:** - -```sql -CREATE ROUTE item GET '/items/:id' - PARAM pretty BOOLEAN DEFAULT false - AS -SELECT $id::INTEGER AS id, $pretty AS pretty; -``` +Supported methods are GET, POST, PUT, PATCH, DELETE, and HEAD. Paths begin +with `/`; named path components such as `'/items/:id'` bind to `$id` in the +handler. Query parameters, selected headers/cookies, and request-body values +can also be declared as handler parameters. Values are bound as prepared SQL +parameters. See the [route guide](../guide/routes-and-params.md) for binding +details. -**Object envelope + empty → 404:** +The default response is JSON. `FORMAT` supports only `json`, `ndjson`, and +`csv`; it does not encode Arrow or Parquet. `STATUS` selects the response +status for a successful handler. `EMPTY STATUS` selects the status used when +the handler returns no rows. `BODY SCHEMA` validates a JSON request body and +requires the DuckDB `json` and `json_schema` extensions to be available; route +DDL does not install them. `BODY TYPE` and `PARAM` declare typed inputs. ```sql -CREATE ROUTE health GET '/api/health' - ENVELOPE object - AS SELECT 'ok' AS status, 'sd-shelf' AS service; --- GET → {"status":"ok","service":"sd-shelf"} +CREATE ROUTE item GET '/items/:id' STATUS 200 AS + SELECT $id::INTEGER AS id; -CREATE ROUTE get_doc GET '/documents/:id' - ENVELOPE object - EMPTY STATUS 404 BODY '{"detail":"Not Found"}' - AS SELECT * FROM documents WHERE id = $id::INTEGER; --- 1 row → {…}; 0 rows → 404 {"detail":"Not Found"} +CREATE ROUTE create_item POST '/items' + BODY TYPE JSON + BODY SCHEMA '{"type":"object","required":["name"]}' +AS + SELECT json_extract_string($body, '$.name') AS name; ``` -**Long-running handler (extends socket IO past the serve default):** +`SCHEDULE` accepts a six-field cron expression with seconds first. Scheduled +routes cannot contain path parameters. -```sql -CREATE ROUTE chrome_exhaust POST '/chrome/exhaust' - PARAM url_prefix VARCHAR - WITH (timeout_sec := 180) - AS SELECT chrome_exhaust($url_prefix) AS result; --- equivalent: … TIMEOUT 180 AS SELECT … -``` - -**Response control columns:** `html`, `text`, `location`, `set_cookie` / `set-cookie` — see [headers guide](../guide/headers-cookies-redirects.md). - ---- - -## 2. CREATE AUTH +## `CREATE AUTH` ```sql -CREATE [OR REPLACE] AUTH AS API_KEY [ ( HEADER '' ) ]; +CREATE [OR REPLACE] AUTH AS API_KEY + [(HEADER '')]; -CREATE [OR REPLACE] AUTH AS JWT ( - SECRET '' - [, ALGORITHM HS256] -); +CREATE [OR REPLACE] AUTH AS JWT + (SECRET '' [, ALGORITHM HS256]); DROP AUTH ; ``` -| Kind | Default header | Notes | -|------|----------------|-------| -| `API_KEY` | `X-API-Key` | Keys added via `quackapi_add_api_key` (SHA-256 stored) | -| `JWT` | `Authorization` (Bearer) | **HS256 only** | - -**Example:** - -```sql -CREATE AUTH site AS API_KEY; -CREATE AUTH jwt_auth AS JWT ( SECRET 'conformance-secret' ); -``` - ---- - -## 3. CREATE GROUP / CREATE API GROUP - -```sql -CREATE [OR REPLACE] [API] GROUP - WITH ( - prefix='/absolute/path' - [, auth= | require=] - [, tags='csv,tags'] - ); - --- Keyword form (no WITH): -CREATE [OR REPLACE] API GROUP - PREFIX '/p' - [TAGS 't'] - [REQUIRE ]; - -DROP [API] GROUP ; -``` - -`CREATE GROUP` and `CREATE API GROUP` are synonyms. `auth=` and `require=` are synonyms. - -**Example:** - -```sql -CREATE GROUP v1 WITH (prefix='/api/v1', auth=api, tags='items,v1'); -CREATE ROUTE items_list GET '/items' GROUP v1 AS SELECT 1 AS id; -``` - ---- - -## 4. CREATE API FOR TABLE - -```sql -CREATE [OR REPLACE] API FOR TABLE
- [AT ''] - [KEY '']; -``` - -| Clause | Default | Constraint | -|--------|---------|------------| -| table | — | Bare id or `"quoted""ident"` | -| `AT` | `'/
'` | Quoted path starting with `/` | -| `KEY` | `'id'` | Bare identifier → path param | - -Expands to: - -- `GET ` → `SELECT * FROM
` -- `GET /:` → `SELECT * FROM
WHERE = $` - -Read-only. Generated names: `
_list`, `
_get`. - -**Example:** - -```sql -CREATE API FOR TABLE documents AT '/api/documents' KEY 'doc_id'; -``` - ---- - -## 5. CREATE GRAPHQL FOR TABLE - -Optional allowlist for the built-in thin GraphQL endpoint (`POST /graphql`, `GET /graphql/schema`). Empty allowlist = **open** mode (all main-schema tables). See [graphql-v0.md](../guide/graphql-v0.md). - -```sql -CREATE [OR REPLACE] GRAPHQL FOR TABLE
[,
...]; - -DROP GRAPHQL FOR TABLE
[,
...]; - -DROP GRAPHQL ALL; -- clear allowlist → open mode -``` - -| Piece | Rules | -|-------|--------| -| **table** | Bare id or `"quoted""ident"`; must exist in schema `main` at CREATE | -| **OR REPLACE** | Idempotent re-register (no error if already listed) | -| **DROP … ALL** | Clears entire allowlist | - -Inspect: - -```sql -SELECT * FROM quackapi_graphql_tables(); --- open mode: one row mode='open', table_name NULL --- allowlist: mode='allowlist', one row per table_name -``` - -**Example:** +API keys default to the `X-API-Key` header. Add keys with +`quackapi_add_api_key`; key inspection never returns the secret value. JWT +authentication supports HS256. Apply an auth definition with `REQUIRE` on a +route or as a group's inherited auth. ```sql -CREATE GRAPHQL FOR TABLE users, posts; +CREATE AUTH api AS API_KEY; +SELECT * FROM quackapi_add_api_key('api', 'secret-key', 'user-42'); +CREATE ROUTE me GET '/me' REQUIRE api AS SELECT $claims_sub AS subject; ``` -### CREATE GRAPHQL ROUTE +Route auth decides whether a request may reach its handler. It does not filter +rows or rewrite arbitrary SQL by tenant. Put tenant predicates directly in +the handler or use a scoped view. -Named path mounts independent of the built-in `/graphql` allowlist. See [graphql-v0.md](../guide/graphql-v0.md). +## `CREATE GROUP` ```sql -CREATE [OR REPLACE] GRAPHQL ROUTE - POST '' - FROM
[,
...] - [REQUIRE ] - [LIMIT ]; +CREATE [OR REPLACE] GROUP + WITH (prefix = '' [, auth = ] + [, tags = '[, ...]']); -DROP GRAPHQL ROUTE ; +DROP GROUP ; ``` -| Piece | Rules | -|-------|--------| -| **path** | Absolute; not reserved (`/graphql`, `/graphql/schema`); unique among GraphQL routes | -| **FROM** | Required; tables must exist in `main` | -| **REQUIRE** | Optional; auth scheme must already exist | -| **LIMIT** | Optional 1..100000 (default 100) | - -Inspect: - -```sql -SELECT * FROM quackapi_graphql_routes(); --- name, method, path, tables, require_auth, limit -``` - -HTTP: `POST ` (query body) and `GET /schema`. - ---- - -## 6. CREATE QUEUE - -```sql -CREATE [OR REPLACE] QUEUE - [WITH ( - max_attempts=, -- default 3; range 1..1000 - visibility_timeout='30s'|30, -- s/m/h suffix or seconds - backoff_base_seconds= -- default 2; 0 = immediate - )]; - -DROP QUEUE ; -``` - -Queue name: `[A-Za-z0-9_-]+`. - -Creates durable table **`quackapi_jobs`** on first use. - -**Example:** +A group prefixes route paths and can provide default auth and OpenAPI tags. +A route's explicit `REQUIRE` takes precedence over inherited group auth. ```sql -CREATE QUEUE emails WITH (max_attempts=5, visibility_timeout='30s'); +CREATE GROUP api_v1 WITH (prefix = '/api/v1', auth = api, tags = 'v1,public'); +CREATE ROUTE status GET '/status' GROUP api_v1 AS SELECT 'ok' AS status; ``` ---- - -## 7. CREATE STREAM - -```sql -CREATE [OR REPLACE] STREAM GET '' - [WITH (interval='1s'|2|'250ms')] - AS
- ADD ROW ACCESS POLICY ON ([, ...]); - -DROP ROW ACCESS POLICY ; -``` - -Expression may reference policy columns and `$claims_*`. - -**Example:** - -```sql -CREATE ROW ACCESS POLICY tenant_isolation - AS (tenant_id VARCHAR) RETURNS BOOLEAN - USING (tenant_id = $claims_tenant OR $claims_role = 'admin'); - -ALTER TABLE pol_orders - ADD ROW ACCESS POLICY tenant_isolation ON (tenant_id); -``` - ---- - -## 9. CREATE MASKING POLICY - -```sql -CREATE [OR REPLACE] MASKING POLICY - ON - USING (); - -ALTER TABLE
- MODIFY COLUMN SET MASKING POLICY ; - -DROP MASKING POLICY ; -``` - -Inside `USING`, **`val`** is the original column value. Claims bind as `$claims_*`. - -**Example:** - -```sql -CREATE MASKING POLICY mask_email ON VARCHAR - USING (CASE WHEN $claims_role = 'admin' THEN val ELSE '***' END); - -ALTER TABLE pol_users - MODIFY COLUMN email SET MASKING POLICY mask_email; -``` - ---- - -## 10. CREATE MIDDLEWARE +## `CREATE MIDDLEWARE` ```sql CREATE [OR REPLACE] MIDDLEWARE - BEFORE|AFTER [GROUP ] - AS ; - - - Methods: GET, POST, PUT, DELETE, PATCH, HEAD. - - Path segments `:id` / `{id}` and query/body fields bind to `$id`. - - Cast/constraint failures return **422** with FastAPI-shaped - `{"detail":[{"loc":[...],"msg":...,"type":...}]}`. - - Default JSON response is an **array of row objects** (SQL result-set - semantics). A single column named `html` or `text` returns raw HTML/text; - `location` / `set_cookie` set response headers. - - `quackapi_serve([port], host := …, static_dir := …, cors_origins := …, memory_limit := …)` - uses DuckDB's bundled httplib (background listener + workers). - - Built-in OpenAPI: `GET /openapi.json`, `GET /docs`, `GET /redoc`. - - ## Other DDL / functions - - - **Auth:** `CREATE AUTH … AS API_KEY | JWT (SECRET … [, ALGORITHM HS256])`, - `REQUIRE` on routes, `quackapi_add_api_key`, `quackapi_auths`, claims as - `$claims_*`. JWT verify is self-contained (DuckDB-bundled mbedtls + - `json_*` claims) — does not depend on community `crypto` (that extension - is hash/HMAC only, not JWT parse/exp/nbf/claims). - - **Groups:** `CREATE GROUP … WITH (prefix=…, auth=…, tags=…)` — path prefix - + auth inheritance (`quackapi_groups()`). - - **Table API:** `CREATE API FOR TABLE t [AT '/path'] [KEY 'id']` → GET list - + GET by key only. - - **Queue:** `CREATE QUEUE`, `quackapi_enqueue/dequeue/ack/nack`, table - `quackapi_jobs` (broker-less; compose `cronjob` for workers). - - **SSE:** `CREATE STREAM … GET '/path' AS