Simplify QuackAPI to the HTTP core and bounded provider-backed POST

.github/workflows/MainDistributionPipeline.yml

@@ -46,32 +46,15 @@ jobs:
       extension_name: quackapi
       format_checks: "format"
 
-  # Own job (not the reusable distribution workflow): needs `services:` for
-  # Postgres so require-env QUACKAPI_TEST_PG_DSN tests actually run. Linux
-  # amd64 only — does not publish artifacts or expand the release matrix.
-  linux-amd64-pg-sqllogic:
-    name: Linux amd64 SQLLogic with Postgres
+  # Focused Linux validation for the SQLLogic suite and real HTTP contracts.
+  # The released extension uses DuckDB SQL only; no external database service
+  # is required.
+  linux-amd64-sqllogic:
+    name: Linux amd64 SQLLogic
     runs-on: ubuntu-24.04
     env:
       GEN: ninja
       DUCKDB_GIT_VERSION: v1.5.6
-      ENABLE_EXTENSION_AUTOLOADING: "1"
-      ENABLE_EXTENSION_AUTOINSTALL: "1"
-      QUACKAPI_TEST_PG_DSN: postgresql://quackapi_pg_cache:qa_pg_cache_test@127.0.0.1:5432/quackapi_pg_cache_test
-    services:
-      postgres:
-        image: postgres:16
-        env:
-          POSTGRES_USER: quackapi_pg_cache
-          POSTGRES_PASSWORD: qa_pg_cache_test
-          POSTGRES_DB: quackapi_pg_cache_test
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd "pg_isready -U quackapi_pg_cache -d quackapi_pg_cache_test"
-          --health-interval 5s
-          --health-timeout 5s
-          --health-retries 10
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -92,40 +75,144 @@ jobs:
       - name: Install build deps
         run: |
           sudo apt-get update
-          sudo apt-get install -y ninja-build ccache libpq-dev postgresql-client
+          sudo apt-get install -y ninja-build ccache libssl-dev libcurl4-openssl-dev
 
       - name: Setup Ccache
         uses: hendrikmuhs/ccache-action@main
         continue-on-error: true
         with:
-          key: ccache-quackapi-pg-sqllogic-linux-amd64-v1.5.6
-          restore-keys: ccache-quackapi-pg-sqllogic-linux-amd64-
-
-      - name: Wait for Postgres
-        run: |
-          until pg_isready -h 127.0.0.1 -p 5432 -U quackapi_pg_cache -d quackapi_pg_cache_test; do
-            sleep 1
-          done
+          key: ccache-quackapi-sqllogic-linux-amd64-v1.5.6
+          restore-keys: ccache-quackapi-sqllogic-linux-amd64-
 
       - name: Build extension
         run: make release -j"$(nproc)"
 
       - name: Run quackapi SQLLogic
         run: ./build/release/test/unittest "test/sql/quackapi*"
 
-      - name: Verify outbound worker cleanup
+      - name: Run BODY SCHEMA SQLLogic coverage with local json_schema
         run: |
-          c++ -std=c++17 -pthread -Wall -Wextra -Werror -Isrc/include test/cpp/quackapi_thread_join_test.cpp -o /tmp/quackapi-thread-join-test
-          /tmp/quackapi-thread-join-test
-
-      - name: Verify build without libpq
-        run: c++ -std=c++17 -fsyntax-only -isystem duckdb/src/include -Isrc/include src/quackapi_pg.cpp
+          set -euo pipefail
+          extension_repo="$PWD/build/release/repository"
+          community_extension_dir="$RUNNER_TEMP/quackapi-community-extensions"
+          test_extension_dir="$RUNNER_TEMP/quackapi-test-extensions"
+          test_config="$RUNNER_TEMP/quackapi-test-config.json"
+          mkdir -p "$community_extension_dir" "$test_extension_dir" "$extension_repo/v1.5.6/linux_amd64"
+          ./build/release/duckdb -no-stdin -c "SET extension_directory='$community_extension_dir'; INSTALL json_schema FROM community;"
+          cp "$community_extension_dir/v1.5.6/linux_amd64/json_schema.duckdb_extension" \
+            "$extension_repo/v1.5.6/linux_amd64/"
+          cat > "$test_config" <<EOF
+          {"autoloading":"available","settings":[{"name":"extension_directory","value":"$test_extension_dir"}]}
+          EOF
+          log_file="$(mktemp)"
+          LOCAL_EXTENSION_REPO="$extension_repo" ./build/release/test/unittest \
+            --test-dir "$PWD" --test-config "$test_config" \
+            "test/sql/quackapi_body_schema.test" 2>&1 | tee "$log_file"
+          if grep -Eiq "skip.*require json_schema|require json_schema.*skip" "$log_file"; then
+            echo "BODY SCHEMA coverage was skipped because json_schema was unavailable" >&2
+            exit 1
+          fi
+          if ! grep -Eq "16 assertions" "$log_file"; then
+            echo "Expected all 16 BODY SCHEMA assertions to run" >&2
+            exit 1
+          fi
 
       - name: Verify request database lifetime
         run: |
           c++ -std=c++17 -pthread -Wall -Wextra -Werror -isystem duckdb/src/include test/cpp/quackapi_lifetime_test.cpp -Lbuild/release/src -lduckdb -Wl,-rpath,"$PWD/build/release/src" -o /tmp/quackapi-lifetime-test
           /tmp/quackapi-lifetime-test
 
+      - name: Build clean pinned core for loadable extension contracts
+        run: |
+          cmake -S duckdb -B build/post-clean-core \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DBUILD_SHARED_LIBS=ON \
+            -DBUILD_MAIN_DUCKDB_LIBRARY=ON \
+            -DBUILD_SHELL=OFF \
+            -DBUILD_UNITTESTS=OFF \
+            -DBUILD_BENCHMARKS=OFF \
+            -DEXTENSION_STATIC_BUILD=FALSE \
+            -DDUCKDB_EXTENSION_CONFIGS= \
+            -DDISABLE_BUILTIN_EXTENSIONS=TRUE \
+            -DDISABLE_EXTENSION_LOAD=FALSE \
+            -DOVERRIDE_GIT_DESCRIBE=v1.5.6
+          cmake --build build/post-clean-core --target duckdb -j"$(nproc)"
+          test -f build/post-clean-core/src/libduckdb.so
+
+      - name: Build pinned curl_httpfs POST provider
+        uses: actions/checkout@v4
+        with:
+          repository: asubbarao-ifr/duckdb-curl-filesystem
+          ref: a48f83c6198a0252672f6f696e446e1525745a8a
+          path: provider-curl-httpfs
+          submodules: recursive
+
+      - name: Build pinned HTTPFS-Curl POST provider
+        uses: actions/checkout@v4
+        with:
+          repository: asubbarao-ifr/duckdb-httpfs
+          ref: 748f5ac8dddee531384d9901387d43b0f6a2db43
+          path: provider-httpfs-curl
+          submodules: recursive
+
+      - name: Build provider artifacts against the tested DuckDB core
+        run: |
+          duckdb_core="$PWD/build/release/src/libduckdb_static.a"
+          test -f "$duckdb_core"
+          test "$(git -C provider-curl-httpfs rev-parse HEAD)" = a48f83c6198a0252672f6f696e446e1525745a8a
+          test "$(git -C provider-httpfs-curl rev-parse HEAD)" = 748f5ac8dddee531384d9901387d43b0f6a2db43
+          test "$(git -C provider-curl-httpfs/duckdb rev-parse HEAD)" = 069cc9f9b5be802405797faecc284961b07c70ef
+          test "$(git -C provider-httpfs-curl/duckdb rev-parse HEAD)" = 069cc9f9b5be802405797faecc284961b07c70ef
+          make -C provider-curl-httpfs release -j"$(nproc)" \
+            EXTENSION_STATIC_BUILD=1 \
+            EXT_FLAGS="-DPREBUILT_BINARY=$duckdb_core" \
+            OVERRIDE_GIT_DESCRIBE=v1.5.6
+          make -C provider-httpfs-curl release -j"$(nproc)" \
+            EXTENSION_STATIC_BUILD=1 \
+            EXT_FLAGS="-DPREBUILT_BINARY=$duckdb_core" \
+            OVERRIDE_GIT_DESCRIBE=v1.5.6
+
+      - name: Verify outbound POST over real HTTP with both providers
+        run: |
+          quackapi_artifact="$PWD/build/release/extension/quackapi/quackapi.duckdb_extension"
+          quackapi_version="$(git rev-parse --short HEAD)"
+          duckdb_test_core="$PWD/build/post-clean-core/src/libduckdb.so"
+          curl_provider="$PWD/provider-curl-httpfs/build/release/extension/curl_httpfs/curl_httpfs.duckdb_extension"
+          httpfs_provider="$PWD/provider-httpfs-curl/build/release/extension/httpfs/httpfs.duckdb_extension"
+          cmake -S test/cpp -B build/post_http_test_curl \
+            -DQUACKAPI_EXTENSION="$quackapi_artifact" \
+            -DQUACKAPI_VERSION="$quackapi_version" \
+            -DDUCKDB_LIBRARY="$duckdb_test_core" \
+            -DPOST_PROVIDER_EXTENSION="$curl_provider" \
+            -DPOST_PROVIDER_NAME=curl_httpfs \
+            -DPOST_PROVIDER_VERSION=v0.4.5 \
+            -DENABLE_PROVIDER_REPLACEMENT=ON \
+            -DCURL_HTTPFS_EXTENSION="$curl_provider" \
+            -DCURL_HTTPFS_VERSION=v0.4.5 \
+            -DHTTPFS_EXTENSION="$httpfs_provider" \
+            -DHTTPFS_VERSION=748f5ac
+          cmake --build build/post_http_test_curl -j"$(nproc)"
+          echo "DuckDB CLI identity:"
+          ./build/release/duckdb -c "PRAGMA version"
+          for test_binary in \
+            build/post_http_test_curl/quackapi_post_http_test \
+            build/post_http_test_curl/quackapi_post_tls_test \
+            build/post_http_test_curl/quackapi_provider_replacement_test; do
+            echo "Runtime dependencies for $test_binary:"
+            readelf -d "$test_binary" | grep -E 'NEEDED|RPATH|RUNPATH' || true
+            ldd "$test_binary" | grep -E 'duckdb|not found' || true
+          done
+          ctest --test-dir build/post_http_test_curl --output-on-failure
+          cmake -S test/cpp -B build/post_http_test_httpfs \
+            -DQUACKAPI_EXTENSION="$quackapi_artifact" \
+            -DQUACKAPI_VERSION="$quackapi_version" \
+            -DDUCKDB_LIBRARY="$duckdb_test_core" \
+            -DPOST_PROVIDER_EXTENSION="$httpfs_provider" \
+            -DPOST_PROVIDER_NAME=httpfs \
+            -DPOST_PROVIDER_VERSION=748f5ac
+          cmake --build build/post_http_test_httpfs -j"$(nproc)"
+          ctest --test-dir build/post_http_test_httpfs --output-on-failure
+
       - name: Verify benchmark correctness gates
         run: |
           python3 bench/test_bench_config.py

CMakeLists.txt

@@ -19,22 +19,21 @@ set(EXTENSION_SOURCES
     src/quackapi_extension.cpp
     src/quackapi_registry.cpp
     src/quackapi_server.cpp
+    src/quackapi_request_binding.cpp
+    src/quackapi_response_encoding.cpp
+    src/quackapi_logging.cpp
     src/quackapi_validation.cpp
     src/quackapi_batteries.cpp
     src/quackapi_ddl.cpp
     src/quackapi_auth.cpp
-    src/quackapi_http_fetch.cpp
+    src/quackapi_post_result.cpp
+    src/quackapi_post.cpp
+    src/quackapi_post_executor.cpp
+    src/quackapi_post_provider.cpp
     src/quackapi_limits.cpp
     src/quackapi_middleware.cpp
-    src/quackapi_table_api.cpp
     src/quackapi_openapi.cpp
-    src/quackapi_graphql.cpp
-    src/quackapi_queue.cpp
-    src/quackapi_policy.cpp
-    src/quackapi_stream.cpp
-    src/quackapi_from_x.cpp
     src/quackapi_util.cpp
-    src/quackapi_pg.cpp
 )
 
 build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES})
@@ -50,7 +49,7 @@ target_include_directories(${EXTENSION_NAME} BEFORE PRIVATE ${CMAKE_CURRENT_SOUR
 target_include_directories(${LOADABLE_EXTENSION_NAME} BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/third_party/httplib)
 
 # The vendored httplib default backlog is 5. That drops loopback connections
-# when the outbound pool fans out to its 32-client cap before the server can
+# when outbound clients fan out to configured concurrency before the server can
 # accept them. Use the platform's full listen backlog for both extension forms.
 target_compile_definitions(${EXTENSION_NAME} PRIVATE CPPHTTPLIB_LISTEN_BACKLOG=SOMAXCONN)
 target_compile_definitions(${LOADABLE_EXTENSION_NAME} PRIVATE CPPHTTPLIB_LISTEN_BACKLOG=SOMAXCONN)
@@ -81,42 +80,6 @@ target_include_directories(${LOADABLE_EXTENSION_NAME} PRIVATE
 target_link_libraries(${EXTENSION_NAME} duckdb_mbedtls)
 target_link_libraries(${LOADABLE_EXTENSION_NAME} duckdb_mbedtls)
 
-# Optional libpq — pure-HTTP Postgres path (same idea as FastAPI+psycopg).
-# When pg_dsn is set on quackapi_serve, simple routes run via PQexecParams
-# instead of DuckDB ATTACH. Soft: build still succeeds without libpq.
-#
-# Never link a host-only Homebrew dylib into a cross-arch build (CI osx_amd64
-# on arm64 runners finds /opt/homebrew arm64 libpq → link fails).
-set(QUACKAPI_TRY_LIBPQ ON)
-if (APPLE AND CMAKE_OSX_ARCHITECTURES)
-	if (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" AND
-	    NOT CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|i386")
-		set(QUACKAPI_TRY_LIBPQ OFF)
-		message(STATUS "quackapi: skip libpq (target x86_64, host ${CMAKE_HOST_SYSTEM_PROCESSOR})")
-	elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" AND
-	        NOT CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64|aarch64")
-		set(QUACKAPI_TRY_LIBPQ OFF)
-		message(STATUS "quackapi: skip libpq (target arm64, host ${CMAKE_HOST_SYSTEM_PROCESSOR})")
-	endif()
-endif()
-if (QUACKAPI_TRY_LIBPQ)
-	find_path(QUACKAPI_LIBPQ_INCLUDE libpq-fe.h
-	    HINTS /opt/homebrew/opt/libpq/include /usr/local/opt/libpq/include /usr/include/postgresql)
-	find_library(QUACKAPI_LIBPQ_LIB pq
-	    HINTS /opt/homebrew/opt/libpq/lib /usr/local/opt/libpq/lib)
-endif()
-if (QUACKAPI_TRY_LIBPQ AND QUACKAPI_LIBPQ_INCLUDE AND QUACKAPI_LIBPQ_LIB)
-	message(STATUS "quackapi: libpq found — native Postgres path enabled")
-	target_include_directories(${EXTENSION_NAME} PRIVATE ${QUACKAPI_LIBPQ_INCLUDE})
-	target_include_directories(${LOADABLE_EXTENSION_NAME} PRIVATE ${QUACKAPI_LIBPQ_INCLUDE})
-	target_link_libraries(${EXTENSION_NAME} ${QUACKAPI_LIBPQ_LIB})
-	target_link_libraries(${LOADABLE_EXTENSION_NAME} ${QUACKAPI_LIBPQ_LIB})
-	target_compile_definitions(${EXTENSION_NAME} PRIVATE QUACKAPI_HAS_LIBPQ=1)
-	target_compile_definitions(${LOADABLE_EXTENSION_NAME} PRIVATE QUACKAPI_HAS_LIBPQ=1)
-else()
-	message(STATUS "quackapi: libpq not found — native Postgres path disabled")
-endif()
-
 # Response compression: DuckDB-bundled zstd + miniz (gzip). NO new dependencies.
 # Citations (verified in this tree / upstream):
 #   duckdb/third_party/zstd/CMakeLists.txt → add_library(duckdb_zstd STATIC …)