Add pgsql-http Extension

.github/workflows/publish-pgembed-http.yml

@@ -0,0 +1,190 @@
+# Build + publish the pgembed-http split wheel.
+#
+# Mirrors build-and-test.yml: same runners, same `make` invocation, same
+# trusted-publishing release action. The ONLY difference is EXTENSIONS:
+# base builds the default set, this builds pgsql_http. No PG_MAJOR is set
+# here on purpose — like base, it uses the pgbuild/Makefile default, so the
+# extension can never drift to a different Postgres major than the base wheel.
+#
+# One-time PyPI setup: register a *pending publisher* for project
+# `pgembed-http` with owner/repo = this repo, workflow filename =
+# `publish-pgembed-http.yml`, environment = `pypi`. The first tagged run
+# then creates the project on first upload.
+#
+# Release flow: bump src/pgembed_pgsql_http/pyproject.toml version, push tag
+# `pgembed-http-v<version>`. The workflow asserts tag == package version.
+name: Publish pgembed-http
+
+on:
+  push:
+    tags:
+      - "pgembed-http-v*"
+  pull_request:
+    paths:
+      - "src/pgembed_pgsql_http/**"
+      - "pgbuild/Makefile"
+      - "Makefile"
+      - ".github/workflows/publish-pgembed-http.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build pgembed-http on ${{ matrix.os }}-${{ matrix.arch }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # Same set as the base wheel matrix, minus Windows (pgsql-http's
+        # libcurl support there is unverified).
+        include:
+          - runner: macos-latest
+            os: macos-latest
+            arch: arm64
+            deployment-target: "11.0"
+          - runner: ubuntu-latest
+            os: ubuntu-latest
+            arch: x86_64
+            deployment-target: ""
+          - runner: ubuntu-24.04-arm
+            os: ubuntu-latest
+            arch: aarch64
+            deployment-target: ""
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - name: Check tag matches package version
+        if: startsWith(github.ref, 'refs/tags/pgembed-http-v')
+        run: |
+          TAG_VERSION="${GITHUB_REF_NAME#pgembed-http-v}"
+          PKG_VERSION="$(grep '^version' src/pgembed_pgsql_http/pyproject.toml | sed 's/.*"\(.*\)".*/\1/')"
+          if [ "$TAG_VERSION" != "$PKG_VERSION" ]; then
+            echo "tag version $TAG_VERSION != package version $PKG_VERSION"
+            exit 1
+          fi
+
+      # Reuse the base build cache when available: it holds a full postgres
+      # build from the same Makefile defaults, so `make EXTENSIONS=pgsql_http`
+      # below only needs to compile the http extension itself. Restore-only:
+      # this workflow must never clobber the base cache. Key replicates
+      # build-and-test.yml exactly for an exact-match hit.
+      - name: Restore postgres build from base cache
+        uses: actions/cache/restore@v3
+        with:
+          path: |
+            pgbuild
+            src/pgembed/pginstall
+          key: ${{ runner.os }}-${{ runner.arch }}-build-cache-postgres-${{
+            hashFiles('Makefile', 'pgbuild/Makefile', '.github/workflows/build-and-test.yml') }}
+
+      # Same install flavor as build-and-test.yml, trimmed to what a pure-C
+      # extension + postgres need (no Rust/cargo-pgrx: nothing pgrx-based here).
+      - name: Install build deps (Linux)
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc make pkg-config bison flex \
+            libssl-dev zlib1g-dev libcurl4-openssl-dev
+
+      - name: Install build deps (macOS)
+        if: matrix.os == 'macos-latest'
+        run: brew install lz4
+        # libcurl headers come from the Xcode SDK; no brew curl on purpose —
+        # linking brewed libcurl would tie the .so to a Homebrew path that
+        # target machines may not have.
+
+      - name: Build postgres and pgsql_http
+        shell: bash
+        env:
+          MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment-target }}
+        run: |
+          # Same plain `make` as base (inherits PG_MAJOR default); only the
+          # extension set differs.
+          make EXTENSIONS=pgsql_http
+
+      - name: Stage split-wheel build tree
+        shell: bash
+        run: |
+          # The repo keeps the flat shim layout (like pgembed_pgvector) so the
+          # root install keeps working; the wheel needs a real package dir, so
+          # assemble it in a staging tree and build there.
+          rm -rf build-pgembed-http
+          mkdir -p build-pgembed-http/pgembed_pgsql_http
+          cp src/pgembed_pgsql_http/pyproject.toml \
+             src/pgembed_pgsql_http/setup.py \
+             src/pgembed_pgsql_http/README.md build-pgembed-http/
+          cp src/pgembed_pgsql_http/__init__.py build-pgembed-http/pgembed_pgsql_http/
+          BIN=src/pgembed/pginstall/bin
+          LIB="$("$BIN/pg_config" --pkglibdir)"
+          SHARE="$("$BIN/pg_config" --sharedir)/postgresql/extension"
+          ls "$LIB"/http.so "$SHARE"/http.control "$SHARE"/http--*.sql
+          cp "$LIB"/http.so build-pgembed-http/pgembed_pgsql_http/
+          cp "$SHARE"/http.control "$SHARE"/http--*.sql build-pgembed-http/pgembed_pgsql_http/
+          ls -la build-pgembed-http/pgembed_pgsql_http/
+
+      - name: Build wheel
+        run: |
+          python -m pip install --upgrade pip build
+          cd build-pgembed-http
+          python -m build
+          ls dist/
+          # setup.py forces a platform tag; a py3-none-any wheel here would
+          # mean the .so didn't get classified — fail loudly.
+          if ls dist/*-py3-none-any.whl >/dev/null 2>&1; then
+            echo "pure-Python wheel tag on a binary wheel — refusing"
+            exit 1
+          fi
+
+      - name: Smoke test (CREATE EXTENSION http)
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install "psycopg2-binary" "build-pgembed-http/dist/"*.whl
+          python -m pip install pgembed
+          python - <<'EOF'
+          import tempfile
+          import pgembed
+          print(pgembed.list_extensions())
+          assert pgembed.has_extension("pgsql_http")
+          with pgembed.get_server(tempfile.mkdtemp(), cleanup_mode="delete") as pg:
+              out = pg.create_extension("http")
+              assert out.strip() == "CREATE EXTENSION", out
+              print(pg.psql("SELECT extversion FROM pg_extension WHERE extname='http';"))
+          EOF
+
+      - uses: actions/upload-artifact@v4
+        with:
+          path: build-pgembed-http/dist/*
+          name: pgembed-http-${{ matrix.os }}-${{ matrix.arch }}
+          overwrite: true
+
+  publish-to-pypi:
+    # Same gate shape as base (any pgembed-http-v* tag).
+    if: startsWith(github.ref, 'refs/tags/pgembed-http-v')
+    name: Publish pgembed-http to PyPI
+    needs:
+      - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pgembed-http
+    permissions:
+      id-token: write
+    steps:
+      - name: Download all the dists
+        uses: actions/download-artifact@v4
+        with:
+          pattern: pgembed-http-*
+          path: dist/
+          merge-multiple: true
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

README.md

@@ -58,15 +58,17 @@ pip install pgembed
 pip install pgembed-pgvector
 pip install pgembed-pgvectorscale
 pip install pgembed-pgtextsearch
+pip install pgembed-http
 
 # Multiple extensions
-pip install pgembed-pgvector pgembed-pgvectorscale pgembed-pgtextsearch
+pip install pgembed-pgvector pgembed-pgvectorscale pgembed-pgtextsearch pgembed-http
 ```
 
 Available extensions:
 - `pgembed-pgvector`: Vector similarity search (works on all platforms)
 - `pgembed-pgvectorscale`: High-performance vector storage (requires Rust, not available on Alpine/Windows)
 - `pgembed-pgtextsearch`: BM25-based full-text search (requires Rust, not available on Alpine/Windows)
+- `pgembed-http`: HTTP client (`http_get`, `http_post`, …) via pgsql-http (requires libcurl at build time)
 
 ### Checking available extensions
 
@@ -88,6 +90,7 @@ if pgembed.has_extension('pgvector'):
 - **pgvector**: Works on Linux, macOS (Intel & Apple Silicon), Windows
 - **pgvectorscale**: Works on Linux, macOS (Intel & Apple Silicon). NOT available on Alpine Linux or Windows (requires Rust)
 - **pgtextsearch**: Works on Linux, macOS (Intel & Apple Silicon). NOT available on Alpine Linux or Windows (requires Rust)
+- **pgembed-http**: Works on Linux, macOS (Intel & Apple Silicon). Requires libcurl dev headers at build time (`libcurl4-openssl-dev` on Debian/Ubuntu)
 
 ### Building specific extensions
 
@@ -103,6 +106,9 @@ make pgvectorscale
 # Build only pgtextsearch
 make pgtextsearch
 
+# Build only pgsql-http (for the pgembed-http split wheel)
+make EXTENSIONS="pgsql_http" all
+
 # Build specific combination
 make EXTENSIONS="pgvector pgtextsearch" all
 ```

pgbuild/Makefile

@@ -41,12 +41,13 @@ EXTENSIONS := $(filter-out $(UNSUPPORTED_EXTENSIONS),$(REQUESTED_EXTENSIONS))
 .PHONY: all
 all: postgres $(EXTENSIONS)
 
-.PHONY: pgvector pgvectorscale pgtextsearch pg_search pg_duckdb
+.PHONY: pgvector pgvectorscale pgtextsearch pg_search pg_duckdb pgsql_http
 pgvector: postgres $(INSTALL_PREFIX)/lib/vector.so
 pgvectorscale: postgres $(INSTALL_PREFIX)/lib/vectorscale.so
 pgtextsearch: postgres $(INSTALL_PREFIX)/lib/pg_textsearch.so
 pg_search: postgres $(INSTALL_PREFIX)/lib/pg_search.so
 pg_duckdb: postgres $(INSTALL_PREFIX)/lib/pg_duckdb.so
+pgsql_http: postgres $(INSTALL_PREFIX)/lib/http.so
 
 ### postgres
 # Major postgres version to build. Override from the command line or CI, e.g. `make PG_MAJOR=18`.
@@ -98,6 +99,23 @@ $(INSTALL_PREFIX)/lib/vector.so: $(PGVECTOR_DIR)/Makefile $(INSTALL_PREFIX)/bin/
 		&& $(MAKE) -C $(PGVECTOR_DIR) PG_CONFIG=$(INSTALL_PREFIX)/bin/pg_config$(EXE_EXT) -j \
 		&& $(MAKE) -C $(PGVECTOR_DIR) PG_CONFIG=$(INSTALL_PREFIX)/bin/pg_config$(EXE_EXT) install
 
+### pgsql_http
+PGSQL_HTTP_TAG := v1.7.2
+PGSQL_HTTP_URL := https://github.com/pramsey/pgsql-http/archive/refs/tags/$(PGSQL_HTTP_TAG).tar.gz
+PGSQL_HTTP_DIR := pgsql-http-$(PGSQL_HTTP_TAG)
+
+$(PGSQL_HTTP_DIR).tar.gz:
+	curl -L -o $(PGSQL_HTTP_DIR).tar.gz $(PGSQL_HTTP_URL)
+
+$(PGSQL_HTTP_DIR)/Makefile: $(PGSQL_HTTP_DIR).tar.gz
+	mkdir -p $(PGSQL_HTTP_DIR)
+	tar xzf $(PGSQL_HTTP_DIR).tar.gz -C $(PGSQL_HTTP_DIR) --strip-components=1
+	touch $(PGSQL_HTTP_DIR)/Makefile
+
+$(INSTALL_PREFIX)/lib/http.so: $(PGSQL_HTTP_DIR)/Makefile $(INSTALL_PREFIX)/bin/postgres
+	unset MAKELEVEL && unset MAKEFLAGS && unset MFLAGS \
+		&& $(MAKE) -C $(PGSQL_HTTP_DIR) PG_CONFIG=$(INSTALL_PREFIX)/bin/pg_config$(EXE_EXT) -j \
+		&& $(MAKE) -C $(PGSQL_HTTP_DIR) PG_CONFIG=$(INSTALL_PREFIX)/bin/pg_config$(EXE_EXT) install
 ### pg_duckdb
 PG_DUCKDB_TAG := v1.1.1
 PG_DUCKDB_REPO := https://github.com/duckdb/pg_duckdb
@@ -204,6 +222,7 @@ clean:
 	rm -rf $(PG_DUCKDB_DIR)
 	rm -rf $(PGTEXTSEARCH_DIR)
 	rm -rf $(PGSEARCH_DIR)
+	rm -rf $(PGSQL_HTTP_DIR)
 
 clean-all: clean
 	rm -rf *.tar.gz

pyproject.toml

@@ -7,7 +7,7 @@ requires-python = ">=3.12"
 license = {file = "LICENSE.txt"}
 urls={repository="https://github.com/Ladybug-Memory/pgembed"}
 authors=[{ name="Oscar Moll", email="orm@csail.mit.edu" }, { name="Arun Sharma", email="arun@sharma-home.net" }]
-keywords=["postgresql", "pgvector", "pg_duckdb", "pgembed", "pg_textsearch", "rag"]
+keywords=["postgresql", "pgvector", "pg_duckdb", "pgembed", "pg_textsearch", "rag", "pgsql-http"]
 dependencies = [
     "fasteners>=0.19",
     "platformdirs>=4.0.0",

src/pgembed/__init__.py

@@ -33,6 +33,7 @@ def _get_pkg_path():
     "pgvector": "pgembed_pgvector",
     "pgvectorscale": "pgembed_pgvectorscale",
     "pgtextsearch": "pgembed_pgtextsearch",
+    "pgsql_http": "pgembed_pgsql_http"
 }
 
 EXTENSION_SO_FILES = {
@@ -41,6 +42,7 @@ def _get_pkg_path():
     "pgtextsearch": "pg_textsearch.so",
     "pg_search": "pg_search.so",
     "pg_duckdb": "pg_duckdb.so",
+    "pgsql_http": "http.so",
 }
 
 EXTENSION_NAMES = (
@@ -49,6 +51,7 @@ def _get_pkg_path():
     "pgtextsearch",
     "pg_search",
     "pg_duckdb",
+    "pgsql_http"
 )
 
 
@@ -115,6 +118,7 @@ def get_extension_create_name(name: str) -> str:
         "pgtextsearch": "pg_textsearch",
         "pg_search": "pg_search",
         "pg_duckdb": "pg_duckdb",
+        "pgsql_http": "http"
     }
     return create_names.get(name, name)
 

src/pgembed/postgres_server.py

@@ -331,6 +331,7 @@ def create_extension(self, extension_name: str) -> str:
             "pg_textsearch": "pgtextsearch",
             "pg_search": "pg_search",
             "pg_duckdb": "pg_duckdb",
+            "http": "pgsql_http",
         }
 
         pkg_name = extension_map.get(extension_name, extension_name)

src/pgembed_pgsql_http/README.md

@@ -0,0 +1,28 @@
+# pgembed-http
+
+`pgsql-http` ([pramsey/pgsql-http](https://github.com/pramsey/pgsql-http))
+as a separate `pgembed` extension package.
+
+Install alongside the base package (which stays HTTP-free):
+
+```bash
+pip install pgembed pgembed-http
+```
+
+```python
+import pgembed
+
+assert pgembed.has_extension("pgsql_http")
+
+server = pgembed.get_server("/path/to/data")
+server.create_extension("http")   # CREATE EXTENSION http
+```
+
+Available functions depend on the bundled pgsql-http version
+(`SELECT * FROM pg_available_extensions WHERE name = 'http'` after install).
+Typical entry points: `http_get`, `http_post`, `http_header`, `http_response`,
+`http_response_code`.
+
+Requires libcurl dev headers at *build* time only
+(`libcurl4-openssl-dev` on Debian/Ubuntu); the wheel ships a prebuilt
+`http.so`, one file per OS/arch.