diff --git a/.github/workflows/publish-pgembed-http.yml b/.github/workflows/publish-pgembed-http.yml new file mode 100644 index 0000000..af40f50 --- /dev/null +++ b/.github/workflows/publish-pgembed-http.yml @@ -0,0 +1,190 @@ +# Build + publish the pgembed-http split wheel. +# +# Mirrors build-and-test.yml: same runners, same `make` invocation, same +# trusted-publishing release action. The ONLY difference is EXTENSIONS: +# base builds the default set, this builds pgsql_http. No PG_MAJOR is set +# here on purpose — like base, it uses the pgbuild/Makefile default, so the +# extension can never drift to a different Postgres major than the base wheel. +# +# One-time PyPI setup: register a *pending publisher* for project +# `pgembed-http` with owner/repo = this repo, workflow filename = +# `publish-pgembed-http.yml`, environment = `pypi`. The first tagged run +# then creates the project on first upload. +# +# Release flow: bump src/pgembed_pgsql_http/pyproject.toml version, push tag +# `pgembed-http-v`. The workflow asserts tag == package version. +name: Publish pgembed-http + +on: + push: + tags: + - "pgembed-http-v*" + pull_request: + paths: + - "src/pgembed_pgsql_http/**" + - "pgbuild/Makefile" + - "Makefile" + - ".github/workflows/publish-pgembed-http.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Build pgembed-http on ${{ matrix.os }}-${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + # Same set as the base wheel matrix, minus Windows (pgsql-http's + # libcurl support there is unverified). + include: + - runner: macos-latest + os: macos-latest + arch: arm64 + deployment-target: "11.0" + - runner: ubuntu-latest + os: ubuntu-latest + arch: x86_64 + deployment-target: "" + - runner: ubuntu-24.04-arm + os: ubuntu-latest + arch: aarch64 + deployment-target: "" + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Check tag matches package version + if: startsWith(github.ref, 'refs/tags/pgembed-http-v') + run: | + TAG_VERSION="${GITHUB_REF_NAME#pgembed-http-v}" + PKG_VERSION="$(grep '^version' src/pgembed_pgsql_http/pyproject.toml | sed 's/.*"\(.*\)".*/\1/')" + if [ "$TAG_VERSION" != "$PKG_VERSION" ]; then + echo "tag version $TAG_VERSION != package version $PKG_VERSION" + exit 1 + fi + + # Reuse the base build cache when available: it holds a full postgres + # build from the same Makefile defaults, so `make EXTENSIONS=pgsql_http` + # below only needs to compile the http extension itself. Restore-only: + # this workflow must never clobber the base cache. Key replicates + # build-and-test.yml exactly for an exact-match hit. + - name: Restore postgres build from base cache + uses: actions/cache/restore@v3 + with: + path: | + pgbuild + src/pgembed/pginstall + key: ${{ runner.os }}-${{ runner.arch }}-build-cache-postgres-${{ + hashFiles('Makefile', 'pgbuild/Makefile', '.github/workflows/build-and-test.yml') }} + + # Same install flavor as build-and-test.yml, trimmed to what a pure-C + # extension + postgres need (no Rust/cargo-pgrx: nothing pgrx-based here). + - name: Install build deps (Linux) + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get update + sudo apt-get install -y gcc make pkg-config bison flex \ + libssl-dev zlib1g-dev libcurl4-openssl-dev + + - name: Install build deps (macOS) + if: matrix.os == 'macos-latest' + run: brew install lz4 + # libcurl headers come from the Xcode SDK; no brew curl on purpose — + # linking brewed libcurl would tie the .so to a Homebrew path that + # target machines may not have. + + - name: Build postgres and pgsql_http + shell: bash + env: + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment-target }} + run: | + # Same plain `make` as base (inherits PG_MAJOR default); only the + # extension set differs. + make EXTENSIONS=pgsql_http + + - name: Stage split-wheel build tree + shell: bash + run: | + # The repo keeps the flat shim layout (like pgembed_pgvector) so the + # root install keeps working; the wheel needs a real package dir, so + # assemble it in a staging tree and build there. + rm -rf build-pgembed-http + mkdir -p build-pgembed-http/pgembed_pgsql_http + cp src/pgembed_pgsql_http/pyproject.toml \ + src/pgembed_pgsql_http/setup.py \ + src/pgembed_pgsql_http/README.md build-pgembed-http/ + cp src/pgembed_pgsql_http/__init__.py build-pgembed-http/pgembed_pgsql_http/ + BIN=src/pgembed/pginstall/bin + LIB="$("$BIN/pg_config" --pkglibdir)" + SHARE="$("$BIN/pg_config" --sharedir)/postgresql/extension" + ls "$LIB"/http.so "$SHARE"/http.control "$SHARE"/http--*.sql + cp "$LIB"/http.so build-pgembed-http/pgembed_pgsql_http/ + cp "$SHARE"/http.control "$SHARE"/http--*.sql build-pgembed-http/pgembed_pgsql_http/ + ls -la build-pgembed-http/pgembed_pgsql_http/ + + - name: Build wheel + run: | + python -m pip install --upgrade pip build + cd build-pgembed-http + python -m build + ls dist/ + # setup.py forces a platform tag; a py3-none-any wheel here would + # mean the .so didn't get classified — fail loudly. + if ls dist/*-py3-none-any.whl >/dev/null 2>&1; then + echo "pure-Python wheel tag on a binary wheel — refusing" + exit 1 + fi + + - name: Smoke test (CREATE EXTENSION http) + run: | + python -m pip install --upgrade pip + python -m pip install "psycopg2-binary" "build-pgembed-http/dist/"*.whl + python -m pip install pgembed + python - <<'EOF' + import tempfile + import pgembed + print(pgembed.list_extensions()) + assert pgembed.has_extension("pgsql_http") + with pgembed.get_server(tempfile.mkdtemp(), cleanup_mode="delete") as pg: + out = pg.create_extension("http") + assert out.strip() == "CREATE EXTENSION", out + print(pg.psql("SELECT extversion FROM pg_extension WHERE extname='http';")) + EOF + + - uses: actions/upload-artifact@v4 + with: + path: build-pgembed-http/dist/* + name: pgembed-http-${{ matrix.os }}-${{ matrix.arch }} + overwrite: true + + publish-to-pypi: + # Same gate shape as base (any pgembed-http-v* tag). + if: startsWith(github.ref, 'refs/tags/pgembed-http-v') + name: Publish pgembed-http to PyPI + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/pgembed-http + permissions: + id-token: write + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + pattern: pgembed-http-* + path: dist/ + merge-multiple: true + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/README.md b/README.md index 06eee63..b24bf6f 100644 --- a/README.md +++ b/README.md @@ -58,15 +58,17 @@ pip install pgembed pip install pgembed-pgvector pip install pgembed-pgvectorscale pip install pgembed-pgtextsearch +pip install pgembed-http # Multiple extensions -pip install pgembed-pgvector pgembed-pgvectorscale pgembed-pgtextsearch +pip install pgembed-pgvector pgembed-pgvectorscale pgembed-pgtextsearch pgembed-http ``` Available extensions: - `pgembed-pgvector`: Vector similarity search (works on all platforms) - `pgembed-pgvectorscale`: High-performance vector storage (requires Rust, not available on Alpine/Windows) - `pgembed-pgtextsearch`: BM25-based full-text search (requires Rust, not available on Alpine/Windows) +- `pgembed-http`: HTTP client (`http_get`, `http_post`, …) via pgsql-http (requires libcurl at build time) ### Checking available extensions @@ -88,6 +90,7 @@ if pgembed.has_extension('pgvector'): - **pgvector**: Works on Linux, macOS (Intel & Apple Silicon), Windows - **pgvectorscale**: Works on Linux, macOS (Intel & Apple Silicon). NOT available on Alpine Linux or Windows (requires Rust) - **pgtextsearch**: Works on Linux, macOS (Intel & Apple Silicon). NOT available on Alpine Linux or Windows (requires Rust) +- **pgembed-http**: Works on Linux, macOS (Intel & Apple Silicon). Requires libcurl dev headers at build time (`libcurl4-openssl-dev` on Debian/Ubuntu) ### Building specific extensions @@ -103,6 +106,9 @@ make pgvectorscale # Build only pgtextsearch make pgtextsearch +# Build only pgsql-http (for the pgembed-http split wheel) +make EXTENSIONS="pgsql_http" all + # Build specific combination make EXTENSIONS="pgvector pgtextsearch" all ``` diff --git a/pgbuild/Makefile b/pgbuild/Makefile index 015dc8b..4c294db 100644 --- a/pgbuild/Makefile +++ b/pgbuild/Makefile @@ -41,12 +41,13 @@ EXTENSIONS := $(filter-out $(UNSUPPORTED_EXTENSIONS),$(REQUESTED_EXTENSIONS)) .PHONY: all all: postgres $(EXTENSIONS) -.PHONY: pgvector pgvectorscale pgtextsearch pg_search pg_duckdb +.PHONY: pgvector pgvectorscale pgtextsearch pg_search pg_duckdb pgsql_http pgvector: postgres $(INSTALL_PREFIX)/lib/vector.so pgvectorscale: postgres $(INSTALL_PREFIX)/lib/vectorscale.so pgtextsearch: postgres $(INSTALL_PREFIX)/lib/pg_textsearch.so pg_search: postgres $(INSTALL_PREFIX)/lib/pg_search.so pg_duckdb: postgres $(INSTALL_PREFIX)/lib/pg_duckdb.so +pgsql_http: postgres $(INSTALL_PREFIX)/lib/http.so ### postgres # Major postgres version to build. Override from the command line or CI, e.g. `make PG_MAJOR=18`. @@ -98,6 +99,23 @@ $(INSTALL_PREFIX)/lib/vector.so: $(PGVECTOR_DIR)/Makefile $(INSTALL_PREFIX)/bin/ && $(MAKE) -C $(PGVECTOR_DIR) PG_CONFIG=$(INSTALL_PREFIX)/bin/pg_config$(EXE_EXT) -j \ && $(MAKE) -C $(PGVECTOR_DIR) PG_CONFIG=$(INSTALL_PREFIX)/bin/pg_config$(EXE_EXT) install +### pgsql_http +PGSQL_HTTP_TAG := v1.7.2 +PGSQL_HTTP_URL := https://github.com/pramsey/pgsql-http/archive/refs/tags/$(PGSQL_HTTP_TAG).tar.gz +PGSQL_HTTP_DIR := pgsql-http-$(PGSQL_HTTP_TAG) + +$(PGSQL_HTTP_DIR).tar.gz: + curl -L -o $(PGSQL_HTTP_DIR).tar.gz $(PGSQL_HTTP_URL) + +$(PGSQL_HTTP_DIR)/Makefile: $(PGSQL_HTTP_DIR).tar.gz + mkdir -p $(PGSQL_HTTP_DIR) + tar xzf $(PGSQL_HTTP_DIR).tar.gz -C $(PGSQL_HTTP_DIR) --strip-components=1 + touch $(PGSQL_HTTP_DIR)/Makefile + +$(INSTALL_PREFIX)/lib/http.so: $(PGSQL_HTTP_DIR)/Makefile $(INSTALL_PREFIX)/bin/postgres + unset MAKELEVEL && unset MAKEFLAGS && unset MFLAGS \ + && $(MAKE) -C $(PGSQL_HTTP_DIR) PG_CONFIG=$(INSTALL_PREFIX)/bin/pg_config$(EXE_EXT) -j \ + && $(MAKE) -C $(PGSQL_HTTP_DIR) PG_CONFIG=$(INSTALL_PREFIX)/bin/pg_config$(EXE_EXT) install ### pg_duckdb PG_DUCKDB_TAG := v1.1.1 PG_DUCKDB_REPO := https://github.com/duckdb/pg_duckdb @@ -204,6 +222,7 @@ clean: rm -rf $(PG_DUCKDB_DIR) rm -rf $(PGTEXTSEARCH_DIR) rm -rf $(PGSEARCH_DIR) + rm -rf $(PGSQL_HTTP_DIR) clean-all: clean rm -rf *.tar.gz diff --git a/pyproject.toml b/pyproject.toml index 2674a2d..eb9b962 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.12" license = {file = "LICENSE.txt"} urls={repository="https://github.com/Ladybug-Memory/pgembed"} authors=[{ name="Oscar Moll", email="orm@csail.mit.edu" }, { name="Arun Sharma", email="arun@sharma-home.net" }] -keywords=["postgresql", "pgvector", "pg_duckdb", "pgembed", "pg_textsearch", "rag"] +keywords=["postgresql", "pgvector", "pg_duckdb", "pgembed", "pg_textsearch", "rag", "pgsql-http"] dependencies = [ "fasteners>=0.19", "platformdirs>=4.0.0", diff --git a/src/pgembed/__init__.py b/src/pgembed/__init__.py index d9b6978..aa18d80 100644 --- a/src/pgembed/__init__.py +++ b/src/pgembed/__init__.py @@ -33,6 +33,7 @@ def _get_pkg_path(): "pgvector": "pgembed_pgvector", "pgvectorscale": "pgembed_pgvectorscale", "pgtextsearch": "pgembed_pgtextsearch", + "pgsql_http": "pgembed_pgsql_http" } EXTENSION_SO_FILES = { @@ -41,6 +42,7 @@ def _get_pkg_path(): "pgtextsearch": "pg_textsearch.so", "pg_search": "pg_search.so", "pg_duckdb": "pg_duckdb.so", + "pgsql_http": "http.so", } EXTENSION_NAMES = ( @@ -49,6 +51,7 @@ def _get_pkg_path(): "pgtextsearch", "pg_search", "pg_duckdb", + "pgsql_http" ) @@ -115,6 +118,7 @@ def get_extension_create_name(name: str) -> str: "pgtextsearch": "pg_textsearch", "pg_search": "pg_search", "pg_duckdb": "pg_duckdb", + "pgsql_http": "http" } return create_names.get(name, name) diff --git a/src/pgembed/postgres_server.py b/src/pgembed/postgres_server.py index 678b023..b45118e 100644 --- a/src/pgembed/postgres_server.py +++ b/src/pgembed/postgres_server.py @@ -331,6 +331,7 @@ def create_extension(self, extension_name: str) -> str: "pg_textsearch": "pgtextsearch", "pg_search": "pg_search", "pg_duckdb": "pg_duckdb", + "http": "pgsql_http", } pkg_name = extension_map.get(extension_name, extension_name) diff --git a/src/pgembed_pgsql_http/README.md b/src/pgembed_pgsql_http/README.md new file mode 100644 index 0000000..dc62da9 --- /dev/null +++ b/src/pgembed_pgsql_http/README.md @@ -0,0 +1,28 @@ +# pgembed-http + +`pgsql-http` ([pramsey/pgsql-http](https://github.com/pramsey/pgsql-http)) +as a separate `pgembed` extension package. + +Install alongside the base package (which stays HTTP-free): + +```bash +pip install pgembed pgembed-http +``` + +```python +import pgembed + +assert pgembed.has_extension("pgsql_http") + +server = pgembed.get_server("/path/to/data") +server.create_extension("http") # CREATE EXTENSION http +``` + +Available functions depend on the bundled pgsql-http version +(`SELECT * FROM pg_available_extensions WHERE name = 'http'` after install). +Typical entry points: `http_get`, `http_post`, `http_header`, `http_response`, +`http_response_code`. + +Requires libcurl dev headers at *build* time only +(`libcurl4-openssl-dev` on Debian/Ubuntu); the wheel ships a prebuilt +`http.so`, one file per OS/arch. diff --git a/src/pgembed_pgsql_http/__init__.py b/src/pgembed_pgsql_http/__init__.py new file mode 100644 index 0000000..6f23447 --- /dev/null +++ b/src/pgembed_pgsql_http/__init__.py @@ -0,0 +1,50 @@ +__version__ = "0.1.8" + +EXTENSION_NAME = "pgsql_http" +EXTENSION_SO = "http.so" +EXTENSION_CREATE = "http" + + +def get_extension_path(): + from pathlib import Path + + pkg_dir = Path(__file__).parent + so_path = pkg_dir / EXTENSION_SO + if so_path.exists(): + return so_path + + try: + import pgembed + + base_lib = pgembed.EXTENSION_LIB_PATH + bundled = base_lib / "postgresql" / EXTENSION_SO + if bundled.exists(): + return bundled + except ImportError: + pass + + return None + + +def get_extension_share_path(): + from pathlib import Path + + try: + import pgembed + + base_share = ( + Path(__file__).parent / "pginstall" / "share" / "postgresql" / "extension" + ) + control_file = base_share / f"{EXTENSION_CREATE}.control" + if control_file.exists(): + return base_share + + base_share = ( + pgembed.EXTENSION_LIB_PATH.parent / "share" / "postgresql" / "extension" + ) + control_file = base_share / f"{EXTENSION_CREATE}.control" + if control_file.exists(): + return base_share + except ImportError: + pass + return None diff --git a/src/pgembed_pgsql_http/pyproject.toml b/src/pgembed_pgsql_http/pyproject.toml new file mode 100644 index 0000000..3a47fc3 --- /dev/null +++ b/src/pgembed_pgsql_http/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "pgembed-http" +version = "0.1.8" +description = "pgsql-http extension for pgembed" +readme = "README.md" +requires-python = ">=3.12" +dependencies = ["pgembed>=0.1.8"] + +[project.optional-dependencies] +test = ["pytest"] + +[tool.setuptools.packages.find] +where = ["."] +include = ["pgembed_pgsql_http"] + +# The prebuilt extension binaries are staged into build-pgembed-http/ by CI +# (see .github/workflows/publish-pgembed-http.yml) before building. +[tool.setuptools.package-data] +pgembed_pgsql_http = ["http.so", "http.control", "http--*.sql"] + +[build-system] +requires = ["setuptools>=58.0.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/src/pgembed_pgsql_http/setup.py b/src/pgembed_pgsql_http/setup.py new file mode 100644 index 0000000..877fd85 --- /dev/null +++ b/src/pgembed_pgsql_http/setup.py @@ -0,0 +1,15 @@ +# Forces platform-specific wheel tags. +# +# The wheel ships a prebuilt http.so, so a py3-none-any tag would be wrong +# (pip could install a Linux .so on macOS). Reporting ext modules makes +# bdist_wheel emit cpXX-cpXX- instead. +from setuptools import setup +from setuptools.dist import Distribution + + +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return True + + +setup(distclass=BinaryDistribution) diff --git a/tests/test_pgembed.py b/tests/test_pgembed.py index 61bbc6e..e235162 100644 --- a/tests/test_pgembed.py +++ b/tests/test_pgembed.py @@ -256,6 +256,12 @@ def test_pgvector(tmp_postgres): ret = tmp_postgres.psql("CREATE EXTENSION vector;") assert ret.strip() == "CREATE EXTENSION" +def test_pgsql_http(tmp_postgres): + if not pgembed.has_extension("pgsql_http"): + pytest.skip("pgsql_http extension not available in this build") + ret = tmp_postgres.create_extension("http") + assert ret.strip() == "CREATE EXTENSION" + def test_start_failure_log(caplog): """ Test server log contents are shown in python log when failures """