From 3a33b3092ec6bc54fa131dd28bbb5a593ec4c6e1 Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Mon, 5 Oct 2026 12:33:32 -0700 Subject: [PATCH] Add lightweight CI for pull requests The existing build-and-test workflow only runs on pushes to main/testing and releases, and builds full wheels on four platforms, so PRs get no feedback. Add a fast CI workflow (lint + pytest on 3.12-3.14) that runs on pull requests and pushes to main. Server-dependent tests skip gracefully when the bundled PostgreSQL binaries have not been built (see tests/conftest.py); they still run in build-and-test.yml where the binaries exist. Also fix the 26 pre-existing ruff violations so the lint job is green, and cap test-only sqlalchemy to <2.1 since sqlalchemy-utils 0.42.1 (latest) is incompatible with sqlalchemy 2.1 and breaks test collection on fresh installs. --- .github/workflows/ci.yml | 62 ++++++++++++++++++++++++++++++++++ examples/pgduckdb_example.py | 11 +++--- pyproject.toml | 8 ++++- src/pgembed/__init__.py | 11 +++++- src/pgembed/_commands.py | 2 -- src/pgembed/postgres_server.py | 13 ++++--- src/pgembed/utils.py | 5 +-- tests/conftest.py | 40 ++++++++++++++++++++++ tests/test_pgembed.py | 2 +- 9 files changed, 131 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 tests/conftest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..72d8c7f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,62 @@ +# Lightweight CI for pull requests (and pushes to main). +# +# The full multi-platform wheel build + test suite lives in +# build-and-test.yml and is too heavy to run on every PR, so this +# workflow gives fast feedback instead: +# - lint: ruff check over the whole repo +# - test: install the package with test extras and run pytest. +# Tests that need a running PostgreSQL server (bundled binaries +# built via `make`, see tests/conftest.py) are skipped here; they +# still run in build-and-test.yml where the binaries exist. +name: CI + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: Lint (ruff) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install ruff + run: python -m pip install "ruff==0.9.5" + - name: Ruff check + run: ruff check . + - name: Compile check + run: python -m compileall -q src tests + + test: + name: Tests (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install package with test extras + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[test]" + - name: Smoke import + run: python -c "import pgembed; print(pgembed.list_extensions())" + - name: Run tests + run: python -m pytest tests/ -v diff --git a/examples/pgduckdb_example.py b/examples/pgduckdb_example.py index 8108a93..b2b7653 100644 --- a/examples/pgduckdb_example.py +++ b/examples/pgduckdb_example.py @@ -2,10 +2,7 @@ import pgembed import sqlalchemy as sa from sqlalchemy_utils import database_exists, create_database -import os from pathlib import Path -import subprocess -import time import glob import duckdb @@ -55,10 +52,10 @@ def patched_ensure_pgdata_inited(self): # Only add if not already present if "pg_duckdb" not in config: with open(conf_file, "a") as f: - f.write(f"\n# pg_duckdb configuration\n") - f.write(f"shared_preload_libraries = 'pg_duckdb'\n") + f.write("\n# pg_duckdb configuration\n") + f.write("shared_preload_libraries = 'pg_duckdb'\n") f.write(f"dynamic_library_path = '{pg_duckdb_lib}'\n") - print(f"✓ Configured pg_duckdb in postgresql.conf") + print("✓ Configured pg_duckdb in postgresql.conf") # Apply the patch @@ -491,7 +488,7 @@ def create_migration_function(conn): dbname = parsed.path.lstrip("/").split("?")[0] or database_name socket_path = str(pg.pgdata) - print(f" Connecting to PostgreSQL via Unix socket...") + print(" Connecting to PostgreSQL via Unix socket...") # Attach PostgreSQL using Unix socket duckdb_conn.execute( diff --git a/pyproject.toml b/pyproject.toml index 732f8f3..2674a2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,10 +18,13 @@ dependencies = [ dev = [ "sysv_ipc", ] +# NB: sqlalchemy-utils 0.42.1 (latest) is incompatible with sqlalchemy >= 2.1 +# (uses the removed sqlalchemy.orm.attributes.ScalarAttributeImpl), +# so cap sqlalchemy until sqlalchemy-utils supports 2.1. test = [ "pytest", "psycopg2-binary", - "sqlalchemy>=2", + "sqlalchemy>=2,<2.1", "sqlalchemy-utils" ] @@ -31,6 +34,9 @@ include = ["pgembed*", "pgembed_pg*"] # package names should match these glob p [tool.pytest.ini_options] testpaths = ["tests"] +markers = [ + "no_server_required: test runs without built PostgreSQL binaries (lightweight CI)", +] [tool.cibuildwheel] test-extras = "test" diff --git a/src/pgembed/__init__.py b/src/pgembed/__init__.py index 9eda487..d9b6978 100644 --- a/src/pgembed/__init__.py +++ b/src/pgembed/__init__.py @@ -1,10 +1,19 @@ -from ._commands import * +from ._commands import * # noqa: F403 from .postgres_server import PostgresServer, get_server from pathlib import Path from typing import Optional import logging import importlib.util +__all__ = [ + "PostgresServer", + "get_server", + "has_extension", + "list_extensions", + "get_extension_create_name", + "get_extension_path", +] + _logger = logging.getLogger("pgembed") diff --git a/src/pgembed/_commands.py b/src/pgembed/_commands.py index 0979efc..86b92ee 100644 --- a/src/pgembed/_commands.py +++ b/src/pgembed/_commands.py @@ -20,8 +20,6 @@ def _get_pkg_path(): _postgres_binaries_available = POSTGRES_BIN_PATH.exists() if not _postgres_binaries_available: - import os - _logger = logging.getLogger("pgembed") _logger.warning( f"PostgreSQL binaries not found at {POSTGRES_BIN_PATH}. " diff --git a/src/pgembed/postgres_server.py b/src/pgembed/postgres_server.py index 6e3aa03..678b023 100644 --- a/src/pgembed/postgres_server.py +++ b/src/pgembed/postgres_server.py @@ -16,7 +16,6 @@ find_suitable_socket_dir, DiskList, PostmasterInfo, - process_is_running, ) if platform.system() != "Windows": @@ -33,9 +32,9 @@ def _get_command(name: str): cmd = getattr(_commands, name, None) if cmd is None: raise RuntimeError( - f"PostgreSQL binaries not available. " - f"pgembed was installed without PostgreSQL binaries or they were not built. " - f"Run 'make build' in the pgembed source directory." + "PostgreSQL binaries not available. " + "pgembed was installed without PostgreSQL binaries or they were not built. " + "Run 'make build' in the pgembed source directory." ) return cmd @@ -245,14 +244,14 @@ def ensure_postgres_running(self) -> None: # in Windows, when there is a postmaster.pid, init_ctl seems to return # but the file is not immediately updated, here we wait until the file shows # a new running server. see test_stale_postmaster - _logger.info(f"waiting for postmaster info to show a running process") + _logger.info("waiting for postmaster info to show a running process") pinfo = PostmasterInfo.read_from_pgdata(self.pgdata) _logger.info(f"running... checking if ready {pinfo=}") if pinfo is not None and pinfo.is_running() and pinfo.status == "ready": self._postmaster_info = pinfo break - _logger.info(f"not ready yet... waiting a bit more...") + _logger.info("not ready yet... waiting a bit more...") time.sleep(1.0) _logger.info(f"Now asserting server is running {self._postmaster_info=}") @@ -287,7 +286,7 @@ def _cleanup(self) -> None: pass # somehow the server is already stopped. if not stopped: - _logger.warning(f"Failed to stop server, killing it instead.") + _logger.warning("Failed to stop server, killing it instead.") self._postmaster_info.process.terminate() try: self._postmaster_info.process.wait(2) diff --git a/src/pgembed/utils.py b/src/pgembed/utils.py index 9866c4b..3df8df9 100644 --- a/src/pgembed/utils.py +++ b/src/pgembed/utils.py @@ -1,5 +1,4 @@ from pathlib import Path -import typing from typing import Optional, List, Dict import subprocess import json @@ -10,7 +9,6 @@ import stat import psutil import datetime -import shutil _logger = logging.getLogger("pgembed") @@ -142,14 +140,13 @@ def process_is_running(pid: int) -> bool: if platform.system() != "Windows": + import pwd def ensure_user_exists(username: str) -> Optional["pwd.struct_passwd"]: """Ensure system user `username` exists. Returns their pwentry if user exists, otherwise it creates a user through `useradd`. Assume permissions to add users, eg run as root. """ - import pwd - try: entry = pwd.getpwnam(username) except KeyError: diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..848437b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,40 @@ +"""Pytest configuration for pgembed tests. + +Most tests need a running PostgreSQL server backed by the bundled +binaries (built via `make`). Where those binaries have not been built -- +e.g. a lightweight PR check or a fresh contributor checkout -- the +server-dependent tests are skipped instead of failing. + +Tests that do NOT need a server opt out explicitly with:: + + @pytest.mark.no_server_required +""" + +import importlib.util +from pathlib import Path + +import pytest + + +def _postgres_binaries_available() -> bool: + spec = importlib.util.find_spec("pgembed") + if spec is None or not spec.submodule_search_locations: + return False + bin_dir = Path(spec.submodule_search_locations[0]) / "pginstall" / "bin" + return (bin_dir / "postgres").exists() or (bin_dir / "postgres.exe").exists() + + +POSTGRES_BINARIES_AVAILABLE = _postgres_binaries_available() + +requires_server = pytest.mark.skipif( + not POSTGRES_BINARIES_AVAILABLE, + reason="PostgreSQL binaries not built; run `make` to build them", +) + + +def pytest_collection_modifyitems(items): + if POSTGRES_BINARIES_AVAILABLE: + return + for item in items: + if "no_server_required" not in item.keywords: + item.add_marker(requires_server) diff --git a/tests/test_pgembed.py b/tests/test_pgembed.py index 62aa89b..61bbc6e 100644 --- a/tests/test_pgembed.py +++ b/tests/test_pgembed.py @@ -12,7 +12,6 @@ import psutil import platform import sqlalchemy as sa -import datetime from sqlalchemy_utils import database_exists, create_database import logging import os @@ -88,6 +87,7 @@ def _kill_server(pid : Union[int,psutil.Process,None]) -> None: if proc.is_running(): proc.kill() +@pytest.mark.no_server_required def test_get_port(): address = '127.0.0.1' port = find_suitable_port(address)