From 67200f437c775f3af09e3c7977669066ae211acd Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Wed, 11 Nov 2026 10:25:26 -0600 Subject: [PATCH 1/2] Move server setup into build_server() So that tests can build the server exactly as startup does. No change in behavior. --- papercut.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/papercut.py b/papercut.py index e41034a..729794b 100755 --- a/papercut.py +++ b/papercut.py @@ -915,16 +915,12 @@ def finish(self): print('Closing the request') -if __name__ == '__main__': - # set up signal handler - def sighandler(signum, frame): - if __DEBUG__: print("\nShutting down papercut...") - server.socket.close() - time.sleep(1) - sys.exit(0) - - settings.validate() +def build_server(address, server_class=None): + """Creates the server, with the storage and auth backends from settings. + server_class defaults to NNTPServer (forking on POSIX, threading on + Windows); the tests also run the other one. + """ # dynamic loading of the appropriate storage backend module temp = importlib.import_module('storage.%s' % (settings.storage_backend)) if settings.nntp_cache == 'yes': @@ -938,9 +934,22 @@ def sighandler(signum, frame): temp = importlib.import_module('auth.%s' % (settings.auth_backend)) auth = temp.Papercut_Auth() - signal.signal(signal.SIGINT, sighandler) - print('Papercut %s (%s storage module) - starting up' % (__VERSION__, settings.storage_backend)) - server = NNTPServer((settings.nntp_hostname, settings.nntp_port), NNTPRequestHandler) + server = (server_class or NNTPServer)(address, NNTPRequestHandler) server.backend = backend server.auth = auth + return server + + +if __name__ == '__main__': + # set up signal handler + def sighandler(signum, frame): + if __DEBUG__: print("\nShutting down papercut...") + server.socket.close() + time.sleep(1) + sys.exit(0) + + settings.validate() + signal.signal(signal.SIGINT, sighandler) + print('Papercut %s (%s storage module) - starting up' % (__VERSION__, settings.storage_backend)) + server = build_server((settings.nntp_hostname, settings.nntp_port)) server.serve_forever() From 510c7e2b484d35fae52b129b8c6f0699d165564e Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Wed, 11 Nov 2026 10:40:15 -0600 Subject: [PATCH 2/2] Give each client connection its own backends The storage (and auth) backend was created once at startup, so every client used the same database connection: on POSIX each forked child inherited it, and on Windows every thread shared it. With two clients at once, their queries and results interleaved -- clients got each other's rows (e.g. an author name where an article number was expected), SQL built from the wrong results, or MySQLdb's 'connection is already in use from another thread'. The server now holds factories, and each connection creates its own storage backend (in the forked child on POSIX) and, on first use, its own auth backend; both are closed when the connection ends. If the database can't be reached, the client gets '400 service temporarily unavailable'. build_server() still connects once at startup, so a bad configuration fails immediately. tests/integration/test_concurrency.py runs 6 clients at once against the server as build_server() creates it, forking and threading, on MySQL and PostgreSQL; before this change 5-6 of the 6 clients failed in every combination. --- papercut.py | 87 +++++++++++++++++++------ tests/integration/conftest.py | 23 ++++--- tests/integration/serve.py | 25 +++++++ tests/integration/test_concurrency.py | 83 ++++++++++++++++++++++++ tests/test_protocol.py | 93 +++++++++++++++++++++++++++ 5 files changed, 282 insertions(+), 29 deletions(-) create mode 100644 tests/integration/serve.py create mode 100644 tests/integration/test_concurrency.py diff --git a/papercut.py b/papercut.py index 729794b..a476995 100755 --- a/papercut.py +++ b/papercut.py @@ -75,6 +75,7 @@ STATUS_AUTH_ACCEPTED = '281 Authentication accepted' STATUS_AUTH_CONTINUE = '381 More authentication information required' STATUS_SERVER_VERSION = '200 Papercut %s' % (__VERSION__) +STATUS_UNAVAILABLE = '400 service temporarily unavailable' # the currently supported overview headers overview_headers = ('Subject', 'From', 'Date', 'Message-ID', 'References', 'Bytes', 'Lines', 'Xref') @@ -87,17 +88,30 @@ if os.name == 'posix': class NNTPServer(socketserver.ForkingTCPServer): allow_reuse_address = 1 - # storage and auth backends, set before serve_forever() - backend = None - auth = None + # callables that create the storage and auth backends for each + # connection (set by build_server) + backend_factory = None + auth_factory = None if settings.max_connections: max_children = settings.max_connections else: class NNTPServer(socketserver.ThreadingTCPServer): allow_reuse_address = 1 - # storage and auth backends, set before serve_forever() - backend = None - auth = None + # callables that create the storage and auth backends for each + # connection (set by build_server) + backend_factory = None + auth_factory = None + + +def close_backend(backend): + """Closes a backend's database (or upstream NNTP) connection, if any.""" + try: + if hasattr(backend, 'conn'): + backend.conn.close() + elif hasattr(backend, 'nntp'): + backend.nntp.quit() + except Exception: + pass class NNTPRequestHandler(socketserver.StreamRequestHandler): @@ -105,15 +119,34 @@ class NNTPRequestHandler(socketserver.StreamRequestHandler): # interrupts a blocking read on Python 3 (PEP 475) and never worked on Windows timeout = __TIMEOUT__ - # the backends live on the server, so each server (and each test) can - # use its own - @property - def backend(self): - return self.server.backend + backend = None + _auth = None + + def setup(self): + socketserver.StreamRequestHandler.setup(self) + # Each connection gets its own storage backend, and so its own + # database connection: one shared between forked children or + # threads interleaves their queries and results. (A server with no + # factory, as in the unit tests, shares its 'backend' instance.) + factory = getattr(self.server, 'backend_factory', None) + self.owns_backend = factory is not None + if factory is not None: + try: + self.backend = factory() + except Exception: + settings.logEvent('Error - Could not create the storage backend for %s' % (self.client_address[0])) + settings.logEvent(traceback.format_exc()) + self.backend = None + else: + self.backend = getattr(self.server, 'backend', None) @property def auth(self): - return self.server.auth + # created on first use: most connections never log in + if self._auth is None: + factory = getattr(self.server, 'auth_factory', None) + self._auth = factory() if factory is not None else getattr(self.server, 'auth', None) + return self._auth # this is the list of supported commands commands = ('ARTICLE', 'BODY', 'HEAD', @@ -141,6 +174,9 @@ def auth(self): def handle(self): settings.logEvent('Connection from %s' % (self.client_address[0])) + if self.backend is None: + self.send_response(STATUS_UNAVAILABLE) + return if settings.server_type == 'read-only': self.send_response(STATUS_READYNOPOST % (settings.nntp_hostname, __VERSION__)) else: @@ -910,6 +946,10 @@ def finish(self): self.sending_article = 0 self.auth_username = '' self.article_lines = [] + if getattr(self, 'owns_backend', False): + close_backend(self.backend) + if self._auth is not None and getattr(self.server, 'auth_factory', None) is not None: + close_backend(self._auth) socketserver.StreamRequestHandler.finish(self) if __DEBUG__: print('Closing the request') @@ -922,21 +962,28 @@ def build_server(address, server_class=None): Windows); the tests also run the other one. """ # dynamic loading of the appropriate storage backend module - temp = importlib.import_module('storage.%s' % (settings.storage_backend)) + storage_module = importlib.import_module('storage.%s' % (settings.storage_backend)) if settings.nntp_cache == 'yes': - backend = papercut_cache.Cache(temp, papercut_cache.cache_methods) + def backend_factory(): + return papercut_cache.Cache(storage_module, papercut_cache.cache_methods) else: - backend = temp.Papercut_Storage() + backend_factory = storage_module.Papercut_Storage # now for the authentication module, if needed - auth = None + auth_factory = None if settings.nntp_auth == 'yes': - temp = importlib.import_module('auth.%s' % (settings.auth_backend)) - auth = temp.Papercut_Auth() + auth_factory = importlib.import_module('auth.%s' % (settings.auth_backend)).Papercut_Auth + + # connect once now, so a configuration problem shows up on startup + # rather than on the first client's connection + close_backend(backend_factory()) + if auth_factory is not None: + close_backend(auth_factory()) + # each connection creates its own backends (see NNTPRequestHandler.setup) server = (server_class or NNTPServer)(address, NNTPRequestHandler) - server.backend = backend - server.auth = auth + server.backend_factory = backend_factory + server.auth_factory = auth_factory return server diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b7a7067..fa24c07 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -80,23 +80,28 @@ def query(self, sql, params=None): return query(self.kind, self.database, sql, params) -def start_live(monkeypatch, request, storage_module, kind, database, auth_module=None, **extra_settings): +def settings_for(kind, database, **extra_settings): + """The settings values for running a backend against a test database.""" host = POSTGRES_HOST if kind == "postgres" else MYSQL_HOST values = dict(dbhost=host, dbname=database, dbuser="papercut", dbpass="papercut", nntp_hostname="news.example.com", phorum_settings_path=os.path.join(FIXTURES, "phorum_settings") + "/") values.update(extra_settings) - for name, value in values.items(): + return values + + +def start_live(monkeypatch, request, storage_module, kind, database, auth_module=None, **extra_settings): + for name, value in settings_for(kind, database, **extra_settings).items(): monkeypatch.setattr(settings, name, value) - if kind == "postgres": - # the PostgreSQL user is the database owner (see the postgres fixtures) - monkeypatch.setattr(settings, "dbuser", "papercut") - backend = importlib.import_module(storage_module).Papercut_Storage() - auth = importlib.import_module(auth_module).Papercut_Auth() if auth_module else None - server = fakes.start_server(papercut.NNTPRequestHandler, backend, auth) + # factories, as build_server sets up: each connection gets its own backends + backend_factory = importlib.import_module(storage_module).Papercut_Storage + auth_factory = importlib.import_module(auth_module).Papercut_Auth if auth_module else None + server = fakes.start_server(papercut.NNTPRequestHandler, None) + server.backend_factory = backend_factory + server.auth_factory = auth_factory request.addfinalizer(server.server_close) request.addfinalizer(server.shutdown) - return Live(server, backend, kind, database) + return Live(server, backend_factory, kind, database) @pytest.fixture(autouse=True) diff --git a/tests/integration/serve.py b/tests/integration/serve.py new file mode 100644 index 0000000..c81d6a4 --- /dev/null +++ b/tests/integration/serve.py @@ -0,0 +1,25 @@ +"""Runs a Papercut server in its own process (see test_concurrency.py). + +Imported by multiprocessing's 'spawn' start method, so a server that +crashes (e.g. a DB driver misused from several threads) fails the test +instead of taking pytest down with it. +""" +import socketserver + + +class ThreadingServer(socketserver.ThreadingTCPServer): + """The server class Papercut uses on Windows.""" + allow_reuse_address = True + daemon_threads = True + + +def run(settings_values, mode, log_file, port_queue): + import settings + for name, value in settings_values.items(): + setattr(settings, name, value) + settings.log_file = log_file + import papercut + server_class = ThreadingServer if mode == "threading" else papercut.NNTPServer + server = papercut.build_server(("127.0.0.1", 0), server_class) + port_queue.put(server.server_address[1]) + server.serve_forever() diff --git a/tests/integration/test_concurrency.py b/tests/integration/test_concurrency.py new file mode 100644 index 0000000..f65ac87 --- /dev/null +++ b/tests/integration/test_concurrency.py @@ -0,0 +1,83 @@ +"""Several clients at once, against the server as startup builds it. + +Each client connection must get its own database connection: sharing one +between forked children (POSIX) or threads (Windows) interleaves their +queries and results. +""" +import multiprocessing +import os +import threading +import time +import warnings + +import pytest + +import serve +from conftest import settings_for + +GROUP = "phorum.test" +CLIENTS = 6 +ROUNDS = 15 +TIMEOUT = 15 # per socket operation: a hang counts as a failure +DEADLINE = 90 # for the whole test + +BACKENDS = { + "phorum_mysql": ("mysql", "phorum"), + "phorum_pgsql": ("postgres", "phorum"), +} + + +@pytest.fixture(params=["forking", "threading"]) +def server_port(request, tmp_path): + mode = request.param + storage = request.node.callspec.params["backend"] + kind, database = BACKENDS[storage] + values = settings_for(kind, database, storage_backend=storage, nntp_cache="no", nntp_auth="no") + context = multiprocessing.get_context("spawn") + ports = context.Queue() + process = context.Process(target=serve.run, args=(values, mode, str(tmp_path / "server.log"), ports), daemon=True) + process.start() + port = ports.get(timeout=30) + yield port, tmp_path / "server.log" + process.terminate() + process.join(10) + + +def client_session(port, errors): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + import nntplib + try: + with nntplib.NNTP("127.0.0.1", port, timeout=TIMEOUT) as s: + for _ in range(ROUNDS): + _, count, first, last, _ = s.group(GROUP) + assert (count, first, last) == (3, 1, 4), (count, first, last) + _, overviews = s.over((1, 4)) + assert [n for n, _ in overviews] == [1, 2, 4], overviews + _, info = s.article("2") + assert b"Subject: Re: First post" in info.lines, info.lines[:4] + _, body = s.body("4") + assert body.lines == ["Corpo com acentuação".encode()], body.lines + except Exception as e: # collected, so every client's failure is reported + errors.append("%s: %s" % (type(e).__name__, e)) + + +@pytest.mark.parametrize("backend", sorted(BACKENDS)) +def test_concurrent_clients_get_correct_results(backend, server_port): + port, log = server_port + errors = [] + clients = [threading.Thread(target=client_session, args=(port, errors), daemon=True) for _ in range(CLIENTS)] + for c in clients: + c.start() + deadline = time.time() + DEADLINE + for c in clients: + c.join(max(0, deadline - time.time())) + hung = sum(c.is_alive() for c in clients) + exceptions = sorted({l for l in log.read_text().splitlines() if l and not l.startswith((" ", "[", "Traceback"))}) if log.exists() else [] + server_errors = exceptions[:6] + assert not hung and not errors, "%d of %d clients failed, %d hung; e.g.:\n%s\nserver log:\n%s" % ( + len(errors), CLIENTS, hung, "\n".join(errors[:3]), "\n".join(server_errors)) + # and the work really went through the server + text = log.read_text() + assert text.count("Connection from") == CLIENTS + assert text.count("Received request: GROUP") == CLIENTS * ROUNDS diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 4f9fd09..03b102b 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -669,3 +669,96 @@ def test_login_names_are_not_logged_by_the_server(auth_on, client): assert wait_for_log("Connection closed") assert received_requests() == ["AUTHINFO", "AUTHINFO", "QUIT"] assert "alice" not in log_text() + + +# --- backends per connection --- + +class RecordingConnection: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + +def make_factory(created, fail=False): + def factory(): + if fail: + raise RuntimeError("database unreachable") + backend = fakes.FakeBackend() + backend.conn = RecordingConnection() + created.append(backend) + return backend + return factory + + +@pytest.fixture +def factory_server(): + srv = fakes.start_server(papercut.NNTPRequestHandler, None) + yield srv + srv.shutdown() + srv.server_close() + + +def test_each_connection_gets_its_own_backend(factory_server): + created = [] + factory_server.backend_factory = make_factory(created) + first = fakes.Client(factory_server.server_address[1]) + second = fakes.Client(factory_server.server_address[1]) + first.cmd("GROUP test.group") + second.cmd("GROUP test.group") + assert len(created) == 2 + assert [b.called("get_GROUP") for b in created] == [[("test.group",)], [("test.group",)]] + first.cmd("QUIT") + second.cmd("QUIT") + assert wait_until(lambda: all(b.conn.closed for b in created)) + + +def test_unreachable_database_gets_a_400(factory_server): + factory_server.backend_factory = make_factory([], fail=True) + c = fakes.Client(factory_server.server_address[1]) + assert code(c.greeting) == "400" + assert c.readline_bytes() == b"" + assert wait_for_log("database unreachable") + c.close() + + +def test_auth_backend_is_created_only_on_login(factory_server, monkeypatch): + monkeypatch.setattr(settings, "nntp_auth", "yes") + factory_server.backend_factory = make_factory([]) + logins = [] + factory_server.auth_factory = lambda: logins.append(1) or fakes.FakeAuth({"alice": "s3cret"}) + c = fakes.Client(factory_server.server_address[1]) + c.cmd("MODE READER") + assert logins == [] + c.cmd("AUTHINFO USER alice") + assert code(c.cmd("AUTHINFO PASS s3cret")) == "281" + assert logins == [1] + c.close() + + +def wait_until(condition, seconds=5): + deadline = time.time() + seconds + while time.time() < deadline: + if condition(): + return True + time.sleep(0.02) + return False + + +def test_build_server_uses_factories_and_checks_the_configuration(monkeypatch): + monkeypatch.setattr(settings, "storage_backend", "maildir") + srv = papercut.build_server(("127.0.0.1", 0), fakes.FakeServer) + try: + assert srv.backend_factory is not None and srv.auth_factory is None + assert not hasattr(srv, "backend") or srv.backend is None + finally: + srv.server_close() + + +def test_build_server_fails_on_startup_with_a_bad_configuration(monkeypatch): + monkeypatch.setattr(settings, "storage_backend", "mbox") + monkeypatch.setattr(settings, "nntp_auth", "yes") + monkeypatch.setattr(settings, "auth_backend", "no_such_module") + with pytest.raises(ModuleNotFoundError): + papercut.build_server(("127.0.0.1", 0), fakes.FakeServer)