diff --git a/papercut.py b/papercut.py index e41034a..a476995 100755 --- a/papercut.py +++ b/papercut.py @@ -75,6 +75,7 @@ STATUS_AUTH_ACCEPTED = '281 Authentication accepted' STATUS_AUTH_CONTINUE = '381 More authentication information required' STATUS_SERVER_VERSION = '200 Papercut %s' % (__VERSION__) +STATUS_UNAVAILABLE = '400 service temporarily unavailable' # the currently supported overview headers overview_headers = ('Subject', 'From', 'Date', 'Message-ID', 'References', 'Bytes', 'Lines', 'Xref') @@ -87,17 +88,30 @@ if os.name == 'posix': class NNTPServer(socketserver.ForkingTCPServer): allow_reuse_address = 1 - # storage and auth backends, set before serve_forever() - backend = None - auth = None + # callables that create the storage and auth backends for each + # connection (set by build_server) + backend_factory = None + auth_factory = None if settings.max_connections: max_children = settings.max_connections else: class NNTPServer(socketserver.ThreadingTCPServer): allow_reuse_address = 1 - # storage and auth backends, set before serve_forever() - backend = None - auth = None + # callables that create the storage and auth backends for each + # connection (set by build_server) + backend_factory = None + auth_factory = None + + +def close_backend(backend): + """Closes a backend's database (or upstream NNTP) connection, if any.""" + try: + if hasattr(backend, 'conn'): + backend.conn.close() + elif hasattr(backend, 'nntp'): + backend.nntp.quit() + except Exception: + pass class NNTPRequestHandler(socketserver.StreamRequestHandler): @@ -105,15 +119,34 @@ class NNTPRequestHandler(socketserver.StreamRequestHandler): # interrupts a blocking read on Python 3 (PEP 475) and never worked on Windows timeout = __TIMEOUT__ - # the backends live on the server, so each server (and each test) can - # use its own - @property - def backend(self): - return self.server.backend + backend = None + _auth = None + + def setup(self): + socketserver.StreamRequestHandler.setup(self) + # Each connection gets its own storage backend, and so its own + # database connection: one shared between forked children or + # threads interleaves their queries and results. (A server with no + # factory, as in the unit tests, shares its 'backend' instance.) + factory = getattr(self.server, 'backend_factory', None) + self.owns_backend = factory is not None + if factory is not None: + try: + self.backend = factory() + except Exception: + settings.logEvent('Error - Could not create the storage backend for %s' % (self.client_address[0])) + settings.logEvent(traceback.format_exc()) + self.backend = None + else: + self.backend = getattr(self.server, 'backend', None) @property def auth(self): - return self.server.auth + # created on first use: most connections never log in + if self._auth is None: + factory = getattr(self.server, 'auth_factory', None) + self._auth = factory() if factory is not None else getattr(self.server, 'auth', None) + return self._auth # this is the list of supported commands commands = ('ARTICLE', 'BODY', 'HEAD', @@ -141,6 +174,9 @@ def auth(self): def handle(self): settings.logEvent('Connection from %s' % (self.client_address[0])) + if self.backend is None: + self.send_response(STATUS_UNAVAILABLE) + return if settings.server_type == 'read-only': self.send_response(STATUS_READYNOPOST % (settings.nntp_hostname, __VERSION__)) else: @@ -910,11 +946,47 @@ def finish(self): self.sending_article = 0 self.auth_username = '' self.article_lines = [] + if getattr(self, 'owns_backend', False): + close_backend(self.backend) + if self._auth is not None and getattr(self.server, 'auth_factory', None) is not None: + close_backend(self._auth) socketserver.StreamRequestHandler.finish(self) if __DEBUG__: print('Closing the request') +def build_server(address, server_class=None): + """Creates the server, with the storage and auth backends from settings. + + server_class defaults to NNTPServer (forking on POSIX, threading on + Windows); the tests also run the other one. + """ + # dynamic loading of the appropriate storage backend module + storage_module = importlib.import_module('storage.%s' % (settings.storage_backend)) + if settings.nntp_cache == 'yes': + def backend_factory(): + return papercut_cache.Cache(storage_module, papercut_cache.cache_methods) + else: + backend_factory = storage_module.Papercut_Storage + + # now for the authentication module, if needed + auth_factory = None + if settings.nntp_auth == 'yes': + auth_factory = importlib.import_module('auth.%s' % (settings.auth_backend)).Papercut_Auth + + # connect once now, so a configuration problem shows up on startup + # rather than on the first client's connection + close_backend(backend_factory()) + if auth_factory is not None: + close_backend(auth_factory()) + + # each connection creates its own backends (see NNTPRequestHandler.setup) + server = (server_class or NNTPServer)(address, NNTPRequestHandler) + server.backend_factory = backend_factory + server.auth_factory = auth_factory + return server + + if __name__ == '__main__': # set up signal handler def sighandler(signum, frame): @@ -924,23 +996,7 @@ def sighandler(signum, frame): sys.exit(0) settings.validate() - - # dynamic loading of the appropriate storage backend module - temp = importlib.import_module('storage.%s' % (settings.storage_backend)) - if settings.nntp_cache == 'yes': - backend = papercut_cache.Cache(temp, papercut_cache.cache_methods) - else: - backend = temp.Papercut_Storage() - - # now for the authentication module, if needed - auth = None - if settings.nntp_auth == 'yes': - temp = importlib.import_module('auth.%s' % (settings.auth_backend)) - auth = temp.Papercut_Auth() - signal.signal(signal.SIGINT, sighandler) print('Papercut %s (%s storage module) - starting up' % (__VERSION__, settings.storage_backend)) - server = NNTPServer((settings.nntp_hostname, settings.nntp_port), NNTPRequestHandler) - server.backend = backend - server.auth = auth + server = build_server((settings.nntp_hostname, settings.nntp_port)) server.serve_forever() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b7a7067..fa24c07 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -80,23 +80,28 @@ def query(self, sql, params=None): return query(self.kind, self.database, sql, params) -def start_live(monkeypatch, request, storage_module, kind, database, auth_module=None, **extra_settings): +def settings_for(kind, database, **extra_settings): + """The settings values for running a backend against a test database.""" host = POSTGRES_HOST if kind == "postgres" else MYSQL_HOST values = dict(dbhost=host, dbname=database, dbuser="papercut", dbpass="papercut", nntp_hostname="news.example.com", phorum_settings_path=os.path.join(FIXTURES, "phorum_settings") + "/") values.update(extra_settings) - for name, value in values.items(): + return values + + +def start_live(monkeypatch, request, storage_module, kind, database, auth_module=None, **extra_settings): + for name, value in settings_for(kind, database, **extra_settings).items(): monkeypatch.setattr(settings, name, value) - if kind == "postgres": - # the PostgreSQL user is the database owner (see the postgres fixtures) - monkeypatch.setattr(settings, "dbuser", "papercut") - backend = importlib.import_module(storage_module).Papercut_Storage() - auth = importlib.import_module(auth_module).Papercut_Auth() if auth_module else None - server = fakes.start_server(papercut.NNTPRequestHandler, backend, auth) + # factories, as build_server sets up: each connection gets its own backends + backend_factory = importlib.import_module(storage_module).Papercut_Storage + auth_factory = importlib.import_module(auth_module).Papercut_Auth if auth_module else None + server = fakes.start_server(papercut.NNTPRequestHandler, None) + server.backend_factory = backend_factory + server.auth_factory = auth_factory request.addfinalizer(server.server_close) request.addfinalizer(server.shutdown) - return Live(server, backend, kind, database) + return Live(server, backend_factory, kind, database) @pytest.fixture(autouse=True) diff --git a/tests/integration/serve.py b/tests/integration/serve.py new file mode 100644 index 0000000..c81d6a4 --- /dev/null +++ b/tests/integration/serve.py @@ -0,0 +1,25 @@ +"""Runs a Papercut server in its own process (see test_concurrency.py). + +Imported by multiprocessing's 'spawn' start method, so a server that +crashes (e.g. a DB driver misused from several threads) fails the test +instead of taking pytest down with it. +""" +import socketserver + + +class ThreadingServer(socketserver.ThreadingTCPServer): + """The server class Papercut uses on Windows.""" + allow_reuse_address = True + daemon_threads = True + + +def run(settings_values, mode, log_file, port_queue): + import settings + for name, value in settings_values.items(): + setattr(settings, name, value) + settings.log_file = log_file + import papercut + server_class = ThreadingServer if mode == "threading" else papercut.NNTPServer + server = papercut.build_server(("127.0.0.1", 0), server_class) + port_queue.put(server.server_address[1]) + server.serve_forever() diff --git a/tests/integration/test_concurrency.py b/tests/integration/test_concurrency.py new file mode 100644 index 0000000..f65ac87 --- /dev/null +++ b/tests/integration/test_concurrency.py @@ -0,0 +1,83 @@ +"""Several clients at once, against the server as startup builds it. + +Each client connection must get its own database connection: sharing one +between forked children (POSIX) or threads (Windows) interleaves their +queries and results. +""" +import multiprocessing +import os +import threading +import time +import warnings + +import pytest + +import serve +from conftest import settings_for + +GROUP = "phorum.test" +CLIENTS = 6 +ROUNDS = 15 +TIMEOUT = 15 # per socket operation: a hang counts as a failure +DEADLINE = 90 # for the whole test + +BACKENDS = { + "phorum_mysql": ("mysql", "phorum"), + "phorum_pgsql": ("postgres", "phorum"), +} + + +@pytest.fixture(params=["forking", "threading"]) +def server_port(request, tmp_path): + mode = request.param + storage = request.node.callspec.params["backend"] + kind, database = BACKENDS[storage] + values = settings_for(kind, database, storage_backend=storage, nntp_cache="no", nntp_auth="no") + context = multiprocessing.get_context("spawn") + ports = context.Queue() + process = context.Process(target=serve.run, args=(values, mode, str(tmp_path / "server.log"), ports), daemon=True) + process.start() + port = ports.get(timeout=30) + yield port, tmp_path / "server.log" + process.terminate() + process.join(10) + + +def client_session(port, errors): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + import nntplib + try: + with nntplib.NNTP("127.0.0.1", port, timeout=TIMEOUT) as s: + for _ in range(ROUNDS): + _, count, first, last, _ = s.group(GROUP) + assert (count, first, last) == (3, 1, 4), (count, first, last) + _, overviews = s.over((1, 4)) + assert [n for n, _ in overviews] == [1, 2, 4], overviews + _, info = s.article("2") + assert b"Subject: Re: First post" in info.lines, info.lines[:4] + _, body = s.body("4") + assert body.lines == ["Corpo com acentuação".encode()], body.lines + except Exception as e: # collected, so every client's failure is reported + errors.append("%s: %s" % (type(e).__name__, e)) + + +@pytest.mark.parametrize("backend", sorted(BACKENDS)) +def test_concurrent_clients_get_correct_results(backend, server_port): + port, log = server_port + errors = [] + clients = [threading.Thread(target=client_session, args=(port, errors), daemon=True) for _ in range(CLIENTS)] + for c in clients: + c.start() + deadline = time.time() + DEADLINE + for c in clients: + c.join(max(0, deadline - time.time())) + hung = sum(c.is_alive() for c in clients) + exceptions = sorted({l for l in log.read_text().splitlines() if l and not l.startswith((" ", "[", "Traceback"))}) if log.exists() else [] + server_errors = exceptions[:6] + assert not hung and not errors, "%d of %d clients failed, %d hung; e.g.:\n%s\nserver log:\n%s" % ( + len(errors), CLIENTS, hung, "\n".join(errors[:3]), "\n".join(server_errors)) + # and the work really went through the server + text = log.read_text() + assert text.count("Connection from") == CLIENTS + assert text.count("Received request: GROUP") == CLIENTS * ROUNDS diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 4f9fd09..03b102b 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -669,3 +669,96 @@ def test_login_names_are_not_logged_by_the_server(auth_on, client): assert wait_for_log("Connection closed") assert received_requests() == ["AUTHINFO", "AUTHINFO", "QUIT"] assert "alice" not in log_text() + + +# --- backends per connection --- + +class RecordingConnection: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + +def make_factory(created, fail=False): + def factory(): + if fail: + raise RuntimeError("database unreachable") + backend = fakes.FakeBackend() + backend.conn = RecordingConnection() + created.append(backend) + return backend + return factory + + +@pytest.fixture +def factory_server(): + srv = fakes.start_server(papercut.NNTPRequestHandler, None) + yield srv + srv.shutdown() + srv.server_close() + + +def test_each_connection_gets_its_own_backend(factory_server): + created = [] + factory_server.backend_factory = make_factory(created) + first = fakes.Client(factory_server.server_address[1]) + second = fakes.Client(factory_server.server_address[1]) + first.cmd("GROUP test.group") + second.cmd("GROUP test.group") + assert len(created) == 2 + assert [b.called("get_GROUP") for b in created] == [[("test.group",)], [("test.group",)]] + first.cmd("QUIT") + second.cmd("QUIT") + assert wait_until(lambda: all(b.conn.closed for b in created)) + + +def test_unreachable_database_gets_a_400(factory_server): + factory_server.backend_factory = make_factory([], fail=True) + c = fakes.Client(factory_server.server_address[1]) + assert code(c.greeting) == "400" + assert c.readline_bytes() == b"" + assert wait_for_log("database unreachable") + c.close() + + +def test_auth_backend_is_created_only_on_login(factory_server, monkeypatch): + monkeypatch.setattr(settings, "nntp_auth", "yes") + factory_server.backend_factory = make_factory([]) + logins = [] + factory_server.auth_factory = lambda: logins.append(1) or fakes.FakeAuth({"alice": "s3cret"}) + c = fakes.Client(factory_server.server_address[1]) + c.cmd("MODE READER") + assert logins == [] + c.cmd("AUTHINFO USER alice") + assert code(c.cmd("AUTHINFO PASS s3cret")) == "281" + assert logins == [1] + c.close() + + +def wait_until(condition, seconds=5): + deadline = time.time() + seconds + while time.time() < deadline: + if condition(): + return True + time.sleep(0.02) + return False + + +def test_build_server_uses_factories_and_checks_the_configuration(monkeypatch): + monkeypatch.setattr(settings, "storage_backend", "maildir") + srv = papercut.build_server(("127.0.0.1", 0), fakes.FakeServer) + try: + assert srv.backend_factory is not None and srv.auth_factory is None + assert not hasattr(srv, "backend") or srv.backend is None + finally: + srv.server_close() + + +def test_build_server_fails_on_startup_with_a_bad_configuration(monkeypatch): + monkeypatch.setattr(settings, "storage_backend", "mbox") + monkeypatch.setattr(settings, "nntp_auth", "yes") + monkeypatch.setattr(settings, "auth_backend", "no_such_module") + with pytest.raises(ModuleNotFoundError): + papercut.build_server(("127.0.0.1", 0), fakes.FakeServer)