Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 85 additions & 29 deletions papercut.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
STATUS_AUTH_ACCEPTED = '281 Authentication accepted'
STATUS_AUTH_CONTINUE = '381 More authentication information required'
STATUS_SERVER_VERSION = '200 Papercut %s' % (__VERSION__)
STATUS_UNAVAILABLE = '400 service temporarily unavailable'

# the currently supported overview headers
overview_headers = ('Subject', 'From', 'Date', 'Message-ID', 'References', 'Bytes', 'Lines', 'Xref')
Expand All @@ -87,33 +88,65 @@
if os.name == 'posix':
class NNTPServer(socketserver.ForkingTCPServer):
allow_reuse_address = 1
# storage and auth backends, set before serve_forever()
backend = None
auth = None
# callables that create the storage and auth backends for each
# connection (set by build_server)
backend_factory = None
auth_factory = None
if settings.max_connections:
max_children = settings.max_connections
else:
class NNTPServer(socketserver.ThreadingTCPServer):
allow_reuse_address = 1
# storage and auth backends, set before serve_forever()
backend = None
auth = None
# callables that create the storage and auth backends for each
# connection (set by build_server)
backend_factory = None
auth_factory = None


def close_backend(backend):
"""Closes a backend's database (or upstream NNTP) connection, if any."""
try:
if hasattr(backend, 'conn'):
backend.conn.close()
elif hasattr(backend, 'nntp'):
backend.nntp.quit()
except Exception:
pass


class NNTPRequestHandler(socketserver.StreamRequestHandler):
# socket read timeout; replaces the old SIGALRM approach, which no longer
# interrupts a blocking read on Python 3 (PEP 475) and never worked on Windows
timeout = __TIMEOUT__

# the backends live on the server, so each server (and each test) can
# use its own
@property
def backend(self):
return self.server.backend
backend = None
_auth = None

def setup(self):
socketserver.StreamRequestHandler.setup(self)
# Each connection gets its own storage backend, and so its own
# database connection: one shared between forked children or
# threads interleaves their queries and results. (A server with no
# factory, as in the unit tests, shares its 'backend' instance.)
factory = getattr(self.server, 'backend_factory', None)
self.owns_backend = factory is not None
if factory is not None:
try:
self.backend = factory()
except Exception:
settings.logEvent('Error - Could not create the storage backend for %s' % (self.client_address[0]))
settings.logEvent(traceback.format_exc())
self.backend = None
else:
self.backend = getattr(self.server, 'backend', None)

@property
def auth(self):
return self.server.auth
# created on first use: most connections never log in
if self._auth is None:
factory = getattr(self.server, 'auth_factory', None)
self._auth = factory() if factory is not None else getattr(self.server, 'auth', None)
return self._auth

# this is the list of supported commands
commands = ('ARTICLE', 'BODY', 'HEAD',
Expand Down Expand Up @@ -141,6 +174,9 @@ def auth(self):

def handle(self):
settings.logEvent('Connection from %s' % (self.client_address[0]))
if self.backend is None:
self.send_response(STATUS_UNAVAILABLE)
return
if settings.server_type == 'read-only':
self.send_response(STATUS_READYNOPOST % (settings.nntp_hostname, __VERSION__))
else:
Expand Down Expand Up @@ -910,11 +946,47 @@ def finish(self):
self.sending_article = 0
self.auth_username = ''
self.article_lines = []
if getattr(self, 'owns_backend', False):
close_backend(self.backend)
if self._auth is not None and getattr(self.server, 'auth_factory', None) is not None:
close_backend(self._auth)
socketserver.StreamRequestHandler.finish(self)
if __DEBUG__:
print('Closing the request')


def build_server(address, server_class=None):
"""Creates the server, with the storage and auth backends from settings.

server_class defaults to NNTPServer (forking on POSIX, threading on
Windows); the tests also run the other one.
"""
# dynamic loading of the appropriate storage backend module
storage_module = importlib.import_module('storage.%s' % (settings.storage_backend))
if settings.nntp_cache == 'yes':
def backend_factory():
return papercut_cache.Cache(storage_module, papercut_cache.cache_methods)
else:
backend_factory = storage_module.Papercut_Storage

# now for the authentication module, if needed
auth_factory = None
if settings.nntp_auth == 'yes':
auth_factory = importlib.import_module('auth.%s' % (settings.auth_backend)).Papercut_Auth

# connect once now, so a configuration problem shows up on startup
# rather than on the first client's connection
close_backend(backend_factory())
if auth_factory is not None:
close_backend(auth_factory())

# each connection creates its own backends (see NNTPRequestHandler.setup)
server = (server_class or NNTPServer)(address, NNTPRequestHandler)
server.backend_factory = backend_factory
server.auth_factory = auth_factory
return server


if __name__ == '__main__':
# set up signal handler
def sighandler(signum, frame):
Expand All @@ -924,23 +996,7 @@ def sighandler(signum, frame):
sys.exit(0)

settings.validate()

# dynamic loading of the appropriate storage backend module
temp = importlib.import_module('storage.%s' % (settings.storage_backend))
if settings.nntp_cache == 'yes':
backend = papercut_cache.Cache(temp, papercut_cache.cache_methods)
else:
backend = temp.Papercut_Storage()

# now for the authentication module, if needed
auth = None
if settings.nntp_auth == 'yes':
temp = importlib.import_module('auth.%s' % (settings.auth_backend))
auth = temp.Papercut_Auth()

signal.signal(signal.SIGINT, sighandler)
print('Papercut %s (%s storage module) - starting up' % (__VERSION__, settings.storage_backend))
server = NNTPServer((settings.nntp_hostname, settings.nntp_port), NNTPRequestHandler)
server.backend = backend
server.auth = auth
server = build_server((settings.nntp_hostname, settings.nntp_port))
server.serve_forever()
23 changes: 14 additions & 9 deletions tests/integration/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,23 +80,28 @@ def query(self, sql, params=None):
return query(self.kind, self.database, sql, params)


def start_live(monkeypatch, request, storage_module, kind, database, auth_module=None, **extra_settings):
def settings_for(kind, database, **extra_settings):
"""The settings values for running a backend against a test database."""
host = POSTGRES_HOST if kind == "postgres" else MYSQL_HOST
values = dict(dbhost=host, dbname=database, dbuser="papercut", dbpass="papercut",
nntp_hostname="news.example.com",
phorum_settings_path=os.path.join(FIXTURES, "phorum_settings") + "/")
values.update(extra_settings)
for name, value in values.items():
return values


def start_live(monkeypatch, request, storage_module, kind, database, auth_module=None, **extra_settings):
for name, value in settings_for(kind, database, **extra_settings).items():
monkeypatch.setattr(settings, name, value)
if kind == "postgres":
# the PostgreSQL user is the database owner (see the postgres fixtures)
monkeypatch.setattr(settings, "dbuser", "papercut")
backend = importlib.import_module(storage_module).Papercut_Storage()
auth = importlib.import_module(auth_module).Papercut_Auth() if auth_module else None
server = fakes.start_server(papercut.NNTPRequestHandler, backend, auth)
# factories, as build_server sets up: each connection gets its own backends
backend_factory = importlib.import_module(storage_module).Papercut_Storage
auth_factory = importlib.import_module(auth_module).Papercut_Auth if auth_module else None
server = fakes.start_server(papercut.NNTPRequestHandler, None)
server.backend_factory = backend_factory
server.auth_factory = auth_factory
request.addfinalizer(server.server_close)
request.addfinalizer(server.shutdown)
return Live(server, backend, kind, database)
return Live(server, backend_factory, kind, database)


@pytest.fixture(autouse=True)
Expand Down
25 changes: 25 additions & 0 deletions tests/integration/serve.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
"""Runs a Papercut server in its own process (see test_concurrency.py).

Imported by multiprocessing's 'spawn' start method, so a server that
crashes (e.g. a DB driver misused from several threads) fails the test
instead of taking pytest down with it.
"""
import socketserver


class ThreadingServer(socketserver.ThreadingTCPServer):
"""The server class Papercut uses on Windows."""
allow_reuse_address = True
daemon_threads = True


def run(settings_values, mode, log_file, port_queue):
import settings
for name, value in settings_values.items():
setattr(settings, name, value)
settings.log_file = log_file
import papercut
server_class = ThreadingServer if mode == "threading" else papercut.NNTPServer
server = papercut.build_server(("127.0.0.1", 0), server_class)
port_queue.put(server.server_address[1])
server.serve_forever()
83 changes: 83 additions & 0 deletions tests/integration/test_concurrency.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
"""Several clients at once, against the server as startup builds it.

Each client connection must get its own database connection: sharing one
between forked children (POSIX) or threads (Windows) interleaves their
queries and results.
"""
import multiprocessing
import os
import threading
import time
import warnings

import pytest

import serve
from conftest import settings_for

GROUP = "phorum.test"
CLIENTS = 6
ROUNDS = 15
TIMEOUT = 15 # per socket operation: a hang counts as a failure
DEADLINE = 90 # for the whole test

BACKENDS = {
"phorum_mysql": ("mysql", "phorum"),
"phorum_pgsql": ("postgres", "phorum"),
}


@pytest.fixture(params=["forking", "threading"])
def server_port(request, tmp_path):
mode = request.param
storage = request.node.callspec.params["backend"]
kind, database = BACKENDS[storage]
values = settings_for(kind, database, storage_backend=storage, nntp_cache="no", nntp_auth="no")
context = multiprocessing.get_context("spawn")
ports = context.Queue()
process = context.Process(target=serve.run, args=(values, mode, str(tmp_path / "server.log"), ports), daemon=True)
process.start()
port = ports.get(timeout=30)
yield port, tmp_path / "server.log"
process.terminate()
process.join(10)


def client_session(port, errors):
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
import nntplib
try:
with nntplib.NNTP("127.0.0.1", port, timeout=TIMEOUT) as s:
for _ in range(ROUNDS):
_, count, first, last, _ = s.group(GROUP)
assert (count, first, last) == (3, 1, 4), (count, first, last)
_, overviews = s.over((1, 4))
assert [n for n, _ in overviews] == [1, 2, 4], overviews
_, info = s.article("2")
assert b"Subject: Re: First post" in info.lines, info.lines[:4]
_, body = s.body("4")
assert body.lines == ["Corpo com acentuação".encode()], body.lines
except Exception as e: # collected, so every client's failure is reported
errors.append("%s: %s" % (type(e).__name__, e))


@pytest.mark.parametrize("backend", sorted(BACKENDS))
def test_concurrent_clients_get_correct_results(backend, server_port):
port, log = server_port
errors = []
clients = [threading.Thread(target=client_session, args=(port, errors), daemon=True) for _ in range(CLIENTS)]
for c in clients:
c.start()
deadline = time.time() + DEADLINE
for c in clients:
c.join(max(0, deadline - time.time()))
hung = sum(c.is_alive() for c in clients)
exceptions = sorted({l for l in log.read_text().splitlines() if l and not l.startswith((" ", "[", "Traceback"))}) if log.exists() else []
server_errors = exceptions[:6]
assert not hung and not errors, "%d of %d clients failed, %d hung; e.g.:\n%s\nserver log:\n%s" % (
len(errors), CLIENTS, hung, "\n".join(errors[:3]), "\n".join(server_errors))
# and the work really went through the server
text = log.read_text()
assert text.count("Connection from") == CLIENTS
assert text.count("Received request: GROUP") == CLIENTS * ROUNDS
Loading
Loading