From 2339291cec6a91c8a54b5aef98c1ba64bc01fced Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 06:29:48 -0600 Subject: [PATCH 01/11] Make the mysql backend's schema and queries work on current MySQL - mysql_storage.sql: TYPE=MyISAM was removed in MySQL 5.5 (ENGINE=), and '0000-00-00' datetime defaults are rejected by 5.7+'s default strict mode, so the shipped schema did not load. It now uses InnoDB and utf8mb4 (MyISAM's 1000-byte key limit is too small for utf8mb4 varchar(255) indexes). - 'COUNT(*) AS check': CHECK is a reserved word since MySQL 8.0.16, so group_exists() -- called by almost every command -- was a syntax error. --- storage/mysql.py | 4 ++-- storage/mysql_storage.sql | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/storage/mysql.py b/storage/mysql.py index 10a16b3..76f819d 100755 --- a/storage/mysql.py +++ b/storage/mysql.py @@ -34,7 +34,7 @@ def get_header(self, lines): def group_exists(self, group_name): stmt = """ SELECT - COUNT(*) AS check + COUNT(*) AS total FROM papercut_groups WHERE @@ -46,7 +46,7 @@ def article_exists(self, group_name, style, range): table_name = self.get_table_name(group_name) stmt = """ SELECT - COUNT(*) AS check + COUNT(*) AS total FROM %s WHERE diff --git a/storage/mysql_storage.sql b/storage/mysql_storage.sql index 587e835..ee8e9ea 100755 --- a/storage/mysql_storage.sql +++ b/storage/mysql_storage.sql @@ -8,7 +8,7 @@ CREATE TABLE papercut_groups ( PRIMARY KEY (id), KEY name (name), KEY active (active) -) TYPE=MyISAM; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS papercut_groups_auth; CREATE TABLE papercut_groups_auth ( @@ -20,12 +20,12 @@ CREATE TABLE papercut_groups_auth ( PRIMARY KEY (id), KEY name (name), KEY username (username) -) TYPE=MyISAM; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS papercut_default_table; CREATE TABLE papercut_default_table ( id int(10) unsigned NOT NULL default '0', - datestamp datetime NOT NULL default '0000-00-00 00:00:00', + datestamp datetime NOT NULL default CURRENT_TIMESTAMP, thread int(10) unsigned NOT NULL default '0', parent int(10) unsigned NOT NULL default '0', author varchar(255) NOT NULL default '', @@ -41,4 +41,4 @@ CREATE TABLE papercut_default_table ( KEY subject (subject), KEY thread (thread), KEY parent (parent) -) TYPE=MyISAM; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; From 097074108d8bfab19001d758af07937c0bddb65e Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 06:29:48 -0600 Subject: [PATCH 02/11] Commit every statement and use utf8mb4 on MySQL connections MySQLdb starts with autocommit off. On the MyISAM tables Papercut was written for that didn't matter, but on InnoDB (the default since MySQL 5.5, and what most Phorum/phpBB databases use today) posted articles were never committed, and their locks blocked other connections. Connections also didn't set a character set, so non-ASCII text came back garbled (mojibake). --- auth/mysql.py | 4 +++- auth/phorum_mysql_users.py | 4 +++- auth/phpbb_mysql_users.py | 4 +++- auth/phpnuke_phpbb_mysql_users.py | 4 +++- auth/postnuke_phpbb_mysql_users.py | 4 +++- storage/mysql.py | 4 +++- storage/phorum_mysql.py | 4 +++- storage/phpbb_mysql.py | 4 +++- storage/phpnuke_phpbb_mysql.py | 4 +++- 9 files changed, 27 insertions(+), 9 deletions(-) diff --git a/auth/mysql.py b/auth/mysql.py index 7ef1025..cd1ab0c 100755 --- a/auth/mysql.py +++ b/auth/mysql.py @@ -10,7 +10,9 @@ class Papercut_Auth: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def is_valid_user(self, username, password): diff --git a/auth/phorum_mysql_users.py b/auth/phorum_mysql_users.py index 83cf0f9..4a2ca5c 100755 --- a/auth/phorum_mysql_users.py +++ b/auth/phorum_mysql_users.py @@ -17,7 +17,9 @@ class Papercut_Auth: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def is_valid_user(self, username, password): diff --git a/auth/phpbb_mysql_users.py b/auth/phpbb_mysql_users.py index 84ba067..e863b91 100755 --- a/auth/phpbb_mysql_users.py +++ b/auth/phpbb_mysql_users.py @@ -15,7 +15,9 @@ class Papercut_Auth: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def is_valid_user(self, username, password): diff --git a/auth/phpnuke_phpbb_mysql_users.py b/auth/phpnuke_phpbb_mysql_users.py index d750fe7..1f3b802 100755 --- a/auth/phpnuke_phpbb_mysql_users.py +++ b/auth/phpnuke_phpbb_mysql_users.py @@ -12,7 +12,9 @@ class Papercut_Auth: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def is_valid_user(self, username, password): diff --git a/auth/postnuke_phpbb_mysql_users.py b/auth/postnuke_phpbb_mysql_users.py index cf0f80a..3d24171 100755 --- a/auth/postnuke_phpbb_mysql_users.py +++ b/auth/postnuke_phpbb_mysql_users.py @@ -13,7 +13,9 @@ class Papercut_Auth: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def is_valid_user(self, username, password): diff --git a/storage/mysql.py b/storage/mysql.py index 76f819d..882cc95 100755 --- a/storage/mysql.py +++ b/storage/mysql.py @@ -22,7 +22,9 @@ class Papercut_Storage: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def get_body(self, lines): diff --git a/storage/phorum_mysql.py b/storage/phorum_mysql.py index 87814c4..68b1aa7 100755 --- a/storage/phorum_mysql.py +++ b/storage/phorum_mysql.py @@ -36,7 +36,9 @@ class Papercut_Storage: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def get_message_body(self, headers): diff --git a/storage/phpbb_mysql.py b/storage/phpbb_mysql.py index a2212d6..dfbe620 100755 --- a/storage/phpbb_mysql.py +++ b/storage/phpbb_mysql.py @@ -31,7 +31,9 @@ class Papercut_Storage: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def get_message_body(self, headers): diff --git a/storage/phpnuke_phpbb_mysql.py b/storage/phpnuke_phpbb_mysql.py index 7f3284c..e3142a3 100755 --- a/storage/phpnuke_phpbb_mysql.py +++ b/storage/phpnuke_phpbb_mysql.py @@ -30,7 +30,9 @@ class Papercut_Storage: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def get_message_body(self, headers): From fbf05395b976af14ff6a8570839b59e873452be0 Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 06:29:48 -0600 Subject: [PATCH 03/11] Fix NEWNEWS message-ids in the SQL backends The rows from fetchall() were formatted whole, producing message-ids like <(2,)@group> instead of <2@group>. --- storage/mysql.py | 2 +- storage/phorum_mysql.py | 2 +- storage/phorum_pgsql.py | 2 +- storage/phpbb_mysql.py | 2 +- storage/phpnuke_phpbb_mysql.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/storage/mysql.py b/storage/mysql.py index 882cc95..96856a8 100755 --- a/storage/mysql.py +++ b/storage/mysql.py @@ -131,7 +131,7 @@ def get_NEWNEWS(self, ts, group='*'): continue ids = list(self.cursor.fetchall()) for id in ids: - articles.append("<%s@%s>" % (id, group)) + articles.append("<%s@%s>" % (id[0], group)) if len(articles) == 0: return '' else: diff --git a/storage/phorum_mysql.py b/storage/phorum_mysql.py index 68b1aa7..c7607c3 100755 --- a/storage/phorum_mysql.py +++ b/storage/phorum_mysql.py @@ -271,7 +271,7 @@ def get_NEWNEWS(self, ts, group='*'): continue ids = list(self.cursor.fetchall()) for id in ids: - articles.append("<%s@%s>" % (id, group)) + articles.append("<%s@%s>" % (id[0], group)) if len(articles) == 0: return '' else: diff --git a/storage/phorum_pgsql.py b/storage/phorum_pgsql.py index 75d9914..64b3721 100755 --- a/storage/phorum_pgsql.py +++ b/storage/phorum_pgsql.py @@ -283,7 +283,7 @@ def get_NEWNEWS(self, ts, group='*'): continue ids = list(self.cursor.fetchall()) for id in ids: - articles.append("<%s@%s>" % (id, group)) + articles.append("<%s@%s>" % (id[0], group)) if len(articles) == 0: return '' else: diff --git a/storage/phpbb_mysql.py b/storage/phpbb_mysql.py index dfbe620..1e06935 100755 --- a/storage/phpbb_mysql.py +++ b/storage/phpbb_mysql.py @@ -155,7 +155,7 @@ def get_NEWNEWS(self, ts, group='*'): continue ids = list(self.cursor.fetchall()) for id in ids: - articles.append("<%s@%s>" % (id, group_name)) + articles.append("<%s@%s>" % (id[0], group_name)) if len(articles) == 0: return '' else: diff --git a/storage/phpnuke_phpbb_mysql.py b/storage/phpnuke_phpbb_mysql.py index e3142a3..7aca441 100755 --- a/storage/phpnuke_phpbb_mysql.py +++ b/storage/phpnuke_phpbb_mysql.py @@ -154,7 +154,7 @@ def get_NEWNEWS(self, ts, group='*'): continue ids = list(self.cursor.fetchall()) for id in ids: - articles.append("<%s@%s>" % (id, group_name)) + articles.append("<%s@%s>" % (id[0], group_name)) if len(articles) == 0: return '' else: From 663e9b662ca7e751e65263b1e4d39031faad3eb0 Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 06:29:57 -0600 Subject: [PATCH 04/11] Send Phorum notification emails as proper UTF-8 messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit smtplib's sendmail() only accepts ASCII strings, so a post containing e.g. 'é' raised UnicodeEncodeError after the article was saved: the client was told '441 Posting failed' for a post that went through, and no moderator or mailing-list email was sent. The messages are now built with email.message.EmailMessage (RFC 2047 headers, UTF-8 body). --- storage/phorum_mysql.py | 4 ++-- storage/phorum_pgsql.py | 4 ++-- storage/strutil.py | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/storage/phorum_mysql.py b/storage/phorum_mysql.py index c7607c3..5a0e837 100755 --- a/storage/phorum_mysql.py +++ b/storage/phorum_mysql.py @@ -188,7 +188,7 @@ def send_notifications(self, group_name, msg_id, thread_id, parent_id, msg_autho emails = self.get_notification_emails(forum_id) for recipient in emails: current_msg = msg_tpl % vars() - smtp.sendmail("Phorum <%s>" % (recipient), recipient, current_msg) + strutil.send_text_mail(smtp, "Phorum <%s>" % (recipient), recipient, current_msg) # XXX: Coding blind here. I really don't know much about how Phorum works with # XXX: sending forum postings as emails, but it's here. Let's call this a @@ -236,7 +236,7 @@ def send_notifications(self, group_name, msg_id, thread_id, parent_id, msg_autho from papercut import __VERSION__ phorum_version = settings.phorum_version current_msg = notification_mail_tpl % vars() - smtp.sendmail('%s %s' % (msg_author, msg_email), email_list, current_msg) + strutil.send_text_mail(smtp, '%s %s' % (msg_author, msg_email), email_list, current_msg) smtp.quit() def get_NEWGROUPS(self, ts, group='%'): diff --git a/storage/phorum_pgsql.py b/storage/phorum_pgsql.py index 64b3721..193814f 100755 --- a/storage/phorum_pgsql.py +++ b/storage/phorum_pgsql.py @@ -198,7 +198,7 @@ def send_notifications(self, group_name, msg_id, thread_id, parent_id, msg_autho emails = self.get_notification_emails(forum_id) for recipient in emails: current_msg = msg_tpl % vars() - smtp.sendmail("Phorum <%s>" % (recipient), recipient, current_msg) + strutil.send_text_mail(smtp, "Phorum <%s>" % (recipient), recipient, current_msg) # XXX: Coding blind here. I really don't know much about how Phorum works with # XXX: sending forum postings as emails, but it's here. Let's call this a @@ -247,7 +247,7 @@ def send_notifications(self, group_name, msg_id, thread_id, parent_id, msg_autho from papercut import __VERSION__ phorum_version = settings.phorum_version current_msg = notification_mail_tpl % vars() - smtp.sendmail('%s %s' % (msg_author, msg_email), email_list, current_msg) + strutil.send_text_mail(smtp, '%s %s' % (msg_author, msg_email), email_list, current_msg) smtp.quit() def get_NEWGROUPS(self, ts, group='%'): diff --git a/storage/strutil.py b/storage/strutil.py index 07a303b..0ec9299 100755 --- a/storage/strutil.py +++ b/storage/strutil.py @@ -4,6 +4,7 @@ import time import re import quopri +from email.message import EmailMessage singleline_regexp = re.compile(r"^\.", re.M) @@ -76,6 +77,19 @@ def mime_decode_header(text): """ return encoded_word_regexp.sub(_decode_encoded_word, text) +def send_text_mail(smtp, from_addr, to_addr, text): + """Sends a plain-text email given as 'Header: value' lines, a blank + line and the body. Non-ASCII headers are RFC 2047 encoded and the body + is sent as UTF-8 (smtplib's sendmail() only accepts ASCII strings).""" + head, sep, body = text.replace('\r\n', '\n').partition('\n\n') + msg = EmailMessage() + for line in head.split('\n'): + name, _, value = line.partition(':') + if name and value: + msg[name.strip()] = value.strip() + msg.set_content(body) + smtp.send_message(msg, from_addr, [to_addr]) + def format_wildcards(pattern): return pattern.replace('*', '.*').replace('?', '.*') From b258623f63b61f373fda17c1b4b8906fc3e8ebf2 Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 06:29:57 -0600 Subject: [PATCH 05/11] Fix the PostgreSQL Phorum backend against a real database - get_STAT() returned cursor.execute()'s result, which is always None with psycopg2, so every article was reported missing (STAT, and the server's article lookups) - NEXT/LAST used MySQL's 'LIMIT 1, 0', which is a syntax error in PostgreSQL (and would select no rows in MySQL) - new-thread posts used CURRENT_TIMESTAMP(), invalid in PostgreSQL - the auth module ignored dbhost and dbpass, so it only worked over a local socket without a password --- auth/phorum_pgsql_users.py | 4 +++- storage/phorum_pgsql.py | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/auth/phorum_pgsql_users.py b/auth/phorum_pgsql_users.py index cf48a2b..dbbd572 100755 --- a/auth/phorum_pgsql_users.py +++ b/auth/phorum_pgsql_users.py @@ -17,7 +17,9 @@ class Papercut_Auth: """ def __init__(self): - self.conn = psycopg2.connect(database=settings.dbname, user=settings.dbuser) + # (this used to ignore dbhost and dbpass, so it only worked over a + # local socket without a password) + self.conn = psycopg2.connect(host=settings.dbhost, database=settings.dbname, user=settings.dbuser, password=settings.dbpass) self.conn.autocommit = True self.cursor = self.conn.cursor() diff --git a/storage/phorum_pgsql.py b/storage/phorum_pgsql.py index 193814f..e4bebe1 100755 --- a/storage/phorum_pgsql.py +++ b/storage/phorum_pgsql.py @@ -329,7 +329,10 @@ def get_STAT(self, group_name, id): WHERE approved='Y' AND id=%s""" % (table_name, int(id)) - return self.cursor.execute(stmt) + # (psycopg2's execute() always returns None, so this used to report + # every article as missing) + self.cursor.execute(stmt) + return self.cursor.rowcount > 0 def get_ARTICLE(self, group_name, id): table_name = self.get_table_name(group_name) @@ -383,7 +386,7 @@ def get_LAST(self, group_name, current_id): id < %s ORDER BY id DESC - LIMIT 1, 0""" % (table_name, int(current_id)) + LIMIT 1""" % (table_name, int(current_id)) self.cursor.execute(stmt) num_rows = self.cursor.rowcount if num_rows == 0: @@ -402,7 +405,7 @@ def get_NEXT(self, group_name, current_id): id > %s ORDER BY id ASC - LIMIT 1, 0""" % (table_name, int(current_id)) + LIMIT 1""" % (table_name, int(current_id)) self.cursor.execute(stmt) num_rows = self.cursor.rowcount if num_rows == 0: @@ -694,7 +697,7 @@ def do_POST(self, group_name, lines, ip_address, username=''): stmt = """ SELECT MAX(id) AS next_id, - DATE_PART('epoch', CURRENT_TIMESTAMP()) + DATE_PART('epoch', CURRENT_TIMESTAMP) FROM %s""" % (table_name) self.cursor.execute(stmt) From 665968d58fa6588a7ba81b33e0295c58a74e5795 Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 06:38:07 -0600 Subject: [PATCH 06/11] Make the phpBB and PHP-Nuke backends work on MySQL 5+ - The queries mixed comma joins with LEFT JOIN ('FROM posts A, posts_text B LEFT JOIN users C ON A.poster_id ...'). Since MySQL 5.0.12 JOIN binds tighter than the comma, so A isn't visible in the ON clause and ARTICLE, HEAD, XOVER, XHDR and XPAT all failed. The comma joins are now parenthesized. - cursor.insert_id() only existed in very old MySQLdb versions, so every post failed; it is cursor.lastrowid now. --- storage/phpbb_mysql.py | 22 ++++++++-------------- storage/phpnuke_phpbb_mysql.py | 22 ++++++++-------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/storage/phpbb_mysql.py b/storage/phpbb_mysql.py index 1e06935..ed6fe57 100755 --- a/storage/phpbb_mysql.py +++ b/storage/phpbb_mysql.py @@ -245,8 +245,7 @@ def get_ARTICLE(self, group_name, id): A.post_username, MIN(D.post_id) FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) INNER JOIN %sposts D ON @@ -340,8 +339,7 @@ def get_HEAD(self, group_name, id): A.post_username, MIN(D.post_id) FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) INNER JOIN %stopics E ON @@ -392,8 +390,7 @@ def get_BODY(self, group_name, id): SELECT B.post_text FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) WHERE A.post_id=B.post_id AND A.forum_id=%s AND @@ -418,8 +415,7 @@ def get_XOVER(self, group_name, start_id, end_id='ggg'): B.post_text, A.post_username FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) LEFT JOIN %susers C ON @@ -477,8 +473,7 @@ def get_XPAT(self, group_name, header, pattern, start_id, end_id='ggg'): B.post_text, A.post_username FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) LEFT JOIN %susers C ON @@ -572,8 +567,7 @@ def get_XHDR(self, group_name, header, style, range): B.post_text, A.post_username FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) LEFT JOIN %stopics C ON @@ -683,7 +677,7 @@ def do_POST(self, group_name, lines, ip_address, username=''): 0 )""" % (prefix, forum_id, poster_id) self.cursor.execute(stmt, (subject,)) - thread_id = self.cursor.insert_id() + thread_id = self.cursor.lastrowid stmt = """ INSERT INTO %sposts @@ -711,7 +705,7 @@ def do_POST(self, group_name, lines, ip_address, username=''): 0 )""" % (prefix, thread_id, forum_id, poster_id) self.cursor.execute(stmt, (self.encode_ip(ip_address), post_username)) - new_id = self.cursor.insert_id() + new_id = self.cursor.lastrowid if not new_id: return None else: diff --git a/storage/phpnuke_phpbb_mysql.py b/storage/phpnuke_phpbb_mysql.py index 7aca441..def8c89 100755 --- a/storage/phpnuke_phpbb_mysql.py +++ b/storage/phpnuke_phpbb_mysql.py @@ -218,8 +218,7 @@ def get_ARTICLE(self, group_name, id): A.post_username, MIN(D.post_id) FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) INNER JOIN %sposts D ON @@ -314,8 +313,7 @@ def get_HEAD(self, group_name, id): A.post_username, MIN(D.post_id) FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) INNER JOIN %stopics E ON @@ -366,8 +364,7 @@ def get_BODY(self, group_name, id): SELECT B.post_text FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) WHERE A.post_id=B.post_id AND A.forum_id=%s AND @@ -393,8 +390,7 @@ def get_XOVER(self, group_name, start_id, end_id='ggg'): B.post_text, A.post_username FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) LEFT JOIN %susers C ON @@ -453,8 +449,7 @@ def get_XPAT(self, group_name, header, pattern, start_id, end_id='ggg'): B.post_text, A.post_username FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) LEFT JOIN %susers C ON @@ -549,8 +544,7 @@ def get_XHDR(self, group_name, header, style, range): B.post_text, A.post_username FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) LEFT JOIN %stopics C ON @@ -661,7 +655,7 @@ def do_POST(self, group_name, lines, ip_address, username=''): 0 )""" % (prefix, forum_id, poster_id) self.cursor.execute(stmt, (subject,)) - thread_id = self.cursor.insert_id() + thread_id = self.cursor.lastrowid stmt = """ INSERT INTO %sposts @@ -689,7 +683,7 @@ def do_POST(self, group_name, lines, ip_address, username=''): 0 )""" % (prefix, thread_id, forum_id, poster_id) self.cursor.execute(stmt, (self.encode_ip(ip_address), post_username)) - new_id = self.cursor.insert_id() + new_id = self.cursor.lastrowid if not new_id: return None else: From fcbc432cbaa68f8176b1890dda284a7ff91b9861 Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 06:38:09 -0600 Subject: [PATCH 07/11] Fix XOVER threading in the phpBB and PHP-Nuke backends XOVER used the topic id as the message number of the article a post replies to, so threads were mixed up (it only looked right for topics whose id happened to equal their first post's id). It now references the topic's first post, as HEAD already did. --- storage/phpbb_mysql.py | 2 +- storage/phpnuke_phpbb_mysql.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/phpbb_mysql.py b/storage/phpbb_mysql.py index ed6fe57..b6dcac2 100755 --- a/storage/phpbb_mysql.py +++ b/storage/phpbb_mysql.py @@ -407,7 +407,7 @@ def get_XOVER(self, group_name, start_id, end_id='ggg'): stmt = """ SELECT A.post_id, - A.topic_id, + D.topic_first_post_id, C.username, C.user_email, CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', D.topic_title) ELSE B.post_subject END, diff --git a/storage/phpnuke_phpbb_mysql.py b/storage/phpnuke_phpbb_mysql.py index def8c89..4bbc2be 100755 --- a/storage/phpnuke_phpbb_mysql.py +++ b/storage/phpnuke_phpbb_mysql.py @@ -382,7 +382,7 @@ def get_XOVER(self, group_name, start_id, end_id='ggg'): stmt = """ SELECT A.post_id, - A.topic_id, + D.topic_first_post_id, C.username, C.user_email, CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', D.topic_title) ELSE B.post_subject END, From 069a39737e98dc0a4aed174940bbc49fdcc1fa40 Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 06:38:38 -0600 Subject: [PATCH 08/11] Update the fake DB connection for autocommit() and lastrowid --- tests/test_sql_backends.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_sql_backends.py b/tests/test_sql_backends.py index f807cf1..f39cfcb 100644 --- a/tests/test_sql_backends.py +++ b/tests/test_sql_backends.py @@ -39,6 +39,7 @@ def __init__(self, width): self.width = width # number of columns in every fetched row self.statements = [] self.rowcount = 1 + self.lastrowid = 1 def execute(self, stmt, params=None): self.statements.append((stmt, params)) @@ -57,7 +58,10 @@ def insert_id(self): class FakeConnection: def __init__(self, cursor): self._cursor = cursor - self.autocommit = False + + def autocommit(self, on): + # MySQLdb style; psycopg2 code assigns 'conn.autocommit = True' instead + pass def cursor(self): return self._cursor From 7a58342b0189d3a231b9b8985d77a2c5d8974b56 Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 06:41:29 -0600 Subject: [PATCH 09/11] Accept wildmat patterns in NEWNEWS NEWNEWS takes a wildmat (e.g. comp.*), but the server refused anything except a bare '*' that wasn't an existing group name with 411. The mysql and Phorum backends then compared the pattern with '=' instead of LIKE, so it would not have matched anyway. --- papercut.py | 5 +++-- storage/mysql.py | 2 +- storage/phorum_mysql.py | 2 +- storage/phorum_pgsql.py | 2 +- tests/test_protocol.py | 10 ++++++++++ 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/papercut.py b/papercut.py index 9eca94f..5b2a4fa 100755 --- a/papercut.py +++ b/papercut.py @@ -287,8 +287,9 @@ def do_NEWNEWS(self): if (len(self.tokens) < 4) or (len(self.tokens) > 6): self.send_response(ERR_CMDSYNTAXERROR) return - # check to see if the group exists - if (self.tokens[1] != '*') and (not self.backend.group_exists(self.tokens[1])): + # check to see if the group exists (unless it's a wildmat pattern, + # which the backend matches against the group names) + if (not re.search(r'[*?\[,!]', self.tokens[1])) and (not self.backend.group_exists(self.tokens[1])): self.send_response(ERR_NOSUCHGROUP) return if (len(self.tokens) > 4) and (self.tokens[4] == 'GMT'): diff --git a/storage/mysql.py b/storage/mysql.py index 96856a8..fd5af2c 100755 --- a/storage/mysql.py +++ b/storage/mysql.py @@ -112,7 +112,7 @@ def get_NEWNEWS(self, ts, group='*'): FROM papercut_groups WHERE - name=%s + name LIKE %s ORDER BY name ASC""" self.cursor.execute(stmt, (group.replace('*', '%'),)) diff --git a/storage/phorum_mysql.py b/storage/phorum_mysql.py index 5a0e837..3a5ed1b 100755 --- a/storage/phorum_mysql.py +++ b/storage/phorum_mysql.py @@ -251,7 +251,7 @@ def get_NEWNEWS(self, ts, group='*'): FROM forums WHERE - nntp_group_name=%s + nntp_group_name LIKE %s ORDER BY nntp_group_name ASC""" self.cursor.execute(stmt, (group.replace('*', '%'),)) diff --git a/storage/phorum_pgsql.py b/storage/phorum_pgsql.py index e4bebe1..2255d5f 100755 --- a/storage/phorum_pgsql.py +++ b/storage/phorum_pgsql.py @@ -262,7 +262,7 @@ def get_NEWNEWS(self, ts, group='*'): FROM forums WHERE - nntp_group_name=%s + nntp_group_name LIKE %s ORDER BY nntp_group_name ASC""" self.cursor.execute(stmt, (group.replace('*', '%'),)) diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 4b81931..027c2dd 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -573,3 +573,13 @@ def test_client_disconnect_mid_session_is_handled(client): client.cmd("GROUP test.group") client.close() assert wait_for_log("Connection closed") + + +@pytest.mark.parametrize("pattern", ["comp.*", "*", "comp.lang.?", "comp.*,!comp.test"]) +def test_newnews_accepts_wildmat_patterns(client, backend, pattern): + # regression: any group argument other than '*' had to be an existing + # group, so wildmats like comp.* were refused with 411 + backend.returns["group_exists"] = False + assert code(client.cmd("NEWNEWS %s 20240105 000000" % pattern)) == "230" + client.read_multiline() + assert backend.called("get_NEWNEWS")[0][1] == pattern From 90335512f365235956742ade5f39770b7ee3b5f7 Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 06:41:29 -0600 Subject: [PATCH 10/11] Add Docker integration tests for the database backends docs/integration-testing-plan.md: docker-compose.test.yml runs MySQL 8.4 (and optionally 5.7) and PostgreSQL 16, with fixture schemas and data for the mysql, Phorum, phpBB, PHP-Nuke and PostNuke backends (loading the repo's own schema and patch scripts), Phorum config files and an SMTP sink. 114 tests drive the real server with nntplib, covering reading, posting (including moderator/mailing-list email), logins and injection payloads. Also a CI job and 'make test-integration'. The bugs these found are fixed in the preceding commits. --- .github/workflows/tests.yml | 14 ++ Makefile | 9 + README.md | 3 + docker-compose.test.yml | 62 +++++++ docs/integration-testing-plan.md | 34 ++-- docs/unit-testing-plan.md | 4 +- tests/docker/Dockerfile | 15 ++ tests/fixtures/phorum_settings/1.php | 5 + tests/fixtures/phorum_settings/forums.php | 6 + tests/fixtures/sql/mysql/01-databases.sql | 12 ++ tests/fixtures/sql/mysql/02-papercut.sql | 11 ++ tests/fixtures/sql/mysql/03-phorum.sql | 57 ++++++ tests/fixtures/sql/mysql/04-phpbb.sql | 78 +++++++++ tests/fixtures/sql/mysql/05-nuke.sql | 80 +++++++++ tests/fixtures/sql/mysql/06-postnuke.sql | 9 + tests/fixtures/sql/postgres/01-phorum.sql | 63 +++++++ tests/integration/conftest.py | 136 +++++++++++++++ tests/integration/test_mysql_backend.py | 148 ++++++++++++++++ tests/integration/test_phorum_backends.py | 188 ++++++++++++++++++++ tests/integration/test_phpbb_backends.py | 203 ++++++++++++++++++++++ 20 files changed, 1125 insertions(+), 12 deletions(-) create mode 100644 Makefile create mode 100644 docker-compose.test.yml create mode 100644 tests/docker/Dockerfile create mode 100644 tests/fixtures/phorum_settings/1.php create mode 100644 tests/fixtures/phorum_settings/forums.php create mode 100644 tests/fixtures/sql/mysql/01-databases.sql create mode 100644 tests/fixtures/sql/mysql/02-papercut.sql create mode 100644 tests/fixtures/sql/mysql/03-phorum.sql create mode 100644 tests/fixtures/sql/mysql/04-phpbb.sql create mode 100644 tests/fixtures/sql/mysql/05-nuke.sql create mode 100644 tests/fixtures/sql/mysql/06-postnuke.sql create mode 100644 tests/fixtures/sql/postgres/01-phorum.sql create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/test_mysql_backend.py create mode 100644 tests/integration/test_phorum_backends.py create mode 100644 tests/integration/test_phpbb_backends.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 325b30e..1d15108 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,3 +32,17 @@ jobs: with: name: coverage path: coverage.xml + + integration: + name: integration tests (MySQL 8.4, PostgreSQL 16) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Run the integration tests against real databases + run: docker compose -f docker-compose.test.yml run --rm papercut + - name: Database logs + if: failure() + run: docker compose -f docker-compose.test.yml logs mysql postgres + - name: Stop the databases + if: always() + run: docker compose -f docker-compose.test.yml down -v diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ccd922a --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +test: + python -m pytest + +# needs Docker; see docs/integration-testing-plan.md +test-integration: + docker compose -f docker-compose.test.yml run --rm papercut + docker compose -f docker-compose.test.yml down -v + +.PHONY: test test-integration diff --git a/README.md b/README.md index 568e976..31b2d4a 100755 --- a/README.md +++ b/README.md @@ -29,6 +29,9 @@ See [INSTALL](INSTALL) for step by step instructions. ```sh pip install -r requirements-dev.txt pytest + +# the database backends against real MySQL and PostgreSQL (needs Docker) +make test-integration ``` See [docs/unit-testing-plan.md](docs/unit-testing-plan.md) and diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..d6f9612 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,62 @@ +# Docker integration tests: see docs/integration-testing-plan.md +# +# docker compose -f docker-compose.test.yml run --rm papercut +# +# (add --profile mysql57 to also test against MySQL 5.7) +services: + mysql: + image: mysql:8.4 + environment: + MYSQL_ROOT_PASSWORD: papercut + volumes: + - ./tests/fixtures/sql/mysql:/docker-entrypoint-initdb.d:ro + - ./storage:/papercut-storage:ro + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-ppapercut"] + interval: 2s + timeout: 5s + retries: 60 + + mysql57: + profiles: [mysql57] + image: mysql:5.7 + platform: linux/amd64 + environment: + MYSQL_ROOT_PASSWORD: papercut + volumes: + - ./tests/fixtures/sql/mysql:/docker-entrypoint-initdb.d:ro + - ./storage:/papercut-storage:ro + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-ppapercut"] + interval: 2s + timeout: 5s + retries: 90 + + postgres: + image: postgres:16 + environment: + POSTGRES_PASSWORD: papercut + volumes: + - ./tests/fixtures/sql/postgres:/docker-entrypoint-initdb.d:ro + - ./storage:/papercut-storage:ro + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + interval: 2s + timeout: 5s + retries: 60 + + papercut: + build: + context: . + dockerfile: tests/docker/Dockerfile + volumes: + - .:/src + environment: + PAPERCUT_TEST_MYSQL_HOSTS: mysql + PAPERCUT_TEST_POSTGRES_HOST: postgres + depends_on: + mysql: + condition: service_healthy + postgres: + condition: service_healthy + command: ["python", "-m", "pytest", "-m", "integration", "-p", "no:cacheprovider", "-v"] diff --git a/docs/integration-testing-plan.md b/docs/integration-testing-plan.md index 76a06ea..b9048f6 100644 --- a/docs/integration-testing-plan.md +++ b/docs/integration-testing-plan.md @@ -1,6 +1,6 @@ # Plan: Docker integration tests for the database backends -**Status:** proposed +**Status:** done (MySQL 8.4 and PostgreSQL 16 in CI; MySQL 5.7 optional) **Related:** [unit testing plan](unit-testing-plan.md), PR #2 (Python 3 port and SQL injection fixes) ## Goal @@ -127,19 +127,31 @@ make test-integration requests, or nightly if it proves slow. The `mysql57` target runs only in the nightly job. -## Expected findings +## Findings -The point of this work is to find problems that only a real database shows. -Some are already visible in the code: +Running the backends against real databases found these bugs, all fixed with +the integration test that caught them: -| Backend | Issue | +| Backend | Bug | | --- | --- | -| `mysql` | `COUNT(*) AS check`: `CHECK` is a reserved word since MySQL 8.0.16, so this is a syntax error there | -| `phorum_mysql` | `GROUP BY id` with other columns in the SELECT, which MySQL 5.7+ rejects by default (`ONLY_FULL_GROUP_BY`) | -| `phorum_pgsql` | MySQL-only syntax: `LIMIT 1, 0`, `CURRENT_TIMESTAMP()` with parentheses | -| `phorum_mysql` / `phorum_pgsql` | `get_NEWNEWS` matches the group with `=` against a pattern that uses `%` as a wildcard (it should probably be `LIKE`) | - -Fix each one in its own commit, with the integration test that caught it. +| `mysql` | `storage/mysql_storage.sql` didn't load: `TYPE=MyISAM` (removed in 5.5) and `'0000-00-00'` defaults (rejected by 5.7+ strict mode); MyISAM's key limit is also too small for utf8mb4 | +| `mysql` | `COUNT(*) AS check`: `CHECK` is reserved since MySQL 8.0.16, so almost every command failed | +| all MySQL | autocommit was off, so on InnoDB **posts were never committed** and held locks | +| all MySQL | no connection character set, so non-ASCII text came back as mojibake | +| all SQL | `NEWNEWS` message-ids came out as `<(2,)@group>` | +| Phorum | moderator/mailing-list emails raised `UnicodeEncodeError` for non-ASCII posts, after saving them (the client was told the post failed) | +| `phorum_pgsql` | `get_STAT` reported every article missing (psycopg2's `execute()` returns `None`) | +| `phorum_pgsql` | MySQL-only syntax: `LIMIT 1, 0` (NEXT/LAST), `CURRENT_TIMESTAMP()` (new threads) | +| `phorum_pgsql_users` | ignored `dbhost` and `dbpass` | +| phpBB, PHP-Nuke | comma joins mixed with `LEFT JOIN` fail on MySQL 5.0.12+, breaking ARTICLE/HEAD/XOVER/XHDR/XPAT | +| phpBB, PHP-Nuke | `cursor.insert_id()` no longer exists, so every post failed | +| phpBB, PHP-Nuke | XOVER used the topic id as the replied-to article number, mixing up threads | +| server, `mysql`, Phorum | `NEWNEWS comp.* ...` was refused with 411 (only a bare `*` was allowed), and the backends compared the pattern with `=` instead of `LIKE` | + +Predicted from reading the code, but not borne out: + +- `ONLY_FULL_GROUP_BY`: the Phorum `GROUP BY id` is on the primary key, which + MySQL 5.7.5+ accepts. ## Out of scope diff --git a/docs/unit-testing-plan.md b/docs/unit-testing-plan.md index ad06719..bd0c521 100644 --- a/docs/unit-testing-plan.md +++ b/docs/unit-testing-plan.md @@ -1,6 +1,6 @@ # Plan: Unit tests -**Status:** in progress (steps 1-6 done; Docker integration tests remain) +**Status:** done **Related:** [integration testing plan](integration-testing-plan.md) ## Goal @@ -197,6 +197,8 @@ Mark these `xfail` with a reason, and fix them in follow-up PRs: - `papercut.py` `do_LISTGROUP`: `numbers.split('\r\n')` always returns at least one element, so an empty group never resets the current article. +- `papercut.py` `do_XROVER`: it calls `do_XHDR` with `REFERENCES`, but + `do_XHDR` only accepts `SUBJECT` and `FROM`, so `XROVER` always returns 501. Found and fixed in step 1, with regression tests: diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile new file mode 100644 index 0000000..977afa8 --- /dev/null +++ b/tests/docker/Dockerfile @@ -0,0 +1,15 @@ +# Test image for the Docker integration tests (docs/integration-testing-plan.md) +FROM python:3.13-slim + +# build dependencies for mysqlclient (the MySQLdb module) +RUN apt-get update \ + && apt-get install -y --no-install-recommends gcc pkg-config default-libmysqlclient-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt requirements-dev.txt /tmp/ +RUN pip install --no-cache-dir --root-user-action=ignore \ + -r /tmp/requirements.txt -r /tmp/requirements-dev.txt aiosmtpd + +# the repo is mounted here at run time +WORKDIR /src +ENV PYTHONDONTWRITEBYTECODE=1 diff --git a/tests/fixtures/phorum_settings/1.php b/tests/fixtures/phorum_settings/1.php new file mode 100644 index 0000000..57f9c81 --- /dev/null +++ b/tests/fixtures/phorum_settings/1.php @@ -0,0 +1,5 @@ + diff --git a/tests/fixtures/phorum_settings/forums.php b/tests/fixtures/phorum_settings/forums.php new file mode 100644 index 0000000..977445b --- /dev/null +++ b/tests/fixtures/phorum_settings/forums.php @@ -0,0 +1,6 @@ + diff --git a/tests/fixtures/sql/mysql/01-databases.sql b/tests/fixtures/sql/mysql/01-databases.sql new file mode 100644 index 0000000..ee45b2e --- /dev/null +++ b/tests/fixtures/sql/mysql/01-databases.sql @@ -0,0 +1,12 @@ +-- one database per backend family, and an unprivileged user for Papercut +CREATE DATABASE papercut CHARACTER SET utf8mb4; +CREATE DATABASE phorum CHARACTER SET utf8mb4; +CREATE DATABASE phpbb CHARACTER SET utf8mb4; +CREATE DATABASE nuke CHARACTER SET utf8mb4; +CREATE DATABASE postnuke CHARACTER SET utf8mb4; +CREATE USER 'papercut'@'%' IDENTIFIED BY 'papercut'; +GRANT SELECT, INSERT, UPDATE, DELETE ON papercut.* TO 'papercut'@'%'; +GRANT SELECT, INSERT, UPDATE, DELETE ON phorum.* TO 'papercut'@'%'; +GRANT SELECT, INSERT, UPDATE, DELETE ON phpbb.* TO 'papercut'@'%'; +GRANT SELECT, INSERT, UPDATE, DELETE ON nuke.* TO 'papercut'@'%'; +GRANT SELECT, INSERT, UPDATE, DELETE ON postnuke.* TO 'papercut'@'%'; diff --git a/tests/fixtures/sql/mysql/02-papercut.sql b/tests/fixtures/sql/mysql/02-papercut.sql new file mode 100644 index 0000000..91f6ae1 --- /dev/null +++ b/tests/fixtures/sql/mysql/02-papercut.sql @@ -0,0 +1,11 @@ +SET NAMES utf8mb4; +USE papercut; +-- the 'mysql' storage backend and auth module: the schema shipped in the repo +SOURCE /papercut-storage/mysql_storage.sql; +INSERT INTO papercut_groups (name, active, description, table_name) + VALUES ('papercut.test', 1, 'Test group', 'papercut_default_table'); +INSERT INTO papercut_default_table (id, datestamp, thread, parent, author, subject, bytes, line_num, host, body) VALUES + (1, '2024-01-01 10:00:00', 1, 0, 'Alice', 'First post', 11, 1, '10.0.0.1', 'Hello world'), + (2, '2024-01-02 10:00:00', 1, 1, 'Bob', 'Re: First post', 14, 2, '10.0.0.2', 'Olá, mundo\n.dot'), + (3, '2024-01-03 10:00:00', 3, 0, 'Carol', 'Another thread', 5, 1, '10.0.0.3', 'third'); +INSERT INTO papercut_groups_auth (name, username, password) VALUES ('papercut.test', 'alice', 'secret'); diff --git a/tests/fixtures/sql/mysql/03-phorum.sql b/tests/fixtures/sql/mysql/03-phorum.sql new file mode 100644 index 0000000..206470d --- /dev/null +++ b/tests/fixtures/sql/mysql/03-phorum.sql @@ -0,0 +1,57 @@ +SET NAMES utf8mb4; +USE phorum; +-- Phorum 3 tables (only the columns Papercut uses), then the repo's patch script +CREATE TABLE forums ( + id int unsigned NOT NULL PRIMARY KEY, + name varchar(50) NOT NULL DEFAULT '', + active smallint NOT NULL DEFAULT 1, + description varchar(255) NOT NULL DEFAULT '', + table_name varchar(50) NOT NULL DEFAULT '', + email_list varchar(50) NOT NULL DEFAULT '', + email_return varchar(50) NOT NULL DEFAULT '' +); +CREATE TABLE forum_1 ( + id int unsigned NOT NULL PRIMARY KEY, + datestamp datetime NOT NULL, + thread int unsigned NOT NULL DEFAULT 0, + parent int unsigned NOT NULL DEFAULT 0, + author varchar(50) NOT NULL DEFAULT '', + subject varchar(255) NOT NULL DEFAULT '', + email varchar(200) NOT NULL DEFAULT '', + host varchar(50) NOT NULL DEFAULT '', + email_reply char(1) NOT NULL DEFAULT 'N', + approved char(1) NOT NULL DEFAULT 'N', + msgid varchar(100) NOT NULL DEFAULT '', + modifystamp int unsigned NOT NULL DEFAULT 0, + userid int unsigned NOT NULL DEFAULT 0 +); +CREATE TABLE forum_1_bodies ( + id int unsigned NOT NULL PRIMARY KEY, + body text NOT NULL, + thread int unsigned NOT NULL DEFAULT 0 +); +CREATE TABLE forums_auth ( + id int unsigned NOT NULL PRIMARY KEY, + name varchar(50) NOT NULL DEFAULT '', + username varchar(50) NOT NULL DEFAULT '', + password varchar(50) NOT NULL DEFAULT '', + email varchar(200) NOT NULL DEFAULT '' +); +CREATE TABLE forums_moderators ( + user_id int unsigned NOT NULL, + forum_id int unsigned NOT NULL +); +SOURCE /papercut-storage/phorum_mysql_fix.sql; +INSERT INTO forums (id, nntp_group_name, name, description, table_name, email_list, email_return) + VALUES (1, 'phorum.test', 'Test forum', 'A Phorum test forum', 'forum_1', 'list@example.com', 'return@example.com'); +INSERT INTO forum_1 (id, datestamp, thread, parent, author, subject, email, host, approved, modifystamp) VALUES + (1, '2024-01-01 10:00:00', 1, 0, 'Alice', 'First post', 'alice@example.com', '10.0.0.1', 'Y', 1704103200), + (2, '2024-01-02 10:00:00', 1, 1, 'Bob', 'Re: First post', 'bob@example.com', '10.0.0.2', 'Y', 1704189600), + (3, '2024-01-03 10:00:00', 3, 0, 'Spammer', 'Not approved', 'spam@example.com', '10.0.0.3', 'N', 1704276000), + (4, '2024-01-04 10:00:00', 4, 0, 'Carol', 'Olá mundo', '', '10.0.0.4', 'Y', 1704362400); +INSERT INTO forum_1_bodies (id, body, thread) VALUES + (1, 'Hello world', 1), (2, 'A reply\n.leading dot', 1), (3, 'spam', 3), (4, 'Corpo com acentuação', 4); +INSERT INTO forums_auth (id, name, username, password, email) VALUES + (1, 'Moderator', 'moderator', MD5('secret'), 'mod@example.com'), + (2, 'Old User', 'olduser', 'sefjKaLm7zybE', 'old@example.com'); +INSERT INTO forums_moderators (user_id, forum_id) VALUES (1, 1); diff --git a/tests/fixtures/sql/mysql/04-phpbb.sql b/tests/fixtures/sql/mysql/04-phpbb.sql new file mode 100644 index 0000000..f6959d7 --- /dev/null +++ b/tests/fixtures/sql/mysql/04-phpbb.sql @@ -0,0 +1,78 @@ +SET NAMES utf8mb4; +USE phpbb; +-- phpBB 2 tables (only the columns Papercut uses), then the repo's patch script +CREATE TABLE phpbb_forums ( + forum_id smallint unsigned NOT NULL PRIMARY KEY, + forum_name varchar(150) NOT NULL DEFAULT '', + forum_desc text NOT NULL, + forum_posts int unsigned NOT NULL DEFAULT 0, + forum_topics int unsigned NOT NULL DEFAULT 0, + forum_last_post_id int unsigned NOT NULL DEFAULT 0, + auth_view tinyint NOT NULL DEFAULT 0 +); +CREATE TABLE phpbb_topics ( + topic_id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + forum_id smallint unsigned NOT NULL DEFAULT 0, + topic_title char(60) NOT NULL DEFAULT '', + topic_poster int NOT NULL DEFAULT 0, + topic_time int NOT NULL DEFAULT 0, + topic_replies int unsigned NOT NULL DEFAULT 0, + topic_status tinyint NOT NULL DEFAULT 0, + topic_vote tinyint NOT NULL DEFAULT 0, + topic_type tinyint NOT NULL DEFAULT 0, + topic_first_post_id int unsigned NOT NULL DEFAULT 0, + topic_last_post_id int unsigned NOT NULL DEFAULT 0 +); +CREATE TABLE phpbb_posts ( + post_id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + topic_id int unsigned NOT NULL DEFAULT 0, + forum_id smallint unsigned NOT NULL DEFAULT 0, + poster_id int NOT NULL DEFAULT 0, + post_time int NOT NULL DEFAULT 0, + poster_ip char(8) NOT NULL DEFAULT '', + post_username varchar(25) NOT NULL DEFAULT '', + enable_bbcode tinyint NOT NULL DEFAULT 1, + enable_html tinyint NOT NULL DEFAULT 0, + enable_smilies tinyint NOT NULL DEFAULT 1, + enable_sig tinyint NOT NULL DEFAULT 1 +); +CREATE TABLE phpbb_posts_text ( + post_id int unsigned NOT NULL PRIMARY KEY, + bbcode_uid char(32) NOT NULL DEFAULT '', + post_subject char(60) NOT NULL DEFAULT '', + post_text text NOT NULL +); +CREATE TABLE phpbb_users ( + user_id int NOT NULL PRIMARY KEY, + username varchar(25) NOT NULL DEFAULT '', + user_password varchar(32) NOT NULL DEFAULT '', + user_email varchar(255) NOT NULL DEFAULT '', + user_posts int unsigned NOT NULL DEFAULT 0 +); +CREATE TABLE phpbb_auth_access ( + group_id int NOT NULL DEFAULT 0, + forum_id smallint unsigned NOT NULL DEFAULT 0, + auth_view tinyint NOT NULL DEFAULT 0 +); +CREATE TABLE phpbb_user_group ( + group_id int NOT NULL DEFAULT 0, + user_id int NOT NULL DEFAULT 0 +); +SOURCE /papercut-storage/phpbb_mysql_fix.sql; + +INSERT INTO phpbb_forums (forum_id, forum_name, nntp_group_name, forum_desc, forum_posts, forum_topics, forum_last_post_id, auth_view) + VALUES (1, 'Test forum', 'phpbb.test', 'A phpBB test forum', 3, 2, 3, 0); +INSERT INTO phpbb_users (user_id, username, user_password, user_email, user_posts) VALUES + (-1, 'Anonymous', '', '', 0), + (2, 'alice', MD5('secret'), 'alice@example.com', 2); +INSERT INTO phpbb_topics (topic_id, forum_id, topic_title, topic_poster, topic_time, topic_replies, topic_first_post_id, topic_last_post_id) VALUES + (1, 1, 'First topic', 2, 1704103200, 1, 1, 2), + (2, 1, 'Olá mundo', -1, 1704276000, 0, 3, 3); +INSERT INTO phpbb_posts (post_id, topic_id, forum_id, poster_id, post_time, poster_ip, post_username) VALUES + (1, 1, 1, 2, 1704103200, '0a000001', ''), + (2, 1, 1, 2, 1704189600, '0a000001', ''), + (3, 2, 1, -1, 1704276000, '0a000002', 'Guest'); +INSERT INTO phpbb_posts_text (post_id, bbcode_uid, post_subject, post_text) VALUES + (1, 'abc', 'First topic', 'Hello from phpBB'), + (2, 'abc', '', 'A reply\n.leading dot'), + (3, 'abc', 'Olá mundo', 'Corpo com acentuação'); diff --git a/tests/fixtures/sql/mysql/05-nuke.sql b/tests/fixtures/sql/mysql/05-nuke.sql new file mode 100644 index 0000000..eaebe58 --- /dev/null +++ b/tests/fixtures/sql/mysql/05-nuke.sql @@ -0,0 +1,80 @@ +SET NAMES utf8mb4; +USE nuke; +-- PHP-Nuke: phpBB with the nuke_bb prefix, and users in nuke_users +CREATE TABLE nuke_bbforums ( + forum_id smallint unsigned NOT NULL PRIMARY KEY, + forum_name varchar(150) NOT NULL DEFAULT '', + forum_desc text NOT NULL, + forum_posts int unsigned NOT NULL DEFAULT 0, + forum_topics int unsigned NOT NULL DEFAULT 0, + forum_last_post_id int unsigned NOT NULL DEFAULT 0, + auth_view tinyint NOT NULL DEFAULT 0 +); +CREATE TABLE nuke_bbtopics ( + topic_id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + forum_id smallint unsigned NOT NULL DEFAULT 0, + topic_title char(60) NOT NULL DEFAULT '', + topic_poster int NOT NULL DEFAULT 0, + topic_time int NOT NULL DEFAULT 0, + topic_replies int unsigned NOT NULL DEFAULT 0, + topic_status tinyint NOT NULL DEFAULT 0, + topic_vote tinyint NOT NULL DEFAULT 0, + topic_type tinyint NOT NULL DEFAULT 0, + topic_first_post_id int unsigned NOT NULL DEFAULT 0, + topic_last_post_id int unsigned NOT NULL DEFAULT 0 +); +CREATE TABLE nuke_bbposts ( + post_id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + topic_id int unsigned NOT NULL DEFAULT 0, + forum_id smallint unsigned NOT NULL DEFAULT 0, + poster_id int NOT NULL DEFAULT 0, + post_time int NOT NULL DEFAULT 0, + poster_ip char(8) NOT NULL DEFAULT '', + post_username varchar(25) NOT NULL DEFAULT '', + enable_bbcode tinyint NOT NULL DEFAULT 1, + enable_html tinyint NOT NULL DEFAULT 0, + enable_smilies tinyint NOT NULL DEFAULT 1, + enable_sig tinyint NOT NULL DEFAULT 1 +); +CREATE TABLE nuke_bbposts_text ( + post_id int unsigned NOT NULL PRIMARY KEY, + bbcode_uid char(32) NOT NULL DEFAULT '', + post_subject char(60) NOT NULL DEFAULT '', + post_text text NOT NULL +); +CREATE TABLE nuke_users ( + user_id int NOT NULL PRIMARY KEY, + username varchar(25) NOT NULL DEFAULT '', + user_password varchar(32) NOT NULL DEFAULT '', + user_email varchar(255) NOT NULL DEFAULT '', + user_posts int unsigned NOT NULL DEFAULT 0 +); +CREATE TABLE nuke_bbauth_access ( + group_id int NOT NULL DEFAULT 0, + forum_id smallint unsigned NOT NULL DEFAULT 0, + auth_view tinyint NOT NULL DEFAULT 0 +); +CREATE TABLE nuke_bbuser_group ( + group_id int NOT NULL DEFAULT 0, + user_id int NOT NULL DEFAULT 0 +); +-- (no patch script ships for PHP-Nuke; this is phpbb_mysql_fix.sql for nuke_bbforums) +ALTER TABLE nuke_bbforums ADD nntp_group_name VARCHAR(30) AFTER forum_name; +ALTER TABLE nuke_bbforums ADD UNIQUE (nntp_group_name); + +INSERT INTO nuke_bbforums (forum_id, forum_name, nntp_group_name, forum_desc, forum_posts, forum_topics, forum_last_post_id, auth_view) + VALUES (1, 'Test forum', 'nuke.test', 'A PHP-Nuke test forum', 3, 2, 3, 0); +INSERT INTO nuke_users (user_id, username, user_password, user_email, user_posts) VALUES + (-1, 'Anonymous', '', '', 0), + (2, 'alice', MD5('secret'), 'alice@example.com', 2); +INSERT INTO nuke_bbtopics (topic_id, forum_id, topic_title, topic_poster, topic_time, topic_replies, topic_first_post_id, topic_last_post_id) VALUES + (1, 1, 'First topic', 2, 1704103200, 1, 1, 2), + (2, 1, 'Olá mundo', -1, 1704276000, 0, 3, 3); +INSERT INTO nuke_bbposts (post_id, topic_id, forum_id, poster_id, post_time, poster_ip, post_username) VALUES + (1, 1, 1, 2, 1704103200, '0a000001', ''), + (2, 1, 1, 2, 1704189600, '0a000001', ''), + (3, 2, 1, -1, 1704276000, '0a000002', 'Guest'); +INSERT INTO nuke_bbposts_text (post_id, bbcode_uid, post_subject, post_text) VALUES + (1, 'abc', 'First topic', 'Hello from PHP-Nuke'), + (2, 'abc', '', 'A reply\n.leading dot'), + (3, 'abc', 'Olá mundo', 'Corpo com acentuação'); diff --git a/tests/fixtures/sql/mysql/06-postnuke.sql b/tests/fixtures/sql/mysql/06-postnuke.sql new file mode 100644 index 0000000..7c8fed7 --- /dev/null +++ b/tests/fixtures/sql/mysql/06-postnuke.sql @@ -0,0 +1,9 @@ +SET NAMES utf8mb4; +USE postnuke; +-- the postnuke_phpbb_mysql_users auth module only reads nuke_users +CREATE TABLE nuke_users ( + pn_uid int NOT NULL PRIMARY KEY, + pn_uname varchar(25) NOT NULL DEFAULT '', + pn_pass varchar(40) NOT NULL DEFAULT '' +); +INSERT INTO nuke_users (pn_uid, pn_uname, pn_pass) VALUES (2, 'alice', MD5('secret')); diff --git a/tests/fixtures/sql/postgres/01-phorum.sql b/tests/fixtures/sql/postgres/01-phorum.sql new file mode 100644 index 0000000..a90356f --- /dev/null +++ b/tests/fixtures/sql/postgres/01-phorum.sql @@ -0,0 +1,63 @@ +-- Phorum 3 on PostgreSQL (only the columns Papercut uses), then the repo's patch script +CREATE ROLE papercut LOGIN PASSWORD 'papercut'; +CREATE DATABASE phorum ENCODING 'UTF8'; +\connect phorum + +CREATE TABLE forums ( + id integer NOT NULL PRIMARY KEY, + name varchar(50) NOT NULL DEFAULT '', + active smallint NOT NULL DEFAULT 1, + description varchar(255) NOT NULL DEFAULT '', + table_name varchar(50) NOT NULL DEFAULT '', + email_list varchar(50) NOT NULL DEFAULT '', + email_return varchar(50) NOT NULL DEFAULT '' +); +CREATE TABLE forum_1 ( + id integer NOT NULL PRIMARY KEY, + datestamp timestamp NOT NULL, + thread integer NOT NULL DEFAULT 0, + parent integer NOT NULL DEFAULT 0, + author varchar(50) NOT NULL DEFAULT '', + subject varchar(255) NOT NULL DEFAULT '', + email varchar(200) NOT NULL DEFAULT '', + host varchar(50) NOT NULL DEFAULT '', + email_reply char(1) NOT NULL DEFAULT 'N', + approved char(1) NOT NULL DEFAULT 'N', + msgid varchar(100) NOT NULL DEFAULT '', + modifystamp integer NOT NULL DEFAULT 0, + userid integer NOT NULL DEFAULT 0 +); +CREATE TABLE forum_1_bodies ( + id integer NOT NULL PRIMARY KEY, + body text NOT NULL, + thread integer NOT NULL DEFAULT 0 +); +CREATE TABLE forums_auth ( + id integer NOT NULL PRIMARY KEY, + name varchar(50) NOT NULL DEFAULT '', + username varchar(50) NOT NULL DEFAULT '', + password varchar(50) NOT NULL DEFAULT '', + email varchar(200) NOT NULL DEFAULT '' +); +CREATE TABLE forums_moderators ( + user_id integer NOT NULL, + forum_id integer NOT NULL +); + +\i /papercut-storage/phorum_pgsql_fix.sql + +INSERT INTO forums (id, nntp_group_name, name, description, table_name, email_list, email_return) + VALUES (1, 'phorum.test', 'Test forum', 'A Phorum test forum', 'forum_1', 'list@example.com', 'return@example.com'); +INSERT INTO forum_1 (id, datestamp, thread, parent, author, subject, email, host, approved, modifystamp) VALUES + (1, '2024-01-01 10:00:00', 1, 0, 'Alice', 'First post', 'alice@example.com', '10.0.0.1', 'Y', 1704103200), + (2, '2024-01-02 10:00:00', 1, 1, 'Bob', 'Re: First post', 'bob@example.com', '10.0.0.2', 'Y', 1704189600), + (3, '2024-01-03 10:00:00', 3, 0, 'Spammer', 'Not approved', 'spam@example.com', '10.0.0.3', 'N', 1704276000), + (4, '2024-01-04 10:00:00', 4, 0, 'Carol', 'Olá mundo', '', '10.0.0.4', 'Y', 1704362400); +INSERT INTO forum_1_bodies (id, body, thread) VALUES + (1, 'Hello world', 1), (2, E'A reply\n.leading dot', 1), (3, 'spam', 3), (4, 'Corpo com acentuação', 4); +INSERT INTO forums_auth (id, name, username, password, email) VALUES + (1, 'Moderator', 'moderator', MD5('secret'), 'mod@example.com'), + (2, 'Old User', 'olduser', 'sefjKaLm7zybE', 'old@example.com'); +INSERT INTO forums_moderators (user_id, forum_id) VALUES (1, 1); + +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO papercut; diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..2f218f2 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,136 @@ +"""Docker integration tests: the SQL backends against real databases. + +Run with: docker compose -f docker-compose.test.yml run --rm papercut +(see docs/integration-testing-plan.md) +""" +import importlib +import os +import sys +import warnings + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +import fakes # noqa: E402 +import papercut # noqa: E402 +import settings # noqa: E402 + +with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + import nntplib # noqa: E402 + +FIXTURES = os.path.join(os.path.dirname(__file__), "..", "fixtures") +MYSQL_HOST = os.environ.get("PAPERCUT_TEST_MYSQL_HOSTS", "") +POSTGRES_HOST = os.environ.get("PAPERCUT_TEST_POSTGRES_HOST", "") + + +def pytest_collection_modifyitems(items): + for item in items: + if "integration" in str(item.fspath): + item.add_marker(pytest.mark.integration) + if not (MYSQL_HOST and POSTGRES_HOST): + item.add_marker(pytest.mark.skip(reason="needs the Docker test databases")) + + +def root_connection(kind, database): + """A privileged connection, for setting up and checking data.""" + if kind == "postgres": + import psycopg2 + conn = psycopg2.connect(host=POSTGRES_HOST, dbname=database, user="postgres", password="papercut") + conn.autocommit = True + return conn + import MySQLdb + conn = MySQLdb.connect(host=MYSQL_HOST, db=database, user="root", passwd="papercut", charset="utf8mb4") + conn.autocommit(True) + return conn + + +def query(kind, database, sql, params=None): + conn = root_connection(kind, database) + try: + cursor = conn.cursor() + cursor.execute(sql, params) + # lists of tuples, whichever driver + return [tuple(row) for row in cursor.fetchall()] if cursor.description else None + finally: + conn.close() + + +class Live: + """A running Papercut server for one backend, plus helpers.""" + + def __init__(self, server, backend, kind, database): + self.server = server + self.backend = backend + self.kind = kind + self.database = database + + @property + def port(self): + return self.server.server_address[1] + + def nntp(self, **kwargs): + return nntplib.NNTP("127.0.0.1", self.port, readermode=False, **kwargs) + + def raw(self): + return fakes.Client(self.port) + + def query(self, sql, params=None): + return query(self.kind, self.database, sql, params) + + +def start_live(monkeypatch, request, storage_module, kind, database, auth_module=None, **extra_settings): + host = POSTGRES_HOST if kind == "postgres" else MYSQL_HOST + values = dict(dbhost=host, dbname=database, dbuser="papercut", dbpass="papercut", + nntp_hostname="news.example.com", + phorum_settings_path=os.path.join(FIXTURES, "phorum_settings") + "/") + values.update(extra_settings) + for name, value in values.items(): + monkeypatch.setattr(settings, name, value) + if kind == "postgres": + # the PostgreSQL user is the database owner (see the postgres fixtures) + monkeypatch.setattr(settings, "dbuser", "papercut") + backend = importlib.import_module(storage_module).Papercut_Storage() + auth = importlib.import_module(auth_module).Papercut_Auth() if auth_module else None + server = fakes.start_server(papercut.NNTPRequestHandler, backend, auth) + request.addfinalizer(server.server_close) + request.addfinalizer(server.shutdown) + return Live(server, backend, kind, database) + + +@pytest.fixture(autouse=True) +def show_server_log(): + """Prints the server log after each test (pytest shows it on failure).""" + yield + if os.path.exists(settings.log_file): + with open(settings.log_file) as f: + errors = [line for line in f if "Error" in line or "Traceback" in line or line.startswith((" ", "MySQLdb", "psycopg2"))] + if errors: + print("".join(errors)) + + +class MailSink: + """Collects the messages sent to it over SMTP.""" + + def __init__(self): + self.messages = [] + + async def handle_DATA(self, server, session, envelope): + import email + import email.policy + msg = email.message_from_bytes(envelope.content, policy=email.policy.default) + # decoded headers and body, as a mail client would show them + text = "".join("%s: %s\n" % item for item in msg.items()) + "\n" + msg.get_content() + self.messages.append((envelope.mail_from, list(envelope.rcpt_tos), text)) + return "250 OK" + + +@pytest.fixture +def smtp(): + """An SMTP server on localhost:25, where the Phorum backends send mail.""" + from aiosmtpd.controller import Controller + sink = MailSink() + controller = Controller(sink, hostname="127.0.0.1", port=25) + controller.start() + yield sink + controller.stop() diff --git a/tests/integration/test_mysql_backend.py b/tests/integration/test_mysql_backend.py new file mode 100644 index 0000000..67a3e7d --- /dev/null +++ b/tests/integration/test_mysql_backend.py @@ -0,0 +1,148 @@ +"""The 'mysql' storage backend and auth module (storage/mysql_storage.sql).""" +import datetime +import io + +import pytest + +import settings +from conftest import start_live + +GROUP = "papercut.test" +EVIL = "x' OR '1'='1" + + +@pytest.fixture +def live(monkeypatch, request): + live = start_live(monkeypatch, request, "storage.mysql", "mysql", "papercut", auth_module="auth.mysql") + yield live + live.query("DELETE FROM papercut_default_table WHERE id > 3") + + +def test_list(live): + with live.nntp() as s: + _, groups = s.list() + assert [tuple(g) for g in groups] == [(GROUP, "3", "1", "y")] + + +def test_group(live): + with live.nntp() as s: + _, count, first, last, name = s.group(GROUP) + assert (count, first, last, name) == (3, 1, 3, GROUP) + + +def test_unknown_group(live): + with live.nntp() as s: + with pytest.raises(Exception, match="411"): + s.group("no.such.group") + + +def test_article_head_and_body(live): + with live.nntp() as s: + s.group(GROUP) + _, info = s.article("2") + lines = [l.decode() for l in info.lines] + assert "Subject: Re: First post" in lines + assert "References: <1@%s>" % GROUP in lines + assert lines[-2:] == ["Olá, mundo", ".dot"] + _, head = s.head("1") + assert "From: Alice" in [l.decode() for l in head.lines] + _, body = s.body("1") + assert [l.decode() for l in body.lines] == ["Hello world"] + + +def test_stat_next_last(live): + with live.nntp() as s: + s.group(GROUP) + assert s.stat("1")[1] == 1 + assert s.next()[1] == 2 + assert s.last()[1] == 1 + + +def test_xover(live): + with live.nntp() as s: + s.group(GROUP) + _, overviews = s.over((1, 3)) + assert [n for n, _ in overviews] == [1, 2, 3] + assert overviews[1][1]["subject"] == "Re: First post" + assert overviews[1][1]["references"] == "<1@%s>" % GROUP + + +def test_xhdr(live): + with live.nntp() as s: + s.group(GROUP) + _, headers = s.xhdr("subject", "1-3") + assert headers == [("1", "First post"), ("2", "Re: First post"), ("3", "Another thread")] + + +def test_xpat(live): + with live.nntp() as s: + s.group(GROUP) + _, lines = s._longcmdstring("XPAT Subject 1-3 *First*") + assert lines == ["1 First post", "2 Re: First post"] + + +def test_listgroup(live): + with live.nntp() as s: + _, lines = s._longcmdstring("LISTGROUP " + GROUP) + assert lines == ["1", "2", "3"] + + +def test_list_newsgroups(live): + with live.nntp() as s: + _, descriptions = s.descriptions("papercut.*") + assert descriptions == {GROUP: "Test group"} + + +def test_newnews(live): + with live.nntp() as s: + _, ids = s.newnews(GROUP, datetime.datetime(2024, 1, 2)) + assert ids == ["<2@%s>" % GROUP, "<3@%s>" % GROUP] + + +def test_post_and_read_back(live): + article = ("From: Tester \r\nNewsgroups: %s\r\nSubject: Posted héllo\r\n" + "References: <1@%s>\r\n\r\nposted body ✓\r\n.dot\r\n" % (GROUP, GROUP)) + with live.nntp() as s: + s.post(io.BytesIO(article.encode())) + _, count, first, last, _ = s.group(GROUP) + assert last == 4 + _, info = s.article("4") + lines = [l.decode() for l in info.lines] + assert "Subject: Posted héllo" in lines + assert "References: <1@%s>" % GROUP in lines + assert lines[-2:] == ["posted body ✓", ".dot"] + + +@pytest.fixture +def auth_on(monkeypatch): + monkeypatch.setattr(settings, "nntp_auth", "yes") + + +@pytest.mark.parametrize("username, password, accepted", [ + ("alice", "secret", True), + ("alice", "wrong", False), + ("nobody", "secret", False), + (EVIL, EVIL, False), + ("alice\\", "secret", False), +]) +def test_login(live, auth_on, username, password, accepted): + c = live.raw() + c.cmd("AUTHINFO USER " + username.replace(" ", "")) + result = c.cmd("AUTHINFO PASS " + password.replace(" ", "")) + assert result.startswith("281" if accepted else "502") + c.close() + + +@pytest.mark.parametrize("command", [ + "GROUP " + EVIL.replace(" ", ""), + "LISTGROUP " + EVIL.replace(" ", ""), + "XGTITLE " + EVIL.replace(" ", ""), + "NEWNEWS " + EVIL.replace(" ", "") + " 20240101 000000", +]) +def test_injection_payloads_are_harmless(live, command): + before = live.query("SELECT COUNT(*) FROM papercut_default_table") + c = live.raw() + response = c.cmd(command) + c.close() + assert response[:3] in ("211", "230", "282", "411", "481") + assert live.query("SELECT COUNT(*) FROM papercut_default_table") == before diff --git a/tests/integration/test_phorum_backends.py b/tests/integration/test_phorum_backends.py new file mode 100644 index 0000000..09ec3be --- /dev/null +++ b/tests/integration/test_phorum_backends.py @@ -0,0 +1,188 @@ +"""The Phorum storage and auth backends, on MySQL and PostgreSQL.""" +import datetime +import io + +import pytest + +import settings +from conftest import start_live + +GROUP = "phorum.test" +EVIL = "x' OR '1'='1" + +BACKENDS = { + "mysql": ("storage.phorum_mysql", "auth.phorum_mysql_users", "mysql"), + "pgsql": ("storage.phorum_pgsql", "auth.phorum_pgsql_users", "postgres"), +} + + +@pytest.fixture(params=sorted(BACKENDS)) +def live(monkeypatch, request): + storage, auth, kind = BACKENDS[request.param] + live = start_live(monkeypatch, request, storage, kind, "phorum", auth_module=auth) + yield live + live.query("DELETE FROM forum_1 WHERE id > 4") + live.query("DELETE FROM forum_1_bodies WHERE id > 4") + + +def lines_of(info): + return [l.decode() for l in info.lines] + + +def test_list(live): + with live.nntp() as s: + _, groups = s.list() + assert [tuple(g) for g in groups] == [(GROUP, "4", "1", "y")] + + +def test_group_counts_only_approved_messages(live): + with live.nntp() as s: + _, count, first, last, name = s.group(GROUP) + assert (count, first, last, name) == (3, 1, 4, GROUP) + + +def test_unapproved_messages_are_not_served(live): + with live.nntp() as s: + s.group(GROUP) + with pytest.raises(Exception, match="423"): + s.article("3") + _, numbers = s._longcmdstring("LISTGROUP " + GROUP) + assert numbers == ["1", "2", "4"] + + +def test_article(live): + with live.nntp() as s: + s.group(GROUP) + _, info = s.article("2") + lines = lines_of(info) + assert "From: Bob " in lines + assert "Subject: Re: First post" in lines + assert "References: <1@%s>" % GROUP in lines + assert lines[-2:] == ["A reply", ".leading dot"] + + +def test_non_ascii_content(live): + with live.nntp() as s: + s.group(GROUP) + _, head = s.head("4") + _, body = s.body("4") + assert "Subject: Olá mundo" in lines_of(head) + assert lines_of(body) == ["Corpo com acentuação"] + + +def test_stat_next_last(live): + with live.nntp() as s: + s.group(GROUP) + assert s.stat("1")[1] == 1 + assert s.next()[1] == 2 + assert s.next()[1] == 4 # 3 is not approved + assert s.last()[1] == 2 + + +def test_xover(live): + with live.nntp() as s: + s.group(GROUP) + _, overviews = s.over((1, 4)) + assert [n for n, _ in overviews] == [1, 2, 4] + assert overviews[1][1]["from"] == "Bob " + assert overviews[1][1]["references"] == "<1@%s>" % GROUP + + +def test_xhdr(live): + with live.nntp() as s: + s.group(GROUP) + _, headers = s.xhdr("subject", "1-4") + assert headers == [("1", "First post"), ("2", "Re: First post"), ("4", "Olá mundo")] + + +def test_xpat(live): + with live.nntp() as s: + s.group(GROUP) + _, lines = s._longcmdstring("XPAT Subject 1-4 *First*") + assert lines == ["1 First post", "2 Re: First post"] + + +def test_list_newsgroups(live): + with live.nntp() as s: + _, descriptions = s.descriptions("phorum.*") + assert descriptions == {GROUP: "A Phorum test forum"} + + +def test_newnews(live): + with live.nntp() as s: + _, ids = s.newnews(GROUP, datetime.datetime(2024, 1, 2)) + assert ids == ["<2@%s>" % GROUP, "<4@%s>" % GROUP] + + +def test_post_of_a_new_thread(live, smtp): + article = ("From: Tester \r\nNewsgroups: %s\r\nSubject: New thread\r\n\r\nfirst\r\n" % GROUP) + with live.nntp() as s: + s.post(io.BytesIO(article.encode())) + s.group(GROUP) + _, info = s.article("5") + lines = lines_of(info) + assert "Subject: New thread" in lines + assert not any(line.startswith("References:") for line in lines) + assert live.query("SELECT thread, parent FROM forum_1 WHERE id = 5") == [(5, 0)] + + +@pytest.mark.parametrize("subject, body", [ + ("Plain subject", "plain body"), + ("Assunto com acentuação", "corpo ✓"), +]) +def test_post_saves_and_notifies(live, smtp, subject, body): + article = ("From: Tester \r\nNewsgroups: %s\r\nSubject: %s\r\n" + "References: <1@%s>\r\n\r\n%s\r\n" % (GROUP, subject, GROUP, body)) + with live.nntp() as s: + s.post(io.BytesIO(article.encode())) + _, count, _, last, _ = s.group(GROUP) + assert (count, last) == (4, 5) + _, info = s.article("5") + lines = lines_of(info) + assert "Subject: %s" % subject in lines + assert "References: <1@%s>" % GROUP in lines # threaded under message 1 + assert lines[-1] == body + recipients = sorted(r for _, rcpts, _ in smtp.messages for r in rcpts) + assert recipients == ["list@example.com", "mod@example.com"] + assert all(body in text for _, _, text in smtp.messages) + + +@pytest.fixture +def auth_on(monkeypatch): + monkeypatch.setattr(settings, "nntp_auth", "yes") + + +@pytest.mark.parametrize("username, password, accepted", [ + ("moderator", "secret", True), # MD5 (newer Phorum) + ("olduser", "secret", True), # crypt (older Phorum) + ("moderator", "wrong", False), + ("olduser", "!secret", False), + ("nobody", "secret", False), + (EVIL.replace(" ", ""), "x", False), +]) +def test_login(live, auth_on, username, password, accepted): + c = live.raw() + c.cmd("AUTHINFO USER " + username) + assert c.cmd("AUTHINFO PASS " + password).startswith("281" if accepted else "502") + c.close() + + +@pytest.mark.parametrize("command", [ + "GROUP " + EVIL.replace(" ", ""), + "LISTGROUP " + EVIL.replace(" ", ""), + "XGTITLE " + EVIL.replace(" ", ""), + "NEWNEWS " + EVIL.replace(" ", "") + " 20240101 000000", +]) +def test_injection_payloads_are_harmless(live, command): + before = live.query("SELECT COUNT(*) FROM forum_1") + c = live.raw() + response = c.cmd(command) + c.close() + assert response[:3] in ("211", "230", "282", "411", "481") + assert live.query("SELECT COUNT(*) FROM forum_1") == before + + +def test_newnews_with_a_wildcard(live): + with live.nntp() as s: + _, ids = s.newnews("phorum.*", datetime.datetime(2024, 1, 2)) + assert ids == ["<2@%s>" % GROUP, "<4@%s>" % GROUP] diff --git a/tests/integration/test_phpbb_backends.py b/tests/integration/test_phpbb_backends.py new file mode 100644 index 0000000..8b28ae0 --- /dev/null +++ b/tests/integration/test_phpbb_backends.py @@ -0,0 +1,203 @@ +"""The phpBB and PHP-Nuke storage backends, and the phpBB-family auth modules.""" +import datetime +import io + +import pytest + +import settings +from conftest import start_live + +EVIL = "x'OR'1'='1" + +BACKENDS = { + "phpbb": dict(storage="storage.phpbb_mysql", auth="auth.phpbb_mysql_users", database="phpbb", + group="phpbb.test", prefix="phpbb_", users="phpbb_users", + settings=dict(phpbb_table_prefix="phpbb_")), + "nuke": dict(storage="storage.phpnuke_phpbb_mysql", auth="auth.phpnuke_phpbb_mysql_users", database="nuke", + group="nuke.test", prefix="nuke_bb", users="nuke_users", + settings=dict(phpbb_table_prefix="nuke_bb", nuke_table_prefix="nuke_")), +} + + +@pytest.fixture(params=sorted(BACKENDS)) +def live(monkeypatch, request): + config = BACKENDS[request.param] + live = start_live(monkeypatch, request, config["storage"], "mysql", config["database"], + auth_module=config["auth"], **config["settings"]) + live.group = config["group"] + live.prefix = config["prefix"] + reset(live, config) + yield live + reset(live, config) + + +def reset(live, config): + """Back to the fixture data (posts 1-3, topics 1-2).""" + p = config["prefix"] + live.query("DELETE FROM %sposts WHERE post_id > 3" % p) + live.query("DELETE FROM %sposts_text WHERE post_id > 3" % p) + live.query("DELETE FROM %stopics WHERE topic_id > 2" % p) + # InnoDB keeps counting after deletes; this resets to MAX(id) + 1 + live.query("ALTER TABLE %sposts AUTO_INCREMENT = 1" % p) + live.query("ALTER TABLE %stopics AUTO_INCREMENT = 1" % p) + live.query("UPDATE %sforums SET forum_posts=3, forum_topics=2, forum_last_post_id=3" % p) + live.query("UPDATE %stopics SET topic_replies=1, topic_last_post_id=2 WHERE topic_id=1" % p) + live.query("UPDATE %s SET user_posts=2 WHERE user_id=2" % config["users"]) + + +def lines_of(info): + return [l.decode() for l in info.lines] + + +def test_list(live): + with live.nntp() as s: + _, groups = s.list() + assert [tuple(g) for g in groups] == [(live.group, "3", "1", "y")] + + +def test_group(live): + with live.nntp() as s: + _, count, first, last, name = s.group(live.group) + assert (count, first, last, name) == (3, 1, 3, live.group) + + +def test_article(live): + with live.nntp() as s: + s.group(live.group) + _, info = s.article("2") + lines = lines_of(info) + assert "From: alice " in lines + # an empty post subject becomes 'Re: ' + assert "Subject: Re: First topic" in lines + assert "References: <1@%s>" % live.group in lines + assert lines[-2:] == ["A reply", ".leading dot"] + + +def test_guest_posts_and_non_ascii(live): + with live.nntp() as s: + s.group(live.group) + _, head = s.head("3") + _, body = s.body("3") + head = lines_of(head) + assert "From: Guest" in head + assert "Subject: Olá mundo" in head + assert not any(l.startswith("References:") for l in head) # first post of its topic + assert lines_of(body) == ["Corpo com acentuação"] + + +def test_stat_next_last(live): + with live.nntp() as s: + s.group(live.group) + assert s.stat("1")[1] == 1 + assert s.next()[1] == 2 + assert s.last()[1] == 1 + + +def test_xover(live): + with live.nntp() as s: + s.group(live.group) + _, overviews = s.over((1, 3)) + assert [n for n, _ in overviews] == [1, 2, 3] + assert overviews[1][1]["subject"] == "Re: First topic" + assert overviews[1][1]["references"] == "<1@%s>" % live.group + assert overviews[2][1]["references"] == "" + + +def test_xhdr(live): + with live.nntp() as s: + s.group(live.group) + _, headers = s.xhdr("subject", "1-3") + assert headers == [("1", "First topic"), ("2", "Re: First topic"), ("3", "Olá mundo")] + + +def test_xpat(live): + with live.nntp() as s: + s.group(live.group) + _, lines = s._longcmdstring("XPAT Subject 1-3 *mundo*") + assert lines == ["3 Olá mundo"] + + +def test_listgroup(live): + with live.nntp() as s: + _, lines = s._longcmdstring("LISTGROUP " + live.group) + assert lines == ["1", "2", "3"] + + +def test_list_newsgroups(live): + with live.nntp() as s: + _, descriptions = s.descriptions("*") + assert list(descriptions) == [live.group] + + +def test_newnews(live): + with live.nntp() as s: + _, ids = s.newnews(live.group, datetime.datetime(2024, 1, 2)) + assert ids == ["<2@%s>" % live.group, "<3@%s>" % live.group] + + +def post(live, subject, body, references=None): + lines = ["From: Tester ", "Newsgroups: " + live.group, "Subject: " + subject] + if references: + lines.append("References: " + references) + article = "\r\n".join(lines) + "\r\n\r\n" + body + "\r\n" + with live.nntp() as s: + s.post(io.BytesIO(article.encode())) + _, count, _, last, _ = s.group(live.group) + _, info = s.article(str(last)) + return count, last, lines_of(info) + + +def test_post_a_new_topic(live): + count, last, lines = post(live, "Novo tópico", "corpo ✓") + assert (count, last) == (4, 4) + assert "Subject: Novo tópico" in lines + assert lines[-1] == "corpo ✓" + topic = live.query("SELECT topic_id, topic_title FROM %stopics WHERE topic_first_post_id = 4" % live.prefix) + assert topic == [(3, "Novo tópico")] + + +def test_post_a_reply(live): + count, last, lines = post(live, "Re: First topic", "a reply", references="<1@%s>" % live.group) + assert (count, last) == (4, 4) + assert "References: <1@%s>" % live.group in lines + assert live.query("SELECT topic_id FROM %sposts WHERE post_id = 4" % live.prefix) == [(1,)] + assert live.query("SELECT topic_replies, topic_last_post_id FROM %stopics WHERE topic_id = 1" % live.prefix) == [(2, 4)] + + +@pytest.fixture +def auth_on(monkeypatch): + monkeypatch.setattr(settings, "nntp_auth", "yes") + + +@pytest.mark.parametrize("username, password, accepted", [ + ("alice", "secret", True), + ("alice", "wrong", False), + ("nobody", "secret", False), + (EVIL, EVIL, False), +]) +def test_login(live, auth_on, username, password, accepted): + c = live.raw() + c.cmd("AUTHINFO USER " + username) + assert c.cmd("AUTHINFO PASS " + password).startswith("281" if accepted else "502") + c.close() + + +def test_postnuke_login(monkeypatch, request, auth_on): + live = start_live(monkeypatch, request, "storage.phpbb_mysql", "mysql", "postnuke", + auth_module="auth.postnuke_phpbb_mysql_users") + for username, password, accepted in [("alice", "secret", True), ("alice", "nope", False), (EVIL, "x", False)]: + c = live.raw() + c.cmd("AUTHINFO USER " + username) + assert c.cmd("AUTHINFO PASS " + password).startswith("281" if accepted else "502") + c.close() + + +@pytest.mark.parametrize("command", ["GROUP " + EVIL, "LISTGROUP " + EVIL, "XGTITLE " + EVIL, + "NEWNEWS " + EVIL + " 20240101 000000"]) +def test_injection_payloads_are_harmless(live, command): + before = live.query("SELECT COUNT(*) FROM %sposts" % live.prefix) + c = live.raw() + response = c.cmd(command) + c.close() + assert response[:3] in ("211", "230", "282", "411", "481") + assert live.query("SELECT COUNT(*) FROM %sposts" % live.prefix) == before From eef397fdce3ba3dad1b68c615c5d3c7095c38252 Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 06:43:29 -0600 Subject: [PATCH 11/11] Don't require nntplib to collect the integration tests On Python 3.13+ nntplib comes from standard-nntplib, which only the Docker test image installs, so the unit test jobs failed while collecting tests/integration. --- tests/integration/conftest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2f218f2..b7a7067 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -15,10 +15,6 @@ import papercut # noqa: E402 import settings # noqa: E402 -with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - import nntplib # noqa: E402 - FIXTURES = os.path.join(os.path.dirname(__file__), "..", "fixtures") MYSQL_HOST = os.environ.get("PAPERCUT_TEST_MYSQL_HOSTS", "") POSTGRES_HOST = os.environ.get("PAPERCUT_TEST_POSTGRES_HOST", "") @@ -70,6 +66,11 @@ def port(self): return self.server.server_address[1] def nntp(self, **kwargs): + # imported here: on Python 3.13+ it comes from standard-nntplib, which + # only the Docker image installs (collecting these tests mustn't need it) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + import nntplib return nntplib.NNTP("127.0.0.1", self.port, readermode=False, **kwargs) def raw(self):