diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 325b30e..1d15108 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,3 +32,17 @@ jobs: with: name: coverage path: coverage.xml + + integration: + name: integration tests (MySQL 8.4, PostgreSQL 16) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Run the integration tests against real databases + run: docker compose -f docker-compose.test.yml run --rm papercut + - name: Database logs + if: failure() + run: docker compose -f docker-compose.test.yml logs mysql postgres + - name: Stop the databases + if: always() + run: docker compose -f docker-compose.test.yml down -v diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ccd922a --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +test: + python -m pytest + +# needs Docker; see docs/integration-testing-plan.md +test-integration: + docker compose -f docker-compose.test.yml run --rm papercut + docker compose -f docker-compose.test.yml down -v + +.PHONY: test test-integration diff --git a/README.md b/README.md index 568e976..31b2d4a 100755 --- a/README.md +++ b/README.md @@ -29,6 +29,9 @@ See [INSTALL](INSTALL) for step by step instructions. ```sh pip install -r requirements-dev.txt pytest + +# the database backends against real MySQL and PostgreSQL (needs Docker) +make test-integration ``` See [docs/unit-testing-plan.md](docs/unit-testing-plan.md) and diff --git a/auth/mysql.py b/auth/mysql.py index 7ef1025..cd1ab0c 100755 --- a/auth/mysql.py +++ b/auth/mysql.py @@ -10,7 +10,9 @@ class Papercut_Auth: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def is_valid_user(self, username, password): diff --git a/auth/phorum_mysql_users.py b/auth/phorum_mysql_users.py index 83cf0f9..4a2ca5c 100755 --- a/auth/phorum_mysql_users.py +++ b/auth/phorum_mysql_users.py @@ -17,7 +17,9 @@ class Papercut_Auth: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def is_valid_user(self, username, password): diff --git a/auth/phorum_pgsql_users.py b/auth/phorum_pgsql_users.py index cf48a2b..dbbd572 100755 --- a/auth/phorum_pgsql_users.py +++ b/auth/phorum_pgsql_users.py @@ -17,7 +17,9 @@ class Papercut_Auth: """ def __init__(self): - self.conn = psycopg2.connect(database=settings.dbname, user=settings.dbuser) + # (this used to ignore dbhost and dbpass, so it only worked over a + # local socket without a password) + self.conn = psycopg2.connect(host=settings.dbhost, database=settings.dbname, user=settings.dbuser, password=settings.dbpass) self.conn.autocommit = True self.cursor = self.conn.cursor() diff --git a/auth/phpbb_mysql_users.py b/auth/phpbb_mysql_users.py index 84ba067..e863b91 100755 --- a/auth/phpbb_mysql_users.py +++ b/auth/phpbb_mysql_users.py @@ -15,7 +15,9 @@ class Papercut_Auth: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def is_valid_user(self, username, password): diff --git a/auth/phpnuke_phpbb_mysql_users.py b/auth/phpnuke_phpbb_mysql_users.py index d750fe7..1f3b802 100755 --- a/auth/phpnuke_phpbb_mysql_users.py +++ b/auth/phpnuke_phpbb_mysql_users.py @@ -12,7 +12,9 @@ class Papercut_Auth: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def is_valid_user(self, username, password): diff --git a/auth/postnuke_phpbb_mysql_users.py b/auth/postnuke_phpbb_mysql_users.py index cf0f80a..3d24171 100755 --- a/auth/postnuke_phpbb_mysql_users.py +++ b/auth/postnuke_phpbb_mysql_users.py @@ -13,7 +13,9 @@ class Papercut_Auth: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def is_valid_user(self, username, password): diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..d6f9612 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,62 @@ +# Docker integration tests: see docs/integration-testing-plan.md +# +# docker compose -f docker-compose.test.yml run --rm papercut +# +# (add --profile mysql57 to also test against MySQL 5.7) +services: + mysql: + image: mysql:8.4 + environment: + MYSQL_ROOT_PASSWORD: papercut + volumes: + - ./tests/fixtures/sql/mysql:/docker-entrypoint-initdb.d:ro + - ./storage:/papercut-storage:ro + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-ppapercut"] + interval: 2s + timeout: 5s + retries: 60 + + mysql57: + profiles: [mysql57] + image: mysql:5.7 + platform: linux/amd64 + environment: + MYSQL_ROOT_PASSWORD: papercut + volumes: + - ./tests/fixtures/sql/mysql:/docker-entrypoint-initdb.d:ro + - ./storage:/papercut-storage:ro + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-ppapercut"] + interval: 2s + timeout: 5s + retries: 90 + + postgres: + image: postgres:16 + environment: + POSTGRES_PASSWORD: papercut + volumes: + - ./tests/fixtures/sql/postgres:/docker-entrypoint-initdb.d:ro + - ./storage:/papercut-storage:ro + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + interval: 2s + timeout: 5s + retries: 60 + + papercut: + build: + context: . + dockerfile: tests/docker/Dockerfile + volumes: + - .:/src + environment: + PAPERCUT_TEST_MYSQL_HOSTS: mysql + PAPERCUT_TEST_POSTGRES_HOST: postgres + depends_on: + mysql: + condition: service_healthy + postgres: + condition: service_healthy + command: ["python", "-m", "pytest", "-m", "integration", "-p", "no:cacheprovider", "-v"] diff --git a/docs/integration-testing-plan.md b/docs/integration-testing-plan.md index 76a06ea..b9048f6 100644 --- a/docs/integration-testing-plan.md +++ b/docs/integration-testing-plan.md @@ -1,6 +1,6 @@ # Plan: Docker integration tests for the database backends -**Status:** proposed +**Status:** done (MySQL 8.4 and PostgreSQL 16 in CI; MySQL 5.7 optional) **Related:** [unit testing plan](unit-testing-plan.md), PR #2 (Python 3 port and SQL injection fixes) ## Goal @@ -127,19 +127,31 @@ make test-integration requests, or nightly if it proves slow. The `mysql57` target runs only in the nightly job. -## Expected findings +## Findings -The point of this work is to find problems that only a real database shows. -Some are already visible in the code: +Running the backends against real databases found these bugs, all fixed with +the integration test that caught them: -| Backend | Issue | +| Backend | Bug | | --- | --- | -| `mysql` | `COUNT(*) AS check`: `CHECK` is a reserved word since MySQL 8.0.16, so this is a syntax error there | -| `phorum_mysql` | `GROUP BY id` with other columns in the SELECT, which MySQL 5.7+ rejects by default (`ONLY_FULL_GROUP_BY`) | -| `phorum_pgsql` | MySQL-only syntax: `LIMIT 1, 0`, `CURRENT_TIMESTAMP()` with parentheses | -| `phorum_mysql` / `phorum_pgsql` | `get_NEWNEWS` matches the group with `=` against a pattern that uses `%` as a wildcard (it should probably be `LIKE`) | - -Fix each one in its own commit, with the integration test that caught it. +| `mysql` | `storage/mysql_storage.sql` didn't load: `TYPE=MyISAM` (removed in 5.5) and `'0000-00-00'` defaults (rejected by 5.7+ strict mode); MyISAM's key limit is also too small for utf8mb4 | +| `mysql` | `COUNT(*) AS check`: `CHECK` is reserved since MySQL 8.0.16, so almost every command failed | +| all MySQL | autocommit was off, so on InnoDB **posts were never committed** and held locks | +| all MySQL | no connection character set, so non-ASCII text came back as mojibake | +| all SQL | `NEWNEWS` message-ids came out as `<(2,)@group>` | +| Phorum | moderator/mailing-list emails raised `UnicodeEncodeError` for non-ASCII posts, after saving them (the client was told the post failed) | +| `phorum_pgsql` | `get_STAT` reported every article missing (psycopg2's `execute()` returns `None`) | +| `phorum_pgsql` | MySQL-only syntax: `LIMIT 1, 0` (NEXT/LAST), `CURRENT_TIMESTAMP()` (new threads) | +| `phorum_pgsql_users` | ignored `dbhost` and `dbpass` | +| phpBB, PHP-Nuke | comma joins mixed with `LEFT JOIN` fail on MySQL 5.0.12+, breaking ARTICLE/HEAD/XOVER/XHDR/XPAT | +| phpBB, PHP-Nuke | `cursor.insert_id()` no longer exists, so every post failed | +| phpBB, PHP-Nuke | XOVER used the topic id as the replied-to article number, mixing up threads | +| server, `mysql`, Phorum | `NEWNEWS comp.* ...` was refused with 411 (only a bare `*` was allowed), and the backends compared the pattern with `=` instead of `LIKE` | + +Predicted from reading the code, but not borne out: + +- `ONLY_FULL_GROUP_BY`: the Phorum `GROUP BY id` is on the primary key, which + MySQL 5.7.5+ accepts. ## Out of scope diff --git a/docs/unit-testing-plan.md b/docs/unit-testing-plan.md index ad06719..bd0c521 100644 --- a/docs/unit-testing-plan.md +++ b/docs/unit-testing-plan.md @@ -1,6 +1,6 @@ # Plan: Unit tests -**Status:** in progress (steps 1-6 done; Docker integration tests remain) +**Status:** done **Related:** [integration testing plan](integration-testing-plan.md) ## Goal @@ -197,6 +197,8 @@ Mark these `xfail` with a reason, and fix them in follow-up PRs: - `papercut.py` `do_LISTGROUP`: `numbers.split('\r\n')` always returns at least one element, so an empty group never resets the current article. +- `papercut.py` `do_XROVER`: it calls `do_XHDR` with `REFERENCES`, but + `do_XHDR` only accepts `SUBJECT` and `FROM`, so `XROVER` always returns 501. Found and fixed in step 1, with regression tests: diff --git a/papercut.py b/papercut.py index 9eca94f..5b2a4fa 100755 --- a/papercut.py +++ b/papercut.py @@ -287,8 +287,9 @@ def do_NEWNEWS(self): if (len(self.tokens) < 4) or (len(self.tokens) > 6): self.send_response(ERR_CMDSYNTAXERROR) return - # check to see if the group exists - if (self.tokens[1] != '*') and (not self.backend.group_exists(self.tokens[1])): + # check to see if the group exists (unless it's a wildmat pattern, + # which the backend matches against the group names) + if (not re.search(r'[*?\[,!]', self.tokens[1])) and (not self.backend.group_exists(self.tokens[1])): self.send_response(ERR_NOSUCHGROUP) return if (len(self.tokens) > 4) and (self.tokens[4] == 'GMT'): diff --git a/storage/mysql.py b/storage/mysql.py index 10a16b3..fd5af2c 100755 --- a/storage/mysql.py +++ b/storage/mysql.py @@ -22,7 +22,9 @@ class Papercut_Storage: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def get_body(self, lines): @@ -34,7 +36,7 @@ def get_header(self, lines): def group_exists(self, group_name): stmt = """ SELECT - COUNT(*) AS check + COUNT(*) AS total FROM papercut_groups WHERE @@ -46,7 +48,7 @@ def article_exists(self, group_name, style, range): table_name = self.get_table_name(group_name) stmt = """ SELECT - COUNT(*) AS check + COUNT(*) AS total FROM %s WHERE @@ -110,7 +112,7 @@ def get_NEWNEWS(self, ts, group='*'): FROM papercut_groups WHERE - name=%s + name LIKE %s ORDER BY name ASC""" self.cursor.execute(stmt, (group.replace('*', '%'),)) @@ -129,7 +131,7 @@ def get_NEWNEWS(self, ts, group='*'): continue ids = list(self.cursor.fetchall()) for id in ids: - articles.append("<%s@%s>" % (id, group)) + articles.append("<%s@%s>" % (id[0], group)) if len(articles) == 0: return '' else: diff --git a/storage/mysql_storage.sql b/storage/mysql_storage.sql index 587e835..ee8e9ea 100755 --- a/storage/mysql_storage.sql +++ b/storage/mysql_storage.sql @@ -8,7 +8,7 @@ CREATE TABLE papercut_groups ( PRIMARY KEY (id), KEY name (name), KEY active (active) -) TYPE=MyISAM; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS papercut_groups_auth; CREATE TABLE papercut_groups_auth ( @@ -20,12 +20,12 @@ CREATE TABLE papercut_groups_auth ( PRIMARY KEY (id), KEY name (name), KEY username (username) -) TYPE=MyISAM; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS papercut_default_table; CREATE TABLE papercut_default_table ( id int(10) unsigned NOT NULL default '0', - datestamp datetime NOT NULL default '0000-00-00 00:00:00', + datestamp datetime NOT NULL default CURRENT_TIMESTAMP, thread int(10) unsigned NOT NULL default '0', parent int(10) unsigned NOT NULL default '0', author varchar(255) NOT NULL default '', @@ -41,4 +41,4 @@ CREATE TABLE papercut_default_table ( KEY subject (subject), KEY thread (thread), KEY parent (parent) -) TYPE=MyISAM; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/storage/phorum_mysql.py b/storage/phorum_mysql.py index 87814c4..3a5ed1b 100755 --- a/storage/phorum_mysql.py +++ b/storage/phorum_mysql.py @@ -36,7 +36,9 @@ class Papercut_Storage: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def get_message_body(self, headers): @@ -186,7 +188,7 @@ def send_notifications(self, group_name, msg_id, thread_id, parent_id, msg_autho emails = self.get_notification_emails(forum_id) for recipient in emails: current_msg = msg_tpl % vars() - smtp.sendmail("Phorum <%s>" % (recipient), recipient, current_msg) + strutil.send_text_mail(smtp, "Phorum <%s>" % (recipient), recipient, current_msg) # XXX: Coding blind here. I really don't know much about how Phorum works with # XXX: sending forum postings as emails, but it's here. Let's call this a @@ -234,7 +236,7 @@ def send_notifications(self, group_name, msg_id, thread_id, parent_id, msg_autho from papercut import __VERSION__ phorum_version = settings.phorum_version current_msg = notification_mail_tpl % vars() - smtp.sendmail('%s %s' % (msg_author, msg_email), email_list, current_msg) + strutil.send_text_mail(smtp, '%s %s' % (msg_author, msg_email), email_list, current_msg) smtp.quit() def get_NEWGROUPS(self, ts, group='%'): @@ -249,7 +251,7 @@ def get_NEWNEWS(self, ts, group='*'): FROM forums WHERE - nntp_group_name=%s + nntp_group_name LIKE %s ORDER BY nntp_group_name ASC""" self.cursor.execute(stmt, (group.replace('*', '%'),)) @@ -269,7 +271,7 @@ def get_NEWNEWS(self, ts, group='*'): continue ids = list(self.cursor.fetchall()) for id in ids: - articles.append("<%s@%s>" % (id, group)) + articles.append("<%s@%s>" % (id[0], group)) if len(articles) == 0: return '' else: diff --git a/storage/phorum_pgsql.py b/storage/phorum_pgsql.py index 75d9914..2255d5f 100755 --- a/storage/phorum_pgsql.py +++ b/storage/phorum_pgsql.py @@ -198,7 +198,7 @@ def send_notifications(self, group_name, msg_id, thread_id, parent_id, msg_autho emails = self.get_notification_emails(forum_id) for recipient in emails: current_msg = msg_tpl % vars() - smtp.sendmail("Phorum <%s>" % (recipient), recipient, current_msg) + strutil.send_text_mail(smtp, "Phorum <%s>" % (recipient), recipient, current_msg) # XXX: Coding blind here. I really don't know much about how Phorum works with # XXX: sending forum postings as emails, but it's here. Let's call this a @@ -247,7 +247,7 @@ def send_notifications(self, group_name, msg_id, thread_id, parent_id, msg_autho from papercut import __VERSION__ phorum_version = settings.phorum_version current_msg = notification_mail_tpl % vars() - smtp.sendmail('%s %s' % (msg_author, msg_email), email_list, current_msg) + strutil.send_text_mail(smtp, '%s %s' % (msg_author, msg_email), email_list, current_msg) smtp.quit() def get_NEWGROUPS(self, ts, group='%'): @@ -262,7 +262,7 @@ def get_NEWNEWS(self, ts, group='*'): FROM forums WHERE - nntp_group_name=%s + nntp_group_name LIKE %s ORDER BY nntp_group_name ASC""" self.cursor.execute(stmt, (group.replace('*', '%'),)) @@ -283,7 +283,7 @@ def get_NEWNEWS(self, ts, group='*'): continue ids = list(self.cursor.fetchall()) for id in ids: - articles.append("<%s@%s>" % (id, group)) + articles.append("<%s@%s>" % (id[0], group)) if len(articles) == 0: return '' else: @@ -329,7 +329,10 @@ def get_STAT(self, group_name, id): WHERE approved='Y' AND id=%s""" % (table_name, int(id)) - return self.cursor.execute(stmt) + # (psycopg2's execute() always returns None, so this used to report + # every article as missing) + self.cursor.execute(stmt) + return self.cursor.rowcount > 0 def get_ARTICLE(self, group_name, id): table_name = self.get_table_name(group_name) @@ -383,7 +386,7 @@ def get_LAST(self, group_name, current_id): id < %s ORDER BY id DESC - LIMIT 1, 0""" % (table_name, int(current_id)) + LIMIT 1""" % (table_name, int(current_id)) self.cursor.execute(stmt) num_rows = self.cursor.rowcount if num_rows == 0: @@ -402,7 +405,7 @@ def get_NEXT(self, group_name, current_id): id > %s ORDER BY id ASC - LIMIT 1, 0""" % (table_name, int(current_id)) + LIMIT 1""" % (table_name, int(current_id)) self.cursor.execute(stmt) num_rows = self.cursor.rowcount if num_rows == 0: @@ -694,7 +697,7 @@ def do_POST(self, group_name, lines, ip_address, username=''): stmt = """ SELECT MAX(id) AS next_id, - DATE_PART('epoch', CURRENT_TIMESTAMP()) + DATE_PART('epoch', CURRENT_TIMESTAMP) FROM %s""" % (table_name) self.cursor.execute(stmt) diff --git a/storage/phpbb_mysql.py b/storage/phpbb_mysql.py index a2212d6..b6dcac2 100755 --- a/storage/phpbb_mysql.py +++ b/storage/phpbb_mysql.py @@ -31,7 +31,9 @@ class Papercut_Storage: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def get_message_body(self, headers): @@ -153,7 +155,7 @@ def get_NEWNEWS(self, ts, group='*'): continue ids = list(self.cursor.fetchall()) for id in ids: - articles.append("<%s@%s>" % (id, group_name)) + articles.append("<%s@%s>" % (id[0], group_name)) if len(articles) == 0: return '' else: @@ -243,8 +245,7 @@ def get_ARTICLE(self, group_name, id): A.post_username, MIN(D.post_id) FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) INNER JOIN %sposts D ON @@ -338,8 +339,7 @@ def get_HEAD(self, group_name, id): A.post_username, MIN(D.post_id) FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) INNER JOIN %stopics E ON @@ -390,8 +390,7 @@ def get_BODY(self, group_name, id): SELECT B.post_text FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) WHERE A.post_id=B.post_id AND A.forum_id=%s AND @@ -408,7 +407,7 @@ def get_XOVER(self, group_name, start_id, end_id='ggg'): stmt = """ SELECT A.post_id, - A.topic_id, + D.topic_first_post_id, C.username, C.user_email, CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', D.topic_title) ELSE B.post_subject END, @@ -416,8 +415,7 @@ def get_XOVER(self, group_name, start_id, end_id='ggg'): B.post_text, A.post_username FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) LEFT JOIN %susers C ON @@ -475,8 +473,7 @@ def get_XPAT(self, group_name, header, pattern, start_id, end_id='ggg'): B.post_text, A.post_username FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) LEFT JOIN %susers C ON @@ -570,8 +567,7 @@ def get_XHDR(self, group_name, header, style, range): B.post_text, A.post_username FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) LEFT JOIN %stopics C ON @@ -681,7 +677,7 @@ def do_POST(self, group_name, lines, ip_address, username=''): 0 )""" % (prefix, forum_id, poster_id) self.cursor.execute(stmt, (subject,)) - thread_id = self.cursor.insert_id() + thread_id = self.cursor.lastrowid stmt = """ INSERT INTO %sposts @@ -709,7 +705,7 @@ def do_POST(self, group_name, lines, ip_address, username=''): 0 )""" % (prefix, thread_id, forum_id, poster_id) self.cursor.execute(stmt, (self.encode_ip(ip_address), post_username)) - new_id = self.cursor.insert_id() + new_id = self.cursor.lastrowid if not new_id: return None else: diff --git a/storage/phpnuke_phpbb_mysql.py b/storage/phpnuke_phpbb_mysql.py index 7f3284c..4bbc2be 100755 --- a/storage/phpnuke_phpbb_mysql.py +++ b/storage/phpnuke_phpbb_mysql.py @@ -30,7 +30,9 @@ class Papercut_Storage: """ def __init__(self): - self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass) + self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4") + # commit every statement (InnoDB tables would otherwise never see posts) + self.conn.autocommit(True) self.cursor = self.conn.cursor() def get_message_body(self, headers): @@ -152,7 +154,7 @@ def get_NEWNEWS(self, ts, group='*'): continue ids = list(self.cursor.fetchall()) for id in ids: - articles.append("<%s@%s>" % (id, group_name)) + articles.append("<%s@%s>" % (id[0], group_name)) if len(articles) == 0: return '' else: @@ -216,8 +218,7 @@ def get_ARTICLE(self, group_name, id): A.post_username, MIN(D.post_id) FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) INNER JOIN %sposts D ON @@ -312,8 +313,7 @@ def get_HEAD(self, group_name, id): A.post_username, MIN(D.post_id) FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) INNER JOIN %stopics E ON @@ -364,8 +364,7 @@ def get_BODY(self, group_name, id): SELECT B.post_text FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) WHERE A.post_id=B.post_id AND A.forum_id=%s AND @@ -383,7 +382,7 @@ def get_XOVER(self, group_name, start_id, end_id='ggg'): stmt = """ SELECT A.post_id, - A.topic_id, + D.topic_first_post_id, C.username, C.user_email, CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', D.topic_title) ELSE B.post_subject END, @@ -391,8 +390,7 @@ def get_XOVER(self, group_name, start_id, end_id='ggg'): B.post_text, A.post_username FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) LEFT JOIN %susers C ON @@ -451,8 +449,7 @@ def get_XPAT(self, group_name, header, pattern, start_id, end_id='ggg'): B.post_text, A.post_username FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) LEFT JOIN %susers C ON @@ -547,8 +544,7 @@ def get_XHDR(self, group_name, header, style, range): B.post_text, A.post_username FROM - %sposts A, - %sposts_text B + (%sposts A, %sposts_text B) LEFT JOIN %stopics C ON @@ -659,7 +655,7 @@ def do_POST(self, group_name, lines, ip_address, username=''): 0 )""" % (prefix, forum_id, poster_id) self.cursor.execute(stmt, (subject,)) - thread_id = self.cursor.insert_id() + thread_id = self.cursor.lastrowid stmt = """ INSERT INTO %sposts @@ -687,7 +683,7 @@ def do_POST(self, group_name, lines, ip_address, username=''): 0 )""" % (prefix, thread_id, forum_id, poster_id) self.cursor.execute(stmt, (self.encode_ip(ip_address), post_username)) - new_id = self.cursor.insert_id() + new_id = self.cursor.lastrowid if not new_id: return None else: diff --git a/storage/strutil.py b/storage/strutil.py index 07a303b..0ec9299 100755 --- a/storage/strutil.py +++ b/storage/strutil.py @@ -4,6 +4,7 @@ import time import re import quopri +from email.message import EmailMessage singleline_regexp = re.compile(r"^\.", re.M) @@ -76,6 +77,19 @@ def mime_decode_header(text): """ return encoded_word_regexp.sub(_decode_encoded_word, text) +def send_text_mail(smtp, from_addr, to_addr, text): + """Sends a plain-text email given as 'Header: value' lines, a blank + line and the body. Non-ASCII headers are RFC 2047 encoded and the body + is sent as UTF-8 (smtplib's sendmail() only accepts ASCII strings).""" + head, sep, body = text.replace('\r\n', '\n').partition('\n\n') + msg = EmailMessage() + for line in head.split('\n'): + name, _, value = line.partition(':') + if name and value: + msg[name.strip()] = value.strip() + msg.set_content(body) + smtp.send_message(msg, from_addr, [to_addr]) + def format_wildcards(pattern): return pattern.replace('*', '.*').replace('?', '.*') diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile new file mode 100644 index 0000000..977afa8 --- /dev/null +++ b/tests/docker/Dockerfile @@ -0,0 +1,15 @@ +# Test image for the Docker integration tests (docs/integration-testing-plan.md) +FROM python:3.13-slim + +# build dependencies for mysqlclient (the MySQLdb module) +RUN apt-get update \ + && apt-get install -y --no-install-recommends gcc pkg-config default-libmysqlclient-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt requirements-dev.txt /tmp/ +RUN pip install --no-cache-dir --root-user-action=ignore \ + -r /tmp/requirements.txt -r /tmp/requirements-dev.txt aiosmtpd + +# the repo is mounted here at run time +WORKDIR /src +ENV PYTHONDONTWRITEBYTECODE=1 diff --git a/tests/fixtures/phorum_settings/1.php b/tests/fixtures/phorum_settings/1.php new file mode 100644 index 0000000..57f9c81 --- /dev/null +++ b/tests/fixtures/phorum_settings/1.php @@ -0,0 +1,5 @@ + diff --git a/tests/fixtures/phorum_settings/forums.php b/tests/fixtures/phorum_settings/forums.php new file mode 100644 index 0000000..977445b --- /dev/null +++ b/tests/fixtures/phorum_settings/forums.php @@ -0,0 +1,6 @@ + diff --git a/tests/fixtures/sql/mysql/01-databases.sql b/tests/fixtures/sql/mysql/01-databases.sql new file mode 100644 index 0000000..ee45b2e --- /dev/null +++ b/tests/fixtures/sql/mysql/01-databases.sql @@ -0,0 +1,12 @@ +-- one database per backend family, and an unprivileged user for Papercut +CREATE DATABASE papercut CHARACTER SET utf8mb4; +CREATE DATABASE phorum CHARACTER SET utf8mb4; +CREATE DATABASE phpbb CHARACTER SET utf8mb4; +CREATE DATABASE nuke CHARACTER SET utf8mb4; +CREATE DATABASE postnuke CHARACTER SET utf8mb4; +CREATE USER 'papercut'@'%' IDENTIFIED BY 'papercut'; +GRANT SELECT, INSERT, UPDATE, DELETE ON papercut.* TO 'papercut'@'%'; +GRANT SELECT, INSERT, UPDATE, DELETE ON phorum.* TO 'papercut'@'%'; +GRANT SELECT, INSERT, UPDATE, DELETE ON phpbb.* TO 'papercut'@'%'; +GRANT SELECT, INSERT, UPDATE, DELETE ON nuke.* TO 'papercut'@'%'; +GRANT SELECT, INSERT, UPDATE, DELETE ON postnuke.* TO 'papercut'@'%'; diff --git a/tests/fixtures/sql/mysql/02-papercut.sql b/tests/fixtures/sql/mysql/02-papercut.sql new file mode 100644 index 0000000..91f6ae1 --- /dev/null +++ b/tests/fixtures/sql/mysql/02-papercut.sql @@ -0,0 +1,11 @@ +SET NAMES utf8mb4; +USE papercut; +-- the 'mysql' storage backend and auth module: the schema shipped in the repo +SOURCE /papercut-storage/mysql_storage.sql; +INSERT INTO papercut_groups (name, active, description, table_name) + VALUES ('papercut.test', 1, 'Test group', 'papercut_default_table'); +INSERT INTO papercut_default_table (id, datestamp, thread, parent, author, subject, bytes, line_num, host, body) VALUES + (1, '2024-01-01 10:00:00', 1, 0, 'Alice', 'First post', 11, 1, '10.0.0.1', 'Hello world'), + (2, '2024-01-02 10:00:00', 1, 1, 'Bob', 'Re: First post', 14, 2, '10.0.0.2', 'Olá, mundo\n.dot'), + (3, '2024-01-03 10:00:00', 3, 0, 'Carol', 'Another thread', 5, 1, '10.0.0.3', 'third'); +INSERT INTO papercut_groups_auth (name, username, password) VALUES ('papercut.test', 'alice', 'secret'); diff --git a/tests/fixtures/sql/mysql/03-phorum.sql b/tests/fixtures/sql/mysql/03-phorum.sql new file mode 100644 index 0000000..206470d --- /dev/null +++ b/tests/fixtures/sql/mysql/03-phorum.sql @@ -0,0 +1,57 @@ +SET NAMES utf8mb4; +USE phorum; +-- Phorum 3 tables (only the columns Papercut uses), then the repo's patch script +CREATE TABLE forums ( + id int unsigned NOT NULL PRIMARY KEY, + name varchar(50) NOT NULL DEFAULT '', + active smallint NOT NULL DEFAULT 1, + description varchar(255) NOT NULL DEFAULT '', + table_name varchar(50) NOT NULL DEFAULT '', + email_list varchar(50) NOT NULL DEFAULT '', + email_return varchar(50) NOT NULL DEFAULT '' +); +CREATE TABLE forum_1 ( + id int unsigned NOT NULL PRIMARY KEY, + datestamp datetime NOT NULL, + thread int unsigned NOT NULL DEFAULT 0, + parent int unsigned NOT NULL DEFAULT 0, + author varchar(50) NOT NULL DEFAULT '', + subject varchar(255) NOT NULL DEFAULT '', + email varchar(200) NOT NULL DEFAULT '', + host varchar(50) NOT NULL DEFAULT '', + email_reply char(1) NOT NULL DEFAULT 'N', + approved char(1) NOT NULL DEFAULT 'N', + msgid varchar(100) NOT NULL DEFAULT '', + modifystamp int unsigned NOT NULL DEFAULT 0, + userid int unsigned NOT NULL DEFAULT 0 +); +CREATE TABLE forum_1_bodies ( + id int unsigned NOT NULL PRIMARY KEY, + body text NOT NULL, + thread int unsigned NOT NULL DEFAULT 0 +); +CREATE TABLE forums_auth ( + id int unsigned NOT NULL PRIMARY KEY, + name varchar(50) NOT NULL DEFAULT '', + username varchar(50) NOT NULL DEFAULT '', + password varchar(50) NOT NULL DEFAULT '', + email varchar(200) NOT NULL DEFAULT '' +); +CREATE TABLE forums_moderators ( + user_id int unsigned NOT NULL, + forum_id int unsigned NOT NULL +); +SOURCE /papercut-storage/phorum_mysql_fix.sql; +INSERT INTO forums (id, nntp_group_name, name, description, table_name, email_list, email_return) + VALUES (1, 'phorum.test', 'Test forum', 'A Phorum test forum', 'forum_1', 'list@example.com', 'return@example.com'); +INSERT INTO forum_1 (id, datestamp, thread, parent, author, subject, email, host, approved, modifystamp) VALUES + (1, '2024-01-01 10:00:00', 1, 0, 'Alice', 'First post', 'alice@example.com', '10.0.0.1', 'Y', 1704103200), + (2, '2024-01-02 10:00:00', 1, 1, 'Bob', 'Re: First post', 'bob@example.com', '10.0.0.2', 'Y', 1704189600), + (3, '2024-01-03 10:00:00', 3, 0, 'Spammer', 'Not approved', 'spam@example.com', '10.0.0.3', 'N', 1704276000), + (4, '2024-01-04 10:00:00', 4, 0, 'Carol', 'Olá mundo', '', '10.0.0.4', 'Y', 1704362400); +INSERT INTO forum_1_bodies (id, body, thread) VALUES + (1, 'Hello world', 1), (2, 'A reply\n.leading dot', 1), (3, 'spam', 3), (4, 'Corpo com acentuação', 4); +INSERT INTO forums_auth (id, name, username, password, email) VALUES + (1, 'Moderator', 'moderator', MD5('secret'), 'mod@example.com'), + (2, 'Old User', 'olduser', 'sefjKaLm7zybE', 'old@example.com'); +INSERT INTO forums_moderators (user_id, forum_id) VALUES (1, 1); diff --git a/tests/fixtures/sql/mysql/04-phpbb.sql b/tests/fixtures/sql/mysql/04-phpbb.sql new file mode 100644 index 0000000..f6959d7 --- /dev/null +++ b/tests/fixtures/sql/mysql/04-phpbb.sql @@ -0,0 +1,78 @@ +SET NAMES utf8mb4; +USE phpbb; +-- phpBB 2 tables (only the columns Papercut uses), then the repo's patch script +CREATE TABLE phpbb_forums ( + forum_id smallint unsigned NOT NULL PRIMARY KEY, + forum_name varchar(150) NOT NULL DEFAULT '', + forum_desc text NOT NULL, + forum_posts int unsigned NOT NULL DEFAULT 0, + forum_topics int unsigned NOT NULL DEFAULT 0, + forum_last_post_id int unsigned NOT NULL DEFAULT 0, + auth_view tinyint NOT NULL DEFAULT 0 +); +CREATE TABLE phpbb_topics ( + topic_id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + forum_id smallint unsigned NOT NULL DEFAULT 0, + topic_title char(60) NOT NULL DEFAULT '', + topic_poster int NOT NULL DEFAULT 0, + topic_time int NOT NULL DEFAULT 0, + topic_replies int unsigned NOT NULL DEFAULT 0, + topic_status tinyint NOT NULL DEFAULT 0, + topic_vote tinyint NOT NULL DEFAULT 0, + topic_type tinyint NOT NULL DEFAULT 0, + topic_first_post_id int unsigned NOT NULL DEFAULT 0, + topic_last_post_id int unsigned NOT NULL DEFAULT 0 +); +CREATE TABLE phpbb_posts ( + post_id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + topic_id int unsigned NOT NULL DEFAULT 0, + forum_id smallint unsigned NOT NULL DEFAULT 0, + poster_id int NOT NULL DEFAULT 0, + post_time int NOT NULL DEFAULT 0, + poster_ip char(8) NOT NULL DEFAULT '', + post_username varchar(25) NOT NULL DEFAULT '', + enable_bbcode tinyint NOT NULL DEFAULT 1, + enable_html tinyint NOT NULL DEFAULT 0, + enable_smilies tinyint NOT NULL DEFAULT 1, + enable_sig tinyint NOT NULL DEFAULT 1 +); +CREATE TABLE phpbb_posts_text ( + post_id int unsigned NOT NULL PRIMARY KEY, + bbcode_uid char(32) NOT NULL DEFAULT '', + post_subject char(60) NOT NULL DEFAULT '', + post_text text NOT NULL +); +CREATE TABLE phpbb_users ( + user_id int NOT NULL PRIMARY KEY, + username varchar(25) NOT NULL DEFAULT '', + user_password varchar(32) NOT NULL DEFAULT '', + user_email varchar(255) NOT NULL DEFAULT '', + user_posts int unsigned NOT NULL DEFAULT 0 +); +CREATE TABLE phpbb_auth_access ( + group_id int NOT NULL DEFAULT 0, + forum_id smallint unsigned NOT NULL DEFAULT 0, + auth_view tinyint NOT NULL DEFAULT 0 +); +CREATE TABLE phpbb_user_group ( + group_id int NOT NULL DEFAULT 0, + user_id int NOT NULL DEFAULT 0 +); +SOURCE /papercut-storage/phpbb_mysql_fix.sql; + +INSERT INTO phpbb_forums (forum_id, forum_name, nntp_group_name, forum_desc, forum_posts, forum_topics, forum_last_post_id, auth_view) + VALUES (1, 'Test forum', 'phpbb.test', 'A phpBB test forum', 3, 2, 3, 0); +INSERT INTO phpbb_users (user_id, username, user_password, user_email, user_posts) VALUES + (-1, 'Anonymous', '', '', 0), + (2, 'alice', MD5('secret'), 'alice@example.com', 2); +INSERT INTO phpbb_topics (topic_id, forum_id, topic_title, topic_poster, topic_time, topic_replies, topic_first_post_id, topic_last_post_id) VALUES + (1, 1, 'First topic', 2, 1704103200, 1, 1, 2), + (2, 1, 'Olá mundo', -1, 1704276000, 0, 3, 3); +INSERT INTO phpbb_posts (post_id, topic_id, forum_id, poster_id, post_time, poster_ip, post_username) VALUES + (1, 1, 1, 2, 1704103200, '0a000001', ''), + (2, 1, 1, 2, 1704189600, '0a000001', ''), + (3, 2, 1, -1, 1704276000, '0a000002', 'Guest'); +INSERT INTO phpbb_posts_text (post_id, bbcode_uid, post_subject, post_text) VALUES + (1, 'abc', 'First topic', 'Hello from phpBB'), + (2, 'abc', '', 'A reply\n.leading dot'), + (3, 'abc', 'Olá mundo', 'Corpo com acentuação'); diff --git a/tests/fixtures/sql/mysql/05-nuke.sql b/tests/fixtures/sql/mysql/05-nuke.sql new file mode 100644 index 0000000..eaebe58 --- /dev/null +++ b/tests/fixtures/sql/mysql/05-nuke.sql @@ -0,0 +1,80 @@ +SET NAMES utf8mb4; +USE nuke; +-- PHP-Nuke: phpBB with the nuke_bb prefix, and users in nuke_users +CREATE TABLE nuke_bbforums ( + forum_id smallint unsigned NOT NULL PRIMARY KEY, + forum_name varchar(150) NOT NULL DEFAULT '', + forum_desc text NOT NULL, + forum_posts int unsigned NOT NULL DEFAULT 0, + forum_topics int unsigned NOT NULL DEFAULT 0, + forum_last_post_id int unsigned NOT NULL DEFAULT 0, + auth_view tinyint NOT NULL DEFAULT 0 +); +CREATE TABLE nuke_bbtopics ( + topic_id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + forum_id smallint unsigned NOT NULL DEFAULT 0, + topic_title char(60) NOT NULL DEFAULT '', + topic_poster int NOT NULL DEFAULT 0, + topic_time int NOT NULL DEFAULT 0, + topic_replies int unsigned NOT NULL DEFAULT 0, + topic_status tinyint NOT NULL DEFAULT 0, + topic_vote tinyint NOT NULL DEFAULT 0, + topic_type tinyint NOT NULL DEFAULT 0, + topic_first_post_id int unsigned NOT NULL DEFAULT 0, + topic_last_post_id int unsigned NOT NULL DEFAULT 0 +); +CREATE TABLE nuke_bbposts ( + post_id int unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, + topic_id int unsigned NOT NULL DEFAULT 0, + forum_id smallint unsigned NOT NULL DEFAULT 0, + poster_id int NOT NULL DEFAULT 0, + post_time int NOT NULL DEFAULT 0, + poster_ip char(8) NOT NULL DEFAULT '', + post_username varchar(25) NOT NULL DEFAULT '', + enable_bbcode tinyint NOT NULL DEFAULT 1, + enable_html tinyint NOT NULL DEFAULT 0, + enable_smilies tinyint NOT NULL DEFAULT 1, + enable_sig tinyint NOT NULL DEFAULT 1 +); +CREATE TABLE nuke_bbposts_text ( + post_id int unsigned NOT NULL PRIMARY KEY, + bbcode_uid char(32) NOT NULL DEFAULT '', + post_subject char(60) NOT NULL DEFAULT '', + post_text text NOT NULL +); +CREATE TABLE nuke_users ( + user_id int NOT NULL PRIMARY KEY, + username varchar(25) NOT NULL DEFAULT '', + user_password varchar(32) NOT NULL DEFAULT '', + user_email varchar(255) NOT NULL DEFAULT '', + user_posts int unsigned NOT NULL DEFAULT 0 +); +CREATE TABLE nuke_bbauth_access ( + group_id int NOT NULL DEFAULT 0, + forum_id smallint unsigned NOT NULL DEFAULT 0, + auth_view tinyint NOT NULL DEFAULT 0 +); +CREATE TABLE nuke_bbuser_group ( + group_id int NOT NULL DEFAULT 0, + user_id int NOT NULL DEFAULT 0 +); +-- (no patch script ships for PHP-Nuke; this is phpbb_mysql_fix.sql for nuke_bbforums) +ALTER TABLE nuke_bbforums ADD nntp_group_name VARCHAR(30) AFTER forum_name; +ALTER TABLE nuke_bbforums ADD UNIQUE (nntp_group_name); + +INSERT INTO nuke_bbforums (forum_id, forum_name, nntp_group_name, forum_desc, forum_posts, forum_topics, forum_last_post_id, auth_view) + VALUES (1, 'Test forum', 'nuke.test', 'A PHP-Nuke test forum', 3, 2, 3, 0); +INSERT INTO nuke_users (user_id, username, user_password, user_email, user_posts) VALUES + (-1, 'Anonymous', '', '', 0), + (2, 'alice', MD5('secret'), 'alice@example.com', 2); +INSERT INTO nuke_bbtopics (topic_id, forum_id, topic_title, topic_poster, topic_time, topic_replies, topic_first_post_id, topic_last_post_id) VALUES + (1, 1, 'First topic', 2, 1704103200, 1, 1, 2), + (2, 1, 'Olá mundo', -1, 1704276000, 0, 3, 3); +INSERT INTO nuke_bbposts (post_id, topic_id, forum_id, poster_id, post_time, poster_ip, post_username) VALUES + (1, 1, 1, 2, 1704103200, '0a000001', ''), + (2, 1, 1, 2, 1704189600, '0a000001', ''), + (3, 2, 1, -1, 1704276000, '0a000002', 'Guest'); +INSERT INTO nuke_bbposts_text (post_id, bbcode_uid, post_subject, post_text) VALUES + (1, 'abc', 'First topic', 'Hello from PHP-Nuke'), + (2, 'abc', '', 'A reply\n.leading dot'), + (3, 'abc', 'Olá mundo', 'Corpo com acentuação'); diff --git a/tests/fixtures/sql/mysql/06-postnuke.sql b/tests/fixtures/sql/mysql/06-postnuke.sql new file mode 100644 index 0000000..7c8fed7 --- /dev/null +++ b/tests/fixtures/sql/mysql/06-postnuke.sql @@ -0,0 +1,9 @@ +SET NAMES utf8mb4; +USE postnuke; +-- the postnuke_phpbb_mysql_users auth module only reads nuke_users +CREATE TABLE nuke_users ( + pn_uid int NOT NULL PRIMARY KEY, + pn_uname varchar(25) NOT NULL DEFAULT '', + pn_pass varchar(40) NOT NULL DEFAULT '' +); +INSERT INTO nuke_users (pn_uid, pn_uname, pn_pass) VALUES (2, 'alice', MD5('secret')); diff --git a/tests/fixtures/sql/postgres/01-phorum.sql b/tests/fixtures/sql/postgres/01-phorum.sql new file mode 100644 index 0000000..a90356f --- /dev/null +++ b/tests/fixtures/sql/postgres/01-phorum.sql @@ -0,0 +1,63 @@ +-- Phorum 3 on PostgreSQL (only the columns Papercut uses), then the repo's patch script +CREATE ROLE papercut LOGIN PASSWORD 'papercut'; +CREATE DATABASE phorum ENCODING 'UTF8'; +\connect phorum + +CREATE TABLE forums ( + id integer NOT NULL PRIMARY KEY, + name varchar(50) NOT NULL DEFAULT '', + active smallint NOT NULL DEFAULT 1, + description varchar(255) NOT NULL DEFAULT '', + table_name varchar(50) NOT NULL DEFAULT '', + email_list varchar(50) NOT NULL DEFAULT '', + email_return varchar(50) NOT NULL DEFAULT '' +); +CREATE TABLE forum_1 ( + id integer NOT NULL PRIMARY KEY, + datestamp timestamp NOT NULL, + thread integer NOT NULL DEFAULT 0, + parent integer NOT NULL DEFAULT 0, + author varchar(50) NOT NULL DEFAULT '', + subject varchar(255) NOT NULL DEFAULT '', + email varchar(200) NOT NULL DEFAULT '', + host varchar(50) NOT NULL DEFAULT '', + email_reply char(1) NOT NULL DEFAULT 'N', + approved char(1) NOT NULL DEFAULT 'N', + msgid varchar(100) NOT NULL DEFAULT '', + modifystamp integer NOT NULL DEFAULT 0, + userid integer NOT NULL DEFAULT 0 +); +CREATE TABLE forum_1_bodies ( + id integer NOT NULL PRIMARY KEY, + body text NOT NULL, + thread integer NOT NULL DEFAULT 0 +); +CREATE TABLE forums_auth ( + id integer NOT NULL PRIMARY KEY, + name varchar(50) NOT NULL DEFAULT '', + username varchar(50) NOT NULL DEFAULT '', + password varchar(50) NOT NULL DEFAULT '', + email varchar(200) NOT NULL DEFAULT '' +); +CREATE TABLE forums_moderators ( + user_id integer NOT NULL, + forum_id integer NOT NULL +); + +\i /papercut-storage/phorum_pgsql_fix.sql + +INSERT INTO forums (id, nntp_group_name, name, description, table_name, email_list, email_return) + VALUES (1, 'phorum.test', 'Test forum', 'A Phorum test forum', 'forum_1', 'list@example.com', 'return@example.com'); +INSERT INTO forum_1 (id, datestamp, thread, parent, author, subject, email, host, approved, modifystamp) VALUES + (1, '2024-01-01 10:00:00', 1, 0, 'Alice', 'First post', 'alice@example.com', '10.0.0.1', 'Y', 1704103200), + (2, '2024-01-02 10:00:00', 1, 1, 'Bob', 'Re: First post', 'bob@example.com', '10.0.0.2', 'Y', 1704189600), + (3, '2024-01-03 10:00:00', 3, 0, 'Spammer', 'Not approved', 'spam@example.com', '10.0.0.3', 'N', 1704276000), + (4, '2024-01-04 10:00:00', 4, 0, 'Carol', 'Olá mundo', '', '10.0.0.4', 'Y', 1704362400); +INSERT INTO forum_1_bodies (id, body, thread) VALUES + (1, 'Hello world', 1), (2, E'A reply\n.leading dot', 1), (3, 'spam', 3), (4, 'Corpo com acentuação', 4); +INSERT INTO forums_auth (id, name, username, password, email) VALUES + (1, 'Moderator', 'moderator', MD5('secret'), 'mod@example.com'), + (2, 'Old User', 'olduser', 'sefjKaLm7zybE', 'old@example.com'); +INSERT INTO forums_moderators (user_id, forum_id) VALUES (1, 1); + +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO papercut; diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..b7a7067 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,137 @@ +"""Docker integration tests: the SQL backends against real databases. + +Run with: docker compose -f docker-compose.test.yml run --rm papercut +(see docs/integration-testing-plan.md) +""" +import importlib +import os +import sys +import warnings + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +import fakes # noqa: E402 +import papercut # noqa: E402 +import settings # noqa: E402 + +FIXTURES = os.path.join(os.path.dirname(__file__), "..", "fixtures") +MYSQL_HOST = os.environ.get("PAPERCUT_TEST_MYSQL_HOSTS", "") +POSTGRES_HOST = os.environ.get("PAPERCUT_TEST_POSTGRES_HOST", "") + + +def pytest_collection_modifyitems(items): + for item in items: + if "integration" in str(item.fspath): + item.add_marker(pytest.mark.integration) + if not (MYSQL_HOST and POSTGRES_HOST): + item.add_marker(pytest.mark.skip(reason="needs the Docker test databases")) + + +def root_connection(kind, database): + """A privileged connection, for setting up and checking data.""" + if kind == "postgres": + import psycopg2 + conn = psycopg2.connect(host=POSTGRES_HOST, dbname=database, user="postgres", password="papercut") + conn.autocommit = True + return conn + import MySQLdb + conn = MySQLdb.connect(host=MYSQL_HOST, db=database, user="root", passwd="papercut", charset="utf8mb4") + conn.autocommit(True) + return conn + + +def query(kind, database, sql, params=None): + conn = root_connection(kind, database) + try: + cursor = conn.cursor() + cursor.execute(sql, params) + # lists of tuples, whichever driver + return [tuple(row) for row in cursor.fetchall()] if cursor.description else None + finally: + conn.close() + + +class Live: + """A running Papercut server for one backend, plus helpers.""" + + def __init__(self, server, backend, kind, database): + self.server = server + self.backend = backend + self.kind = kind + self.database = database + + @property + def port(self): + return self.server.server_address[1] + + def nntp(self, **kwargs): + # imported here: on Python 3.13+ it comes from standard-nntplib, which + # only the Docker image installs (collecting these tests mustn't need it) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + import nntplib + return nntplib.NNTP("127.0.0.1", self.port, readermode=False, **kwargs) + + def raw(self): + return fakes.Client(self.port) + + def query(self, sql, params=None): + return query(self.kind, self.database, sql, params) + + +def start_live(monkeypatch, request, storage_module, kind, database, auth_module=None, **extra_settings): + host = POSTGRES_HOST if kind == "postgres" else MYSQL_HOST + values = dict(dbhost=host, dbname=database, dbuser="papercut", dbpass="papercut", + nntp_hostname="news.example.com", + phorum_settings_path=os.path.join(FIXTURES, "phorum_settings") + "/") + values.update(extra_settings) + for name, value in values.items(): + monkeypatch.setattr(settings, name, value) + if kind == "postgres": + # the PostgreSQL user is the database owner (see the postgres fixtures) + monkeypatch.setattr(settings, "dbuser", "papercut") + backend = importlib.import_module(storage_module).Papercut_Storage() + auth = importlib.import_module(auth_module).Papercut_Auth() if auth_module else None + server = fakes.start_server(papercut.NNTPRequestHandler, backend, auth) + request.addfinalizer(server.server_close) + request.addfinalizer(server.shutdown) + return Live(server, backend, kind, database) + + +@pytest.fixture(autouse=True) +def show_server_log(): + """Prints the server log after each test (pytest shows it on failure).""" + yield + if os.path.exists(settings.log_file): + with open(settings.log_file) as f: + errors = [line for line in f if "Error" in line or "Traceback" in line or line.startswith((" ", "MySQLdb", "psycopg2"))] + if errors: + print("".join(errors)) + + +class MailSink: + """Collects the messages sent to it over SMTP.""" + + def __init__(self): + self.messages = [] + + async def handle_DATA(self, server, session, envelope): + import email + import email.policy + msg = email.message_from_bytes(envelope.content, policy=email.policy.default) + # decoded headers and body, as a mail client would show them + text = "".join("%s: %s\n" % item for item in msg.items()) + "\n" + msg.get_content() + self.messages.append((envelope.mail_from, list(envelope.rcpt_tos), text)) + return "250 OK" + + +@pytest.fixture +def smtp(): + """An SMTP server on localhost:25, where the Phorum backends send mail.""" + from aiosmtpd.controller import Controller + sink = MailSink() + controller = Controller(sink, hostname="127.0.0.1", port=25) + controller.start() + yield sink + controller.stop() diff --git a/tests/integration/test_mysql_backend.py b/tests/integration/test_mysql_backend.py new file mode 100644 index 0000000..67a3e7d --- /dev/null +++ b/tests/integration/test_mysql_backend.py @@ -0,0 +1,148 @@ +"""The 'mysql' storage backend and auth module (storage/mysql_storage.sql).""" +import datetime +import io + +import pytest + +import settings +from conftest import start_live + +GROUP = "papercut.test" +EVIL = "x' OR '1'='1" + + +@pytest.fixture +def live(monkeypatch, request): + live = start_live(monkeypatch, request, "storage.mysql", "mysql", "papercut", auth_module="auth.mysql") + yield live + live.query("DELETE FROM papercut_default_table WHERE id > 3") + + +def test_list(live): + with live.nntp() as s: + _, groups = s.list() + assert [tuple(g) for g in groups] == [(GROUP, "3", "1", "y")] + + +def test_group(live): + with live.nntp() as s: + _, count, first, last, name = s.group(GROUP) + assert (count, first, last, name) == (3, 1, 3, GROUP) + + +def test_unknown_group(live): + with live.nntp() as s: + with pytest.raises(Exception, match="411"): + s.group("no.such.group") + + +def test_article_head_and_body(live): + with live.nntp() as s: + s.group(GROUP) + _, info = s.article("2") + lines = [l.decode() for l in info.lines] + assert "Subject: Re: First post" in lines + assert "References: <1@%s>" % GROUP in lines + assert lines[-2:] == ["Olá, mundo", ".dot"] + _, head = s.head("1") + assert "From: Alice" in [l.decode() for l in head.lines] + _, body = s.body("1") + assert [l.decode() for l in body.lines] == ["Hello world"] + + +def test_stat_next_last(live): + with live.nntp() as s: + s.group(GROUP) + assert s.stat("1")[1] == 1 + assert s.next()[1] == 2 + assert s.last()[1] == 1 + + +def test_xover(live): + with live.nntp() as s: + s.group(GROUP) + _, overviews = s.over((1, 3)) + assert [n for n, _ in overviews] == [1, 2, 3] + assert overviews[1][1]["subject"] == "Re: First post" + assert overviews[1][1]["references"] == "<1@%s>" % GROUP + + +def test_xhdr(live): + with live.nntp() as s: + s.group(GROUP) + _, headers = s.xhdr("subject", "1-3") + assert headers == [("1", "First post"), ("2", "Re: First post"), ("3", "Another thread")] + + +def test_xpat(live): + with live.nntp() as s: + s.group(GROUP) + _, lines = s._longcmdstring("XPAT Subject 1-3 *First*") + assert lines == ["1 First post", "2 Re: First post"] + + +def test_listgroup(live): + with live.nntp() as s: + _, lines = s._longcmdstring("LISTGROUP " + GROUP) + assert lines == ["1", "2", "3"] + + +def test_list_newsgroups(live): + with live.nntp() as s: + _, descriptions = s.descriptions("papercut.*") + assert descriptions == {GROUP: "Test group"} + + +def test_newnews(live): + with live.nntp() as s: + _, ids = s.newnews(GROUP, datetime.datetime(2024, 1, 2)) + assert ids == ["<2@%s>" % GROUP, "<3@%s>" % GROUP] + + +def test_post_and_read_back(live): + article = ("From: Tester \r\nNewsgroups: %s\r\nSubject: Posted héllo\r\n" + "References: <1@%s>\r\n\r\nposted body ✓\r\n.dot\r\n" % (GROUP, GROUP)) + with live.nntp() as s: + s.post(io.BytesIO(article.encode())) + _, count, first, last, _ = s.group(GROUP) + assert last == 4 + _, info = s.article("4") + lines = [l.decode() for l in info.lines] + assert "Subject: Posted héllo" in lines + assert "References: <1@%s>" % GROUP in lines + assert lines[-2:] == ["posted body ✓", ".dot"] + + +@pytest.fixture +def auth_on(monkeypatch): + monkeypatch.setattr(settings, "nntp_auth", "yes") + + +@pytest.mark.parametrize("username, password, accepted", [ + ("alice", "secret", True), + ("alice", "wrong", False), + ("nobody", "secret", False), + (EVIL, EVIL, False), + ("alice\\", "secret", False), +]) +def test_login(live, auth_on, username, password, accepted): + c = live.raw() + c.cmd("AUTHINFO USER " + username.replace(" ", "")) + result = c.cmd("AUTHINFO PASS " + password.replace(" ", "")) + assert result.startswith("281" if accepted else "502") + c.close() + + +@pytest.mark.parametrize("command", [ + "GROUP " + EVIL.replace(" ", ""), + "LISTGROUP " + EVIL.replace(" ", ""), + "XGTITLE " + EVIL.replace(" ", ""), + "NEWNEWS " + EVIL.replace(" ", "") + " 20240101 000000", +]) +def test_injection_payloads_are_harmless(live, command): + before = live.query("SELECT COUNT(*) FROM papercut_default_table") + c = live.raw() + response = c.cmd(command) + c.close() + assert response[:3] in ("211", "230", "282", "411", "481") + assert live.query("SELECT COUNT(*) FROM papercut_default_table") == before diff --git a/tests/integration/test_phorum_backends.py b/tests/integration/test_phorum_backends.py new file mode 100644 index 0000000..09ec3be --- /dev/null +++ b/tests/integration/test_phorum_backends.py @@ -0,0 +1,188 @@ +"""The Phorum storage and auth backends, on MySQL and PostgreSQL.""" +import datetime +import io + +import pytest + +import settings +from conftest import start_live + +GROUP = "phorum.test" +EVIL = "x' OR '1'='1" + +BACKENDS = { + "mysql": ("storage.phorum_mysql", "auth.phorum_mysql_users", "mysql"), + "pgsql": ("storage.phorum_pgsql", "auth.phorum_pgsql_users", "postgres"), +} + + +@pytest.fixture(params=sorted(BACKENDS)) +def live(monkeypatch, request): + storage, auth, kind = BACKENDS[request.param] + live = start_live(monkeypatch, request, storage, kind, "phorum", auth_module=auth) + yield live + live.query("DELETE FROM forum_1 WHERE id > 4") + live.query("DELETE FROM forum_1_bodies WHERE id > 4") + + +def lines_of(info): + return [l.decode() for l in info.lines] + + +def test_list(live): + with live.nntp() as s: + _, groups = s.list() + assert [tuple(g) for g in groups] == [(GROUP, "4", "1", "y")] + + +def test_group_counts_only_approved_messages(live): + with live.nntp() as s: + _, count, first, last, name = s.group(GROUP) + assert (count, first, last, name) == (3, 1, 4, GROUP) + + +def test_unapproved_messages_are_not_served(live): + with live.nntp() as s: + s.group(GROUP) + with pytest.raises(Exception, match="423"): + s.article("3") + _, numbers = s._longcmdstring("LISTGROUP " + GROUP) + assert numbers == ["1", "2", "4"] + + +def test_article(live): + with live.nntp() as s: + s.group(GROUP) + _, info = s.article("2") + lines = lines_of(info) + assert "From: Bob " in lines + assert "Subject: Re: First post" in lines + assert "References: <1@%s>" % GROUP in lines + assert lines[-2:] == ["A reply", ".leading dot"] + + +def test_non_ascii_content(live): + with live.nntp() as s: + s.group(GROUP) + _, head = s.head("4") + _, body = s.body("4") + assert "Subject: Olá mundo" in lines_of(head) + assert lines_of(body) == ["Corpo com acentuação"] + + +def test_stat_next_last(live): + with live.nntp() as s: + s.group(GROUP) + assert s.stat("1")[1] == 1 + assert s.next()[1] == 2 + assert s.next()[1] == 4 # 3 is not approved + assert s.last()[1] == 2 + + +def test_xover(live): + with live.nntp() as s: + s.group(GROUP) + _, overviews = s.over((1, 4)) + assert [n for n, _ in overviews] == [1, 2, 4] + assert overviews[1][1]["from"] == "Bob " + assert overviews[1][1]["references"] == "<1@%s>" % GROUP + + +def test_xhdr(live): + with live.nntp() as s: + s.group(GROUP) + _, headers = s.xhdr("subject", "1-4") + assert headers == [("1", "First post"), ("2", "Re: First post"), ("4", "Olá mundo")] + + +def test_xpat(live): + with live.nntp() as s: + s.group(GROUP) + _, lines = s._longcmdstring("XPAT Subject 1-4 *First*") + assert lines == ["1 First post", "2 Re: First post"] + + +def test_list_newsgroups(live): + with live.nntp() as s: + _, descriptions = s.descriptions("phorum.*") + assert descriptions == {GROUP: "A Phorum test forum"} + + +def test_newnews(live): + with live.nntp() as s: + _, ids = s.newnews(GROUP, datetime.datetime(2024, 1, 2)) + assert ids == ["<2@%s>" % GROUP, "<4@%s>" % GROUP] + + +def test_post_of_a_new_thread(live, smtp): + article = ("From: Tester \r\nNewsgroups: %s\r\nSubject: New thread\r\n\r\nfirst\r\n" % GROUP) + with live.nntp() as s: + s.post(io.BytesIO(article.encode())) + s.group(GROUP) + _, info = s.article("5") + lines = lines_of(info) + assert "Subject: New thread" in lines + assert not any(line.startswith("References:") for line in lines) + assert live.query("SELECT thread, parent FROM forum_1 WHERE id = 5") == [(5, 0)] + + +@pytest.mark.parametrize("subject, body", [ + ("Plain subject", "plain body"), + ("Assunto com acentuação", "corpo ✓"), +]) +def test_post_saves_and_notifies(live, smtp, subject, body): + article = ("From: Tester \r\nNewsgroups: %s\r\nSubject: %s\r\n" + "References: <1@%s>\r\n\r\n%s\r\n" % (GROUP, subject, GROUP, body)) + with live.nntp() as s: + s.post(io.BytesIO(article.encode())) + _, count, _, last, _ = s.group(GROUP) + assert (count, last) == (4, 5) + _, info = s.article("5") + lines = lines_of(info) + assert "Subject: %s" % subject in lines + assert "References: <1@%s>" % GROUP in lines # threaded under message 1 + assert lines[-1] == body + recipients = sorted(r for _, rcpts, _ in smtp.messages for r in rcpts) + assert recipients == ["list@example.com", "mod@example.com"] + assert all(body in text for _, _, text in smtp.messages) + + +@pytest.fixture +def auth_on(monkeypatch): + monkeypatch.setattr(settings, "nntp_auth", "yes") + + +@pytest.mark.parametrize("username, password, accepted", [ + ("moderator", "secret", True), # MD5 (newer Phorum) + ("olduser", "secret", True), # crypt (older Phorum) + ("moderator", "wrong", False), + ("olduser", "!secret", False), + ("nobody", "secret", False), + (EVIL.replace(" ", ""), "x", False), +]) +def test_login(live, auth_on, username, password, accepted): + c = live.raw() + c.cmd("AUTHINFO USER " + username) + assert c.cmd("AUTHINFO PASS " + password).startswith("281" if accepted else "502") + c.close() + + +@pytest.mark.parametrize("command", [ + "GROUP " + EVIL.replace(" ", ""), + "LISTGROUP " + EVIL.replace(" ", ""), + "XGTITLE " + EVIL.replace(" ", ""), + "NEWNEWS " + EVIL.replace(" ", "") + " 20240101 000000", +]) +def test_injection_payloads_are_harmless(live, command): + before = live.query("SELECT COUNT(*) FROM forum_1") + c = live.raw() + response = c.cmd(command) + c.close() + assert response[:3] in ("211", "230", "282", "411", "481") + assert live.query("SELECT COUNT(*) FROM forum_1") == before + + +def test_newnews_with_a_wildcard(live): + with live.nntp() as s: + _, ids = s.newnews("phorum.*", datetime.datetime(2024, 1, 2)) + assert ids == ["<2@%s>" % GROUP, "<4@%s>" % GROUP] diff --git a/tests/integration/test_phpbb_backends.py b/tests/integration/test_phpbb_backends.py new file mode 100644 index 0000000..8b28ae0 --- /dev/null +++ b/tests/integration/test_phpbb_backends.py @@ -0,0 +1,203 @@ +"""The phpBB and PHP-Nuke storage backends, and the phpBB-family auth modules.""" +import datetime +import io + +import pytest + +import settings +from conftest import start_live + +EVIL = "x'OR'1'='1" + +BACKENDS = { + "phpbb": dict(storage="storage.phpbb_mysql", auth="auth.phpbb_mysql_users", database="phpbb", + group="phpbb.test", prefix="phpbb_", users="phpbb_users", + settings=dict(phpbb_table_prefix="phpbb_")), + "nuke": dict(storage="storage.phpnuke_phpbb_mysql", auth="auth.phpnuke_phpbb_mysql_users", database="nuke", + group="nuke.test", prefix="nuke_bb", users="nuke_users", + settings=dict(phpbb_table_prefix="nuke_bb", nuke_table_prefix="nuke_")), +} + + +@pytest.fixture(params=sorted(BACKENDS)) +def live(monkeypatch, request): + config = BACKENDS[request.param] + live = start_live(monkeypatch, request, config["storage"], "mysql", config["database"], + auth_module=config["auth"], **config["settings"]) + live.group = config["group"] + live.prefix = config["prefix"] + reset(live, config) + yield live + reset(live, config) + + +def reset(live, config): + """Back to the fixture data (posts 1-3, topics 1-2).""" + p = config["prefix"] + live.query("DELETE FROM %sposts WHERE post_id > 3" % p) + live.query("DELETE FROM %sposts_text WHERE post_id > 3" % p) + live.query("DELETE FROM %stopics WHERE topic_id > 2" % p) + # InnoDB keeps counting after deletes; this resets to MAX(id) + 1 + live.query("ALTER TABLE %sposts AUTO_INCREMENT = 1" % p) + live.query("ALTER TABLE %stopics AUTO_INCREMENT = 1" % p) + live.query("UPDATE %sforums SET forum_posts=3, forum_topics=2, forum_last_post_id=3" % p) + live.query("UPDATE %stopics SET topic_replies=1, topic_last_post_id=2 WHERE topic_id=1" % p) + live.query("UPDATE %s SET user_posts=2 WHERE user_id=2" % config["users"]) + + +def lines_of(info): + return [l.decode() for l in info.lines] + + +def test_list(live): + with live.nntp() as s: + _, groups = s.list() + assert [tuple(g) for g in groups] == [(live.group, "3", "1", "y")] + + +def test_group(live): + with live.nntp() as s: + _, count, first, last, name = s.group(live.group) + assert (count, first, last, name) == (3, 1, 3, live.group) + + +def test_article(live): + with live.nntp() as s: + s.group(live.group) + _, info = s.article("2") + lines = lines_of(info) + assert "From: alice " in lines + # an empty post subject becomes 'Re: ' + assert "Subject: Re: First topic" in lines + assert "References: <1@%s>" % live.group in lines + assert lines[-2:] == ["A reply", ".leading dot"] + + +def test_guest_posts_and_non_ascii(live): + with live.nntp() as s: + s.group(live.group) + _, head = s.head("3") + _, body = s.body("3") + head = lines_of(head) + assert "From: Guest" in head + assert "Subject: Olá mundo" in head + assert not any(l.startswith("References:") for l in head) # first post of its topic + assert lines_of(body) == ["Corpo com acentuação"] + + +def test_stat_next_last(live): + with live.nntp() as s: + s.group(live.group) + assert s.stat("1")[1] == 1 + assert s.next()[1] == 2 + assert s.last()[1] == 1 + + +def test_xover(live): + with live.nntp() as s: + s.group(live.group) + _, overviews = s.over((1, 3)) + assert [n for n, _ in overviews] == [1, 2, 3] + assert overviews[1][1]["subject"] == "Re: First topic" + assert overviews[1][1]["references"] == "<1@%s>" % live.group + assert overviews[2][1]["references"] == "" + + +def test_xhdr(live): + with live.nntp() as s: + s.group(live.group) + _, headers = s.xhdr("subject", "1-3") + assert headers == [("1", "First topic"), ("2", "Re: First topic"), ("3", "Olá mundo")] + + +def test_xpat(live): + with live.nntp() as s: + s.group(live.group) + _, lines = s._longcmdstring("XPAT Subject 1-3 *mundo*") + assert lines == ["3 Olá mundo"] + + +def test_listgroup(live): + with live.nntp() as s: + _, lines = s._longcmdstring("LISTGROUP " + live.group) + assert lines == ["1", "2", "3"] + + +def test_list_newsgroups(live): + with live.nntp() as s: + _, descriptions = s.descriptions("*") + assert list(descriptions) == [live.group] + + +def test_newnews(live): + with live.nntp() as s: + _, ids = s.newnews(live.group, datetime.datetime(2024, 1, 2)) + assert ids == ["<2@%s>" % live.group, "<3@%s>" % live.group] + + +def post(live, subject, body, references=None): + lines = ["From: Tester ", "Newsgroups: " + live.group, "Subject: " + subject] + if references: + lines.append("References: " + references) + article = "\r\n".join(lines) + "\r\n\r\n" + body + "\r\n" + with live.nntp() as s: + s.post(io.BytesIO(article.encode())) + _, count, _, last, _ = s.group(live.group) + _, info = s.article(str(last)) + return count, last, lines_of(info) + + +def test_post_a_new_topic(live): + count, last, lines = post(live, "Novo tópico", "corpo ✓") + assert (count, last) == (4, 4) + assert "Subject: Novo tópico" in lines + assert lines[-1] == "corpo ✓" + topic = live.query("SELECT topic_id, topic_title FROM %stopics WHERE topic_first_post_id = 4" % live.prefix) + assert topic == [(3, "Novo tópico")] + + +def test_post_a_reply(live): + count, last, lines = post(live, "Re: First topic", "a reply", references="<1@%s>" % live.group) + assert (count, last) == (4, 4) + assert "References: <1@%s>" % live.group in lines + assert live.query("SELECT topic_id FROM %sposts WHERE post_id = 4" % live.prefix) == [(1,)] + assert live.query("SELECT topic_replies, topic_last_post_id FROM %stopics WHERE topic_id = 1" % live.prefix) == [(2, 4)] + + +@pytest.fixture +def auth_on(monkeypatch): + monkeypatch.setattr(settings, "nntp_auth", "yes") + + +@pytest.mark.parametrize("username, password, accepted", [ + ("alice", "secret", True), + ("alice", "wrong", False), + ("nobody", "secret", False), + (EVIL, EVIL, False), +]) +def test_login(live, auth_on, username, password, accepted): + c = live.raw() + c.cmd("AUTHINFO USER " + username) + assert c.cmd("AUTHINFO PASS " + password).startswith("281" if accepted else "502") + c.close() + + +def test_postnuke_login(monkeypatch, request, auth_on): + live = start_live(monkeypatch, request, "storage.phpbb_mysql", "mysql", "postnuke", + auth_module="auth.postnuke_phpbb_mysql_users") + for username, password, accepted in [("alice", "secret", True), ("alice", "nope", False), (EVIL, "x", False)]: + c = live.raw() + c.cmd("AUTHINFO USER " + username) + assert c.cmd("AUTHINFO PASS " + password).startswith("281" if accepted else "502") + c.close() + + +@pytest.mark.parametrize("command", ["GROUP " + EVIL, "LISTGROUP " + EVIL, "XGTITLE " + EVIL, + "NEWNEWS " + EVIL + " 20240101 000000"]) +def test_injection_payloads_are_harmless(live, command): + before = live.query("SELECT COUNT(*) FROM %sposts" % live.prefix) + c = live.raw() + response = c.cmd(command) + c.close() + assert response[:3] in ("211", "230", "282", "411", "481") + assert live.query("SELECT COUNT(*) FROM %sposts" % live.prefix) == before diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 4b81931..027c2dd 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -573,3 +573,13 @@ def test_client_disconnect_mid_session_is_handled(client): client.cmd("GROUP test.group") client.close() assert wait_for_log("Connection closed") + + +@pytest.mark.parametrize("pattern", ["comp.*", "*", "comp.lang.?", "comp.*,!comp.test"]) +def test_newnews_accepts_wildmat_patterns(client, backend, pattern): + # regression: any group argument other than '*' had to be an existing + # group, so wildmats like comp.* were refused with 411 + backend.returns["group_exists"] = False + assert code(client.cmd("NEWNEWS %s 20240105 000000" % pattern)) == "230" + client.read_multiline() + assert backend.called("get_NEWNEWS")[0][1] == pattern diff --git a/tests/test_sql_backends.py b/tests/test_sql_backends.py index f807cf1..f39cfcb 100644 --- a/tests/test_sql_backends.py +++ b/tests/test_sql_backends.py @@ -39,6 +39,7 @@ def __init__(self, width): self.width = width # number of columns in every fetched row self.statements = [] self.rowcount = 1 + self.lastrowid = 1 def execute(self, stmt, params=None): self.statements.append((stmt, params)) @@ -57,7 +58,10 @@ def insert_id(self): class FakeConnection: def __init__(self, cursor): self._cursor = cursor - self.autocommit = False + + def autocommit(self, on): + # MySQLdb style; psycopg2 code assigns 'conn.autocommit = True' instead + pass def cursor(self): return self._cursor