Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,17 @@ jobs:
with:
name: coverage
path: coverage.xml

integration:
name: integration tests (MySQL 8.4, PostgreSQL 16)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Run the integration tests against real databases
run: docker compose -f docker-compose.test.yml run --rm papercut
- name: Database logs
if: failure()
run: docker compose -f docker-compose.test.yml logs mysql postgres
- name: Stop the databases
if: always()
run: docker compose -f docker-compose.test.yml down -v
9 changes: 9 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
test:
python -m pytest

# needs Docker; see docs/integration-testing-plan.md
test-integration:
docker compose -f docker-compose.test.yml run --rm papercut
docker compose -f docker-compose.test.yml down -v

.PHONY: test test-integration
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ See [INSTALL](INSTALL) for step by step instructions.
```sh
pip install -r requirements-dev.txt
pytest

# the database backends against real MySQL and PostgreSQL (needs Docker)
make test-integration
```

See [docs/unit-testing-plan.md](docs/unit-testing-plan.md) and
Expand Down
4 changes: 3 additions & 1 deletion auth/mysql.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ class Papercut_Auth:
"""

def __init__(self):
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass)
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4")
# commit every statement (InnoDB tables would otherwise never see posts)
self.conn.autocommit(True)
self.cursor = self.conn.cursor()

def is_valid_user(self, username, password):
Expand Down
4 changes: 3 additions & 1 deletion auth/phorum_mysql_users.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ class Papercut_Auth:
"""

def __init__(self):
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass)
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4")
# commit every statement (InnoDB tables would otherwise never see posts)
self.conn.autocommit(True)
self.cursor = self.conn.cursor()

def is_valid_user(self, username, password):
Expand Down
4 changes: 3 additions & 1 deletion auth/phorum_pgsql_users.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ class Papercut_Auth:
"""

def __init__(self):
self.conn = psycopg2.connect(database=settings.dbname, user=settings.dbuser)
# (this used to ignore dbhost and dbpass, so it only worked over a
# local socket without a password)
self.conn = psycopg2.connect(host=settings.dbhost, database=settings.dbname, user=settings.dbuser, password=settings.dbpass)
self.conn.autocommit = True
self.cursor = self.conn.cursor()

Expand Down
4 changes: 3 additions & 1 deletion auth/phpbb_mysql_users.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ class Papercut_Auth:
"""

def __init__(self):
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass)
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4")
# commit every statement (InnoDB tables would otherwise never see posts)
self.conn.autocommit(True)
self.cursor = self.conn.cursor()

def is_valid_user(self, username, password):
Expand Down
4 changes: 3 additions & 1 deletion auth/phpnuke_phpbb_mysql_users.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@ class Papercut_Auth:
"""

def __init__(self):
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass)
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4")
# commit every statement (InnoDB tables would otherwise never see posts)
self.conn.autocommit(True)
self.cursor = self.conn.cursor()

def is_valid_user(self, username, password):
Expand Down
4 changes: 3 additions & 1 deletion auth/postnuke_phpbb_mysql_users.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ class Papercut_Auth:
"""

def __init__(self):
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass)
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4")
# commit every statement (InnoDB tables would otherwise never see posts)
self.conn.autocommit(True)
self.cursor = self.conn.cursor()

def is_valid_user(self, username, password):
Expand Down
62 changes: 62 additions & 0 deletions docker-compose.test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
# Docker integration tests: see docs/integration-testing-plan.md
#
# docker compose -f docker-compose.test.yml run --rm papercut
#
# (add --profile mysql57 to also test against MySQL 5.7)
services:
mysql:
image: mysql:8.4
environment:
MYSQL_ROOT_PASSWORD: papercut
volumes:
- ./tests/fixtures/sql/mysql:/docker-entrypoint-initdb.d:ro
- ./storage:/papercut-storage:ro
healthcheck:
test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-ppapercut"]
interval: 2s
timeout: 5s
retries: 60

mysql57:
profiles: [mysql57]
image: mysql:5.7
platform: linux/amd64
environment:
MYSQL_ROOT_PASSWORD: papercut
volumes:
- ./tests/fixtures/sql/mysql:/docker-entrypoint-initdb.d:ro
- ./storage:/papercut-storage:ro
healthcheck:
test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-ppapercut"]
interval: 2s
timeout: 5s
retries: 90

postgres:
image: postgres:16
environment:
POSTGRES_PASSWORD: papercut
volumes:
- ./tests/fixtures/sql/postgres:/docker-entrypoint-initdb.d:ro
- ./storage:/papercut-storage:ro
healthcheck:
test: ["CMD", "pg_isready", "-U", "postgres"]
interval: 2s
timeout: 5s
retries: 60

papercut:
build:
context: .
dockerfile: tests/docker/Dockerfile
volumes:
- .:/src
environment:
PAPERCUT_TEST_MYSQL_HOSTS: mysql
PAPERCUT_TEST_POSTGRES_HOST: postgres
depends_on:
mysql:
condition: service_healthy
postgres:
condition: service_healthy
command: ["python", "-m", "pytest", "-m", "integration", "-p", "no:cacheprovider", "-v"]
34 changes: 23 additions & 11 deletions docs/integration-testing-plan.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Plan: Docker integration tests for the database backends

**Status:** proposed
**Status:** done (MySQL 8.4 and PostgreSQL 16 in CI; MySQL 5.7 optional)
**Related:** [unit testing plan](unit-testing-plan.md), PR #2 (Python 3 port and SQL injection fixes)

## Goal
Expand Down Expand Up @@ -127,19 +127,31 @@ make test-integration
requests, or nightly if it proves slow. The `mysql57` target runs only in the
nightly job.

## Expected findings
## Findings

The point of this work is to find problems that only a real database shows.
Some are already visible in the code:
Running the backends against real databases found these bugs, all fixed with
the integration test that caught them:

| Backend | Issue |
| Backend | Bug |
| --- | --- |
| `mysql` | `COUNT(*) AS check`: `CHECK` is a reserved word since MySQL 8.0.16, so this is a syntax error there |
| `phorum_mysql` | `GROUP BY id` with other columns in the SELECT, which MySQL 5.7+ rejects by default (`ONLY_FULL_GROUP_BY`) |
| `phorum_pgsql` | MySQL-only syntax: `LIMIT 1, 0`, `CURRENT_TIMESTAMP()` with parentheses |
| `phorum_mysql` / `phorum_pgsql` | `get_NEWNEWS` matches the group with `=` against a pattern that uses `%` as a wildcard (it should probably be `LIKE`) |

Fix each one in its own commit, with the integration test that caught it.
| `mysql` | `storage/mysql_storage.sql` didn't load: `TYPE=MyISAM` (removed in 5.5) and `'0000-00-00'` defaults (rejected by 5.7+ strict mode); MyISAM's key limit is also too small for utf8mb4 |
| `mysql` | `COUNT(*) AS check`: `CHECK` is reserved since MySQL 8.0.16, so almost every command failed |
| all MySQL | autocommit was off, so on InnoDB **posts were never committed** and held locks |
| all MySQL | no connection character set, so non-ASCII text came back as mojibake |
| all SQL | `NEWNEWS` message-ids came out as `<(2,)@group>` |
| Phorum | moderator/mailing-list emails raised `UnicodeEncodeError` for non-ASCII posts, after saving them (the client was told the post failed) |
| `phorum_pgsql` | `get_STAT` reported every article missing (psycopg2's `execute()` returns `None`) |
| `phorum_pgsql` | MySQL-only syntax: `LIMIT 1, 0` (NEXT/LAST), `CURRENT_TIMESTAMP()` (new threads) |
| `phorum_pgsql_users` | ignored `dbhost` and `dbpass` |
| phpBB, PHP-Nuke | comma joins mixed with `LEFT JOIN` fail on MySQL 5.0.12+, breaking ARTICLE/HEAD/XOVER/XHDR/XPAT |
| phpBB, PHP-Nuke | `cursor.insert_id()` no longer exists, so every post failed |
| phpBB, PHP-Nuke | XOVER used the topic id as the replied-to article number, mixing up threads |
| server, `mysql`, Phorum | `NEWNEWS comp.* ...` was refused with 411 (only a bare `*` was allowed), and the backends compared the pattern with `=` instead of `LIKE` |

Predicted from reading the code, but not borne out:

- `ONLY_FULL_GROUP_BY`: the Phorum `GROUP BY id` is on the primary key, which
MySQL 5.7.5+ accepts.

## Out of scope

Expand Down
4 changes: 3 additions & 1 deletion docs/unit-testing-plan.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Plan: Unit tests

**Status:** in progress (steps 1-6 done; Docker integration tests remain)
**Status:** done
**Related:** [integration testing plan](integration-testing-plan.md)

## Goal
Expand Down Expand Up @@ -197,6 +197,8 @@ Mark these `xfail` with a reason, and fix them in follow-up PRs:

- `papercut.py` `do_LISTGROUP`: `numbers.split('\r\n')` always returns at
least one element, so an empty group never resets the current article.
- `papercut.py` `do_XROVER`: it calls `do_XHDR` with `REFERENCES`, but
`do_XHDR` only accepts `SUBJECT` and `FROM`, so `XROVER` always returns 501.

Found and fixed in step 1, with regression tests:

Expand Down
5 changes: 3 additions & 2 deletions papercut.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,8 +287,9 @@ def do_NEWNEWS(self):
if (len(self.tokens) < 4) or (len(self.tokens) > 6):
self.send_response(ERR_CMDSYNTAXERROR)
return
# check to see if the group exists
if (self.tokens[1] != '*') and (not self.backend.group_exists(self.tokens[1])):
# check to see if the group exists (unless it's a wildmat pattern,
# which the backend matches against the group names)
if (not re.search(r'[*?\[,!]', self.tokens[1])) and (not self.backend.group_exists(self.tokens[1])):
self.send_response(ERR_NOSUCHGROUP)
return
if (len(self.tokens) > 4) and (self.tokens[4] == 'GMT'):
Expand Down
12 changes: 7 additions & 5 deletions storage/mysql.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ class Papercut_Storage:
"""

def __init__(self):
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass)
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4")
# commit every statement (InnoDB tables would otherwise never see posts)
self.conn.autocommit(True)
self.cursor = self.conn.cursor()

def get_body(self, lines):
Expand All @@ -34,7 +36,7 @@ def get_header(self, lines):
def group_exists(self, group_name):
stmt = """
SELECT
COUNT(*) AS check
COUNT(*) AS total
FROM
papercut_groups
WHERE
Expand All @@ -46,7 +48,7 @@ def article_exists(self, group_name, style, range):
table_name = self.get_table_name(group_name)
stmt = """
SELECT
COUNT(*) AS check
COUNT(*) AS total
FROM
%s
WHERE
Expand Down Expand Up @@ -110,7 +112,7 @@ def get_NEWNEWS(self, ts, group='*'):
FROM
papercut_groups
WHERE
name=%s
name LIKE %s
ORDER BY
name ASC"""
self.cursor.execute(stmt, (group.replace('*', '%'),))
Expand All @@ -129,7 +131,7 @@ def get_NEWNEWS(self, ts, group='*'):
continue
ids = list(self.cursor.fetchall())
for id in ids:
articles.append("<%s@%s>" % (id, group))
articles.append("<%s@%s>" % (id[0], group))
if len(articles) == 0:
return ''
else:
Expand Down
8 changes: 4 additions & 4 deletions storage/mysql_storage.sql
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ CREATE TABLE papercut_groups (
PRIMARY KEY (id),
KEY name (name),
KEY active (active)
) TYPE=MyISAM;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

DROP TABLE IF EXISTS papercut_groups_auth;
CREATE TABLE papercut_groups_auth (
Expand All @@ -20,12 +20,12 @@ CREATE TABLE papercut_groups_auth (
PRIMARY KEY (id),
KEY name (name),
KEY username (username)
) TYPE=MyISAM;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

DROP TABLE IF EXISTS papercut_default_table;
CREATE TABLE papercut_default_table (
id int(10) unsigned NOT NULL default '0',
datestamp datetime NOT NULL default '0000-00-00 00:00:00',
datestamp datetime NOT NULL default CURRENT_TIMESTAMP,
thread int(10) unsigned NOT NULL default '0',
parent int(10) unsigned NOT NULL default '0',
author varchar(255) NOT NULL default '',
Expand All @@ -41,4 +41,4 @@ CREATE TABLE papercut_default_table (
KEY subject (subject),
KEY thread (thread),
KEY parent (parent)
) TYPE=MyISAM;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
12 changes: 7 additions & 5 deletions storage/phorum_mysql.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ class Papercut_Storage:
"""

def __init__(self):
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass)
self.conn = MySQLdb.connect(host=settings.dbhost, db=settings.dbname, user=settings.dbuser, passwd=settings.dbpass, charset="utf8mb4")
# commit every statement (InnoDB tables would otherwise never see posts)
self.conn.autocommit(True)
self.cursor = self.conn.cursor()

def get_message_body(self, headers):
Expand Down Expand Up @@ -186,7 +188,7 @@ def send_notifications(self, group_name, msg_id, thread_id, parent_id, msg_autho
emails = self.get_notification_emails(forum_id)
for recipient in emails:
current_msg = msg_tpl % vars()
smtp.sendmail("Phorum <%s>" % (recipient), recipient, current_msg)
strutil.send_text_mail(smtp, "Phorum <%s>" % (recipient), recipient, current_msg)

# XXX: Coding blind here. I really don't know much about how Phorum works with
# XXX: sending forum postings as emails, but it's here. Let's call this a
Expand Down Expand Up @@ -234,7 +236,7 @@ def send_notifications(self, group_name, msg_id, thread_id, parent_id, msg_autho
from papercut import __VERSION__
phorum_version = settings.phorum_version
current_msg = notification_mail_tpl % vars()
smtp.sendmail('%s %s' % (msg_author, msg_email), email_list, current_msg)
strutil.send_text_mail(smtp, '%s %s' % (msg_author, msg_email), email_list, current_msg)
smtp.quit()

def get_NEWGROUPS(self, ts, group='%'):
Expand All @@ -249,7 +251,7 @@ def get_NEWNEWS(self, ts, group='*'):
FROM
forums
WHERE
nntp_group_name=%s
nntp_group_name LIKE %s
ORDER BY
nntp_group_name ASC"""
self.cursor.execute(stmt, (group.replace('*', '%'),))
Expand All @@ -269,7 +271,7 @@ def get_NEWNEWS(self, ts, group='*'):
continue
ids = list(self.cursor.fetchall())
for id in ids:
articles.append("<%s@%s>" % (id, group))
articles.append("<%s@%s>" % (id[0], group))
if len(articles) == 0:
return ''
else:
Expand Down
Loading
Loading