From ec653e6bec5d862b6b4cde25ded7cc639fbd284c Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 09:45:47 -0600 Subject: [PATCH 1/3] Fix LISTGROUP on empty groups, and XROVER - LISTGROUP: ''.split('\r\n') is [''], so an empty group made '' the current article instead of clearing it, and the empty list was sent as one blank line. XHDR had the same blank-line problem. - XROVER overwrote its range argument with the header name (and raised IndexError without one), and XHDR only accepted SUBJECT and FROM, so XROVER always returned 501. XHDR now also accepts REFERENCES. --- docs/unit-testing-plan.md | 11 +++++--- papercut.py | 34 +++++++++++++++++-------- tests/test_protocol.py | 53 +++++++++++++++++++++++++++++++++++---- 3 files changed, 79 insertions(+), 19 deletions(-) diff --git a/docs/unit-testing-plan.md b/docs/unit-testing-plan.md index bd0c521..c0b2ed0 100644 --- a/docs/unit-testing-plan.md +++ b/docs/unit-testing-plan.md @@ -193,12 +193,15 @@ One PR each: ## Known bugs these tests will expose -Mark these `xfail` with a reason, and fix them in follow-up PRs: +Marked `xfail` while the tests were written, and fixed afterwards: - `papercut.py` `do_LISTGROUP`: `numbers.split('\r\n')` always returns at - least one element, so an empty group never resets the current article. -- `papercut.py` `do_XROVER`: it calls `do_XHDR` with `REFERENCES`, but - `do_XHDR` only accepts `SUBJECT` and `FROM`, so `XROVER` always returns 501. + least one element, so an empty group never reset the current article (and + its empty list was sent as one blank line). +- `papercut.py` `do_XROVER`: it overwrote its range argument with the header + name, and `do_XHDR` only accepted `SUBJECT` and `FROM`, so `XROVER` always + returned 501. Once reachable, phpBB/PHP-Nuke's XHDR `REFERENCES` turned out + to have the same topic-id bug as their XOVER. Found and fixed in step 1, with regression tests: diff --git a/papercut.py b/papercut.py index 5b2a4fa..f9c300b 100755 --- a/papercut.py +++ b/papercut.py @@ -623,18 +623,19 @@ def do_LISTGROUP(self): self.send_response(ERR_NOGROUPSELECTED) return numbers = self.backend.get_LISTGROUP(self.selected_group) - check = numbers.split('\r\n') - if len(check) > 0: + # a group given as an argument is selected, even if it is empty + if len(self.tokens) == 2: + self.selected_group = self.tokens[1] + check = [number for number in numbers.split('\r\n') if number] + if check: # When a valid group is selected by means of this command, the # internally maintained "current article pointer" is set to the first # article in the group. self.selected_article = check[0] - if len(self.tokens) == 2: - self.selected_group = self.tokens[1] - else: - # If an empty newsgroup is selected, the current article pointer is made invalid. + else: + # If an empty newsgroup is selected, the current article pointer is made invalid. self.selected_article = 'ggg' - self.send_response("%s\r\n%s\r\n." % (STATUS_LISTGROUP % (self.backend.get_group_stats(self.selected_group)), numbers)) + self.send_multiline(STATUS_LISTGROUP % (self.backend.get_group_stats(self.selected_group)), numbers) def do_XGTITLE(self): """ @@ -682,7 +683,13 @@ def do_HDR(self): self.do_XHDR() def do_XROVER(self): - self.tokens[1] = 'REFERENCES' + """ + Syntax: + XROVER [range] + Same as XHDR REFERENCES [range]. + """ + # (this used to overwrite the range argument with the header name) + self.tokens = ['XHDR', 'REFERENCES'] + self.tokens[1:] self.do_XHDR() def do_XHDR(self): @@ -701,7 +708,7 @@ def do_XHDR(self): if self.selected_group == 'ggg': self.send_response(ERR_NOGROUPSELECTED) return - if (self.tokens[1].upper() != 'SUBJECT') and (self.tokens[1].upper() != 'FROM'): + if self.tokens[1].upper() not in ('SUBJECT', 'FROM', 'REFERENCES'): self.send_response(ERR_CMDSYNTAXERROR) return if len(self.tokens) == 2: @@ -724,7 +731,7 @@ def do_XHDR(self): if info == None: self.send_response(ERR_NOTCAPABLE) else: - self.send_response("%s\r\n%s\r\n." % (STATUS_XHDR, info)) + self.send_multiline(STATUS_XHDR, info) def do_DATE(self): """ @@ -880,6 +887,13 @@ def get_timestamp(self, date, times, gmt='yes'): # backends turn this back into an epoch with time.mktime() return time.localtime(ts) + def send_multiline(self, status, body): + """Sends a status line and a multi-line block (body may be empty).""" + if body: + self.send_response("%s\r\n%s\r\n." % (status, body)) + else: + self.send_response("%s\r\n." % (status)) + def send_response(self, message): if __DEBUG__: print("server>", message) diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 027c2dd..6df06ba 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -287,9 +287,37 @@ def test_xhdr_ranges(client, backend, command, expected_args): assert backend.called("get_XHDR") == [expected_args] -def test_xhdr_only_supports_subject_and_from(client): +def test_xhdr_supports_subject_from_and_references(client): client.cmd("GROUP test.group") assert code(client.cmd("XHDR date 1-2")) == "501" + assert code(client.cmd("XHDR references 1-2")) == "221" + client.read_multiline() + + +def test_xhdr_with_no_results_sends_an_empty_list(client, backend): + backend.returns["get_XHDR"] = "" + client.cmd("GROUP test.group") + assert code(client.cmd("XHDR subject 1-2")) == "221" + assert client.read_multiline() == [] + + +@pytest.mark.parametrize("command, expected_args", [ + ("XROVER 1-5", ("test.group", "REFERENCES", "range", ("1", "5"))), + ("XROVER 3-", ("test.group", "REFERENCES", "range", "3")), + ("XROVER", ("test.group", "REFERENCES", "unique", "2")), +]) +def test_xrover_is_xhdr_references(client, backend, command, expected_args): + # regression: XROVER overwrote its range argument with the header name, + # and XHDR then refused REFERENCES, so XROVER always failed + client.cmd("GROUP test.group") + client.cmd("STAT 2") + assert code(client.cmd(command)) == "221" + client.read_multiline() + assert backend.called("get_XHDR") == [expected_args] + + +def test_xrover_needs_a_group(client): + assert code(client.cmd("XROVER 1-5")) == "412" def test_xpat(client, backend): @@ -351,15 +379,30 @@ def test_listgroup_without_a_group(client): assert code(client.cmd("LISTGROUP")) == "412" -@pytest.mark.xfail(reason="do_LISTGROUP: ''.split('\\r\\n') is [''], so an empty " - "group never resets the current article", strict=True) def test_listgroup_of_an_empty_group_clears_the_current_article(client, backend): + # regression: ''.split('\r\n') is [''], so an empty group used to make '' + # the current article backend.returns["get_LISTGROUP"] = "" - client.cmd("LISTGROUP test.group") - client.read_multiline() + client.cmd("GROUP test.group") + client.cmd("STAT 2") + assert code(client.cmd("LISTGROUP test.group")) == "211" + # regression: an empty list used to be sent as one blank line + assert client.read_multiline() == [] assert code(client.cmd("STAT")) == "420" +def test_listgroup_selects_an_empty_group(client, backend): + client.cmd("GROUP test.group") + backend.returns["get_LISTGROUP"] = "" + backend.returns["get_group_stats"] = (0, 0, 0, "empty.group") + assert client.cmd("LISTGROUP empty.group").startswith("211 0 0 0 empty.group") + client.read_multiline() + assert backend.called("get_group_stats") == [("empty.group",)] + client.cmd("XOVER 1-2") + client.read_multiline() + assert backend.called("get_XOVER")[0][0] == "empty.group" + + def test_xgtitle(client, backend): assert code(client.cmd("XGTITLE comp.*")) == "282" assert client.read_multiline() == ["test.group A test group"] From 409b6b5a44e11d5b85a9d45f5ec31cf52387ad09 Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 09:45:48 -0600 Subject: [PATCH 2/3] Fix XHDR REFERENCES in the phpBB and PHP-Nuke backends Reachable now that XROVER works: like XOVER before it, XHDR used the topic id as the replied-to article number, and gave every post a reference, including the first post of each topic. --- storage/phpbb_mysql.py | 6 ++++-- storage/phpnuke_phpbb_mysql.py | 6 ++++-- tests/integration/test_mysql_backend.py | 7 +++++++ tests/integration/test_phorum_backends.py | 7 +++++++ tests/integration/test_phpbb_backends.py | 9 +++++++++ 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/storage/phpbb_mysql.py b/storage/phpbb_mysql.py index b6dcac2..1a40d98 100755 --- a/storage/phpbb_mysql.py +++ b/storage/phpbb_mysql.py @@ -559,7 +559,7 @@ def get_XHDR(self, group_name, header, style, range): stmt = """ SELECT A.post_id, - A.topic_id, + C.topic_first_post_id, D.username, D.user_email, CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', C.topic_title) ELSE B.post_subject END, @@ -598,7 +598,9 @@ def get_XHDR(self, group_name, header, style, range): hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[5])))) elif header.upper() == 'MESSAGE-ID': hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name)) - elif (header.upper() == 'REFERENCES') and (row[1] != 0): + elif (header.upper() == 'REFERENCES') and (row[1] != row[0]): + # replies reference the first post of their topic (this used + # to use the topic id as an article number) hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name)) elif header.upper() == 'BYTES': hdrs.append('%s %s' % (row[0], len(row[6]))) diff --git a/storage/phpnuke_phpbb_mysql.py b/storage/phpnuke_phpbb_mysql.py index 4bbc2be..45473cc 100755 --- a/storage/phpnuke_phpbb_mysql.py +++ b/storage/phpnuke_phpbb_mysql.py @@ -536,7 +536,7 @@ def get_XHDR(self, group_name, header, style, range): stmt = """ SELECT A.post_id, - A.topic_id, + C.topic_first_post_id, D.username, D.user_email, CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', C.topic_title) ELSE B.post_subject END, @@ -575,7 +575,9 @@ def get_XHDR(self, group_name, header, style, range): hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[5])))) elif header.upper() == 'MESSAGE-ID': hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name)) - elif (header.upper() == 'REFERENCES') and (row[1] != 0): + elif (header.upper() == 'REFERENCES') and (row[1] != row[0]): + # replies reference the first post of their topic (this used + # to use the topic id as an article number) hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name)) elif header.upper() == 'BYTES': hdrs.append('%s %s' % (row[0], len(row[6]))) diff --git a/tests/integration/test_mysql_backend.py b/tests/integration/test_mysql_backend.py index 67a3e7d..b282025 100644 --- a/tests/integration/test_mysql_backend.py +++ b/tests/integration/test_mysql_backend.py @@ -146,3 +146,10 @@ def test_injection_payloads_are_harmless(live, command): c.close() assert response[:3] in ("211", "230", "282", "411", "481") assert live.query("SELECT COUNT(*) FROM papercut_default_table") == before + + +def test_xrover(live): + with live.nntp() as s: + s.group(GROUP) + _, lines = s._longcmdstring("XROVER 1-3") + assert lines == ["2 <1@%s>" % GROUP] diff --git a/tests/integration/test_phorum_backends.py b/tests/integration/test_phorum_backends.py index 09ec3be..f8d5644 100644 --- a/tests/integration/test_phorum_backends.py +++ b/tests/integration/test_phorum_backends.py @@ -186,3 +186,10 @@ def test_newnews_with_a_wildcard(live): with live.nntp() as s: _, ids = s.newnews("phorum.*", datetime.datetime(2024, 1, 2)) assert ids == ["<2@%s>" % GROUP, "<4@%s>" % GROUP] + + +def test_xrover(live): + with live.nntp() as s: + s.group(GROUP) + _, lines = s._longcmdstring("XROVER 1-4") + assert lines == ["2 <1@%s>" % GROUP] diff --git a/tests/integration/test_phpbb_backends.py b/tests/integration/test_phpbb_backends.py index 8b28ae0..532ae7f 100644 --- a/tests/integration/test_phpbb_backends.py +++ b/tests/integration/test_phpbb_backends.py @@ -201,3 +201,12 @@ def test_injection_payloads_are_harmless(live, command): c.close() assert response[:3] in ("211", "230", "282", "411", "481") assert live.query("SELECT COUNT(*) FROM %sposts" % live.prefix) == before + + +def test_xrover(live): + # only post 2 is a reply; post 3 starts topic 2 (the topic id used to be + # reported as the replied-to article) + with live.nntp() as s: + s.group(live.group) + _, lines = s._longcmdstring("XROVER 1-3") + assert lines == ["2 <1@%s>" % live.group] From 8b9a5436526881dc73cec2ca4674991a042a56bb Mon Sep 17 00:00:00 2001 From: Joao Prado Maia Date: Tue, 10 Nov 2026 09:46:21 -0600 Subject: [PATCH 3/3] Log only the command names of requests Every line a client sent was written to the log, including the full text of posted articles, search patterns, group names and login names (passwords were already filtered). Requests are now logged by command name only ('unknown command' for anything else), and article lines are not logged at all. --- papercut.py | 13 +++++++------ tests/test_protocol.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/papercut.py b/papercut.py index f9c300b..e41034a 100755 --- a/papercut.py +++ b/papercut.py @@ -83,8 +83,6 @@ # so let's create them just once and re-use as needed newsgroups_regexp = re.compile("^Newsgroups:(.*)", re.M) contenttype_regexp = re.compile("^Content-Type:(.*);", re.M) -# NNTP commands are case-insensitive, so this must be too -authinfo_regexp = re.compile(r"^\s*AUTHINFO\s+PASS\b", re.I) if os.name == 'posix': class NNTPServer(socketserver.ForkingTCPServer): @@ -179,10 +177,13 @@ def handle(self): self.tokens = line.split(' ') # NNTP commands are case-insensitive command = self.tokens[0].upper() - # don't save the password in the log file - match = authinfo_regexp.search(line) - if not match: - settings.logEvent('Received request: %s' % (line)) + # log only the command name: arguments can hold passwords, user + # names or search patterns, and article lines are message content + if not self.sending_article: + if command in self.commands: + settings.logEvent('Received request: %s' % (command)) + else: + settings.logEvent('Received request: unknown command') if command == 'POST': if settings.server_type == 'read-only': settings.logEvent('Error - Read-only server received a post request from \'%s\'' % self.client_address[0]) diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 6df06ba..4f9fd09 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -626,3 +626,46 @@ def test_newnews_accepts_wildmat_patterns(client, backend, pattern): assert code(client.cmd("NEWNEWS %s 20240105 000000" % pattern)) == "230" client.read_multiline() assert backend.called("get_NEWNEWS")[0][1] == pattern + + +# --- logging --- + +def received_requests(): + return [line.split("Received request: ", 1)[1].strip() + for line in log_text().splitlines() if "Received request: " in line] + + +def test_only_command_names_are_logged(client): + client.cmd("GROUP secret.group.name") + client.cmd("XHDR subject 1-2") + client.read_multiline() + client.cmd("quit") + assert wait_for_log("Connection closed") + assert received_requests() == ["GROUP", "XHDR", "QUIT"] + assert "secret.group.name" not in log_text() + + +def test_unknown_commands_are_not_logged_verbatim(client): + client.cmd("SOMETHING private-looking-text") + client.cmd("QUIT") + assert wait_for_log("Connection closed") + assert received_requests() == ["unknown command", "QUIT"] + assert "private-looking-text" not in log_text() + + +def test_article_lines_are_not_logged(client): + post(client, ARTICLE_LINES[:3] + ["", "confidential body text"]) + client.cmd("QUIT") + assert wait_for_log("Connection closed") + assert received_requests() == ["POST", "QUIT"] + assert "confidential body text" not in log_text() + assert "Tester" not in log_text() + + +def test_login_names_are_not_logged_by_the_server(auth_on, client): + client.cmd("AUTHINFO USER alice") + client.cmd("AUTHINFO PASS s3cret") + client.cmd("QUIT") + assert wait_for_log("Connection closed") + assert received_requests() == ["AUTHINFO", "AUTHINFO", "QUIT"] + assert "alice" not in log_text()