diff --git a/docs/unit-testing-plan.md b/docs/unit-testing-plan.md index bd0c521..c0b2ed0 100644 --- a/docs/unit-testing-plan.md +++ b/docs/unit-testing-plan.md @@ -193,12 +193,15 @@ One PR each: ## Known bugs these tests will expose -Mark these `xfail` with a reason, and fix them in follow-up PRs: +Marked `xfail` while the tests were written, and fixed afterwards: - `papercut.py` `do_LISTGROUP`: `numbers.split('\r\n')` always returns at - least one element, so an empty group never resets the current article. -- `papercut.py` `do_XROVER`: it calls `do_XHDR` with `REFERENCES`, but - `do_XHDR` only accepts `SUBJECT` and `FROM`, so `XROVER` always returns 501. + least one element, so an empty group never reset the current article (and + its empty list was sent as one blank line). +- `papercut.py` `do_XROVER`: it overwrote its range argument with the header + name, and `do_XHDR` only accepted `SUBJECT` and `FROM`, so `XROVER` always + returned 501. Once reachable, phpBB/PHP-Nuke's XHDR `REFERENCES` turned out + to have the same topic-id bug as their XOVER. Found and fixed in step 1, with regression tests: diff --git a/papercut.py b/papercut.py index 5b2a4fa..e41034a 100755 --- a/papercut.py +++ b/papercut.py @@ -83,8 +83,6 @@ # so let's create them just once and re-use as needed newsgroups_regexp = re.compile("^Newsgroups:(.*)", re.M) contenttype_regexp = re.compile("^Content-Type:(.*);", re.M) -# NNTP commands are case-insensitive, so this must be too -authinfo_regexp = re.compile(r"^\s*AUTHINFO\s+PASS\b", re.I) if os.name == 'posix': class NNTPServer(socketserver.ForkingTCPServer): @@ -179,10 +177,13 @@ def handle(self): self.tokens = line.split(' ') # NNTP commands are case-insensitive command = self.tokens[0].upper() - # don't save the password in the log file - match = authinfo_regexp.search(line) - if not match: - settings.logEvent('Received request: %s' % (line)) + # log only the command name: arguments can hold passwords, user + # names or search patterns, and article lines are message content + if not self.sending_article: + if command in self.commands: + settings.logEvent('Received request: %s' % (command)) + else: + settings.logEvent('Received request: unknown command') if command == 'POST': if settings.server_type == 'read-only': settings.logEvent('Error - Read-only server received a post request from \'%s\'' % self.client_address[0]) @@ -623,18 +624,19 @@ def do_LISTGROUP(self): self.send_response(ERR_NOGROUPSELECTED) return numbers = self.backend.get_LISTGROUP(self.selected_group) - check = numbers.split('\r\n') - if len(check) > 0: + # a group given as an argument is selected, even if it is empty + if len(self.tokens) == 2: + self.selected_group = self.tokens[1] + check = [number for number in numbers.split('\r\n') if number] + if check: # When a valid group is selected by means of this command, the # internally maintained "current article pointer" is set to the first # article in the group. self.selected_article = check[0] - if len(self.tokens) == 2: - self.selected_group = self.tokens[1] - else: - # If an empty newsgroup is selected, the current article pointer is made invalid. + else: + # If an empty newsgroup is selected, the current article pointer is made invalid. self.selected_article = 'ggg' - self.send_response("%s\r\n%s\r\n." % (STATUS_LISTGROUP % (self.backend.get_group_stats(self.selected_group)), numbers)) + self.send_multiline(STATUS_LISTGROUP % (self.backend.get_group_stats(self.selected_group)), numbers) def do_XGTITLE(self): """ @@ -682,7 +684,13 @@ def do_HDR(self): self.do_XHDR() def do_XROVER(self): - self.tokens[1] = 'REFERENCES' + """ + Syntax: + XROVER [range] + Same as XHDR REFERENCES [range]. + """ + # (this used to overwrite the range argument with the header name) + self.tokens = ['XHDR', 'REFERENCES'] + self.tokens[1:] self.do_XHDR() def do_XHDR(self): @@ -701,7 +709,7 @@ def do_XHDR(self): if self.selected_group == 'ggg': self.send_response(ERR_NOGROUPSELECTED) return - if (self.tokens[1].upper() != 'SUBJECT') and (self.tokens[1].upper() != 'FROM'): + if self.tokens[1].upper() not in ('SUBJECT', 'FROM', 'REFERENCES'): self.send_response(ERR_CMDSYNTAXERROR) return if len(self.tokens) == 2: @@ -724,7 +732,7 @@ def do_XHDR(self): if info == None: self.send_response(ERR_NOTCAPABLE) else: - self.send_response("%s\r\n%s\r\n." % (STATUS_XHDR, info)) + self.send_multiline(STATUS_XHDR, info) def do_DATE(self): """ @@ -880,6 +888,13 @@ def get_timestamp(self, date, times, gmt='yes'): # backends turn this back into an epoch with time.mktime() return time.localtime(ts) + def send_multiline(self, status, body): + """Sends a status line and a multi-line block (body may be empty).""" + if body: + self.send_response("%s\r\n%s\r\n." % (status, body)) + else: + self.send_response("%s\r\n." % (status)) + def send_response(self, message): if __DEBUG__: print("server>", message) diff --git a/storage/phpbb_mysql.py b/storage/phpbb_mysql.py index b6dcac2..1a40d98 100755 --- a/storage/phpbb_mysql.py +++ b/storage/phpbb_mysql.py @@ -559,7 +559,7 @@ def get_XHDR(self, group_name, header, style, range): stmt = """ SELECT A.post_id, - A.topic_id, + C.topic_first_post_id, D.username, D.user_email, CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', C.topic_title) ELSE B.post_subject END, @@ -598,7 +598,9 @@ def get_XHDR(self, group_name, header, style, range): hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[5])))) elif header.upper() == 'MESSAGE-ID': hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name)) - elif (header.upper() == 'REFERENCES') and (row[1] != 0): + elif (header.upper() == 'REFERENCES') and (row[1] != row[0]): + # replies reference the first post of their topic (this used + # to use the topic id as an article number) hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name)) elif header.upper() == 'BYTES': hdrs.append('%s %s' % (row[0], len(row[6]))) diff --git a/storage/phpnuke_phpbb_mysql.py b/storage/phpnuke_phpbb_mysql.py index 4bbc2be..45473cc 100755 --- a/storage/phpnuke_phpbb_mysql.py +++ b/storage/phpnuke_phpbb_mysql.py @@ -536,7 +536,7 @@ def get_XHDR(self, group_name, header, style, range): stmt = """ SELECT A.post_id, - A.topic_id, + C.topic_first_post_id, D.username, D.user_email, CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', C.topic_title) ELSE B.post_subject END, @@ -575,7 +575,9 @@ def get_XHDR(self, group_name, header, style, range): hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[5])))) elif header.upper() == 'MESSAGE-ID': hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name)) - elif (header.upper() == 'REFERENCES') and (row[1] != 0): + elif (header.upper() == 'REFERENCES') and (row[1] != row[0]): + # replies reference the first post of their topic (this used + # to use the topic id as an article number) hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name)) elif header.upper() == 'BYTES': hdrs.append('%s %s' % (row[0], len(row[6]))) diff --git a/tests/integration/test_mysql_backend.py b/tests/integration/test_mysql_backend.py index 67a3e7d..b282025 100644 --- a/tests/integration/test_mysql_backend.py +++ b/tests/integration/test_mysql_backend.py @@ -146,3 +146,10 @@ def test_injection_payloads_are_harmless(live, command): c.close() assert response[:3] in ("211", "230", "282", "411", "481") assert live.query("SELECT COUNT(*) FROM papercut_default_table") == before + + +def test_xrover(live): + with live.nntp() as s: + s.group(GROUP) + _, lines = s._longcmdstring("XROVER 1-3") + assert lines == ["2 <1@%s>" % GROUP] diff --git a/tests/integration/test_phorum_backends.py b/tests/integration/test_phorum_backends.py index 09ec3be..f8d5644 100644 --- a/tests/integration/test_phorum_backends.py +++ b/tests/integration/test_phorum_backends.py @@ -186,3 +186,10 @@ def test_newnews_with_a_wildcard(live): with live.nntp() as s: _, ids = s.newnews("phorum.*", datetime.datetime(2024, 1, 2)) assert ids == ["<2@%s>" % GROUP, "<4@%s>" % GROUP] + + +def test_xrover(live): + with live.nntp() as s: + s.group(GROUP) + _, lines = s._longcmdstring("XROVER 1-4") + assert lines == ["2 <1@%s>" % GROUP] diff --git a/tests/integration/test_phpbb_backends.py b/tests/integration/test_phpbb_backends.py index 8b28ae0..532ae7f 100644 --- a/tests/integration/test_phpbb_backends.py +++ b/tests/integration/test_phpbb_backends.py @@ -201,3 +201,12 @@ def test_injection_payloads_are_harmless(live, command): c.close() assert response[:3] in ("211", "230", "282", "411", "481") assert live.query("SELECT COUNT(*) FROM %sposts" % live.prefix) == before + + +def test_xrover(live): + # only post 2 is a reply; post 3 starts topic 2 (the topic id used to be + # reported as the replied-to article) + with live.nntp() as s: + s.group(live.group) + _, lines = s._longcmdstring("XROVER 1-3") + assert lines == ["2 <1@%s>" % live.group] diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 027c2dd..4f9fd09 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -287,9 +287,37 @@ def test_xhdr_ranges(client, backend, command, expected_args): assert backend.called("get_XHDR") == [expected_args] -def test_xhdr_only_supports_subject_and_from(client): +def test_xhdr_supports_subject_from_and_references(client): client.cmd("GROUP test.group") assert code(client.cmd("XHDR date 1-2")) == "501" + assert code(client.cmd("XHDR references 1-2")) == "221" + client.read_multiline() + + +def test_xhdr_with_no_results_sends_an_empty_list(client, backend): + backend.returns["get_XHDR"] = "" + client.cmd("GROUP test.group") + assert code(client.cmd("XHDR subject 1-2")) == "221" + assert client.read_multiline() == [] + + +@pytest.mark.parametrize("command, expected_args", [ + ("XROVER 1-5", ("test.group", "REFERENCES", "range", ("1", "5"))), + ("XROVER 3-", ("test.group", "REFERENCES", "range", "3")), + ("XROVER", ("test.group", "REFERENCES", "unique", "2")), +]) +def test_xrover_is_xhdr_references(client, backend, command, expected_args): + # regression: XROVER overwrote its range argument with the header name, + # and XHDR then refused REFERENCES, so XROVER always failed + client.cmd("GROUP test.group") + client.cmd("STAT 2") + assert code(client.cmd(command)) == "221" + client.read_multiline() + assert backend.called("get_XHDR") == [expected_args] + + +def test_xrover_needs_a_group(client): + assert code(client.cmd("XROVER 1-5")) == "412" def test_xpat(client, backend): @@ -351,15 +379,30 @@ def test_listgroup_without_a_group(client): assert code(client.cmd("LISTGROUP")) == "412" -@pytest.mark.xfail(reason="do_LISTGROUP: ''.split('\\r\\n') is [''], so an empty " - "group never resets the current article", strict=True) def test_listgroup_of_an_empty_group_clears_the_current_article(client, backend): + # regression: ''.split('\r\n') is [''], so an empty group used to make '' + # the current article backend.returns["get_LISTGROUP"] = "" - client.cmd("LISTGROUP test.group") - client.read_multiline() + client.cmd("GROUP test.group") + client.cmd("STAT 2") + assert code(client.cmd("LISTGROUP test.group")) == "211" + # regression: an empty list used to be sent as one blank line + assert client.read_multiline() == [] assert code(client.cmd("STAT")) == "420" +def test_listgroup_selects_an_empty_group(client, backend): + client.cmd("GROUP test.group") + backend.returns["get_LISTGROUP"] = "" + backend.returns["get_group_stats"] = (0, 0, 0, "empty.group") + assert client.cmd("LISTGROUP empty.group").startswith("211 0 0 0 empty.group") + client.read_multiline() + assert backend.called("get_group_stats") == [("empty.group",)] + client.cmd("XOVER 1-2") + client.read_multiline() + assert backend.called("get_XOVER")[0][0] == "empty.group" + + def test_xgtitle(client, backend): assert code(client.cmd("XGTITLE comp.*")) == "282" assert client.read_multiline() == ["test.group A test group"] @@ -583,3 +626,46 @@ def test_newnews_accepts_wildmat_patterns(client, backend, pattern): assert code(client.cmd("NEWNEWS %s 20240105 000000" % pattern)) == "230" client.read_multiline() assert backend.called("get_NEWNEWS")[0][1] == pattern + + +# --- logging --- + +def received_requests(): + return [line.split("Received request: ", 1)[1].strip() + for line in log_text().splitlines() if "Received request: " in line] + + +def test_only_command_names_are_logged(client): + client.cmd("GROUP secret.group.name") + client.cmd("XHDR subject 1-2") + client.read_multiline() + client.cmd("quit") + assert wait_for_log("Connection closed") + assert received_requests() == ["GROUP", "XHDR", "QUIT"] + assert "secret.group.name" not in log_text() + + +def test_unknown_commands_are_not_logged_verbatim(client): + client.cmd("SOMETHING private-looking-text") + client.cmd("QUIT") + assert wait_for_log("Connection closed") + assert received_requests() == ["unknown command", "QUIT"] + assert "private-looking-text" not in log_text() + + +def test_article_lines_are_not_logged(client): + post(client, ARTICLE_LINES[:3] + ["", "confidential body text"]) + client.cmd("QUIT") + assert wait_for_log("Connection closed") + assert received_requests() == ["POST", "QUIT"] + assert "confidential body text" not in log_text() + assert "Tester" not in log_text() + + +def test_login_names_are_not_logged_by_the_server(auth_on, client): + client.cmd("AUTHINFO USER alice") + client.cmd("AUTHINFO PASS s3cret") + client.cmd("QUIT") + assert wait_for_log("Connection closed") + assert received_requests() == ["AUTHINFO", "AUTHINFO", "QUIT"] + assert "alice" not in log_text()