Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions docs/unit-testing-plan.md
Original file line number Diff line number Diff line change
Expand Up @@ -193,12 +193,15 @@ One PR each:

## Known bugs these tests will expose

Mark these `xfail` with a reason, and fix them in follow-up PRs:
Marked `xfail` while the tests were written, and fixed afterwards:

- `papercut.py` `do_LISTGROUP`: `numbers.split('\r\n')` always returns at
least one element, so an empty group never resets the current article.
- `papercut.py` `do_XROVER`: it calls `do_XHDR` with `REFERENCES`, but
`do_XHDR` only accepts `SUBJECT` and `FROM`, so `XROVER` always returns 501.
least one element, so an empty group never reset the current article (and
its empty list was sent as one blank line).
- `papercut.py` `do_XROVER`: it overwrote its range argument with the header
name, and `do_XHDR` only accepted `SUBJECT` and `FROM`, so `XROVER` always
returned 501. Once reachable, phpBB/PHP-Nuke's XHDR `REFERENCES` turned out
to have the same topic-id bug as their XOVER.

Found and fixed in step 1, with regression tests:

Expand Down
47 changes: 31 additions & 16 deletions papercut.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,6 @@
# so let's create them just once and re-use as needed
newsgroups_regexp = re.compile("^Newsgroups:(.*)", re.M)
contenttype_regexp = re.compile("^Content-Type:(.*);", re.M)
# NNTP commands are case-insensitive, so this must be too
authinfo_regexp = re.compile(r"^\s*AUTHINFO\s+PASS\b", re.I)

if os.name == 'posix':
class NNTPServer(socketserver.ForkingTCPServer):
Expand Down Expand Up @@ -179,10 +177,13 @@ def handle(self):
self.tokens = line.split(' ')
# NNTP commands are case-insensitive
command = self.tokens[0].upper()
# don't save the password in the log file
match = authinfo_regexp.search(line)
if not match:
settings.logEvent('Received request: %s' % (line))
# log only the command name: arguments can hold passwords, user
# names or search patterns, and article lines are message content
if not self.sending_article:
if command in self.commands:
settings.logEvent('Received request: %s' % (command))
else:
settings.logEvent('Received request: unknown command')
if command == 'POST':
if settings.server_type == 'read-only':
settings.logEvent('Error - Read-only server received a post request from \'%s\'' % self.client_address[0])
Expand Down Expand Up @@ -623,18 +624,19 @@ def do_LISTGROUP(self):
self.send_response(ERR_NOGROUPSELECTED)
return
numbers = self.backend.get_LISTGROUP(self.selected_group)
check = numbers.split('\r\n')
if len(check) > 0:
# a group given as an argument is selected, even if it is empty
if len(self.tokens) == 2:
self.selected_group = self.tokens[1]
check = [number for number in numbers.split('\r\n') if number]
if check:
# When a valid group is selected by means of this command, the
# internally maintained "current article pointer" is set to the first
# article in the group.
self.selected_article = check[0]
if len(self.tokens) == 2:
self.selected_group = self.tokens[1]
else:
# If an empty newsgroup is selected, the current article pointer is made invalid.
else:
# If an empty newsgroup is selected, the current article pointer is made invalid.
self.selected_article = 'ggg'
self.send_response("%s\r\n%s\r\n." % (STATUS_LISTGROUP % (self.backend.get_group_stats(self.selected_group)), numbers))
self.send_multiline(STATUS_LISTGROUP % (self.backend.get_group_stats(self.selected_group)), numbers)

def do_XGTITLE(self):
"""
Expand Down Expand Up @@ -682,7 +684,13 @@ def do_HDR(self):
self.do_XHDR()

def do_XROVER(self):
self.tokens[1] = 'REFERENCES'
"""
Syntax:
XROVER [range]
Same as XHDR REFERENCES [range].
"""
# (this used to overwrite the range argument with the header name)
self.tokens = ['XHDR', 'REFERENCES'] + self.tokens[1:]
self.do_XHDR()

def do_XHDR(self):
Expand All @@ -701,7 +709,7 @@ def do_XHDR(self):
if self.selected_group == 'ggg':
self.send_response(ERR_NOGROUPSELECTED)
return
if (self.tokens[1].upper() != 'SUBJECT') and (self.tokens[1].upper() != 'FROM'):
if self.tokens[1].upper() not in ('SUBJECT', 'FROM', 'REFERENCES'):
self.send_response(ERR_CMDSYNTAXERROR)
return
if len(self.tokens) == 2:
Expand All @@ -724,7 +732,7 @@ def do_XHDR(self):
if info == None:
self.send_response(ERR_NOTCAPABLE)
else:
self.send_response("%s\r\n%s\r\n." % (STATUS_XHDR, info))
self.send_multiline(STATUS_XHDR, info)

def do_DATE(self):
"""
Expand Down Expand Up @@ -880,6 +888,13 @@ def get_timestamp(self, date, times, gmt='yes'):
# backends turn this back into an epoch with time.mktime()
return time.localtime(ts)

def send_multiline(self, status, body):
"""Sends a status line and a multi-line block (body may be empty)."""
if body:
self.send_response("%s\r\n%s\r\n." % (status, body))
else:
self.send_response("%s\r\n." % (status))

def send_response(self, message):
if __DEBUG__:
print("server>", message)
Expand Down
6 changes: 4 additions & 2 deletions storage/phpbb_mysql.py
Original file line number Diff line number Diff line change
Expand Up @@ -559,7 +559,7 @@ def get_XHDR(self, group_name, header, style, range):
stmt = """
SELECT
A.post_id,
A.topic_id,
C.topic_first_post_id,
D.username,
D.user_email,
CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', C.topic_title) ELSE B.post_subject END,
Expand Down Expand Up @@ -598,7 +598,9 @@ def get_XHDR(self, group_name, header, style, range):
hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[5]))))
elif header.upper() == 'MESSAGE-ID':
hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name))
elif (header.upper() == 'REFERENCES') and (row[1] != 0):
elif (header.upper() == 'REFERENCES') and (row[1] != row[0]):
# replies reference the first post of their topic (this used
# to use the topic id as an article number)
hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name))
elif header.upper() == 'BYTES':
hdrs.append('%s %s' % (row[0], len(row[6])))
Expand Down
6 changes: 4 additions & 2 deletions storage/phpnuke_phpbb_mysql.py
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,7 @@ def get_XHDR(self, group_name, header, style, range):
stmt = """
SELECT
A.post_id,
A.topic_id,
C.topic_first_post_id,
D.username,
D.user_email,
CASE WHEN B.post_subject = '' THEN CONCAT('Re: ', C.topic_title) ELSE B.post_subject END,
Expand Down Expand Up @@ -575,7 +575,9 @@ def get_XHDR(self, group_name, header, style, range):
hdrs.append('%s %s' % (row[0], strutil.get_formatted_time(time.localtime(result[5]))))
elif header.upper() == 'MESSAGE-ID':
hdrs.append('%s <%s@%s>' % (row[0], row[0], group_name))
elif (header.upper() == 'REFERENCES') and (row[1] != 0):
elif (header.upper() == 'REFERENCES') and (row[1] != row[0]):
# replies reference the first post of their topic (this used
# to use the topic id as an article number)
hdrs.append('%s <%s@%s>' % (row[0], row[1], group_name))
elif header.upper() == 'BYTES':
hdrs.append('%s %s' % (row[0], len(row[6])))
Expand Down
7 changes: 7 additions & 0 deletions tests/integration/test_mysql_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -146,3 +146,10 @@ def test_injection_payloads_are_harmless(live, command):
c.close()
assert response[:3] in ("211", "230", "282", "411", "481")
assert live.query("SELECT COUNT(*) FROM papercut_default_table") == before


def test_xrover(live):
with live.nntp() as s:
s.group(GROUP)
_, lines = s._longcmdstring("XROVER 1-3")
assert lines == ["2 <1@%s>" % GROUP]
7 changes: 7 additions & 0 deletions tests/integration/test_phorum_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,3 +186,10 @@ def test_newnews_with_a_wildcard(live):
with live.nntp() as s:
_, ids = s.newnews("phorum.*", datetime.datetime(2024, 1, 2))
assert ids == ["<2@%s>" % GROUP, "<4@%s>" % GROUP]


def test_xrover(live):
with live.nntp() as s:
s.group(GROUP)
_, lines = s._longcmdstring("XROVER 1-4")
assert lines == ["2 <1@%s>" % GROUP]
9 changes: 9 additions & 0 deletions tests/integration/test_phpbb_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,3 +201,12 @@ def test_injection_payloads_are_harmless(live, command):
c.close()
assert response[:3] in ("211", "230", "282", "411", "481")
assert live.query("SELECT COUNT(*) FROM %sposts" % live.prefix) == before


def test_xrover(live):
# only post 2 is a reply; post 3 starts topic 2 (the topic id used to be
# reported as the replied-to article)
with live.nntp() as s:
s.group(live.group)
_, lines = s._longcmdstring("XROVER 1-3")
assert lines == ["2 <1@%s>" % live.group]
96 changes: 91 additions & 5 deletions tests/test_protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,9 +287,37 @@ def test_xhdr_ranges(client, backend, command, expected_args):
assert backend.called("get_XHDR") == [expected_args]


def test_xhdr_only_supports_subject_and_from(client):
def test_xhdr_supports_subject_from_and_references(client):
client.cmd("GROUP test.group")
assert code(client.cmd("XHDR date 1-2")) == "501"
assert code(client.cmd("XHDR references 1-2")) == "221"
client.read_multiline()


def test_xhdr_with_no_results_sends_an_empty_list(client, backend):
backend.returns["get_XHDR"] = ""
client.cmd("GROUP test.group")
assert code(client.cmd("XHDR subject 1-2")) == "221"
assert client.read_multiline() == []


@pytest.mark.parametrize("command, expected_args", [
("XROVER 1-5", ("test.group", "REFERENCES", "range", ("1", "5"))),
("XROVER 3-", ("test.group", "REFERENCES", "range", "3")),
("XROVER", ("test.group", "REFERENCES", "unique", "2")),
])
def test_xrover_is_xhdr_references(client, backend, command, expected_args):
# regression: XROVER overwrote its range argument with the header name,
# and XHDR then refused REFERENCES, so XROVER always failed
client.cmd("GROUP test.group")
client.cmd("STAT 2")
assert code(client.cmd(command)) == "221"
client.read_multiline()
assert backend.called("get_XHDR") == [expected_args]


def test_xrover_needs_a_group(client):
assert code(client.cmd("XROVER 1-5")) == "412"


def test_xpat(client, backend):
Expand Down Expand Up @@ -351,15 +379,30 @@ def test_listgroup_without_a_group(client):
assert code(client.cmd("LISTGROUP")) == "412"


@pytest.mark.xfail(reason="do_LISTGROUP: ''.split('\\r\\n') is [''], so an empty "
"group never resets the current article", strict=True)
def test_listgroup_of_an_empty_group_clears_the_current_article(client, backend):
# regression: ''.split('\r\n') is [''], so an empty group used to make ''
# the current article
backend.returns["get_LISTGROUP"] = ""
client.cmd("LISTGROUP test.group")
client.read_multiline()
client.cmd("GROUP test.group")
client.cmd("STAT 2")
assert code(client.cmd("LISTGROUP test.group")) == "211"
# regression: an empty list used to be sent as one blank line
assert client.read_multiline() == []
assert code(client.cmd("STAT")) == "420"


def test_listgroup_selects_an_empty_group(client, backend):
client.cmd("GROUP test.group")
backend.returns["get_LISTGROUP"] = ""
backend.returns["get_group_stats"] = (0, 0, 0, "empty.group")
assert client.cmd("LISTGROUP empty.group").startswith("211 0 0 0 empty.group")
client.read_multiline()
assert backend.called("get_group_stats") == [("empty.group",)]
client.cmd("XOVER 1-2")
client.read_multiline()
assert backend.called("get_XOVER")[0][0] == "empty.group"


def test_xgtitle(client, backend):
assert code(client.cmd("XGTITLE comp.*")) == "282"
assert client.read_multiline() == ["test.group A test group"]
Expand Down Expand Up @@ -583,3 +626,46 @@ def test_newnews_accepts_wildmat_patterns(client, backend, pattern):
assert code(client.cmd("NEWNEWS %s 20240105 000000" % pattern)) == "230"
client.read_multiline()
assert backend.called("get_NEWNEWS")[0][1] == pattern


# --- logging ---

def received_requests():
return [line.split("Received request: ", 1)[1].strip()
for line in log_text().splitlines() if "Received request: " in line]


def test_only_command_names_are_logged(client):
client.cmd("GROUP secret.group.name")
client.cmd("XHDR subject 1-2")
client.read_multiline()
client.cmd("quit")
assert wait_for_log("Connection closed")
assert received_requests() == ["GROUP", "XHDR", "QUIT"]
assert "secret.group.name" not in log_text()


def test_unknown_commands_are_not_logged_verbatim(client):
client.cmd("SOMETHING private-looking-text")
client.cmd("QUIT")
assert wait_for_log("Connection closed")
assert received_requests() == ["unknown command", "QUIT"]
assert "private-looking-text" not in log_text()


def test_article_lines_are_not_logged(client):
post(client, ARTICLE_LINES[:3] + ["", "confidential body text"])
client.cmd("QUIT")
assert wait_for_log("Connection closed")
assert received_requests() == ["POST", "QUIT"]
assert "confidential body text" not in log_text()
assert "Tester" not in log_text()


def test_login_names_are_not_logged_by_the_server(auth_on, client):
client.cmd("AUTHINFO USER alice")
client.cmd("AUTHINFO PASS s3cret")
client.cmd("QUIT")
assert wait_for_log("Connection closed")
assert received_requests() == ["AUTHINFO", "AUTHINFO", "QUIT"]
assert "alice" not in log_text()
Loading