From d941aa1facb82e7aff53b9fb93e9600169e36341 Mon Sep 17 00:00:00 2001 From: Jacob Coffee Date: Wed, 12 Aug 2026 11:01:28 -0500 Subject: [PATCH] AI: swap to regex from pyparsing --- linehaul/events/parser.py | 177 ++++++------ tests/unit/events/fixtures/download.yml | 340 ++++++++++++++++++++++++ tests/unit/events/fixtures/simple.yml | 100 +++++++ tests/unit/events/test_parser.py | 20 +- 4 files changed, 542 insertions(+), 95 deletions(-) create mode 100644 tests/unit/events/fixtures/simple.yml diff --git a/linehaul/events/parser.py b/linehaul/events/parser.py index 1295fae..8b8cd2c 100644 --- a/linehaul/events/parser.py +++ b/linehaul/events/parser.py @@ -13,6 +13,7 @@ import enum import logging import posixpath +import re from datetime import datetime, timezone from typing import Optional @@ -21,10 +22,6 @@ import attr.validators import cattr -from pyparsing import Literal as L, Word, Optional as OptionalItem -from pyparsing import printables as _printables, rest_of_line -from pyparsing import ParseException - from linehaul.ua import UserAgent, parser as user_agents @@ -51,79 +48,63 @@ class UnparseableEvent(Exception): pass -class _NullValue: - pass - - -NullValue = _NullValue() - - -printables = "".join(set(_printables + " " + "\t") - {"|", "@"}) - -PIPE = L("|").suppress() - -NULL = L("(null)") -NULL.set_parse_action(lambda s, l, t: NullValue) - -TIMESTAMP = Word(printables).set_name("Timestamp") -TIMESTAMP = TIMESTAMP.set_results_name("timestamp") - -COUNTRY_CODE = Word(printables).set_name("Country Code") -COUNTRY_CODE = COUNTRY_CODE.set_results_name("country_code") - -URL = Word(printables).set_name("URL") -URL = URL.set_results_name("url") +# These two regexes replace what used to be a pyparsing grammar. The wire format +# is pipe delimited with a fixed number of fields, so a grammar was more machinery +# than the job needs -- pyparsing accounted for ~60% of the per line cost. They are +# a deliberately faithful translation of that grammar, quirks included: +# +# * A field is pyparsing's ``Word(printables)`` minus "|" and "@", plus space and +# tab. So "@" anywhere in the first nine fields rejects the whole line, and +# spaces/tabs *inside* a field are legal and retained. Non ASCII, DEL and +# control characters reject. +# * pyparsing skips its whitespace set (" \n\t\r") before each token, so leading +# whitespace on a field is stripped from the captured value while trailing +# whitespace is kept, and whitespace is tolerated around every delimiter. +# * The nullable fields use an ordered alternation with a negative lookahead so +# that "(null)x" rejects (pyparsing committed to the "(null)" literal and then +# failed on the missing pipe) while "(nullx" still matches as a plain word. +# * The user agent was ``rest_of_line``, i.e. everything up to a newline, taken +# verbatim -- it may contain "|" and "@" -- and ``parse_all=True`` allowed only +# trailing whitespace after it. +# +# NB: ``parse_string`` expanded tabs before parsing, so ``parse`` below must call +# ``str.expandtabs()`` to keep captured values byte for byte identical. +# +# The quantifiers are possessive (3.11+). A field may contain spaces and is also +# preceded and followed by optional whitespace, so an ordinary greedy quantifier +# leaves the split between "whitespace" and "field content" ambiguous; with nine +# such fields a line that fails late (say, an invalid package type) backtracks +# through every combination and takes exponential time. pyparsing had no such +# problem because its Word is maximal munch and never gives characters back -- +# which is exactly what a possessive quantifier expresses. +_WS = r"[ \t\n\r]*+" +_WORD = r"[!-?A-{}~][ \t!-?A-{}~]*+" -REQUEST = TIMESTAMP + PIPE + OptionalItem(COUNTRY_CODE) + PIPE + URL -PROJECT_NAME = NULL | Word(printables) -PROJECT_NAME = PROJECT_NAME.set_results_name("project_name") -PROJECT_NAME.set_name("Project Name") +def _nullable(name): + return rf"(?:\(null\)|(?P<{name}>(?!\(null\)){_WORD}))" -VERSION = NULL | Word(printables) -VERSION = VERSION.set_results_name("version") -VERSION.set_name("Version") -PACKAGE_TYPE = NULL | ( - L("sdist") - | L("bdist_wheel") - | L("bdist_dmg") - | L("bdist_dumb") - | L("bdist_egg") - | L("bdist_msi") - | L("bdist_rpm") - | L("bdist_wininst") +_PACKAGE_TYPE = ( + r"(?:\(null\)|(?Psdist|bdist_wheel|bdist_dmg|bdist_dumb" + r"|bdist_egg|bdist_msi|bdist_rpm|bdist_wininst))" ) -PACKAGE_TYPE = PACKAGE_TYPE.set_results_name("package_type") -PACKAGE_TYPE.set_name("Package Type") - -PROJECT = PROJECT_NAME + PIPE + VERSION + PIPE + PACKAGE_TYPE - -TLS_PROTOCOL = NULL | Word(printables) -TLS_PROTOCOL = TLS_PROTOCOL.set_results_name("tls_protocol") -TLS_PROTOCOL.set_name("TLS Protocol") - -TLS_CIPHER = NULL | Word(printables) -TLS_CIPHER = TLS_CIPHER.set_results_name("tls_cipher") -TLS_CIPHER.set_name("TLS Cipher") - -TLS = TLS_PROTOCOL + PIPE + TLS_CIPHER - -USER_AGENT = rest_of_line -USER_AGENT = USER_AGENT.set_results_name("user_agent") -USER_AGENT.set_name("UserAgent") - -V3_HEADER = L("download") -MESSAGE_v3 = ( - V3_HEADER + PIPE + REQUEST + PIPE + TLS + PIPE + PROJECT + PIPE + USER_AGENT +_TAIL = r"(?P[^\n]*+)(?:\n[ \t\n\r]*+)?\Z" +_COMMON = ( + rf"{_WS}\|{_WS}(?P{_WORD}){_WS}\|" + rf"(?:{_WS}(?P{_WORD}))?{_WS}\|" + rf"{_WS}(?P{_WORD}){_WS}\|" + rf"{_WS}{_nullable('tls_protocol')}{_WS}\|" + rf"{_WS}{_nullable('tls_cipher')}{_WS}\|" ) -SIMPLE_HEADER = L("simple") -MESSAGE_SIMPLE = ( - SIMPLE_HEADER + PIPE + REQUEST + PIPE + TLS + PIPE + PIPE + PIPE + PIPE + USER_AGENT +# Two separate patterns rather than one alternation: duplicate group names across +# branches are a syntax error before Python 3.12, and we target 3.11. +MESSAGE_v3 = re.compile( + rf"\A{_WS}download{_COMMON}{_WS}{_nullable('project_name')}{_WS}\|" + rf"{_WS}{_nullable('version')}{_WS}\|{_WS}{_PACKAGE_TYPE}{_WS}\|{_TAIL}" ) - -MESSAGE = MESSAGE_SIMPLE | MESSAGE_v3 +MESSAGE_SIMPLE = re.compile(rf"\A{_WS}simple{_COMMON}{_WS}\|{_WS}\|{_WS}\|{_TAIL}") @enum.unique @@ -188,44 +169,54 @@ class Simple: def _value_or_none(value): - if value is NullValue or value == "": + # A missing regex group (an absent country code, or a field that matched the + # "(null)" literal) comes back as None already; a field that is only present in + # one of the two message shapes is looked up with a "" default. + if value is None or value == "": return None else: return value def parse(message): - try: - parsed = MESSAGE.parse_string(message, parse_all=True) - except ParseException as exc: - raise UnparseableEvent("{!r} {}".format(message, exc)) from None + # parse_string() used to expandtabs() the input before parsing, so every + # captured value -- including the user agent -- had its tabs expanded. Keep + # doing it, or stored values would silently change. + expanded = message.expandtabs() + + simple = True + parsed = MESSAGE_SIMPLE.match(expanded) + if parsed is None: + simple = False + parsed = MESSAGE_v3.match(expanded) + if parsed is None: + raise UnparseableEvent("{!r} does not match a known event".format(message)) + + parsed = parsed.groupdict() + + url = parsed["url"] data = {} - data["timestamp"] = parsed.timestamp - data["tls_protocol"] = _value_or_none(parsed.tls_protocol) - data["tls_cipher"] = _value_or_none(parsed.tls_cipher) - data["country_code"] = _value_or_none(parsed.country_code) - data["url"] = parsed.url + data["timestamp"] = parsed["timestamp"] + data["tls_protocol"] = _value_or_none(parsed["tls_protocol"]) + data["tls_cipher"] = _value_or_none(parsed["tls_cipher"]) + data["country_code"] = _value_or_none(parsed["country_code"]) + data["url"] = url data["file"] = {} - data["file"]["filename"] = posixpath.basename(parsed.url) - data["file"]["project"] = _value_or_none(parsed.project_name) - data["file"]["version"] = _value_or_none(parsed.version) - data["file"]["type"] = _value_or_none(parsed.package_type) + data["file"]["filename"] = posixpath.basename(url) + data["file"]["project"] = _value_or_none(parsed.get("project_name")) + data["file"]["version"] = _value_or_none(parsed.get("version")) + data["file"]["type"] = _value_or_none(parsed.get("package_type")) - if parsed[0] == "download": - data["project"] = _value_or_none(parsed.project_name) - result = _cattr.structure(data, Download) - elif parsed[0] == "simple": - data["project"] = parsed.url.split("/")[2] + if simple: + data["project"] = url.split("/")[2] result = _cattr.structure(data, Simple) else: - # MESSAGE can only match a "download" or "simple" header today, but guard - # against a future grammar being added without a matching branch here -- - # fail cleanly instead of an UnboundLocalError on `result` below. - raise UnparseableEvent("{!r} unexpected event header {!r}".format(message, parsed[0])) + data["project"] = _value_or_none(parsed["project_name"]) + result = _cattr.structure(data, Download) try: - ua = user_agents.parse(parsed.user_agent) + ua = user_agents.parse(parsed["user_agent"]) if ua is None: return # Ignored user agents mean we'll skip trying to log this event except user_agents.UnknownUserAgentError: diff --git a/tests/unit/events/fixtures/download.yml b/tests/unit/events/fixtures/download.yml index a195028..2c8a75f 100644 --- a/tests/unit/events/fixtures/download.yml +++ b/tests/unit/events/fixtures/download.yml @@ -69,3 +69,343 @@ tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 country_code: JP details: null + + +# The user agent is everything after the 9th pipe, taken verbatim -- it may +# contain pipes and "@", both of which are illegal in every other field. +- event: download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|Wget/1.20.3 | curl@example.com | thing + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: + installer: + name: Browser + +# Leading whitespace on the user agent is kept (so this is an unknown UA), unlike +# every other field, where it is stripped. +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist| bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: null + +# An empty user agent is allowed by the grammar, it just doesn't parse as a UA. +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: null + + +# "@" is not a legal character in any of the first nine fields, so a line +# containing one anywhere but in the user agent is unparseable. +- event: download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/foo@bar/x-1.0.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + result: !!python/name:linehaul.events.parser.UnparseableEvent '' +- event: download|Fri, 20 Jul 2018 02:19:19 GMT|J@|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + result: !!python/name:linehaul.events.parser.UnparseableEvent '' + + +# Leading whitespace inside a field is stripped, trailing whitespace is kept. +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT| US|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: US + details: + installer: + name: bandersnatch + version: 2.2.1 +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|US |/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: "US " + details: + installer: + name: bandersnatch + version: 2.2.1 + +# A whitespace only country code parses as an absent country code. +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT| |/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: null + details: + installer: + name: bandersnatch + version: 2.2.1 + +# The country code is *not* nullable, so a literal "(null)" is kept as a string. +- event: download|Fri, 20 Jul 2018 02:19:19 GMT|(null)|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: "(null)" + details: + installer: + name: bandersnatch + version: 2.2.1 + + +# Spaces are legal *inside* a field. +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: + installer: + name: bandersnatch + version: 2.2.1 + + +# Tabs are expanded (tabsize 8, column aware) before the line is parsed, in every +# field including the user agent. +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ab\tcd/x-1.0.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: "/packages/ab cd/x-1.0.tar.gz" + project: cfn-flip + file: + filename: x-1.0.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: + installer: + name: bandersnatch + version: 2.2.1 +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|pip/20.0.2 {\"cpu\":\"x86\t64\",\"python\":\"3.8.5\"}" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: + python: "3.8.5" + cpu: "x86 64" + + +# A nullable field that starts with "(null)" but doesn't end there is rejected, +# but a value that merely looks like the start of it parses as an ordinary value. +- event: download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|(null)x|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + result: !!python/name:linehaul.events.parser.UnparseableEvent '' +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|(null) x|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)" + result: !!python/name:linehaul.events.parser.UnparseableEvent '' +- event: download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|(nullx|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: "(nullx" + file: + filename: cfn_flip-1.0.3.tar.gz + project: "(nullx" + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: + installer: + name: bandersnatch + version: 2.2.1 + + +# The package type is a closed set of literals, so anything else is unparseable, +# including a known literal with junk appended. +- event: download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|wheelie|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + result: !!python/name:linehaul.events.parser.UnparseableEvent '' +- event: download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdistfoo|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + result: !!python/name:linehaul.events.parser.UnparseableEvent '' + +# Whitespace around the package type literal is tolerated but not captured, which +# is asymmetric with the ordinary word fields above (they keep trailing spaces). +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3| sdist |bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: + installer: + name: bandersnatch + version: 2.2.1 + + +# Leading whitespace on the line as a whole is skipped. +- event: " download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: + installer: + name: bandersnatch + version: 2.2.1 + +# Whitespace is tolerated around the delimiters, though here it leaves a trailing +# space on the timestamp, which the timestamp hook then chokes on. +- event: "download | Fri, 20 Jul 2018 02:19:19 GMT | JP | /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz | TLSv1.2 | ECDHE-RSA-AES128-GCM-SHA256 | cfn-flip | 1.0.3 | sdist |bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)" + result: !!python/name:builtins.ValueError '' + + +# Trailing whitespace after the user agent is allowed, anything else is not. +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)\n" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: + installer: + name: bandersnatch + version: 2.2.1 +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)\n\n" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: + installer: + name: bandersnatch + version: 2.2.1 + +# A "\r\n" line ending is accepted too, but the "\r" lands *inside* the user agent +# (it only ever stops at a "\n"), which makes the UA unrecognisable. +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)\r\n" + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz + project: cfn-flip + file: + filename: cfn_flip-1.0.3.tar.gz + project: cfn-flip + version: 1.0.3 + type: sdist + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: null + +# Anything other than whitespace after the newline is unparseable. +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)\nmore" + result: !!python/name:linehaul.events.parser.UnparseableEvent '' + + +# The required fields cannot be empty. +- event: download|Fri, 20 Jul 2018 02:19:19 GMT|JP||TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + result: !!python/name:linehaul.events.parser.UnparseableEvent '' +- event: download||JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|cfn-flip|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + result: !!python/name:linehaul.events.parser.UnparseableEvent '' + +# Non ASCII characters are not legal in any of the first nine fields. +- event: "download|Fri, 20 Jul 2018 02:19:19 GMT|JP|/packages/ba/c8/a928c55457441c87366eb2423efca9aa0f46380994fd8a476153493c319a/cfn_flip-1.0.3.tar.gz|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|caf\xE9|1.0.3|sdist|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)" + result: !!python/name:linehaul.events.parser.UnparseableEvent '' diff --git a/tests/unit/events/fixtures/simple.yml b/tests/unit/events/fixtures/simple.yml new file mode 100644 index 0000000..3798d55 --- /dev/null +++ b/tests/unit/events/fixtures/simple.yml @@ -0,0 +1,100 @@ +# Basic Example +- event: simple|Fri, 20 Jul 2018 02:19:19 GMT|JP|/simple/cfn-flip/|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256||||bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + type: simple + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /simple/cfn-flip/ + project: cfn-flip + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: + installer: + name: bandersnatch + version: 2.2.1 + + +# Country Code should be optional, and a whitespace only one counts as absent. +- event: simple|Fri, 20 Jul 2018 02:19:19 GMT||/simple/cfn-flip/|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256||||bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + type: simple + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /simple/cfn-flip/ + project: cfn-flip + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: null + details: + installer: + name: bandersnatch + version: 2.2.1 +- event: "simple|Fri, 20 Jul 2018 02:19:19 GMT| |/simple/cfn-flip/|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256||||bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)" + type: simple + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /simple/cfn-flip/ + project: cfn-flip + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: null + details: + installer: + name: bandersnatch + version: 2.2.1 + + +# The TLS details are nullable. +- event: simple|Fri, 20 Jul 2018 02:19:19 GMT|JP|/simple/cfn-flip/|(null)|(null)||||bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + type: simple + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /simple/cfn-flip/ + project: cfn-flip + tls_protocol: null + tls_cipher: null + country_code: JP + details: + installer: + name: bandersnatch + version: 2.2.1 + + +# The user agent may contain pipes, they don't split the message. +- event: simple|Fri, 20 Jul 2018 02:19:19 GMT|JP|/simple/cfn-flip/|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256||||Wget/1.20.3 | thing + type: simple + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /simple/cfn-flip/ + project: cfn-flip + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: + installer: + name: Browser + + +# A trailing newline is allowed. +- event: "simple|Fri, 20 Jul 2018 02:19:19 GMT|JP|/simple/cfn-flip/|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256||||bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64)\n" + type: simple + result: + timestamp: Fri, 20 Jul 2018 02:19:19 GMT + url: /simple/cfn-flip/ + project: cfn-flip + tls_protocol: TLSv1.2 + tls_cipher: ECDHE-RSA-AES128-GCM-SHA256 + country_code: JP + details: + installer: + name: bandersnatch + version: 2.2.1 + + +# The project, version and package type fields must be empty for a simple message. +- event: simple|Fri, 20 Jul 2018 02:19:19 GMT|JP|/simple/cfn-flip/|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256|||x|bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + result: !!python/name:linehaul.events.parser.UnparseableEvent '' + + +# "@" is illegal in the URL here too. +- event: simple|Fri, 20 Jul 2018 02:19:19 GMT|JP|/simple/foo@bar/|TLSv1.2|ECDHE-RSA-AES128-GCM-SHA256||||bandersnatch/2.2.1 (cpython 3.7.0-final0, Darwin x86_64) + result: !!python/name:linehaul.events.parser.UnparseableEvent '' diff --git a/tests/unit/events/test_parser.py b/tests/unit/events/test_parser.py index d61d2e4..13d38b5 100644 --- a/tests/unit/events/test_parser.py +++ b/tests/unit/events/test_parser.py @@ -13,17 +13,20 @@ import inspect import os import os.path +import time import pytest import yaml from hypothesis import given, strategies as st -from linehaul.events.parser import Download, UnparseableEvent, parse, _cattr +from linehaul.events.parser import Download, Simple, UnparseableEvent, parse, _cattr FIXTURE_DIR = os.path.join(os.path.dirname(__file__), "fixtures") +EVENT_TYPES = {"download": Download, "simple": Simple} + def _load_event_fixtures(fixture_dir): fixtures = os.listdir(fixture_dir) @@ -33,8 +36,9 @@ def _load_event_fixtures(fixture_dir): for fixture in fixtures: event = fixture.pop("event") result = fixture.pop("result") + event_type = EVENT_TYPES[fixture.pop("type", "download")] expected = ( - _cattr.structure(result, Download) + _cattr.structure(result, event_type) if isinstance(result, dict) else result ) @@ -55,3 +59,15 @@ def test_download_parsing(event_data, expected): def test_invalid_event(data): with pytest.raises(UnparseableEvent): parse(data) + + +def test_rejects_pathological_line_quickly(): + # Every field may contain spaces and is also surrounded by optional + # whitespace, so without possessive quantifiers this line -- which only fails + # at the very last field -- backtracks exponentially. It used to take minutes. + event = "download|" + ("a" + " " * 20 + "|") * 8 + "some user agent" + + start = time.perf_counter() + with pytest.raises(UnparseableEvent): + parse(event) + assert time.perf_counter() - start < 5