Skip to content

AI: swap to regex from pyparsing #322

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
JacobCoffee wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

AI: swap to regex from pyparsing #322

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
177 changes: 84 additions & 93 deletions linehaul/events/parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import enum
import logging
import posixpath
import re

from datetime import datetime, timezone
from typing import Optional
Expand All @@ -21,10 +22,6 @@
import attr.validators
import cattr

from pyparsing import Literal as L, Word, Optional as OptionalItem
from pyparsing import printables as _printables, rest_of_line
from pyparsing import ParseException

from linehaul.ua import UserAgent, parser as user_agents


Expand All @@ -51,79 +48,63 @@ class UnparseableEvent(Exception):
pass


class _NullValue:
pass


NullValue = _NullValue()


printables = "".join(set(_printables + " " + "\t") - {"|", "@"})

PIPE = L("|").suppress()

NULL = L("(null)")
NULL.set_parse_action(lambda s, l, t: NullValue)

TIMESTAMP = Word(printables).set_name("Timestamp")
TIMESTAMP = TIMESTAMP.set_results_name("timestamp")

COUNTRY_CODE = Word(printables).set_name("Country Code")
COUNTRY_CODE = COUNTRY_CODE.set_results_name("country_code")

URL = Word(printables).set_name("URL")
URL = URL.set_results_name("url")
# These two regexes replace what used to be a pyparsing grammar. The wire format
# is pipe delimited with a fixed number of fields, so a grammar was more machinery
# than the job needs -- pyparsing accounted for ~60% of the per line cost. They are
# a deliberately faithful translation of that grammar, quirks included:
#
# * A field is pyparsing's ``Word(printables)`` minus "|" and "@", plus space and
# tab. So "@" anywhere in the first nine fields rejects the whole line, and
# spaces/tabs *inside* a field are legal and retained. Non ASCII, DEL and
# control characters reject.
# * pyparsing skips its whitespace set (" \n\t\r") before each token, so leading
# whitespace on a field is stripped from the captured value while trailing
# whitespace is kept, and whitespace is tolerated around every delimiter.
# * The nullable fields use an ordered alternation with a negative lookahead so
# that "(null)x" rejects (pyparsing committed to the "(null)" literal and then
# failed on the missing pipe) while "(nullx" still matches as a plain word.
# * The user agent was ``rest_of_line``, i.e. everything up to a newline, taken
# verbatim -- it may contain "|" and "@" -- and ``parse_all=True`` allowed only
# trailing whitespace after it.
#
# NB: ``parse_string`` expanded tabs before parsing, so ``parse`` below must call
# ``str.expandtabs()`` to keep captured values byte for byte identical.
#
# The quantifiers are possessive (3.11+). A field may contain spaces and is also
# preceded and followed by optional whitespace, so an ordinary greedy quantifier
# leaves the split between "whitespace" and "field content" ambiguous; with nine
# such fields a line that fails late (say, an invalid package type) backtracks
# through every combination and takes exponential time. pyparsing had no such
# problem because its Word is maximal munch and never gives characters back --
# which is exactly what a possessive quantifier expresses.
_WS = r"[ \t\n\r]*+"
_WORD = r"[!-?A-{}~][ \t!-?A-{}~]*+"

REQUEST = TIMESTAMP + PIPE + OptionalItem(COUNTRY_CODE) + PIPE + URL

PROJECT_NAME = NULL | Word(printables)
PROJECT_NAME = PROJECT_NAME.set_results_name("project_name")
PROJECT_NAME.set_name("Project Name")
def _nullable(name):
return rf"(?:\(null\)|(?P<{name}>(?!\(null\)){_WORD}))"

VERSION = NULL | Word(printables)
VERSION = VERSION.set_results_name("version")
VERSION.set_name("Version")

PACKAGE_TYPE = NULL | (
L("sdist")
| L("bdist_wheel")
| L("bdist_dmg")
| L("bdist_dumb")
| L("bdist_egg")
| L("bdist_msi")
| L("bdist_rpm")
| L("bdist_wininst")
_PACKAGE_TYPE = (
r"(?:\(null\)|(?P<package_type>sdist|bdist_wheel|bdist_dmg|bdist_dumb"
r"|bdist_egg|bdist_msi|bdist_rpm|bdist_wininst))"
)
PACKAGE_TYPE = PACKAGE_TYPE.set_results_name("package_type")
PACKAGE_TYPE.set_name("Package Type")

PROJECT = PROJECT_NAME + PIPE + VERSION + PIPE + PACKAGE_TYPE

TLS_PROTOCOL = NULL | Word(printables)
TLS_PROTOCOL = TLS_PROTOCOL.set_results_name("tls_protocol")
TLS_PROTOCOL.set_name("TLS Protocol")

TLS_CIPHER = NULL | Word(printables)
TLS_CIPHER = TLS_CIPHER.set_results_name("tls_cipher")
TLS_CIPHER.set_name("TLS Cipher")

TLS = TLS_PROTOCOL + PIPE + TLS_CIPHER

USER_AGENT = rest_of_line
USER_AGENT = USER_AGENT.set_results_name("user_agent")
USER_AGENT.set_name("UserAgent")

V3_HEADER = L("download")
MESSAGE_v3 = (
V3_HEADER + PIPE + REQUEST + PIPE + TLS + PIPE + PROJECT + PIPE + USER_AGENT
_TAIL = r"(?P<user_agent>[^\n]*+)(?:\n[ \t\n\r]*+)?\Z"
_COMMON = (
rf"{_WS}\|{_WS}(?P<timestamp>{_WORD}){_WS}\|"
rf"(?:{_WS}(?P<country_code>{_WORD}))?{_WS}\|"
rf"{_WS}(?P<url>{_WORD}){_WS}\|"
rf"{_WS}{_nullable('tls_protocol')}{_WS}\|"
rf"{_WS}{_nullable('tls_cipher')}{_WS}\|"
)

SIMPLE_HEADER = L("simple")
MESSAGE_SIMPLE = (
SIMPLE_HEADER + PIPE + REQUEST + PIPE + TLS + PIPE + PIPE + PIPE + PIPE + USER_AGENT
# Two separate patterns rather than one alternation: duplicate group names across
# branches are a syntax error before Python 3.12, and we target 3.11.
MESSAGE_v3 = re.compile(
rf"\A{_WS}download{_COMMON}{_WS}{_nullable('project_name')}{_WS}\|"
rf"{_WS}{_nullable('version')}{_WS}\|{_WS}{_PACKAGE_TYPE}{_WS}\|{_TAIL}"
)

MESSAGE = MESSAGE_SIMPLE | MESSAGE_v3
MESSAGE_SIMPLE = re.compile(rf"\A{_WS}simple{_COMMON}{_WS}\|{_WS}\|{_WS}\|{_TAIL}")


@enum.unique
Expand Down Expand Up @@ -188,44 +169,54 @@ class Simple:


def _value_or_none(value):
if value is NullValue or value == "":
# A missing regex group (an absent country code, or a field that matched the
# "(null)" literal) comes back as None already; a field that is only present in
# one of the two message shapes is looked up with a "" default.
if value is None or value == "":
return None
else:
return value


def parse(message):
try:
parsed = MESSAGE.parse_string(message, parse_all=True)
except ParseException as exc:
raise UnparseableEvent("{!r} {}".format(message, exc)) from None
# parse_string() used to expandtabs() the input before parsing, so every
# captured value -- including the user agent -- had its tabs expanded. Keep
# doing it, or stored values would silently change.
expanded = message.expandtabs()

simple = True
parsed = MESSAGE_SIMPLE.match(expanded)
if parsed is None:
simple = False
parsed = MESSAGE_v3.match(expanded)
if parsed is None:
raise UnparseableEvent("{!r} does not match a known event".format(message))

parsed = parsed.groupdict()

url = parsed["url"]

data = {}
data["timestamp"] = parsed.timestamp
data["tls_protocol"] = _value_or_none(parsed.tls_protocol)
data["tls_cipher"] = _value_or_none(parsed.tls_cipher)
data["country_code"] = _value_or_none(parsed.country_code)
data["url"] = parsed.url
data["timestamp"] = parsed["timestamp"]
data["tls_protocol"] = _value_or_none(parsed["tls_protocol"])
data["tls_cipher"] = _value_or_none(parsed["tls_cipher"])
data["country_code"] = _value_or_none(parsed["country_code"])
data["url"] = url
data["file"] = {}
data["file"]["filename"] = posixpath.basename(parsed.url)
data["file"]["project"] = _value_or_none(parsed.project_name)
data["file"]["version"] = _value_or_none(parsed.version)
data["file"]["type"] = _value_or_none(parsed.package_type)
data["file"]["filename"] = posixpath.basename(url)
data["file"]["project"] = _value_or_none(parsed.get("project_name"))
data["file"]["version"] = _value_or_none(parsed.get("version"))
data["file"]["type"] = _value_or_none(parsed.get("package_type"))

if parsed[0] == "download":
data["project"] = _value_or_none(parsed.project_name)
result = _cattr.structure(data, Download)
elif parsed[0] == "simple":
data["project"] = parsed.url.split("/")[2]
if simple:
data["project"] = url.split("/")[2]
result = _cattr.structure(data, Simple)
else:
# MESSAGE can only match a "download" or "simple" header today, but guard
# against a future grammar being added without a matching branch here --
# fail cleanly instead of an UnboundLocalError on `result` below.
raise UnparseableEvent("{!r} unexpected event header {!r}".format(message, parsed[0]))
data["project"] = _value_or_none(parsed["project_name"])
result = _cattr.structure(data, Download)

try:
ua = user_agents.parse(parsed.user_agent)
ua = user_agents.parse(parsed["user_agent"])
if ua is None:
return # Ignored user agents mean we'll skip trying to log this event
except user_agents.UnknownUserAgentError:
Expand Down
Loading
Loading