From a7b2f635c6e29fc7943ebfe1011f62525b146ede Mon Sep 17 00:00:00 2001 From: Vivian Wang Date: Mon, 17 Aug 2026 18:48:18 +0800 Subject: [PATCH] tools: patchoulene: {mail-check,mail-match}: New commands Add the mail-check and mail-match commands to look for new versions of patches on mailing lists, using b4. Signed-off-by: Vivian Wang --- patchoulene/README.md | 148 +++++++++++++ patchoulene/src/patchoulene/__init__.py | 280 ++++++++++++++++++++++++ patchoulene/src/patchoulene/b4.py | 94 ++++++++ 3 files changed, 522 insertions(+) create mode 100644 patchoulene/src/patchoulene/b4.py diff --git a/patchoulene/README.md b/patchoulene/README.md index 2a621be..7b7a007 100644 --- a/patchoulene/README.md +++ b/patchoulene/README.md @@ -137,6 +137,154 @@ $ ./patl diff Compare patches between `rev1` and `rev2`, using information in the database. +### `mail-check` + +```console +$ ./patl mail-check +``` + +Check mailing lists for new versions of patches and patch series, and save to `mail-check.json`. +Use `./patl mail-match` to interactively add replacement information from `mail-check.json` + +### `mail-match` + +```console +$ ./patl mail-match `file` +``` + +The `file` argument is optional; if omitted, it defaults to `mail-check.json`, which is the default output file name of `./patl mail-check`. + +Interactively merge data from the `mail-check.json` file into the patch database. +For each patch or patch series version update, a replacement view and a prompt would appear. +The patches are matched heuristically by subject initially. +You can edit, accept, discard each replacement item on each prompt. + +
+ +Detailed example + +The view is divided into four sections. +Firstly, the current version shows the current known version of the series. +Patches are shown with their currently matched replacements, if any. + +Some patches may not be eligible for replacement. +These patches are shown with parentheses around their subjects. +The reason will be shown as well. + +``` +Current version: + (A1) [PATCH 1/4] riscv: defconfig: enable RFKILL and RFKILL_GPIO + mail:20260716213314.3027969-2-aurelien@aurel32.net + -> (B1) [PATCH v2 1/4] riscv: defconfig: enable RFKILL and RFKILL_GPIO + ([PATCH 2/4] riscv: dts: spacemit: k3: add rfkill node for Bluetooth on Pico-ITX board) (has replacement) + -> riscv: dts: spacemit: k3: add rfkill node for Bluetooth on Pico-ITX board + mail:20260729172450.1660418-3-aurelien@aurel32.net + (A3) [PATCH 3/4] riscv: dts: spacemit: k3: add USB3 B and C controllers for Pico-ITX board + mail:20260716213314.3027969-4-aurelien@aurel32.net + -> (B3) [PATCH v2 3/4] riscv: dts: spacemit: k3: add USB3 B and C controllers for Pico-ITX board + (A4) [PATCH 4/4] riscv: dts: spacemit: k3: add rfkill node for WLAN on Pico-ITX board + mail:20260716213314.3027969-5-aurelien@aurel32.net + (no replacement) +``` + +Secondly, the replacement series is shown. + +``` +Replacement version: + (B1) [PATCH v2 1/4] riscv: defconfig: enable RFKILL and RFKILL_GPIO + mail:20260729172450.1660418-2-aurelien@aurel32.net + (B2) [PATCH v2 2/4] riscv: dts: spacemit: k3: add rfkill node for Bluetooth on Pico-ITX board + mail:20260729172450.1660418-3-aurelien@aurel32.net + (B3) [PATCH v2 3/4] riscv: dts: spacemit: k3: add USB3 B and C controllers for Pico-ITX board + mail:20260729172450.1660418-4-aurelien@aurel32.net + (B4) [PATCH v2 4/4] riscv: dts: spacemit: k3: add rfkill node for WLAN + mail:20260729172450.1660418-5-aurelien@aurel32.net +``` + +Then the current replacement is shown again, concisely as a matrix. +Here, it means that patch A1 corresponds to B1, and A3 corresponds to B3. +Patch A4 has no replacement. + +``` +Matrix: + 1 = 1 + 3 = 3 + 4 = + +1/3 patches missing replacement +``` + +Finally, the prompt shows available actions. You can type a letter and enter to perform one of these actions + +- `y`: Accept the replacements as shown. + This is also the default action performed if you type enter without a letter. +- `n`: Discard the entire matrix. No replacements will be recorded. +- `c`: If both series have the same number of patches, this creates a replacement matrix that matches patch A1 with B1, A2 with B2, etc. + +You can also type a matrix row to edit the replacement of a certain patch. +For example, to say that patch A2 corresponds to B2 and B3, type `2=2,3`. +To delete the replacement for patch A1, type `1=`. + +``` +(Y)es, accept this replacement matrix +(N)o, decline this replacement matrix +Match (c)orresponding patches by number +[Y/n/c/(matrix row)]? +``` + +(If there's no replacement for this series at all, the prompt will be different to highlight this fact.) + +``` +(Y)es, accept that no replacements will be recorded +[Y/(matrix row)]? +``` + +In this case, we can verify that patch A4 corresponds to B4. +Therefore, we type `4=4`. +Alternatively, since both series have four patches, `c` also performs the same action. +This results in: + +``` +Current version: + (A1) [PATCH 1/4] riscv: defconfig: enable RFKILL and RFKILL_GPIO + mail:20260716213314.3027969-2-aurelien@aurel32.net + -> (B1) [PATCH v2 1/4] riscv: defconfig: enable RFKILL and RFKILL_GPIO + ([PATCH 2/4] riscv: dts: spacemit: k3: add rfkill node for Bluetooth on Pico-ITX board) (has replacement) + -> riscv: dts: spacemit: k3: add rfkill node for Bluetooth on Pico-ITX board + mail:20260729172450.1660418-3-aurelien@aurel32.net + (A3) [PATCH 3/4] riscv: dts: spacemit: k3: add USB3 B and C controllers for Pico-ITX board + mail:20260716213314.3027969-4-aurelien@aurel32.net + -> (B3) [PATCH v2 3/4] riscv: dts: spacemit: k3: add USB3 B and C controllers for Pico-ITX board + (A4) [PATCH 4/4] riscv: dts: spacemit: k3: add rfkill node for WLAN on Pico-ITX board + mail:20260716213314.3027969-5-aurelien@aurel32.net + -> (B4) [PATCH v2 4/4] riscv: dts: spacemit: k3: add rfkill node for WLAN + +Replacement version: + (B1) [PATCH v2 1/4] riscv: defconfig: enable RFKILL and RFKILL_GPIO + mail:20260729172450.1660418-2-aurelien@aurel32.net + (B2) [PATCH v2 2/4] riscv: dts: spacemit: k3: add rfkill node for Bluetooth on Pico-ITX board + mail:20260729172450.1660418-3-aurelien@aurel32.net + (B3) [PATCH v2 3/4] riscv: dts: spacemit: k3: add USB3 B and C controllers for Pico-ITX board + mail:20260729172450.1660418-4-aurelien@aurel32.net + (B4) [PATCH v2 4/4] riscv: dts: spacemit: k3: add rfkill node for WLAN + mail:20260729172450.1660418-5-aurelien@aurel32.net + +Matrix: + 1 = 1 + 3 = 3 + 4 = 4 + +(Y)es, accept this replacement matrix +(N)o, decline this replacement matrix +Match (c)orresponding patches by number +[Y/n/c/(matrix row)]? +``` + +The replacement matrix is now complete and correct. +Type enter or `y` then enter to accept this replacement. + +
+ ## Patch series and identifiers (TODO) diff --git a/patchoulene/src/patchoulene/__init__.py b/patchoulene/src/patchoulene/__init__.py index f23915e..f486246 100644 --- a/patchoulene/src/patchoulene/__init__.py +++ b/patchoulene/src/patchoulene/__init__.py @@ -7,6 +7,7 @@ import subprocess import sys +from . import b4 from .message import * from .git import GitCommit, GitRepo @@ -302,6 +303,266 @@ def msg_replacement( print() +def do_mail_check(db: dict[str, dict]) -> list[dict[str, list[dict[str, str]]]]: + def msg_thread_replacement( + this_thread: list[b4.PatchHeader], latest_thread: list[b4.PatchHeader] + ) -> str: + msgs = [f"Patches:"] + for m in this_thread: + msgs.append(f" {m.clean_subject()}") + msgs.append(f" https://patch.msgid.link/{m.message_id()}") + + msgs.append(f"... may have replacement:") + for m in latest_thread: + msgs.append(f" {m.clean_subject()}") + msgs.append(f" https://patch.msgid.link/{m.message_id()}") + + return "\n".join(msgs) + + result = [] + checked = set() + + for pid, data in db.items(): + if data.get("replacement", None): + continue + + if (m := re.fullmatch(r"mail:(\S+)", pid)) is None: + continue + + msgid = m[1] + + status = "Already checked" if msgid in checked else "Checking" + print(f'{status} "{data.get("subject", "???")}" {msgid}', file=sys.stderr) + + if msgid in checked: + continue + + if (res := b4.check_msgid(msgid)) is None: + continue + + this_thread, latest_thread = res + this_ids = [m.message_id() for m in this_thread] + latest_ids = [m.message_id() for m in latest_thread] + + checked.update(this_ids) + if set(this_ids) == set(latest_ids): + continue + + print(msg_thread_replacement(this_thread, latest_thread), file=sys.stderr) + + result.append( + { + "current": [m.headers for m in this_thread], + "replacement": [m.headers for m in latest_thread], + } + ) + + return result + + +def do_mail_match(data: list[dict]): + db = read_patch_db() + for entry in data: + current: list[b4.PatchHeader] + replacement: list[b4.PatchHeader] + proc = lambda ps: [b4.PatchHeader(h) for h in ps] + current, replacement = proc(entry["current"]), proc(entry["replacement"]) + + ids = [f"mail:{h.message_id()}" for h in current] + if all(p not in db or db[p].get("replacement", None) for p in ids): + continue + + curr_keys = [h.clean_subject() for h in current] + rep_keys = [h.clean_subject() for h in replacement] + if len(rep_keys) != len(set(rep_keys)): + print("Duplicates in patch list?", file=sys.stderr) + rep_map = {subj: idx for idx, subj in enumerate(rep_keys)} + + valid = [pid in db and not db[pid].get("replacement", None) for pid in ids] + matrix: list[list[int] | None] + matrix = [None] * len(current) + + # Guess an initial match based on subjects + for i in range(len(curr_keys)): + if valid[i] and curr_keys[i] in rep_map: + matrix[i] = [rep_map[curr_keys[i]]] + + print("-" * 60, file=sys.stderr) + while True: + print("Current version:", file=sys.stderr) + for idxa, ha in enumerate(current): + if not valid[idxa]: + if ids[idxa] not in db: + why_invalid = "(not in db)" + else: + why_invalid = "(has replacement)" + print( + f" {dim(f'({ha.subject()}) {why_invalid}')}", + file=sys.stderr, + ) + + if ids[idxa] in db: + assert "replacement" in db[ids[idxa]] + rs = db[ids[idxa]]["replacement"] + if not isinstance(rs, list): + rs = [rs] + for r in rs: + if info := db.get(r): + print( + f" {dim('->')} {dim(info.get('subject', '???'))}", + file=sys.stderr, + ) + print(f" {dim(r)}", file=sys.stderr) + else: + print( + f" {dim('->')} {dim(r)} {dim('(Unknown patch???)')}", + file=sys.stderr, + ) + continue + print(f" {f'(A{idxa + 1})':>6} {ha.subject()}", file=sys.stderr) + print(f" {dim(f'mail:{ha.message_id()}')}", file=sys.stderr) + row = matrix[idxa] + if row is not None: + for idxb in row: + subj_b = replacement[idxb].subject() + print(f" -> (B{idxb + 1}) {subj_b}", file=sys.stderr) + else: + print(f" (no replacement)", file=sys.stderr) + + print(file=sys.stderr) + print("Replacement version:", file=sys.stderr) + for idxb, hb in enumerate(replacement): + print(f" {f'(B{idxb + 1})':>6} {hb.subject()}", file=sys.stderr) + print(f" {dim(f'mail:{hb.message_id()}')}", file=sys.stderr) + + print(file=sys.stderr) + print("Matrix:", file=sys.stderr) + for idxa in range(len(current)): + if not valid[idxa]: + continue + if matrix[idxa] is not None: + assert matrix[idxa] + mx = matrix[idxa] or [] + mstr = f"{idxa + 1} ={''.join(' ' + str(ib + 1) for ib in mx)}" + print(f" {mstr}", file=sys.stderr) + + print(file=sys.stderr) + + valids = sum(valid) + missing = sum(valid[i] and not matrix[i] for i in range(len(current))) + + if missing == valids: + print(f"None of {valids} patch(es) have replacement", file=sys.stderr) + print(file=sys.stderr) + elif missing: + print( + f"{missing}/{valids} patches missing replacement", file=sys.stderr + ) + print(file=sys.stderr) + + valid_options = [] + valid_keys = [] + + if any(matrix): + valid_options.append("(Y)es, accept this replacement matrix") + valid_keys.append("y") + valid_options.append("(N)o, decline this replacement matrix") + valid_keys.append("n") + else: + valid_options.append( + "(Y)es, accept that no replacements will be recorded" + ) + valid_keys.append("y") + + if len(current) == len(replacement): + valid_options.append("Match (c)orresponding patches by number") + valid_keys.append("c") + + for opt in valid_options: + print(opt, file=sys.stderr) + + while True: + match_row = None + more_keys = "".join("/" + k for k in valid_keys if k != "y") + print(f"[Y{more_keys}/(matrix row)]? ", end="", file=sys.stderr) + key = input().strip() + + if key == "": + assert "y" in valid_keys + key = "y" + break + + if key.lower() in valid_keys: + key = key.lower() + break + + if match_row := re.fullmatch(r"(\d+)\s*=((?:[,\s]*(?:\d+))*)", key): + a = int(match_row[1]) + bs = match_row[2].strip() + bs = re.split(r"[,\s]", bs) if bs else [] + bs = [int(x) for x in bs] + if not (1 <= a <= len(current)) or not valid[a - 1]: + print(f"Invalid row number {a}", file=sys.stderr) + continue + bad = False + for b in bs: + if not (1 <= b <= len(replacement)): + print(f"Invalid replacement {b}", file=sys.stderr) + bad = True + if bad: + continue + a, bs = a - 1, [b - 1 for b in bs] + + break + + print(f"Unrecognized action {key!r}", file=sys.stderr) + + if match_row: + if bs: + matrix[a] = sorted(set(bs)) + else: + matrix[a] = None + continue + + assert key in valid_keys + + match key: + case "y": + print(f"(Accepted)", file=sys.stderr) + for i in range(len(current)): + row = matrix[i] + + if not valid[i] or not row: + continue + + cur_pid = f"mail:{current[i].message_id()}" + rs = [replacement[r] for r in row] + + for r in rs: + rid = f"mail:{r.message_id()}" + if rid not in db: + db[rid] = {"subject": r.clean_subject()} + + rids = [f"mail:{r.message_id()}" for r in rs] + assert "replacement" not in db[cur_pid] + if len(rids) == 1: + db[cur_pid]["replacement"] = rids[0] + else: + db[cur_pid]["replacement"] = rids + break + case "n": + print(f"(Ignored)", file=sys.stderr) + break + case "c": + for i in range(len(current)): + if valid[i]: + matrix[i] = [i] + + print(file=sys.stderr) + + write_patch_db(db) + + def main(): repo = GitRepo(".") match sys.argv[1:]: @@ -315,6 +576,25 @@ def main(): commits = repo.commit_list([f"^refs/tags/{base}", rev]) db = read_patch_db() do_status(db, commits, rev) + case ["mail-check"]: + db = read_patch_db() + result_file = "mail-check.json" + result = do_mail_check(db) + if result: + with open(result_file, "w") as f: + print(f"Writing to {result_file}", file=sys.stderr) + json.dump(result, f, indent=4) + f.write("\n") + else: + print("No updates found", file=sys.stderr) + case ["mail-match"]: + with open("mail-check.json", "rb") as f: + data = json.load(f) + do_mail_match(data) + case ["mail-match", jsonfile]: + with open(jsonfile, "rb") as f: + data = json.load(f) + do_mail_match(data) case ["diff", rev1, rev2]: base1 = guess_base(repo, rev1) print(f"Base for {rev1} is {base1}", file=sys.stderr) diff --git a/patchoulene/src/patchoulene/b4.py b/patchoulene/src/patchoulene/b4.py new file mode 100644 index 0000000..be44d9e --- /dev/null +++ b/patchoulene/src/patchoulene/b4.py @@ -0,0 +1,94 @@ +import re +import shlex +import subprocess +import sys + + +class PatchHeader: + headers: dict[str, str] + + def __init__(self, headers: dict[str, str]): + self.headers = headers + + def __repr__(self): + return f"PatchHeader({self.headers!r})" + + def message_id(self) -> str: + msgid_hdr = self.headers.get("message-id", None) + assert msgid_hdr is not None + assert len(msgid_hdr) > 2 and msgid_hdr[0] == "<" and msgid_hdr[-1] == ">" + return msgid_hdr[1:-1] + + def subject(self) -> str: + subject = self.headers.get("subject", None) + assert subject is not None + return subject + + def clean_subject(self) -> str: + subject = self.subject() + return re.sub(r"^(?:\[[^]]+\])*\s+", "", subject) + + +def parse_mbox_headers(mbox: str) -> list[PatchHeader]: + parsing, last_key = True, None + mails = [] + for line in mbox.splitlines(): + if line.startswith("From "): + parsing, last_key = True, None + mails.append({}) + elif parsing: + if line.isspace() or not line: + parsing = False + elif m := re.match(r"^([A-Za-z0-9-]+): (.*)", line): + assert mails + mails[-1][m[1].lower()] = m[2] + last_key = m[1].lower() + elif line[0] == " ": + assert mails + assert last_key is not None + assert last_key in mails[-1] + mails[-1][last_key] += " " + line[1:] + else: + raise ValueError(f"Weird line {line!r}") + return [PatchHeader(h) for h in mails] + + +def call_process(args: list[str]) -> str: + try: + return subprocess.check_output(args, encoding="utf-8", stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + e.add_note(e.stderr) + raise + + +B4_USER_AGENT = "github.com/openRuyi-Project/kernel-team-tools" + + +def check_msgid(msgid: str) -> tuple[list[PatchHeader], list[PatchHeader]] | None: + CMD = [ + *shlex.split("b4 -c"), + f"lore.useragentplus={B4_USER_AGENT}", + *shlex.split("am --no-add-trailers -o -"), + ] + + try: + this_thread = call_process([*CMD, "--", msgid]) + latest_thread = call_process([*CMD, "--check-newer-revisions", "--", msgid]) + except subprocess.CalledProcessError as e: + print("*** b4 failed vvvvvv", file=sys.stderr) + for note in e.__notes__: + print(note, file=sys.stderr) + print("*** b4 failed ^^^^^^", file=sys.stderr) + + return None + + this_thread_info = parse_mbox_headers(this_thread) + latest_thread_info = parse_mbox_headers(latest_thread) + + this_ids = [m.message_id() for m in this_thread_info] + + if msgid not in this_ids: + print(f"Can't handle weird threading in {msgid}", file=sys.stderr) + return None + + return (this_thread_info, latest_thread_info)