From 427b756b5f15878b81c6954437c1e387a6f24aa9 Mon Sep 17 00:00:00 2001 From: KHeartz Date: Fri, 16 Oct 2026 03:05:46 -0400 Subject: [PATCH 1/2] Scripts: Stop pattern-table rebuilds duplicating addresses A rebuild reused addresses from every block for the same image, so a table holding two such blocks recorded each match once per block, doubling them on every rebuild. Reuse now dedupes, and an old block for an image that was just rebuilt is dropped whatever its tag. An executable named by path was tagged file:, which a game's Steam and Epic copies share, so naming one dropped the other's block as superseded. A path inside a Steam library or an Epic Games Launcher install now gets that store's tag. --- scripts/build_pattern_table.py | 53 +++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/scripts/build_pattern_table.py b/scripts/build_pattern_table.py index 0e468bcac..f49a6a069 100644 --- a/scripts/build_pattern_table.py +++ b/scripts/build_pattern_table.py @@ -34,11 +34,13 @@ The executables are named by --exe (repeatable), by the environment variable named with --exe-env, discovered from a Steam install with --steam-app/--steam-relative, and from a -Microsoft Store / Xbox app install with --msstore-family/--msstore-relative. A block for an +Microsoft Store / Xbox app install with --msstore-family/--msstore-relative. A path inside a +Steam library or an Epic Games Launcher install is tagged with that store. A block for an executable this machine does not have is carried over from the existing table. """ import argparse +import json import os import re import struct @@ -86,8 +88,8 @@ def source_tag(source: str) -> int: - """The storefront an executable came from ("steam:", "msstore:", or - "file:" for one named directly), as stored in its block. A rebuild replaces a + """The storefront an executable came from ("steam:", "msstore:", "epic:", + or "file:"; see named_storefront), as stored in its block. A rebuild replaces a storefront's block -- a patched build supersedes the one before it -- instead of accumulating one per patch.""" return zlib.crc32(source.lower().encode("utf-8")) & 0xFFFFFFFF @@ -386,6 +388,35 @@ def locate_msstore_game(family: str, relative: str): return None +def named_storefront(exe: Path) -> str: + """The storefront of an executable named by path: "steam:" inside a Steam library, + "epic:" inside an Epic Games Launcher install, otherwise "file:". Two stores' + copies of a game share a file name, and a shared tag would read one store's block as the + other's superseded build and drop it.""" + path = exe.resolve() + parts = [part.lower() for part in path.parts] + for i in range(len(parts) - 3): + if parts[i] == "steamapps" and parts[i + 1] == "common": + for manifest in Path(*path.parts[:i + 1]).glob("appmanifest_*.acf"): + text = manifest.read_text(encoding="utf-8", errors="replace") + installdir = re.search(r'"installdir"\s*"([^"]+)"', text) + appid = re.search(r'"appid"\s*"(\d+)"', text) + if installdir and appid and installdir.group(1).lower() == parts[i + 2]: + return "steam:%s" % appid.group(1) + break + + installed = Path(os.environ.get("ProgramData", "C:/ProgramData")) / "Epic" / "UnrealEngineLauncher" / "LauncherInstalled.dat" + try: + installs = json.loads(installed.read_text(encoding="utf-8-sig")).get("InstallationList", []) + except (OSError, ValueError, AttributeError): + installs = [] + for install in installs: + location, app = install.get("InstallLocation"), install.get("AppName") + if location and app and Path(location).resolve() in path.parents: + return "epic:%s" % app + return "file:%s" % exe.name + + def resolve_exes(exes=(), exe_env=None, steam_app=None, steam_relative=None, msstore_family=None, msstore_relative=None): """Every executable this machine has to build the table from, as (path, storefront) pairs: a Steam install, a Microsoft Store install, the environment and the explicit @@ -399,7 +430,7 @@ def resolve_exes(exes=(), exe_env=None, steam_app=None, steam_relative=None, mss named = [Path(exe) for exe in exes or () if exe] if exe_env and os.environ.get(exe_env): named.append(Path(os.environ[exe_env])) - found.extend((exe, "file:%s" % exe.name) for exe in named) + found.extend((exe, named_storefront(exe)) for exe in named) out, seen = [], set() for exe, source in found: @@ -418,17 +449,19 @@ def build_block(exe: Path, source: str, literals, preps, known, jobs: int, requi image, image_base, size_of_image, file_size, file_crc = map_image(exe) print("%s: %d bytes mapped to 0x%X..0x%X" % (exe, file_size, image_base, image_base + size_of_image)) + # A set: an older table can hold two blocks for this image, and a list would record each of + # their matches once per block, doubling the entries on every rebuild. reusable = {} for block in known: if (block.image_base, block.size_of_image, block.file_size, block.file_crc) == (image_base, size_of_image, file_size, file_crc) and file_crc: for h, rva in block.entries: - reusable.setdefault(h, []).append(rva) + reusable.setdefault(h, set()).add(rva) hits, pending = {}, [] for lit in literals: recorded = reusable.get(framework_hash(lit, False)) if recorded is not None and all(matches_at(image, preps[lit], rva) for rva in recorded): - hits[lit] = recorded + hits[lit] = sorted(recorded) else: pending.append(lit) if reusable: @@ -498,15 +531,19 @@ def build_table(exes, patterns: Path, out: Path, style: str = "call", jobs: int # A storefront this machine has no executable for keeps its block from the existing table. # Its patterns were not checked against the current set, so any it lacks are left to the # client's scan. A storefront built here replaces its old block: a patched build supersedes - # the one before it. An untagged block cannot be told apart, so it is dropped. + # the one before it. An untagged block cannot be told apart, so it is dropped. A block for an + # image built here goes whatever its tag: the client serves only the first block per image. built = {b.source for b in blocks} + rebuilt = {(b.image_base, b.size_of_image, b.file_size) for b in blocks} for block in known: + if (block.image_base, block.size_of_image, block.file_size) in rebuilt: + continue identity = "base 0x%X size 0x%X file %d" % (block.image_base, block.size_of_image, block.file_size) if block.source and block.source not in built: print("kept the block for a storefront not on this machine (%s): its patterns were not re-checked" % identity, file=sys.stderr) blocks.append(block) - elif not any((b.image_base, b.size_of_image, b.file_size) == (block.image_base, block.size_of_image, block.file_size) for b in blocks): + else: print("dropped the superseded block (%s)" % identity) payload = b"" From 4c5f80bb25c0b3ae9874c91f185772e6edff6003 Mon Sep 17 00:00:00 2001 From: KHeartz Date: Fri, 16 Oct 2026 03:13:21 -0400 Subject: [PATCH 2/2] Scripts: Fall back to a file tag on unreadable store metadata An unreadable Steam app manifest, or an Epic install list holding something other than install records, raised out of named_storefront and stopped the rebuild. Both now leave the path tagged by file name. --- scripts/build_pattern_table.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/build_pattern_table.py b/scripts/build_pattern_table.py index f49a6a069..78e21274b 100644 --- a/scripts/build_pattern_table.py +++ b/scripts/build_pattern_table.py @@ -398,7 +398,10 @@ def named_storefront(exe: Path) -> str: for i in range(len(parts) - 3): if parts[i] == "steamapps" and parts[i + 1] == "common": for manifest in Path(*path.parts[:i + 1]).glob("appmanifest_*.acf"): - text = manifest.read_text(encoding="utf-8", errors="replace") + try: + text = manifest.read_text(encoding="utf-8", errors="replace") + except OSError: + continue installdir = re.search(r'"installdir"\s*"([^"]+)"', text) appid = re.search(r'"appid"\s*"(\d+)"', text) if installdir and appid and installdir.group(1).lower() == parts[i + 2]: @@ -410,9 +413,12 @@ def named_storefront(exe: Path) -> str: installs = json.loads(installed.read_text(encoding="utf-8-sig")).get("InstallationList", []) except (OSError, ValueError, AttributeError): installs = [] - for install in installs: + # The launcher's file is outside our control: skip anything that isn't an install record. + for install in installs if isinstance(installs, list) else []: + if not isinstance(install, dict): + continue location, app = install.get("InstallLocation"), install.get("AppName") - if location and app and Path(location).resolve() in path.parents: + if isinstance(location, str) and location and app and Path(location).resolve() in path.parents: return "epic:%s" % app return "file:%s" % exe.name