PS5: Find patches for fan translations and rebuilt discs by the game's own serial

AI-PR-NOTICE.md

@@ -0,0 +1 @@
+This issue or PR was made via an AI agent and likely has not been reviewed by a human at all, your time may be entirely wasted.

ps5/frontend/fe_patchdl.cpp

@@ -5,6 +5,8 @@
 
 #include "fe_patchdl.h"
 
+#include <algorithm>
+#include <cctype>
 #include <cerrno>
 #include <chrono>
 #include <cstdio>
@@ -130,10 +132,12 @@ bool IsPatchLine(const std::string& line)
 const std::vector<OnlinePatchSource>& OnlinePatchSources()
 {
 	static const std::vector<OnlinePatchSource> sources = {
-		{"PCSX2", "https://raw.githubusercontent.com/PCSX2/pcsx2_patches/main/patches/%s.pnach", false, "", ""},
+		{"PCSX2", "https://raw.githubusercontent.com/PCSX2/pcsx2_patches/main/patches/%s.pnach", false, "", "",
+			"https://api.github.com/repos/PCSX2/pcsx2_patches/git/trees/main:patches"},
 		{"Gabominated", "https://raw.githubusercontent.com/Gabominated/PCSX2/main/PCSX2%20Patches/%s.pnach", false, "_gabominated",
-			"Gabominated"},
-		{"cheats", "https://raw.githubusercontent.com/xs1l3n7x/pcsx2_cheats_collection/main/cheats/%s.pnach", true, "", ""},
+			"Gabominated", "https://api.github.com/repos/Gabominated/PCSX2/git/trees/main:PCSX2%20Patches"},
+		{"cheats", "https://raw.githubusercontent.com/xs1l3n7x/pcsx2_cheats_collection/main/cheats/%s.pnach", true, "", "",
+			"https://api.github.com/repos/xs1l3n7x/pcsx2_cheats_collection/git/trees/main:cheats"},
 	};
 	return sources;
 }
@@ -225,6 +229,65 @@ std::string TagPnachGroups(const std::string& text, const std::string& tag, cons
 	return out;
 }
 
+std::vector<std::string> ExecutableSerials(const std::vector<uint8_t>& elf)
+{
+	// "B" + region (A America, E Europe, I Asia) + "SLUS-20500": the memory card folder's name, as the game writes it.
+	std::vector<std::string> out;
+	const auto upper = [](uint8_t c) { return c >= 'A' && c <= 'Z'; };
+	const auto digit = [](uint8_t c) { return c >= '0' && c <= '9'; };
+	for (size_t i = 0; i + 12 <= elf.size(); i++)
+	{
+		const uint8_t* p = elf.data() + i;
+		if (p[0] != 'B' || (p[1] != 'A' && p[1] != 'E' && p[1] != 'I') || p[6] != '-')
+			continue;
+		bool ok = upper(p[2]) && upper(p[3]) && upper(p[4]) && upper(p[5]);
+		for (int d = 7; ok && d < 12; d++)
+			ok = digit(p[d]);
+		// what follows isn't a 6th digit ("BASLUS-205001" is no serial)
+		if (!ok || (i + 12 < elf.size() && digit(p[12])))
+			continue;
+		const std::string serial(reinterpret_cast<const char*>(p + 2), 10);
+		if (std::find(out.begin(), out.end(), serial) == out.end())
+			out.push_back(serial);
+	}
+	return out;
+}
+
+std::vector<std::string> TreeFilesForSerial(const std::string& json, const std::string& serial)
+{
+	// No JSON parser here: each "path" key's string value (GitHub writes "path": "SLUS-20500_E169BAF8.pnach", a space after the
+	// colon), names that need no escaping.
+	std::vector<std::string> out;
+	for (size_t at = json.find("\"path\""); at != std::string::npos; at = json.find("\"path\"", at + 6))
+	{
+		size_t start = at + 6;
+		while (start < json.size() && (json[start] == ' ' || json[start] == '\t' || json[start] == '\r' || json[start] == '\n'))
+			start++;
+		if (start >= json.size() || json[start] != ':')
+			continue;
+		start++;
+		while (start < json.size() && (json[start] == ' ' || json[start] == '\t' || json[start] == '\r' || json[start] == '\n'))
+			start++;
+		if (start >= json.size() || json[start] != '"')
+			continue;
+		start++;
+		const size_t end = json.find('"', start);
+		if (end == std::string::npos)
+			break;
+		const std::string name = json.substr(start, end - start);
+		// "<serial>_<8 hex digits>.pnach" and nothing else
+		if (name.size() != serial.size() + 1 + 8 + 6 || name.compare(0, serial.size() + 1, serial + "_") != 0 ||
+			name.compare(name.size() - 6, 6, ".pnach") != 0)
+			continue;
+		bool hex = true;
+		for (size_t k = serial.size() + 1; hex && k < serial.size() + 9; k++)
+			hex = std::isxdigit(static_cast<unsigned char>(name[k])) != 0;
+		if (hex)
+			out.push_back(name);
+	}
+	return out;
+}
+
 int CountPnachGroups(const std::string& text, bool* unnamed)
 {
 	int groups = 0;
@@ -394,14 +457,146 @@ uint32_t OnlinePatches::DiscCrc(const std::string& image, std::string& error)
 OnlinePatchStatus OnlinePatches::Fetch(const GameInfo& game)
 {
 	SetStatus(game.serial, OnlinePatchStatus::State::Working, "Reading the game's CRC");
-	std::string error;
-	const uint32_t crc = DiscCrc(game.path, error);
-	if (crc == 0)
+	std::string name;
+	std::vector<uint8_t> elf;
+	if (!ReadAchievementExecutable(game.path, name, elf) || elf.size() < 4)
 	{
-		Log(game.serial + ": " + error);
+		Log(game.serial + ": the game's executable couldn't be read from the disc image");
 		return {OnlinePatchStatus::State::Failed, "Couldn't read the game's CRC"};
 	}
-	return FetchWithCrc(game.serial, crc);
+	return FetchWithExecutable(game.serial, ElfCrc(elf), elf);
+}
+
+OnlinePatchStatus OnlinePatches::FetchWithExecutable(const std::string& serial, uint32_t crc, const std::vector<uint8_t>& elf)
+{
+	const OnlinePatchStatus own = FetchWithCrc(serial, crc);
+	if (own.state != OnlinePatchStatus::State::Done || own.message.rfind("Nothing online", 0) != 0)
+		return own;
+	// The disc's own serial first (a translation that kept it: same game, another CRC), then the ones the executable names.
+	std::vector<std::string> serials = {serial};
+	for (const std::string& s : ExecutableSerials(elf))
+		if (s != serial)
+			serials.push_back(s);
+	std::string names;
+	for (const std::string& s : serials)
+		names += (names.empty() ? "" : ", ") + s;
+	Log(serial + ": nothing for the disc's own CRC; looking by serial: " + names);
+	const OnlinePatchStatus found = FetchBySerial(serial, crc, serials);
+	return found.state == OnlinePatchStatus::State::Idle ? own : found;
+}
+
+const std::string* OnlinePatches::SourceTree(const OnlinePatchSource& src, int& status)
+{
+	const auto it = m_trees.find(src.name);
+	if (it != m_trees.end())
+	{
+		status = 200;
+		return &it->second;
+	}
+	std::string body;
+	status = m_platform.get_text(src.tree, body);
+	// a list that isn't one (an HTML page, GitHub's 403 once its 60 unauthenticated requests an hour are used) is not kept:
+	// the next game asks again
+	if (status != 200 || body.find("\"tree\"") == std::string::npos)
+		return nullptr;
+	return &(m_trees[src.name] = std::move(body));
+}
+
+OnlinePatchStatus OnlinePatches::FetchBySerial(const std::string& disc_serial, uint32_t crc, const std::vector<std::string>& serials)
+{
+	char disc_key[48];
+	std::snprintf(disc_key, sizeof(disc_key), "%s_%08X", disc_serial.c_str(), crc);
+	int found_patches = 0, found_cheats = 0, failed = 0;
+	std::string why, from;
+	// The first serial anything is found for wins: a later one is another release (Red Dead Revolver's rebuild names the PAL
+	// release's memory card folder too, but its code is the US one's).
+	for (const std::string& serial : serials)
+	{
+		if (found_patches || found_cheats)
+			break;
+		for (const OnlinePatchSource& src : OnlinePatchSources())
+		{
+			SetStatus(disc_serial, OnlinePatchStatus::State::Working, std::string("Asking ") + src.name + " for " + serial);
+			int status = 0;
+			const std::string* tree = SourceTree(src, status);
+			if (!tree)
+			{
+				if (serial == serials.front())
+				{
+					failed++;
+					why = std::string(src.name) + "'s file list: " + (status < 0 ? "no connection" : "HTTP " + std::to_string(status));
+					Log(std::string(disc_key) + ": " + why);
+				}
+				continue;
+			}
+			const std::vector<std::string> files = TreeFilesForSerial(*tree, serial);
+			if (files.size() != 1)
+			{
+				// none, or two releases of the game (which one the disc was made from isn't known): left alone
+				if (!files.empty())
+					Log(std::string(disc_key) + ": " + src.name + ": " + std::to_string(files.size()) + " files for " + serial +
+						" (more than one release): none taken");
+				continue;
+			}
+			const std::string key = files[0].substr(0, files[0].size() - 6);
+			std::string body;
+			const int got = m_platform.get_text(OnlinePatchUrl(src, key), body);
+			if (got != 200 || ClassifyPnach(body) != PnachKind::Patches)
+			{
+				Log(std::string(disc_key) + ": " + src.name + ": " + key + ": " +
+					(got == 200 ? "nothing to apply in it" : "HTTP " + std::to_string(got)));
+				continue;
+			}
+			// Under the disc's own name, so PCSX2 loads it for this game; every group named after where it came from, and off.
+			const std::string tag = src.tag[0] != '\0' ? std::string(src.tag) + ", from " + key : "from " + key;
+			std::string text = "// PS5SX2: " + std::string(src.name) + "'s " + files[0] + ", for the game's own serial (" + serial +
+				               ") found in this disc's executable. Made for another executable: test each group before you keep it on.\n" +
+				               TagPnachGroups(body, tag, tag);
+			if (text.back() != '\n')
+				text += "\n";
+			const std::string dir = src.cheats ? m_cheats : m_patches;
+			MakeDir(dir);
+			const std::string path = dir + "/" + disc_key + src.suffix + "_" + serial + ".pnach";
+			std::string current;
+			const bool exists = ReadWhole(path, current);
+			const auto mine = m_manifest.find(path);
+			const int groups = CountPnachGroups(text);
+			if (exists && current != text && (mine == m_manifest.end() || mine->second != Fingerprint(current)))
+			{
+				Log(std::string(disc_key) + ": " + src.name + ": " + path + " is there and wasn't written by this app: left as it is");
+				(src.cheats ? found_cheats : found_patches) += groups;
+				continue;
+			}
+			if (!exists || current != text)
+			{
+				std::string werr;
+				if (!WriteWhole(path, text, werr))
+				{
+					failed++;
+					why = werr;
+					Log(std::string(disc_key) + ": " + src.name + ": " + werr);
+					continue;
+				}
+				m_manifest[path] = Fingerprint(text);
+				SaveManifest();
+				Log(std::string(disc_key) + ": " + src.name + ": " + key + " -> " + path + " (" + std::to_string(groups) + " group(s))");
+			}
+			(src.cheats ? found_cheats : found_patches) += groups;
+			if (from.find(key) == std::string::npos)
+				from += (from.empty() ? "" : ", ") + key;
+		}
+	}
+	if (found_patches || found_cheats)
+	{
+		const std::string message = std::to_string(found_patches) + (found_patches == 1 ? " patch" : " patches") + ", " +
+			                        std::to_string(found_cheats) + (found_cheats == 1 ? " cheat" : " cheats") + " from " + from +
+			                        " (made for that release: test before use)";
+		Log(std::string(disc_key) + ": " + message);
+		return {OnlinePatchStatus::State::Done, message};
+	}
+	if (failed)
+		return {OnlinePatchStatus::State::Failed, why};
+	return {}; // Idle: nothing by serial either (the caller keeps its own "Nothing online")
 }
 
 OnlinePatchStatus OnlinePatches::FetchWithCrc(const std::string& serial, uint32_t crc)

ps5/frontend/fe_patchdl.h

@@ -14,6 +14,18 @@
 // wrote). The sheet then lists the groups under Patches and Cheats, each off until turned on. Needs proper testing on the
 // console.
 //
+// 2.16 (AI-assisted): when no source has a file for the disc's own "<serial>_<crc>" (a fan translation or a rebuilt disc:
+// its executable differs, so its CRC does, and some boot "NORG_RUS.00" instead of "SLUS_205.00", which gives no serial at
+// all), the game's own serial is looked for inside its executable: a PS2 game names its memory card folder after it
+// ("BASLUS-20500", "BESLES-52478"). For each one found, each patch source's file list (GitHub's tree API, one request a
+// source) gives the files of that serial; when it has just one, that file is fetched and kept under the DISC's name
+// ("<disc serial>_<disc crc>_<serial>.pnach"), so PCSX2 loads it for this game. Every group gets "(from <serial>_<crc>)" in
+// its name and stays off: a patch made for another executable may not fit this one (a translation that moved code would
+// crash or misdraw), so turning one on is the player's own test. Two files for one serial (two releases) are left alone,
+// and the first serial anything is found for wins. The disc's own serial is tried first (a translation that kept it).
+// Asked only when the disc's own file wasn't found; each list once a run (GitHub allows 60 such requests an hour without
+// an account). Not tried on a console yet: needs proper testing there.
+//
 // Copyright (C) 2026 swordpdf
 // SPDX-License-Identifier: GPL-3.0-or-later
 
@@ -70,6 +82,7 @@ struct OnlinePatchSource
 	bool cheats;          // into the cheats folder (else the patches folder)
 	const char* suffix;   // the local file's name after "<serial>_<crc>" ("" or "_gabominated")
 	const char* tag;      // groups renamed "<name> (<tag>)" ("" leaves them)
+	const char* tree;     // 2.16: GitHub's tree API for the source's folder (its file names), for the fallback by serial
 };
 const std::vector<OnlinePatchSource>& OnlinePatchSources();
 // A source's URL for "<serial>_<crc>": its first "%s" replaced by `key`. vk-285-134b: not through snprintf, since the URL
@@ -95,6 +108,12 @@ PnachKind ClassifyPnach(const std::string& text);
 std::string TagPnachGroups(const std::string& text, const std::string& tag, const std::string& unnamed);
 // How many groups ("[...]" lines) a pnach text has, and whether it has patch lines outside any group.
 int CountPnachGroups(const std::string& text, bool* unnamed = nullptr);
+// 2.16 (AI-assisted): the serials a PS2 executable names its memory card folders after ("BASLUS-20500" -> "SLUS-20500";
+// region letter A, E or I, then the 4 letters, a dash and 5 digits), in the order found, each once. Most games have one;
+// some have two regions' (Red Dead Revolver's Russian rebuild: SLUS-20500 and SLES-52478).
+std::vector<std::string> ExecutableSerials(const std::vector<uint8_t>& elf);
+// 2.16: the "<serial>_<CRC>.pnach" names in a GitHub tree API answer ({"tree":[{"path":"..."}]}) for one serial, in order.
+std::vector<std::string> TreeFilesForSerial(const std::string& json, const std::string& serial);
 
 class OnlinePatches
 {
@@ -117,8 +136,13 @@ class OnlinePatches
 	static uint32_t DiscCrc(const std::string& image, std::string& error);
 	// Fetch with a CRC already known (the test's games have no executable).
 	OnlinePatchStatus FetchWithCrc(const std::string& serial, uint32_t crc);
+	// 2.16: FetchWithCrc, then, when it found nothing, the fallback by the serials in the executable (see above).
+	OnlinePatchStatus FetchWithExecutable(const std::string& serial, uint32_t crc, const std::vector<uint8_t>& elf);
 
 private:
+	OnlinePatchStatus FetchBySerial(const std::string& disc_serial, uint32_t crc, const std::vector<std::string>& serials);
+	// One source's file list, asked once a run (the same lists serve every game asked after it).
+	const std::string* SourceTree(const OnlinePatchSource& src, int& status);
 	void Run();
 	void SetStatus(const std::string& serial, OnlinePatchStatus::State state, const std::string& message);
 	void Log(const std::string& line) const;
@@ -134,5 +158,6 @@ class OnlinePatches
 	bool m_stop = false, m_running = false;
 	std::deque<GameInfo> m_queue;
 	std::map<std::string, OnlinePatchStatus> m_status; // by serial
+	std::map<std::string, std::string> m_trees;        // 2.16: a source's tree API answer, by source name (worker thread only)
 };
 } // namespace fe

ps5/frontend/fe_ps5.cpp

@@ -2147,8 +2147,10 @@ std::string orbis_frontend_run(const OrbisFrontendPaths& paths, const char* buil
 		OnlinePatchPlatform op;
 		op.get_text = [](const std::string& url, std::string& body) {
 			body.clear();
-			return g_patchhttp.Get(url, false, 0, 0, [&body](const void* d, size_t n) {
-				if (body.size() + n > (2u << 20))
+			// 2.16: a source's file list (GitHub's tree API, 1.4 MB for PCSX2's in October 2026, and growing) may be bigger than a pnach
+			const size_t cap = url.rfind("https://api.github.com/", 0) == 0 ? (8u << 20) : (2u << 20);
+			return g_patchhttp.Get(url, false, 0, 0, [&body, cap](const void* d, size_t n) {
+				if (body.size() + n > cap)
 					return false;
 				body.append(static_cast<const char*>(d), n);
 				return true;