From 6f1464c0747f34656575d47c35566210aef22242 Mon Sep 17 00:00:00 2001 From: alexispradol Date: Thu, 29 Oct 2026 04:36:24 -0300 Subject: [PATCH 01/11] feat(user_scan): add 2 modules gaming --- .../user_scan/gaming/planetminecraft.py | 35 +++++++++++++ user_scanner/user_scan/gaming/typeracer.py | 52 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 user_scanner/user_scan/gaming/planetminecraft.py create mode 100644 user_scanner/user_scan/gaming/typeracer.py diff --git a/user_scanner/user_scan/gaming/planetminecraft.py b/user_scanner/user_scan/gaming/planetminecraft.py new file mode 100644 index 00000000..b087967d --- /dev/null +++ b/user_scanner/user_scan/gaming/planetminecraft.py @@ -0,0 +1,35 @@ +import re + +from user_scanner.core.impersonate import impersonate_validate +from user_scanner.core.result import Result + + +def validate_planetminecraft(user: str) -> Result: + url = f"https://www.planetminecraft.com/member/{user}" + + def process(response): + if response.status_code == 410: + return Result.available() + if response.status_code == 200: + m = re.search(r"([^<]+) on Planet Minecraft", response.text) + if m and m.group(1).strip().lower() == user.lower(): + extra, media = {}, {} + h = re.search(r"]*>([^<]+)", response.text) + if h: extra["username"] = h.group(1).strip() + lv = re.search(r"Level (\d+)", response.text) + if lv: extra["level"] = lv.group(1) + rk = re.search(r'member-rank-class">([^<]+)', response.text) + if rk: extra["rank"] = rk.group(1).strip() + mid = re.search(r"data-mid=['\"]?(\d+)", response.text) + if mid: extra["member_id"] = mid.group(1) + sub = re.search(r'num_subscribers">(\d+)', response.text) + if sub: extra["followers"] = sub.group(1) + av = re.search( + r"https://static\.planetminecraft\.com/files/avatar/\d+_\d+\.png", + response.text, + ) + if av: media["avatar"] = av.group(0) + return Result.taken(extra=extra, media=media) + return Result.error(f"Unexpected response status: {response.status_code}") + + return impersonate_validate(url, process, show_url=url) diff --git a/user_scanner/user_scan/gaming/typeracer.py b/user_scanner/user_scan/gaming/typeracer.py new file mode 100644 index 00000000..b24fd74f --- /dev/null +++ b/user_scanner/user_scan/gaming/typeracer.py @@ -0,0 +1,52 @@ +import re + +from user_scanner.core.orchestrator import generic_validate, Result + + +def validate_typeracer(user: str) -> Result: + url = "https://data.typeracer.com/pit/profile" + show_url = f"https://data.typeracer.com/pit/profile?user={user}" + + def process(response): + if response.status_code == 200: + m = re.search( + r"([^<(]+) \(([^)]+)\) \(TypeRacer Profile\)", + response.text, + ) + if m and m.group(2).strip().lower() == user.lower(): + extra, media = {}, {} + extra["fullname"] = m.group(1).strip() + stats = { + "Full Avg.": "avg_wpm", + "Best Race": "best_wpm", + "Races": "races", + "WPM %": "wpm_percentile", + "Skill Level": "skill_level", + "Exp Level": "exp_level", + } + for value, label in re.findall( + r'Stat__Top">\s*([^<]+?)\s*\s*' + r'([^<]+)', + response.text, + ): + key = stats.get(label.strip()) + if key: + extra[key] = value.strip() + s = re.search(r"Racing Since:\s*([^<]+)", response.text) + if s: extra["racing_since"] = s.group(1).strip() + s = re.search(r"Location:\s*\s*(\S[^<]*?)\s*", response.text) + if s: extra["location"] = s.group(1).strip() + p = re.search(r'class="Plan__(Basic|Premium)"', response.text) + if p: extra["plan"] = p.group(1).lower() + s = re.search( + r"profileCar[\s\S]*?url\([\"'](/public/images/avatars/[^)\"']+)", + response.text, + ) + if s: + media["avatar"] = f"https://data.typeracer.com{s.group(1)}" + return Result.taken(extra=extra, media=media) + if "Profile Not Found (TypeRacer Pit Stop)" in response.text: + return Result.available() + return Result.error(f"Unexpected response status: {response.status_code}") + + return generic_validate(url, process, show_url=show_url, params={"user": user}) From 060a4e734de180f21f2f679176fdd01b56046c0d Mon Sep 17 00:00:00 2001 From: alexispradol Date: Thu, 29 Oct 2026 04:37:36 -0300 Subject: [PATCH 02/11] feat(user_scan): add 2 modules music --- user_scanner/user_scan/music/mixlr.py | 47 +++++++++++++++++++ .../user_scan/music/ultimate_guitar.py | 45 ++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 user_scanner/user_scan/music/mixlr.py create mode 100644 user_scanner/user_scan/music/ultimate_guitar.py diff --git a/user_scanner/user_scan/music/mixlr.py b/user_scanner/user_scan/music/mixlr.py new file mode 100644 index 00000000..16b7afb6 --- /dev/null +++ b/user_scanner/user_scan/music/mixlr.py @@ -0,0 +1,47 @@ +from user_scanner.core.helpers import get_random_user_agent +from user_scanner.core.orchestrator import generic_validate +from user_scanner.core.result import Result + + +def validate_mixlr(user: str) -> Result: + url = f"https://api.mixlr.com/users/{user.lower()}" + show_url = f"https://mixlr.com/{user}" + + headers = { + "User-Agent": get_random_user_agent(), + "Accept": "application/json", + } + + def process(response): + if response.status_code == 404: + return Result.available() + + if response.status_code == 200: + try: + data = response.json() + except Exception: + return Result.error("200 response status with no recognizable data, report it via GitHub issues") + if not isinstance(data, dict) or "id" not in data or "slug" not in data: + return Result.error("Unexpected API response body, report it via GitHub issues") + + extra = {} + media = {} + if username := data.get("username"): extra["username"] = username + if slug := data.get("slug"): extra["slug"] = slug + if role := data.get("role"): extra["role"] = role + if tz := data.get("time_zone"): extra["time_zone"] = tz + plan = data.get("plan") or {} + if plan.get("reference"): extra["plan"] = plan["reference"] + elif plan.get("name"): extra["plan"] = plan["name"] + if data.get("is_premium"): extra["premium"] = "true" + if data.get("is_live"): extra["live"] = "true" + channel = data.get("channel") or {} + if channel.get("name"): extra["channel"] = channel["name"] + if avatar := data.get("profile_image_url"): + if "fallback-user" not in avatar: media["avatar"] = avatar + + return Result.taken(extra=extra, media=media) + + return Result.error(f"Unexpected status code: {response.status_code}") + + return generic_validate(url, process, show_url=show_url, headers=headers) diff --git a/user_scanner/user_scan/music/ultimate_guitar.py b/user_scanner/user_scan/music/ultimate_guitar.py new file mode 100644 index 00000000..6bcc75e5 --- /dev/null +++ b/user_scanner/user_scan/music/ultimate_guitar.py @@ -0,0 +1,45 @@ +import re +from datetime import datetime, timezone + +from user_scanner.core.impersonate import impersonate_validate +from user_scanner.core.result import Result + + +def validate_ultimate_guitar(user: str) -> Result: + url = f"https://www.ultimate-guitar.com/u/{user}" + + def process(response): + if response.status_code == 410: + return Result.available() + if response.status_code == 200: + m = re.search(r"([^<]+) \| Ultimate-Guitar\.Com", response.text) + if m and m.group(1).strip().lower() == user.lower(): + extra, media = {}, {} + t = re.search(r"usertitle":"([^&]+)"", response.text) + if t: extra["usertitle"] = t.group(1) + fc = re.search(r"follower_count":(\d+)", response.text) + if fc: extra["followers"] = fc.group(1) + fg = re.search(r"following_count":(\d+)", response.text) + if fg: extra["following"] = fg.group(1) + iq = re.search(r"iq":(\d+)", response.text) + if iq: extra["iq"] = iq.group(1) + dr = re.search(r"date_registered":(\d+)", response.text) + if dr: + extra["joined"] = datetime.fromtimestamp( + int(dr.group(1)), tz=timezone.utc + ).strftime("%Y-%m-%d") + city = re.search(r"city":"([^&]*)"", response.text) + if city and city.group(1): extra["city"] = city.group(1) + country = re.search(r"country_name":"([^&]*)"", response.text) + if country and country.group(1): extra["location"] = country.group(1) + va = re.search(r"is_avatar_exists":(true|false)", response.text) + if va and va.group(1) == "true": + av = re.search( + r"avatar":"(https://www\.ultimate-guitar\.com/static[^\s"]+)"", + response.text, + ) + if av: media["avatar"] = av.group(1).replace("&", "&") + return Result.taken(extra=extra, media=media) + return Result.error(f"Unexpected response status: {response.status_code}") + + return impersonate_validate(url, process, show_url=url, allow_redirects=True) From 6fa53c6083b9df921798d58ff986defcbb3d1284 Mon Sep 17 00:00:00 2001 From: alexispradol Date: Thu, 29 Oct 2026 04:38:13 -0300 Subject: [PATCH 03/11] feat(user_scan): add ccm --- user_scanner/user_scan/community/ccm.py | 39 +++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 user_scanner/user_scan/community/ccm.py diff --git a/user_scanner/user_scan/community/ccm.py b/user_scanner/user_scan/community/ccm.py new file mode 100644 index 00000000..bb9c0663 --- /dev/null +++ b/user_scanner/user_scan/community/ccm.py @@ -0,0 +1,39 @@ +import re + +from user_scanner.core.orchestrator import generic_validate, Result + + +def validate_ccm(user: str) -> Result: + url = f"https://ccm.net/profile/user/{user}" + show_url = f"https://ccm.net/profile/user/{user}" + + def process(response): + if response.status_code == 404: + return Result.available() + if response.status_code == 200: + m = re.search(r"Profil de ([^<]+) - CCM", response.text) + if m and m.group(1).strip().lower() == user.lower(): + extra = {} + h = re.search(r'\s*([^<]+?)\s*', response.text) + if h: extra["username"] = h.group(1).strip() + s = re.search(r'pro_header__status">\s*([^<]+)', response.text) + if s: extra["status"] = s.group(1).strip() + for label, key in [ + ("Registration date", "joined"), + ("Gender", "gender"), + ("Location", "location"), + ("Birthday", "birthday"), + ("Website", "website"), + ("Contributions", "contributions"), + ]: + dd = re.search( + rf"
{label}
\s*]*>(.*?)", response.text, re.S + ) + if dd: + val = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", dd.group(1))).strip() + if val and val.lower() not in ("not entered", "n/a"): + extra[key] = val + return Result.taken(extra=extra) + return Result.error(f"Unexpected response status: {response.status_code}") + + return generic_validate(url, process, show_url=show_url, follow_redirects=True) From 5a1337f03151bcddc2a18c56fc8eb778cc36493f Mon Sep 17 00:00:00 2001 From: alexispradol Date: Thu, 29 Oct 2026 04:39:08 -0300 Subject: [PATCH 04/11] feat(user_scan): add listal --- .../user_scan/entertainment/listal.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 user_scanner/user_scan/entertainment/listal.py diff --git a/user_scanner/user_scan/entertainment/listal.py b/user_scanner/user_scan/entertainment/listal.py new file mode 100644 index 00000000..ea8d9498 --- /dev/null +++ b/user_scanner/user_scan/entertainment/listal.py @@ -0,0 +1,29 @@ +import re + +from user_scanner.core.orchestrator import generic_validate, Result + + +def validate_listal(user: str) -> Result: + url = f"https://{user}.listal.com/" + show_url = f"https://{user}.listal.com/" + + def process(response): + if response.status_code != 200: + return Result.error(f"Unexpected response status: {response.status_code}") + m = re.search(r"([^<]+) on Listal", response.text) + if m and m.group(1).strip().lower() == user.lower(): + extra, media = {}, {} + h = re.search(r"]*>([^<]+)", response.text) + if h: extra["username"] = h.group(1).strip() + pt = re.search(r"Points:\s*(?:]*>)?(\d+)", response.text) + if pt: extra["points"] = pt.group(1) + fl = re.search(r"(\d+) Followers", response.text) + if fl: extra["followers"] = fl.group(1) + av = re.search(r']*src="(https://lthumb\.lisimg\.com/[^"]+)"', response.text) + if av: media["avatar"] = av.group(1) + return Result.taken(extra=extra, media=media) + if "Listal - List the stuff you love!" in response.text: + return Result.available() + return Result.error("Unexpected response body, report it via GitHub issues") + + return generic_validate(url, process, show_url=show_url, follow_redirects=True) From a28d0e9c92bbcc9318307a2a405fa360cf312bf7 Mon Sep 17 00:00:00 2001 From: alexispradol Date: Thu, 29 Oct 2026 04:39:50 -0300 Subject: [PATCH 05/11] feat(user_scan): add hackthisshite --- user_scanner/user_scan/dev/hackthissite.py | 45 ++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 user_scanner/user_scan/dev/hackthissite.py diff --git a/user_scanner/user_scan/dev/hackthissite.py b/user_scanner/user_scan/dev/hackthissite.py new file mode 100644 index 00000000..f2b8d38f --- /dev/null +++ b/user_scanner/user_scan/dev/hackthissite.py @@ -0,0 +1,45 @@ +import re + +from user_scanner.core.orchestrator import generic_validate, Result + + +def validate_hackthissite(user: str) -> Result: + url = f"https://www.hackthissite.org/user/view/{user}" + + def process(response): + if response.status_code == 200: + m = re.search(r"Hack This Site :: View User: ([^<]+)", response.text) + if m and m.group(1).strip().lower() == user.lower(): + extra, media = {}, {} + r = re.search(r"Rank:\s*([^<(\n]+?)\s*\((\d+)\s*Points\)", response.text) + if r: + extra["rank"] = r.group(1).strip() + extra["points"] = r.group(2) + s = re.search(r"Status:\s*([^<\n]+){label}:\s*([^<]*)Hack This Site" in response.text: + return Result.available() + return Result.error(f"Unexpected response status: {response.status_code}") + + return generic_validate(url, process, show_url=url) From 3dd4e89529de516bb44cd6509d25ce7d4a1e21f4 Mon Sep 17 00:00:00 2001 From: alexispradol Date: Thu, 29 Oct 2026 14:32:49 -0300 Subject: [PATCH 06/11] fix(mixlr): fixing --- user_scanner/user_scan/music/mixlr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_scanner/user_scan/music/mixlr.py b/user_scanner/user_scan/music/mixlr.py index 16b7afb6..f129dfb3 100644 --- a/user_scanner/user_scan/music/mixlr.py +++ b/user_scanner/user_scan/music/mixlr.py @@ -13,7 +13,7 @@ def validate_mixlr(user: str) -> Result: } def process(response): - if response.status_code == 404: + if response.status_code == 404 and "Resource not found" in response.text: return Result.available() if response.status_code == 200: From 12b0937254dceee49e13453cdf640f635ea480ee Mon Sep 17 00:00:00 2001 From: alexispradol Date: Thu, 29 Oct 2026 14:37:36 -0300 Subject: [PATCH 07/11] Update regex for profile title extraction --- user_scanner/user_scan/community/ccm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_scanner/user_scan/community/ccm.py b/user_scanner/user_scan/community/ccm.py index bb9c0663..a2cdaedc 100644 --- a/user_scanner/user_scan/community/ccm.py +++ b/user_scanner/user_scan/community/ccm.py @@ -11,7 +11,7 @@ def process(response): if response.status_code == 404: return Result.available() if response.status_code == 200: - m = re.search(r"Profil de ([^<]+) - CCM", response.text) + m = re.search(r"(?:Profil de\s+)?([^<']+?)(?:'s profile)?\s*-\s*CCM", response.text) if m and m.group(1).strip().lower() == user.lower(): extra = {} h = re.search(r'\s*([^<]+?)\s*', response.text) From 9f1baa3ac4effe0fec6c8f1d79e16c839c0d51f8 Mon Sep 17 00:00:00 2001 From: alexispradol Date: Thu, 29 Oct 2026 14:39:30 -0300 Subject: [PATCH 08/11] Update regex for title extraction in listal.py --- user_scanner/user_scan/entertainment/listal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_scanner/user_scan/entertainment/listal.py b/user_scanner/user_scan/entertainment/listal.py index ea8d9498..690c61dc 100644 --- a/user_scanner/user_scan/entertainment/listal.py +++ b/user_scanner/user_scan/entertainment/listal.py @@ -10,7 +10,7 @@ def validate_listal(user: str) -> Result: def process(response): if response.status_code != 200: return Result.error(f"Unexpected response status: {response.status_code}") - m = re.search(r"([^<]+) on Listal", response.text) + m = re.search(r"([^<]+?)(?:'s stuff)? on Listal", response.text) if m and m.group(1).strip().lower() == user.lower(): extra, media = {}, {} h = re.search(r"]*>([^<]+)", response.text) From 5f95ec8055fc8f7cd76815d612b6117c95d05023 Mon Sep 17 00:00:00 2001 From: alexispradol Date: Thu, 29 Oct 2026 14:40:25 -0300 Subject: [PATCH 09/11] Update bad values and avatar image regex --- user_scanner/user_scan/dev/hackthissite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/user_scanner/user_scan/dev/hackthissite.py b/user_scanner/user_scan/dev/hackthissite.py index f2b8d38f..c1ecee01 100644 --- a/user_scanner/user_scan/dev/hackthissite.py +++ b/user_scanner/user_scan/dev/hackthissite.py @@ -17,7 +17,7 @@ def process(response): extra["points"] = r.group(2) s = re.search(r"Status:\s*([^<\n]+)Hack This Site" in response.text: From d261fb7f9440cbb416bafa65eed3363dd9272097 Mon Sep 17 00:00:00 2001 From: alexispradol Date: Thu, 29 Oct 2026 14:41:33 -0300 Subject: [PATCH 10/11] Improve 410 response handling in ultimate_guitar.py Handle specific 410 response case for user not found. --- user_scanner/user_scan/music/ultimate_guitar.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/user_scanner/user_scan/music/ultimate_guitar.py b/user_scanner/user_scan/music/ultimate_guitar.py index 6bcc75e5..470d257f 100644 --- a/user_scanner/user_scan/music/ultimate_guitar.py +++ b/user_scanner/user_scan/music/ultimate_guitar.py @@ -10,7 +10,9 @@ def validate_ultimate_guitar(user: str) -> Result: def process(response): if response.status_code == 410: - return Result.available() + if "Oops! We couldn't find that page." in response.text: + return Result.available() + return Result.error("410 response without not-found marker, report it via GitHub issues") if response.status_code == 200: m = re.search(r"([^<]+) \| Ultimate-Guitar\.Com", response.text) if m and m.group(1).strip().lower() == user.lower(): From 8a5db32f28acf32cb4be3c4c430a18f52e51b8b6 Mon Sep 17 00:00:00 2001 From: Kaif <98528577+kaifcodec@users.noreply.github.com> Date: Sat, 7 Nov 2026 13:22:09 +0530 Subject: [PATCH 11/11] fix(user_scan): harden ccm, hackthissite, listal, mixlr, ultimate_guitar and remove blocked modules --- user_scanner/user_scan/community/ccm.py | 42 +++++++---- user_scanner/user_scan/dev/hackthissite.py | 42 ++++++++--- .../user_scan/entertainment/listal.py | 39 +++++++--- .../user_scan/gaming/planetminecraft.py | 35 --------- user_scanner/user_scan/gaming/typeracer.py | 52 ------------- user_scanner/user_scan/music/mixlr.py | 73 +++++++++++++------ .../user_scan/music/ultimate_guitar.py | 48 ++++++++---- 7 files changed, 173 insertions(+), 158 deletions(-) delete mode 100644 user_scanner/user_scan/gaming/planetminecraft.py delete mode 100644 user_scanner/user_scan/gaming/typeracer.py diff --git a/user_scanner/user_scan/community/ccm.py b/user_scanner/user_scan/community/ccm.py index a2cdaedc..48508aac 100644 --- a/user_scanner/user_scan/community/ccm.py +++ b/user_scanner/user_scan/community/ccm.py @@ -1,39 +1,55 @@ import re +import urllib.parse -from user_scanner.core.orchestrator import generic_validate, Result +from user_scanner.core.orchestrator import Result, generic_validate def validate_ccm(user: str) -> Result: - url = f"https://ccm.net/profile/user/{user}" - show_url = f"https://ccm.net/profile/user/{user}" + encoded_user = urllib.parse.quote(user) + url = f"https://ccm.net/profile/user/{encoded_user}" + show_url = f"https://ccm.net/profile/user/{encoded_user}" - def process(response): + def process(response) -> Result: if response.status_code == 404: - return Result.available() + if ( + re.search(r".*?(?:404|erreur|not found).*?", response.text, re.I) + or "404" in response.text + ): + return Result.available() + return Result.error("404 response received without expected error marker") + if response.status_code == 200: - m = re.search(r"(?:Profil de\s+)?([^<']+?)(?:'s profile)?\s*-\s*CCM", response.text) + m = re.search( + r"(?:Profil de\s+)?([^<']+?)(?:'s profile)?\s*-\s*CCM", + response.text, + ) if m and m.group(1).strip().lower() == user.lower(): - extra = {} - h = re.search(r'\s*([^<]+?)\s*', response.text) - if h: extra["username"] = h.group(1).strip() + extra: dict[str, str] = {} s = re.search(r'pro_header__status">\s*([^<]+)', response.text) - if s: extra["status"] = s.group(1).strip() - for label, key in [ + if s: + extra["status"] = s.group(1).strip() + + labels = [ ("Registration date", "joined"), ("Gender", "gender"), ("Location", "location"), ("Birthday", "birthday"), ("Website", "website"), ("Contributions", "contributions"), - ]: + ] + for label, key in labels: dd = re.search( - rf"
{label}
\s*]*>(.*?)", response.text, re.S + rf"
{label}
\s*]*>(.*?)", + response.text, + re.S, ) if dd: val = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", dd.group(1))).strip() if val and val.lower() not in ("not entered", "n/a"): extra[key] = val + return Result.taken(extra=extra) + return Result.error(f"Unexpected response status: {response.status_code}") return generic_validate(url, process, show_url=show_url, follow_redirects=True) diff --git a/user_scanner/user_scan/dev/hackthissite.py b/user_scanner/user_scan/dev/hackthissite.py index c1ecee01..b4858892 100644 --- a/user_scanner/user_scan/dev/hackthissite.py +++ b/user_scanner/user_scan/dev/hackthissite.py @@ -1,45 +1,63 @@ import re +import urllib.parse -from user_scanner.core.orchestrator import generic_validate, Result +from user_scanner.core.impersonate import impersonate_validate +from user_scanner.core.orchestrator import Result def validate_hackthissite(user: str) -> Result: - url = f"https://www.hackthissite.org/user/view/{user}" + encoded_user = urllib.parse.quote(user) + url = f"https://www.hackthissite.org/user/view/{encoded_user}" + show_url = f"https://www.hackthissite.org/user/view/{encoded_user}" - def process(response): + def process(response) -> Result: if response.status_code == 200: - m = re.search(r"Hack This Site :: View User: ([^<]+)", response.text) + m = re.search( + r"Hack This Site :: View User: ([^<]+)", + response.text, + ) if m and m.group(1).strip().lower() == user.lower(): - extra, media = {}, {} + extra: dict[str, str] = {} + media: dict[str, str] = {} + r = re.search(r"Rank:\s*([^<(\n]+?)\s*\((\d+)\s*Points\)", response.text) if r: extra["rank"] = r.group(1).strip() extra["points"] = r.group(2) + s = re.search(r"Status:\s*([^<\n]+){label}:\s*([^<]*)Hack This Site" in response.text: + + if "Hack This Site" in response.text and "View User:" not in response.text: return Result.available() + return Result.error(f"Unexpected response status: {response.status_code}") - return generic_validate(url, process, show_url=url) + return impersonate_validate(url, process, show_url=show_url, impersonate="chrome") diff --git a/user_scanner/user_scan/entertainment/listal.py b/user_scanner/user_scan/entertainment/listal.py index 690c61dc..ee8fab57 100644 --- a/user_scanner/user_scan/entertainment/listal.py +++ b/user_scanner/user_scan/entertainment/listal.py @@ -1,29 +1,48 @@ import re -from user_scanner.core.orchestrator import generic_validate, Result +from user_scanner.core.impersonate import impersonate_validate +from user_scanner.core.orchestrator import Result def validate_listal(user: str) -> Result: + if not re.match(r"^[A-Za-z0-9_-]+$", user): + return Result.error("Invalid username format for Listal") + url = f"https://{user}.listal.com/" show_url = f"https://{user}.listal.com/" - def process(response): + def process(response) -> Result: if response.status_code != 200: return Result.error(f"Unexpected response status: {response.status_code}") + m = re.search(r"([^<]+?)(?:'s stuff)? on Listal", response.text) if m and m.group(1).strip().lower() == user.lower(): - extra, media = {}, {} - h = re.search(r"]*>([^<]+)", response.text) - if h: extra["username"] = h.group(1).strip() + extra: dict[str, str] = {} + media: dict[str, str] = {} + pt = re.search(r"Points:\s*(?:]*>)?(\d+)", response.text) - if pt: extra["points"] = pt.group(1) + if pt: + extra["points"] = pt.group(1) + fl = re.search(r"(\d+) Followers", response.text) - if fl: extra["followers"] = fl.group(1) + if fl: + extra["followers"] = fl.group(1) + av = re.search(r']*src="(https://lthumb\.lisimg\.com/[^"]+)"', response.text) - if av: media["avatar"] = av.group(1) + if av: + media["avatar"] = av.group(1) + return Result.taken(extra=extra, media=media) + if "Listal - List the stuff you love!" in response.text: return Result.available() - return Result.error("Unexpected response body, report it via GitHub issues") - return generic_validate(url, process, show_url=show_url, follow_redirects=True) + return Result.error("Unexpected response body from Listal") + + return impersonate_validate( + url, + process, + show_url=show_url, + impersonate="chrome", + allow_redirects=True, + ) diff --git a/user_scanner/user_scan/gaming/planetminecraft.py b/user_scanner/user_scan/gaming/planetminecraft.py deleted file mode 100644 index b087967d..00000000 --- a/user_scanner/user_scan/gaming/planetminecraft.py +++ /dev/null @@ -1,35 +0,0 @@ -import re - -from user_scanner.core.impersonate import impersonate_validate -from user_scanner.core.result import Result - - -def validate_planetminecraft(user: str) -> Result: - url = f"https://www.planetminecraft.com/member/{user}" - - def process(response): - if response.status_code == 410: - return Result.available() - if response.status_code == 200: - m = re.search(r"([^<]+) on Planet Minecraft", response.text) - if m and m.group(1).strip().lower() == user.lower(): - extra, media = {}, {} - h = re.search(r"]*>([^<]+)", response.text) - if h: extra["username"] = h.group(1).strip() - lv = re.search(r"Level (\d+)", response.text) - if lv: extra["level"] = lv.group(1) - rk = re.search(r'member-rank-class">([^<]+)', response.text) - if rk: extra["rank"] = rk.group(1).strip() - mid = re.search(r"data-mid=['\"]?(\d+)", response.text) - if mid: extra["member_id"] = mid.group(1) - sub = re.search(r'num_subscribers">(\d+)', response.text) - if sub: extra["followers"] = sub.group(1) - av = re.search( - r"https://static\.planetminecraft\.com/files/avatar/\d+_\d+\.png", - response.text, - ) - if av: media["avatar"] = av.group(0) - return Result.taken(extra=extra, media=media) - return Result.error(f"Unexpected response status: {response.status_code}") - - return impersonate_validate(url, process, show_url=url) diff --git a/user_scanner/user_scan/gaming/typeracer.py b/user_scanner/user_scan/gaming/typeracer.py deleted file mode 100644 index b24fd74f..00000000 --- a/user_scanner/user_scan/gaming/typeracer.py +++ /dev/null @@ -1,52 +0,0 @@ -import re - -from user_scanner.core.orchestrator import generic_validate, Result - - -def validate_typeracer(user: str) -> Result: - url = "https://data.typeracer.com/pit/profile" - show_url = f"https://data.typeracer.com/pit/profile?user={user}" - - def process(response): - if response.status_code == 200: - m = re.search( - r"([^<(]+) \(([^)]+)\) \(TypeRacer Profile\)", - response.text, - ) - if m and m.group(2).strip().lower() == user.lower(): - extra, media = {}, {} - extra["fullname"] = m.group(1).strip() - stats = { - "Full Avg.": "avg_wpm", - "Best Race": "best_wpm", - "Races": "races", - "WPM %": "wpm_percentile", - "Skill Level": "skill_level", - "Exp Level": "exp_level", - } - for value, label in re.findall( - r'Stat__Top">\s*([^<]+?)\s*\s*' - r'([^<]+)', - response.text, - ): - key = stats.get(label.strip()) - if key: - extra[key] = value.strip() - s = re.search(r"Racing Since:\s*([^<]+)", response.text) - if s: extra["racing_since"] = s.group(1).strip() - s = re.search(r"Location:\s*\s*(\S[^<]*?)\s*", response.text) - if s: extra["location"] = s.group(1).strip() - p = re.search(r'class="Plan__(Basic|Premium)"', response.text) - if p: extra["plan"] = p.group(1).lower() - s = re.search( - r"profileCar[\s\S]*?url\([\"'](/public/images/avatars/[^)\"']+)", - response.text, - ) - if s: - media["avatar"] = f"https://data.typeracer.com{s.group(1)}" - return Result.taken(extra=extra, media=media) - if "Profile Not Found (TypeRacer Pit Stop)" in response.text: - return Result.available() - return Result.error(f"Unexpected response status: {response.status_code}") - - return generic_validate(url, process, show_url=show_url, params={"user": user}) diff --git a/user_scanner/user_scan/music/mixlr.py b/user_scanner/user_scan/music/mixlr.py index f129dfb3..be72234c 100644 --- a/user_scanner/user_scan/music/mixlr.py +++ b/user_scanner/user_scan/music/mixlr.py @@ -1,10 +1,12 @@ +import urllib.parse + from user_scanner.core.helpers import get_random_user_agent -from user_scanner.core.orchestrator import generic_validate -from user_scanner.core.result import Result +from user_scanner.core.orchestrator import Result, generic_validate def validate_mixlr(user: str) -> Result: - url = f"https://api.mixlr.com/users/{user.lower()}" + encoded_user = urllib.parse.quote(user.lower()) + url = f"https://api.mixlr.com/users/{encoded_user}" show_url = f"https://mixlr.com/{user}" headers = { @@ -12,36 +14,61 @@ def validate_mixlr(user: str) -> Result: "Accept": "application/json", } - def process(response): - if response.status_code == 404 and "Resource not found" in response.text: - return Result.available() + def process(response) -> Result: + if response.status_code == 404: + try: + data = response.json() + if data.get("error") == "Resource not found": + return Result.available() + except Exception: + pass + if "Resource not found" in response.text: + return Result.available() + return Result.error("404 response received without expected error payload") if response.status_code == 200: try: data = response.json() except Exception: - return Result.error("200 response status with no recognizable data, report it via GitHub issues") - if not isinstance(data, dict) or "id" not in data or "slug" not in data: - return Result.error("Unexpected API response body, report it via GitHub issues") - - extra = {} - media = {} - if username := data.get("username"): extra["username"] = username - if slug := data.get("slug"): extra["slug"] = slug - if role := data.get("role"): extra["role"] = role - if tz := data.get("time_zone"): extra["time_zone"] = tz + return Result.error("Invalid JSON in 200 response") + + if not isinstance(data, dict) or "id" not in data: + return Result.error("Unexpected API response structure") + + extra: dict[str, str] = {} + media: dict[str, str] = {} + + if "id" in data: + extra["id"] = str(data["id"]) + + if role := data.get("role"): + extra["role"] = str(role) + + if tz := data.get("time_zone"): + extra["time_zone"] = str(tz) + plan = data.get("plan") or {} - if plan.get("reference"): extra["plan"] = plan["reference"] - elif plan.get("name"): extra["plan"] = plan["name"] - if data.get("is_premium"): extra["premium"] = "true" - if data.get("is_live"): extra["live"] = "true" + if plan.get("reference"): + extra["plan"] = str(plan["reference"]) + elif plan.get("name"): + extra["plan"] = str(plan["name"]) + + if data.get("is_premium"): + extra["premium"] = "true" + + if data.get("is_live"): + extra["live"] = "true" + channel = data.get("channel") or {} - if channel.get("name"): extra["channel"] = channel["name"] + if channel.get("name"): + extra["channel"] = str(channel["name"]) + if avatar := data.get("profile_image_url"): - if "fallback-user" not in avatar: media["avatar"] = avatar + if "fallback-user" not in avatar: + media["avatar"] = str(avatar) return Result.taken(extra=extra, media=media) - return Result.error(f"Unexpected status code: {response.status_code}") + return Result.error(f"Unexpected response status: {response.status_code}") return generic_validate(url, process, show_url=show_url, headers=headers) diff --git a/user_scanner/user_scan/music/ultimate_guitar.py b/user_scanner/user_scan/music/ultimate_guitar.py index 470d257f..aab7c4e5 100644 --- a/user_scanner/user_scan/music/ultimate_guitar.py +++ b/user_scanner/user_scan/music/ultimate_guitar.py @@ -1,47 +1,69 @@ import re +import urllib.parse from datetime import datetime, timezone from user_scanner.core.impersonate import impersonate_validate -from user_scanner.core.result import Result +from user_scanner.core.orchestrator import Result def validate_ultimate_guitar(user: str) -> Result: - url = f"https://www.ultimate-guitar.com/u/{user}" + encoded_user = urllib.parse.quote(user) + url = f"https://www.ultimate-guitar.com/u/{encoded_user}" + show_url = f"https://www.ultimate-guitar.com/u/{encoded_user}" - def process(response): + def process(response) -> Result: if response.status_code == 410: if "Oops! We couldn't find that page." in response.text: return Result.available() - return Result.error("410 response without not-found marker, report it via GitHub issues") + return Result.error("410 response received without expected not-found marker") + if response.status_code == 200: m = re.search(r"([^<]+) \| Ultimate-Guitar\.Com", response.text) if m and m.group(1).strip().lower() == user.lower(): - extra, media = {}, {} + extra: dict[str, str] = {} + media: dict[str, str] = {} + t = re.search(r"usertitle":"([^&]+)"", response.text) - if t: extra["usertitle"] = t.group(1) + if t: + extra["usertitle"] = t.group(1) + fc = re.search(r"follower_count":(\d+)", response.text) - if fc: extra["followers"] = fc.group(1) + if fc: + extra["followers"] = fc.group(1) + fg = re.search(r"following_count":(\d+)", response.text) - if fg: extra["following"] = fg.group(1) + if fg: + extra["following"] = fg.group(1) + iq = re.search(r"iq":(\d+)", response.text) - if iq: extra["iq"] = iq.group(1) + if iq: + extra["iq"] = iq.group(1) + dr = re.search(r"date_registered":(\d+)", response.text) if dr: extra["joined"] = datetime.fromtimestamp( int(dr.group(1)), tz=timezone.utc ).strftime("%Y-%m-%d") + city = re.search(r"city":"([^&]*)"", response.text) - if city and city.group(1): extra["city"] = city.group(1) + if city and city.group(1): + extra["city"] = city.group(1) + country = re.search(r"country_name":"([^&]*)"", response.text) - if country and country.group(1): extra["location"] = country.group(1) + if country and country.group(1): + extra["location"] = country.group(1) + va = re.search(r"is_avatar_exists":(true|false)", response.text) if va and va.group(1) == "true": av = re.search( r"avatar":"(https://www\.ultimate-guitar\.com/static[^\s"]+)"", response.text, ) - if av: media["avatar"] = av.group(1).replace("&", "&") + if av: + media["avatar"] = av.group(1).replace("&", "&") + return Result.taken(extra=extra, media=media) + return Result.error(f"Unexpected response status: {response.status_code}") - return impersonate_validate(url, process, show_url=url, allow_redirects=True) + return impersonate_validate(url, process, show_url=show_url, allow_redirects=True)