From d0393d3c6d0075b60f9a9ff426b423772af2ec11 Mon Sep 17 00:00:00 2001 From: Nick Andrade Date: Tue, 10 Nov 2026 14:08:12 -0800 Subject: [PATCH] Sync shared to 9449e9f shared.lock: ea07617 -> 9449e9f (scripts/sync_shared.py --update) - scripts/sync_shared.py - shared.lock - src/harmony_mcp/logsafe.py - src/harmony_mcp/remote.py - tests/test_logsafe.py - tests/test_remote.py --- scripts/sync_shared.py | 77 ++++++++++++++++++++++++++++++++----- shared.lock | 2 +- src/harmony_mcp/logsafe.py | 24 +++++++++--- src/harmony_mcp/remote.py | 29 ++++++++++---- tests/test_logsafe.py | 79 +++++++++++++++++++++++++++++++++++++- tests/test_remote.py | 45 +++++++++++++++++++++- 6 files changed, 232 insertions(+), 24 deletions(-) diff --git a/scripts/sync_shared.py b/scripts/sync_shared.py index d4e78c5..05786cd 100644 --- a/scripts/sync_shared.py +++ b/scripts/sync_shared.py @@ -96,12 +96,17 @@ _SHA = re.compile(r"[0-9a-f]{40}") _REPO = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+") _PACKAGE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +# A release tag of mcp-server-shared, as the lock's `version` carries it: v0.1.0, v1.2.0-rc1. Strict +# on purpose: it is written into the lock's TOML as is, and anything else on a commit (a +# moving tag, a typo) isn't a version. +_VERSION = re.compile(r"v(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.]+))?") LOCK_HEADER = """\ # Which mcp-server-shared commit this repo's shared files come from, and this # repo's package name (it replaces __PKG__). Written by -# `python scripts/sync_shared.py --update`; see that script. An optional -# `exclude = ["remote"]` names shared modules this repo doesn't take. +# `python scripts/sync_shared.py --update`; see that script. `version`, when +# present, is the tag on that commit: a label only, the commit is the pin. An +# optional `exclude = ["remote"]` names shared modules this repo doesn't take. """ @@ -116,8 +121,11 @@ class Lock: commit: str package: str exclude: tuple[str, ...] = () # shared modules this repo doesn't take + version: str | None = None # the release tag on `commit`, if it has one (written by --update) - KEYS = ("repo", "commit", "package", "exclude") # what read_lock understands; anything else is a typo + # What read_lock understands; anything else is a typo. "version" was added after "exclude": a + # script from before it warns about the key and otherwise ignores it. + KEYS = ("repo", "commit", "version", "package", "exclude") def validate(self) -> Lock: # The regex allows "." and "-" (real repo names have them), but a segment that *is* "." or ".." @@ -131,10 +139,20 @@ def validate(self) -> Lock: for name in self.exclude: if not _PACKAGE.fullmatch(name): raise SyncError(f'{LOCK}: exclude must list module names like "remote" (got {name!r})') + if self.version is not None and not _VERSION.fullmatch(self.version): + raise SyncError(f'{LOCK}: version must be a tag like "v0.1.0" (got {self.version!r})') return self + @property + def at(self) -> str: + """For messages: "1a2b3c4" or "1a2b3c4 (v0.1.0)".""" + return _at(self.commit, self.version) + def dump(self) -> str: - text = f'{LOCK_HEADER}repo = "{self.repo}"\ncommit = "{self.commit}"\npackage = "{self.package}"\n' + text = f'{LOCK_HEADER}repo = "{self.repo}"\ncommit = "{self.commit}"\n' + if self.version: + text += f'version = "{self.version}"\n' + text += f'package = "{self.package}"\n' if self.exclude: text += "exclude = [" + ", ".join(f'"{name}"' for name in self.exclude) + "]\n" return text @@ -160,12 +178,16 @@ def read_lock(root: Path) -> Lock: exclude = data.get("exclude", []) if not isinstance(exclude, list) or not all(isinstance(name, str) for name in exclude): raise SyncError(f'{LOCK}: exclude must be a list of module names, like ["remote"] (got {exclude!r})') + version = data.get("version") + if version is not None and not isinstance(version, str): + raise SyncError(f'{LOCK}: version must be a tag like "v0.1.0" (got {version!r})') try: lock = Lock( str(data.get("repo", DEFAULT_REPO)), str(data["commit"]), str(data["package"]), tuple(dict.fromkeys(exclude)), # drop repeats, keep the order written + version, ) except KeyError as exc: raise SyncError(f"{LOCK} is missing {exc.args[0]!r}") from None @@ -263,6 +285,33 @@ def lock_is_on_main(repo: str, commit: str) -> str | None: return f"couldn't check whether the locked commit is on main (unexpected compare status {status!r})" +def _at(commit: str, version: str | None) -> str: + return f"{commit[:7]} ({version})" if version else commit[:7] + + +def version_tag(repo: str, commit: str) -> str | None: + """The release tag on `commit` (the highest, if several), or None when it has none. + + One call to GitHub's tags endpoint, first page of 100: plenty for this repo's release cadence. + Raises SyncError when GitHub can't be asked; the caller decides what that means. + """ + url = f"https://api.github.com/repos/{repo}/tags?per_page=100" + try: + data = json.loads(_get(url, "application/vnd.github+json")) + except ValueError: + raise SyncError("GitHub's tag list isn't JSON") from None + tags = data if isinstance(data, list) else [] + names = [ + tag.get("name") + for tag in tags + if isinstance(tag, dict) and isinstance(tag.get("commit"), dict) and tag["commit"].get("sha") == commit + ] + versions = [m for name in names if isinstance(name, str) and (m := _VERSION.fullmatch(name))] + # Highest first by number; a release sorts after its own pre-releases (v1.0.0 > v1.0.0-rc1). + best = max(versions, key=lambda m: (int(m[1]), int(m[2]), int(m[3]), m[4] is None, m[4] or ""), default=None) + return best[0] if best else None + + def _warn_if_lock_is_off_main(lock: Lock) -> None: reason = lock_is_on_main(lock.repo, lock.commit) if reason: @@ -274,10 +323,12 @@ class GitHub: repo: str commit: str why: str # "locked" or "main" + version: str | None = None # the lock's version, for the label @property def label(self) -> str: - return f"{self.repo.split('/')[-1]} at {self.commit[:7]} ({self.why})" + version = f" {self.version}" if self.version else "" + return f"{self.repo.split('/')[-1]} at {self.commit[:7]}{version} ({self.why})" def read(self, path: str) -> bytes: return _get(f"https://raw.githubusercontent.com/{self.repo}/{self.commit}/{path}") @@ -648,7 +699,7 @@ def run(args: argparse.Namespace) -> int: print(f"(shared.lock is at {lock.commit[:7]}, main at {main[:7]}, with no shared file changed)") _report_orphans(root, package, files, exclude) return status - source = GitHub(lock.repo, lock.commit, "locked") + source = GitHub(lock.repo, lock.commit, "locked", lock.version) files = wanted_files(source, package, writing=False, root=root, exclude=exclude) status = report(root, files, source.label + _excluding(exclude), "python scripts/sync_shared.py") _report_orphans(root, package, files, exclude) @@ -660,16 +711,24 @@ def run(args: argparse.Namespace) -> int: source = GitHub(lock.repo, main, "main") files = wanted_files(source, package, writing=not dry_run, root=root, exclude=exclude) changed = write_files(root, files, dry_run=dry_run, force=force) - new_lock = Lock(lock.repo, main, lock.package, lock.exclude) # exclude carries over unchanged + try: + version = version_tag(lock.repo, main) + except SyncError as exc: + # The version is a label; the SHA is the pin. Keep the label it had if the commit didn't + # move, else write none rather than guess. + version = lock.version if main == lock.commit else None + kept = f"keeping {version}" if version else "writing none" + print(f"sync_shared.py: warning: couldn't look up a version tag ({exc}); {kept}", file=sys.stderr) + new_lock = Lock(lock.repo, main, lock.package, lock.exclude, version) # exclude carries over unchanged if new_lock != lock and not dry_run: write_atomic(root / LOCK, new_lock.dump().encode("utf-8")) # last: only once every file is in - moved = f"{lock.commit[:7]} -> {main[:7]}" if main != lock.commit else f"already at {main[:7]}" + moved = f"{lock.at} -> {new_lock.at}" if main != lock.commit else f"already at {new_lock.at}" prefix = "dry run, shared.lock" if dry_run else "shared.lock" print(f"{prefix}: {moved}{_excluding(exclude)}; " + (", ".join(changed) if changed else "no files changed")) _report_orphans(root, package, files, exclude) return 1 if (dry_run and (changed or new_lock != lock)) else 0 - source = GitHub(lock.repo, lock.commit, "locked") + source = GitHub(lock.repo, lock.commit, "locked", lock.version) files = wanted_files(source, package, writing=not dry_run, root=root, exclude=exclude) changed = write_files(root, files, dry_run=dry_run, force=force) label = source.label + _excluding(exclude) diff --git a/shared.lock b/shared.lock index ea50775..e1ea9b6 100644 --- a/shared.lock +++ b/shared.lock @@ -3,5 +3,5 @@ # `python scripts/sync_shared.py --update`; see that script. An optional # `exclude = ["remote"]` names shared modules this repo doesn't take. repo = "SDNick484/mcp-server-shared" -commit = "ea0761783b863e61b3aaabe917acbb9af415e37b" +commit = "9449e9fc66db4c1594e018cdc92928002347cac9" package = "harmony_mcp" diff --git a/src/harmony_mcp/logsafe.py b/src/harmony_mcp/logsafe.py index e4ba204..4ee33f0 100644 --- a/src/harmony_mcp/logsafe.py +++ b/src/harmony_mcp/logsafe.py @@ -12,6 +12,7 @@ 2001:db8::1, fe80::1ff:fe00:1 -> x:x::1 (IPv6: last group kept, likewise) AA:BB:CC:DD:EE:FF -> xx:xx:xx:xx:EE:FF AABBCCDDEEFF, 0009b0f76cfd -> xxxxxxxxEEFF, xxxxxxxx6cfd (bare hex, as in MQTT topics) + user@example.com -> u***@example.com (domain kept: it says which IdP) mqtt://user:secret@host -> mqtt://user:***@host http://host/x?token=abc&key=def -> http://host/x?token=***&key=*** /cb?access_token=abc&code=def -> /cb?access_token=***&code=*** (OAuth callbacks) @@ -26,6 +27,9 @@ Loopback and unspecified addresses (127.x, 0.0.0.0, ::1, ::) are left alone: they say nothing about your network and matter when debugging the simulator. +So are things shaped like an e-mail address that aren't one: URL userinfo +(ssh://git@host, mqtt://user:***@host), scp-style remotes (git@github.com:x), +decorators, and hosts without a letters-only TLD (user@localhost, admin@10.0.0.1). Each server's CLI turns redaction off with --no-redact or its _LOG_UNREDACTED=1, e.g. when you are the only reader. Secrets (JWTs, @@ -61,7 +65,17 @@ # groups out of a longer colon run such as an eight-group IPv6 address. _MAC = re.compile(r"(? None: def mask_email(address: str) -> str: """``user@example.com`` -> ``u***@example.com``: enough to tell identities apart, not enough to paste. - For messages that name an identity on purpose (an Access login that isn't on the - allow-list) rather than for ``redact()``, which leaves e-mail addresses alone: they - are rare in library output, and a server's own messages decide what they say. - Anything without an ``@`` comes back unchanged. + ``redact()`` applies this to every address it finds in a log line; call it directly + for text that isn't logged (an error returned to a client) or that is logged + with address redaction off. Anything without an ``@`` comes back unchanged. """ if not address or not isinstance(address, str) or "@" not in address: return address @@ -266,6 +279,7 @@ def redact(text: str) -> str: text = redact_secrets(text) for pattern, replacement in tuple(_IDENTIFIER_PATTERNS): text = pattern.sub(replacement, text) + text = _EMAIL.sub(lambda m: mask_email(m.group(0)), text) # IPv6 before MACs: an eight-group address of two-digit groups is an address, not a MAC. text = _IPV6.sub(_ipv6, text) text = _IPV4.sub(_ip, text) diff --git a/src/harmony_mcp/remote.py b/src/harmony_mcp/remote.py index 1976b53..41188dd 100644 --- a/src/harmony_mcp/remote.py +++ b/src/harmony_mcp/remote.py @@ -250,11 +250,28 @@ def __init__(self, policy: AccessPolicy, key_resolver: KeyResolver | None = None self.policy = policy if key_resolver is None: client = jwt.PyJWKClient(policy.certs_url, cache_jwk_set=True, lifespan=3600, timeout=10) + # One lock around the whole lookup, network fetch included, so every verification waits + # while a refetch is in flight (up to the 10 s timeout). That is the point: a burst of + # tokens naming a key we don't have yet makes one fetch, not one each. Fetches are rare + # (cache expiry, key rotation), and a hit in the cache holds the lock for microseconds. lock = anyio.Lock() + # When PyJWKClient last went to the network, successful or not: the throttle below + # counts fetches, so a steady stream of tokens served from the cache must not move it, + # or a rotated key would be refused until the cache expired. fetch_data is the one + # method that does network I/O, so wrapping it on this instance catches every fetch. last_fetch: float | None = None + fetch_data = client.fetch_data - async def resolve(token: str) -> Any: # pragma: no cover + def fetch_and_record() -> Any: nonlocal last_fetch + try: + return fetch_data() + finally: + last_fetch = time.monotonic() + + client.fetch_data = fetch_and_record # type: ignore[method-assign] + + async def resolve(token: str) -> Any: async with lock: header = jwt.get_unverified_header(token) kid = header.get("kid") @@ -269,10 +286,7 @@ async def resolve(token: str) -> Any: # pragma: no cover and (now - last_fetch) < self.min_refetch_interval ): raise jwt.PyJWKClientError(f"Unable to find a signing key that matches {kid!r}") - try: - key = await anyio.to_thread.run_sync(client.get_signing_key_from_jwt, token) - finally: - last_fetch = time.monotonic() + key = await anyio.to_thread.run_sync(client.get_signing_key_from_jwt, token) return key.key key_resolver = resolve @@ -296,8 +310,9 @@ async def verify(self, token: str) -> dict[str, Any]: email = str(claims.get("email", "")).lower() if email not in self.policy.emails: # The identity is somebody Access let through but this server doesn't want; name it - # only enough to tell logins apart. logsafe doesn't redact e-mail addresses (and this - # module can't import it: shared modules stay self-contained), so mask it here. + # only enough to tell logins apart. logsafe's redact() would mask it in a log line, but + # not with redaction off, and this module can't import logsafe's mask_email (shared + # modules stay self-contained), so mask it here the same way. local, _, domain = email.partition("@") shown = f"{local[:1]}***@{domain}" if domain else "this identity" raise AccessDenied(f"{shown} is not in MCP_ALLOWED_EMAILS") diff --git a/tests/test_logsafe.py b/tests/test_logsafe.py index 172a0ef..e29ae4a 100644 --- a/tests/test_logsafe.py +++ b/tests/test_logsafe.py @@ -9,6 +9,7 @@ import logging import re +import time import pytest @@ -199,6 +200,61 @@ def test_mask_email(): assert mask_email("") == "" +@pytest.mark.parametrize( + ("line", "expected"), + [ + ("denied: user@example.com", "denied: u***@example.com"), + ("from=first.last+tag@mail.example.co.uk,", "from=f***@mail.example.co.uk,"), + ("", ""), + ("send to user@example.com.", "send to u***@example.com."), + ("user@example.com: not allowed", "u***@example.com: not allowed"), + ("mailto:user@example.com", "mailto:u***@example.com"), + ('{"email": "user@example.com"}', '{"email": "u***@example.com"}'), + ("MCP_ALLOWED_EMAILS=a@example.com,bob@example.com", "MCP_ALLOWED_EMAILS=a***@example.com,b***@example.com"), + ("https://example.com/users/user@example.com/", "https://example.com/users/u***@example.com/"), + ], +) +def test_email_addresses_are_masked(line, expected): + assert redact(line) == expected + + +@pytest.mark.parametrize( + ("line", "expected"), + [ + # URL userinfo: a password is a secret (_URL_PASSWORD); a bare user name isn't an address. + ("mqtt://user:hunter2@broker.example.com:1883", "mqtt://user:***@broker.example.com:1883"), + ("ssh://git@git.example.com/repo.git", "ssh://git@git.example.com/repo.git"), + ("https://user@example.com/path", "https://user@example.com/path"), + # scp-style git remotes. + ("git@github.com:owner/repo.git", "git@github.com:owner/repo.git"), + # Decorators in a traceback, and a JWT that happens to sit next to an @. + (' @app.route("/x")', ' @app.route("/x")'), + ("@functools.wraps(fn)", "@functools.wraps(fn)"), + ("eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIxIn0.c2lnbmF0dXJl@example.com", "[jwt redacted]@example.com"), + ("dGVzdA==@example.com", "dGVzdA==@example.com"), + # Not a domain: no letters-only TLD. + ("admin@192.0.2.5:22", "admin@x.x.x.5:22"), + ("user@localhost", "user@localhost"), + ], +) +def test_email_rule_leaves_look_alikes_alone(line, expected): + assert redact(line) == expected + + +@pytest.mark.parametrize("blob", ["a" * 100_000, "a" * 100_000 + "@", "0" * 100_000 + "://", "a." * 50_000 + "@"]) +def test_a_long_token_is_redacted_in_linear_time(blob): + # Logging runs on the event loop: a rule that backtracks over a long hex/base64 blob stalls the + # server. 100 KB took 20 s with the old unanchored URL-password rule; linear rules take ms. + start = time.perf_counter() + redact(f"payload {blob} end") + assert time.perf_counter() - start < 2.0 + + +def test_email_addresses_show_when_address_redaction_is_off(): + # An address identifies a person; it isn't a credential, so --no-redact shows it like an IP. + assert redact_secrets("denied: user@example.com") == "denied: user@example.com" + + # --- property-based: the rules must be idempotent and leave no address behind ------------------ # hypothesis is optional for the siblings (it isn't in their dev dependencies): without it only # these tests are skipped, never the module (a module-level importorskip would skip everything). @@ -214,7 +270,7 @@ def test_mask_email(): @needs_hypothesis @_settings - @hypothesis.given(st.text(alphabet=st.sampled_from("0123456789abcdefABCDEF:.-x /?&=@eyJ,;"), max_size=40)) + @hypothesis.given(st.text(alphabet=st.sampled_from("0123456789abcdefABCDEF:.-x /?&=@eyJ,;+_*"), max_size=40)) def test_redact_is_idempotent(text): """Redacting a redacted line changes nothing: a line can safely pass through two handlers.""" once = redact(text) @@ -233,8 +289,29 @@ def test_no_ipv4_survives_redact(pre, ip, post): @hypothesis.given(st.ip_addresses(v=6), st.text(alphabet="abcxyz ,.;", max_size=8)) def test_no_ipv6_survives_redact(ip, around): hypothesis.assume(not (ip.is_loopback or ip.is_unspecified)) + # A dotted tail of 0.x or 127.x ("::ffff:0.0.0.1") is kept on purpose, as the bare 0.x and + # 127.x addresses are (test_no_ipv4_survives_redact makes the same assumption). + hypothesis.assume(not str(ip).rsplit(":", 1)[1].startswith(("127.", "0."))) assert str(ip) not in redact(f"{around} {ip} {around}") + @needs_hypothesis + @_settings + @hypothesis.given(st.emails(), st.sampled_from([" ", ", ", "<", "(", "=", ": ", '"'])) + def test_no_email_local_part_survives_redact(email, sep): + local, _, domain = email.rpartition("@") + hypothesis.assume(len(local) > 1 and "." in domain and domain.rsplit(".", 1)[1].isalpha()) + hypothesis.assume(all(c.isalnum() or c in "._%+-" for c in local)) + once = redact(f"to{sep}{email} ok") + assert f"{local}@" not in once + assert redact(once) == once + + @needs_hypothesis + @_settings + @hypothesis.given(st.text(alphabet=st.sampled_from("ab.@:/+-_=*xyz.com"), max_size=40)) + def test_email_rule_is_idempotent(text): + once = redact(text) + assert redact(once) == once + def test_identifier_patterns(custom_patterns): register_identifier_pattern(r"device-[0-9a-f]{4}") diff --git a/tests/test_remote.py b/tests/test_remote.py index a201c97..842ed8b 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -490,9 +490,13 @@ class FakePyJWKClient: def __init__(self, *args: object, **kwargs: object) -> None: self.jwk_set_cache = FakeCache(FakeJWKSet([FakeKey("known-kid")])) - def get_signing_key_from_jwt(self, _token: str) -> object: + def fetch_data(self) -> object: nonlocal calls calls += 1 + return {"keys": []} + + def get_signing_key_from_jwt(self, _token: str) -> object: + self.fetch_data() # as PyJWKClient does on an unknown kid raise jwt.PyJWKClientError("unknown kid") monkeypatch.setattr(jwt, "PyJWKClient", FakePyJWKClient) @@ -526,3 +530,42 @@ def get_signing_key_from_jwt(self, _token: str) -> object: with pytest.raises(remote.AccessDenied, match="unknown kid"): await verifier.verify(tok3) assert calls == 2 + + +@pytest.mark.anyio +async def test_jwks_throttle_counts_network_fetches_not_cache_hits(monkeypatch): + # The real PyJWKClient, with only its HTTP request faked. Tokens served from the cache must not + # count as fetches: if they did, steady traffic on the old key would keep a rotated key refused + # until the cache expired. + jwk = jwt.algorithms.RSAAlgorithm.to_jwk(KEY.public_key(), as_dict=True) + published = [{**jwk, "kid": "old-kid", "use": "sig"}] + fetches = 0 + + def fetch_data(self: jwt.PyJWKClient) -> object: + nonlocal fetches + fetches += 1 + return {"keys": list(published)} + + monkeypatch.setattr(jwt.PyJWKClient, "fetch_data", fetch_data) + fake_now = [1000.0] + monkeypatch.setattr(time, "monotonic", lambda: fake_now[0]) + verifier = remote.AccessVerifier(POLICY) + + def signed(kid: str) -> str: + now = int(time.time()) + claims = {"iss": f"https://{TEAM}", "aud": [AUD], "iat": now, "exp": now + 300, "email": "user@example.com"} + return jwt.encode(claims, KEY, algorithm="RS256", headers={"kid": kid}) + + await verifier.verify(signed("old-kid")) # empty cache: one fetch + assert fetches == 1 + + fake_now[0] = 1100.0 + await verifier.verify(signed("old-kid")) # a cache hit: no fetch + assert fetches == 1 + + # Cloudflare rotates: the new kid is 10 s after the last cache hit but 110 s after the last + # fetch, so it is fetched, not throttled. + published.append({**jwk, "kid": "new-kid", "use": "sig"}) + fake_now[0] = 1110.0 + await verifier.verify(signed("new-kid")) + assert fetches == 2