Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 68 additions & 9 deletions scripts/sync_shared.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,12 +96,17 @@
_SHA = re.compile(r"[0-9a-f]{40}")
_REPO = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+")
_PACKAGE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
# A release tag of mcp-server-shared, as the lock's `version` carries it: v0.1.0, v1.2.0-rc1. Strict
# on purpose: it is written into the lock's TOML as is, and anything else on a commit (a
# moving tag, a typo) isn't a version.
_VERSION = re.compile(r"v(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.]+))?")

LOCK_HEADER = """\
# Which mcp-server-shared commit this repo's shared files come from, and this
# repo's package name (it replaces __PKG__). Written by
# `python scripts/sync_shared.py --update`; see that script. An optional
# `exclude = ["remote"]` names shared modules this repo doesn't take.
# `python scripts/sync_shared.py --update`; see that script. `version`, when
# present, is the tag on that commit: a label only, the commit is the pin. An
# optional `exclude = ["remote"]` names shared modules this repo doesn't take.
"""


Expand All @@ -116,8 +121,11 @@ class Lock:
commit: str
package: str
exclude: tuple[str, ...] = () # shared modules this repo doesn't take
version: str | None = None # the release tag on `commit`, if it has one (written by --update)

KEYS = ("repo", "commit", "package", "exclude") # what read_lock understands; anything else is a typo
# What read_lock understands; anything else is a typo. "version" was added after "exclude": a
# script from before it warns about the key and otherwise ignores it.
KEYS = ("repo", "commit", "version", "package", "exclude")

def validate(self) -> Lock:
# The regex allows "." and "-" (real repo names have them), but a segment that *is* "." or ".."
Expand All @@ -131,10 +139,20 @@ def validate(self) -> Lock:
for name in self.exclude:
if not _PACKAGE.fullmatch(name):
raise SyncError(f'{LOCK}: exclude must list module names like "remote" (got {name!r})')
if self.version is not None and not _VERSION.fullmatch(self.version):
raise SyncError(f'{LOCK}: version must be a tag like "v0.1.0" (got {self.version!r})')
return self

@property
def at(self) -> str:
"""For messages: "1a2b3c4" or "1a2b3c4 (v0.1.0)"."""
return _at(self.commit, self.version)

def dump(self) -> str:
text = f'{LOCK_HEADER}repo = "{self.repo}"\ncommit = "{self.commit}"\npackage = "{self.package}"\n'
text = f'{LOCK_HEADER}repo = "{self.repo}"\ncommit = "{self.commit}"\n'
if self.version:
text += f'version = "{self.version}"\n'
text += f'package = "{self.package}"\n'
if self.exclude:
text += "exclude = [" + ", ".join(f'"{name}"' for name in self.exclude) + "]\n"
return text
Expand All @@ -160,12 +178,16 @@ def read_lock(root: Path) -> Lock:
exclude = data.get("exclude", [])
if not isinstance(exclude, list) or not all(isinstance(name, str) for name in exclude):
raise SyncError(f'{LOCK}: exclude must be a list of module names, like ["remote"] (got {exclude!r})')
version = data.get("version")
if version is not None and not isinstance(version, str):
raise SyncError(f'{LOCK}: version must be a tag like "v0.1.0" (got {version!r})')
try:
lock = Lock(
str(data.get("repo", DEFAULT_REPO)),
str(data["commit"]),
str(data["package"]),
tuple(dict.fromkeys(exclude)), # drop repeats, keep the order written
version,
)
except KeyError as exc:
raise SyncError(f"{LOCK} is missing {exc.args[0]!r}") from None
Expand Down Expand Up @@ -263,6 +285,33 @@ def lock_is_on_main(repo: str, commit: str) -> str | None:
return f"couldn't check whether the locked commit is on main (unexpected compare status {status!r})"


def _at(commit: str, version: str | None) -> str:
return f"{commit[:7]} ({version})" if version else commit[:7]


def version_tag(repo: str, commit: str) -> str | None:
"""The release tag on `commit` (the highest, if several), or None when it has none.

One call to GitHub's tags endpoint, first page of 100: plenty for this repo's release cadence.
Raises SyncError when GitHub can't be asked; the caller decides what that means.
"""
url = f"https://api.github.com/repos/{repo}/tags?per_page=100"
try:
data = json.loads(_get(url, "application/vnd.github+json"))
except ValueError:
raise SyncError("GitHub's tag list isn't JSON") from None
tags = data if isinstance(data, list) else []
names = [
tag.get("name")
for tag in tags
if isinstance(tag, dict) and isinstance(tag.get("commit"), dict) and tag["commit"].get("sha") == commit
]
versions = [m for name in names if isinstance(name, str) and (m := _VERSION.fullmatch(name))]
# Highest first by number; a release sorts after its own pre-releases (v1.0.0 > v1.0.0-rc1).
best = max(versions, key=lambda m: (int(m[1]), int(m[2]), int(m[3]), m[4] is None, m[4] or ""), default=None)
return best[0] if best else None


def _warn_if_lock_is_off_main(lock: Lock) -> None:
reason = lock_is_on_main(lock.repo, lock.commit)
if reason:
Expand All @@ -274,10 +323,12 @@ class GitHub:
repo: str
commit: str
why: str # "locked" or "main"
version: str | None = None # the lock's version, for the label

@property
def label(self) -> str:
return f"{self.repo.split('/')[-1]} at {self.commit[:7]} ({self.why})"
version = f" {self.version}" if self.version else ""
return f"{self.repo.split('/')[-1]} at {self.commit[:7]}{version} ({self.why})"

def read(self, path: str) -> bytes:
return _get(f"https://raw.githubusercontent.com/{self.repo}/{self.commit}/{path}")
Expand Down Expand Up @@ -648,7 +699,7 @@ def run(args: argparse.Namespace) -> int:
print(f"(shared.lock is at {lock.commit[:7]}, main at {main[:7]}, with no shared file changed)")
_report_orphans(root, package, files, exclude)
return status
source = GitHub(lock.repo, lock.commit, "locked")
source = GitHub(lock.repo, lock.commit, "locked", lock.version)
files = wanted_files(source, package, writing=False, root=root, exclude=exclude)
status = report(root, files, source.label + _excluding(exclude), "python scripts/sync_shared.py")
_report_orphans(root, package, files, exclude)
Expand All @@ -660,16 +711,24 @@ def run(args: argparse.Namespace) -> int:
source = GitHub(lock.repo, main, "main")
files = wanted_files(source, package, writing=not dry_run, root=root, exclude=exclude)
changed = write_files(root, files, dry_run=dry_run, force=force)
new_lock = Lock(lock.repo, main, lock.package, lock.exclude) # exclude carries over unchanged
try:
version = version_tag(lock.repo, main)
except SyncError as exc:
# The version is a label; the SHA is the pin. Keep the label it had if the commit didn't
# move, else write none rather than guess.
version = lock.version if main == lock.commit else None
kept = f"keeping {version}" if version else "writing none"
print(f"sync_shared.py: warning: couldn't look up a version tag ({exc}); {kept}", file=sys.stderr)
new_lock = Lock(lock.repo, main, lock.package, lock.exclude, version) # exclude carries over unchanged
if new_lock != lock and not dry_run:
write_atomic(root / LOCK, new_lock.dump().encode("utf-8")) # last: only once every file is in
moved = f"{lock.commit[:7]} -> {main[:7]}" if main != lock.commit else f"already at {main[:7]}"
moved = f"{lock.at} -> {new_lock.at}" if main != lock.commit else f"already at {new_lock.at}"
prefix = "dry run, shared.lock" if dry_run else "shared.lock"
print(f"{prefix}: {moved}{_excluding(exclude)}; " + (", ".join(changed) if changed else "no files changed"))
_report_orphans(root, package, files, exclude)
return 1 if (dry_run and (changed or new_lock != lock)) else 0

source = GitHub(lock.repo, lock.commit, "locked")
source = GitHub(lock.repo, lock.commit, "locked", lock.version)
files = wanted_files(source, package, writing=not dry_run, root=root, exclude=exclude)
changed = write_files(root, files, dry_run=dry_run, force=force)
label = source.label + _excluding(exclude)
Expand Down
2 changes: 1 addition & 1 deletion shared.lock
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@
# `python scripts/sync_shared.py --update`; see that script. An optional
# `exclude = ["remote"]` names shared modules this repo doesn't take.
repo = "SDNick484/mcp-server-shared"
commit = "ea0761783b863e61b3aaabe917acbb9af415e37b"
commit = "9449e9fc66db4c1594e018cdc92928002347cac9"
package = "harmony_mcp"
24 changes: 19 additions & 5 deletions src/harmony_mcp/logsafe.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
2001:db8::1, fe80::1ff:fe00:1 -> x:x::1 (IPv6: last group kept, likewise)
AA:BB:CC:DD:EE:FF -> xx:xx:xx:xx:EE:FF
AABBCCDDEEFF, 0009b0f76cfd -> xxxxxxxxEEFF, xxxxxxxx6cfd (bare hex, as in MQTT topics)
user@example.com -> u***@example.com (domain kept: it says which IdP)
mqtt://user:secret@host -> mqtt://user:***@host
http://host/x?token=abc&key=def -> http://host/x?token=***&key=***
/cb?access_token=abc&code=def -> /cb?access_token=***&code=*** (OAuth callbacks)
Expand All @@ -26,6 +27,9 @@

Loopback and unspecified addresses (127.x, 0.0.0.0, ::1, ::) are left alone:
they say nothing about your network and matter when debugging the simulator.
So are things shaped like an e-mail address that aren't one: URL userinfo
(ssh://git@host, mqtt://user:***@host), scp-style remotes (git@github.com:x),
decorators, and hosts without a letters-only TLD (user@localhost, admin@10.0.0.1).

Each server's CLI turns redaction off with --no-redact or its
<NAME>_LOG_UNREDACTED=1, e.g. when you are the only reader. Secrets (JWTs,
Expand Down Expand Up @@ -61,7 +65,17 @@
# groups out of a longer colon run such as an eight-group IPv6 address.
_MAC = re.compile(r"(?<![0-9A-Fa-f:])((?:[0-9A-Fa-f]{2}[:-]){4})([0-9A-Fa-f]{2}[:-][0-9A-Fa-f]{2})(?![0-9A-Fa-f:])")
_BARE_MAC = re.compile(r"(?<![0-9A-Za-z-])([0-9A-Fa-f]{8})([0-9A-Fa-f]{4})(?![0-9A-Za-z-])")
_URL_PASSWORD = re.compile(r"(\w+://[^/\s:@?#]+):[^@/\s?#]+@")
# An e-mail address. The local part can't start inside a longer token (the first lookbehind) or
# right after "://", where "user@host" is URL userinfo, not an address. The domain needs a
# letters-only TLD, so "user@localhost" and "admin@10.0.0.1" don't count, and it can't be followed
# by ":" and a non-space, which is an scp-style git remote (git@github.com:owner/repo). Possessive
# quantifiers (++) because a log line can hold a long token: no backtracking into it.
_EMAIL = re.compile(
r"(?<![\w.%+-])(?<!://)([A-Za-z0-9._%+-]++)@((?:[A-Za-z0-9-]++\.)+[A-Za-z]{2,63})(?![A-Za-z0-9-]|\.[A-Za-z0-9]|:\S)"
)
# Anchored at the start of a word: unanchored, a long run of word characters (a hex or base64 blob)
# was rescanned from every offset, quadratic in the length of the line (20 s for 100 KB).
_URL_PASSWORD = re.compile(r"(?<!\w)(\w++://[^/\s:@?#]+):[^@/\s?#]+@")
# The names whose value is a credential wherever they appear: as a query parameter, as a key in a
# JSON object or a Python dict/headers repr, or as a header line. One vocabulary, so a name added
# for one shape (an OAuth callback's ?access_token=) is caught in the others too. `pin` and `psk`
Expand Down Expand Up @@ -147,10 +161,9 @@ def clear_identifier_patterns() -> None:
def mask_email(address: str) -> str:
"""``user@example.com`` -> ``u***@example.com``: enough to tell identities apart, not enough to paste.

For messages that name an identity on purpose (an Access login that isn't on the
allow-list) rather than for ``redact()``, which leaves e-mail addresses alone: they
are rare in library output, and a server's own messages decide what they say.
Anything without an ``@`` comes back unchanged.
``redact()`` applies this to every address it finds in a log line; call it directly
for text that isn't logged (an error returned to a client) or that is logged
with address redaction off. Anything without an ``@`` comes back unchanged.
"""
if not address or not isinstance(address, str) or "@" not in address:
return address
Expand Down Expand Up @@ -266,6 +279,7 @@ def redact(text: str) -> str:
text = redact_secrets(text)
for pattern, replacement in tuple(_IDENTIFIER_PATTERNS):
text = pattern.sub(replacement, text)
text = _EMAIL.sub(lambda m: mask_email(m.group(0)), text)
# IPv6 before MACs: an eight-group address of two-digit groups is an address, not a MAC.
text = _IPV6.sub(_ipv6, text)
text = _IPV4.sub(_ip, text)
Expand Down
29 changes: 22 additions & 7 deletions src/harmony_mcp/remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,11 +250,28 @@ def __init__(self, policy: AccessPolicy, key_resolver: KeyResolver | None = None
self.policy = policy
if key_resolver is None:
client = jwt.PyJWKClient(policy.certs_url, cache_jwk_set=True, lifespan=3600, timeout=10)
# One lock around the whole lookup, network fetch included, so every verification waits
# while a refetch is in flight (up to the 10 s timeout). That is the point: a burst of
# tokens naming a key we don't have yet makes one fetch, not one each. Fetches are rare
# (cache expiry, key rotation), and a hit in the cache holds the lock for microseconds.
lock = anyio.Lock()
# When PyJWKClient last went to the network, successful or not: the throttle below
# counts fetches, so a steady stream of tokens served from the cache must not move it,
# or a rotated key would be refused until the cache expired. fetch_data is the one
# method that does network I/O, so wrapping it on this instance catches every fetch.
last_fetch: float | None = None
fetch_data = client.fetch_data

async def resolve(token: str) -> Any: # pragma: no cover
def fetch_and_record() -> Any:
nonlocal last_fetch
try:
return fetch_data()
finally:
last_fetch = time.monotonic()

client.fetch_data = fetch_and_record # type: ignore[method-assign]

async def resolve(token: str) -> Any:
async with lock:
header = jwt.get_unverified_header(token)
kid = header.get("kid")
Expand All @@ -269,10 +286,7 @@ async def resolve(token: str) -> Any: # pragma: no cover
and (now - last_fetch) < self.min_refetch_interval
):
raise jwt.PyJWKClientError(f"Unable to find a signing key that matches {kid!r}")
try:
key = await anyio.to_thread.run_sync(client.get_signing_key_from_jwt, token)
finally:
last_fetch = time.monotonic()
key = await anyio.to_thread.run_sync(client.get_signing_key_from_jwt, token)
return key.key

key_resolver = resolve
Expand All @@ -296,8 +310,9 @@ async def verify(self, token: str) -> dict[str, Any]:
email = str(claims.get("email", "")).lower()
if email not in self.policy.emails:
# The identity is somebody Access let through but this server doesn't want; name it
# only enough to tell logins apart. logsafe doesn't redact e-mail addresses (and this
# module can't import it: shared modules stay self-contained), so mask it here.
# only enough to tell logins apart. logsafe's redact() would mask it in a log line, but
# not with redaction off, and this module can't import logsafe's mask_email (shared
# modules stay self-contained), so mask it here the same way.
local, _, domain = email.partition("@")
shown = f"{local[:1]}***@{domain}" if domain else "this identity"
raise AccessDenied(f"{shown} is not in MCP_ALLOWED_EMAILS")
Expand Down
Loading
Loading