From bdc255a10cfb4c39b8fb14e53b6d7df008a471f2 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Fri, 9 Oct 2026 12:44:26 -0400 Subject: [PATCH 1/2] Start v5 blocker fix (pypi-pep440-lockonly) Empty commit to open the draft PR. Co-Authored-By: Claude Opus 5.5 (1M context) From 68435385f33b3f407d9312ade2fa6fe9c6f01bb0 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Fri, 9 Oct 2026 15:30:34 -0400 Subject: [PATCH 2/2] Find lock-only patches for PEP 440 pin spellings On a fresh checkout (no venv), requirements.txt discovery sent a pin as written: `six==1.16` became `pkg:pypi/six@1.16`, `==1.16.0.0` became `@1.16.0.0`, `==01.16.0` became `@01.16.0`. pip installs the registry release 1.16.0 for all three, but the patch API keys it `@1.16.0`, so the scan said "No patches available" and the unpatched release was installed. The same file with a venv holding six was patched. Scan now also asks the API for the other PEP 440 spellings of a lockfile-only PyPI pure-release pin (leading zeros dropped, release padded or trimmed to at least three segments). A patch returned under another spelling is counted as that lockfile-only package, so `notInstalled` and the vendored baseline pre-check see it. The rewriters already match `==` pins under PEP 440 (#478). Fixes #604 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../socket-patch-cli/src/commands/scan/mod.rs | 74 +++++++++++++++++-- .../tests/mode_migration_pypi.rs | 53 +++++++++++++ .../tests/scan_requirements_lock_only.rs | 37 ++++++++++ crates/socket-patch-core/src/utils/pep440.rs | 57 ++++++++++++++ .../socket-patch-core/src/utils/purl_key.rs | 37 ++++++++++ docs/testing/uv-compatibility.md | 5 ++ 6 files changed, 257 insertions(+), 6 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index a3d5b08e6..64231b9e8 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -102,6 +102,53 @@ const BATCH_BODY_BYTE_CAP: usize = 256 * 1024; /// authenticated API and [`DEFAULT_PROXY_BATCH_SIZE`] on the public proxy. /// Floored at 1: `--batch-size 0` is otherwise unvalidated and would make /// the chunking below panic, so it degrades to one-package batches. +/// `purls` plus, for each lockfile-only PyPI purl among them, its other +/// PEP 440 spellings of the same release (#604), deduplicated in order. +fn with_pypi_equivalents(purls: &[String], lockfile_only: &HashSet) -> Vec { + let mut out = purls.to_vec(); + let mut seen: HashSet = purls.iter().map(|p| PurlKey::new(p)).collect(); + for purl in purls { + if !lockfile_only_contains(lockfile_only, purl) { + continue; + } + for spelling in socket_patch_core::utils::purl_key::pypi_equivalent_purls(purl) { + if seen.insert(PurlKey::new(&spelling)) { + out.push(spelling); + } + } + } + out +} + +/// Mark each API purl that names a lockfile-only PyPI pin under another +/// PEP 440 spelling (`@1.16.0` for a lock's `@1.16`, #604) as lockfile-only +/// too, so the `notInstalled` flag, the `[NOT INSTALLED]` marker and the +/// vendored baseline pre-check treat it as the package it is. A spelling an +/// installed copy already carries is left alone. +fn adopt_pypi_equivalents( + packages: &[BatchPackagePatches], + scanned: &[String], + lockfile_only: &mut HashSet, +) { + let scanned_keys: HashSet = scanned.iter().map(|p| PurlKey::new(p)).collect(); + let lock_only_pypi: Vec<&String> = scanned + .iter() + .filter(|p| p.starts_with("pkg:pypi/") && lockfile_only_contains(lockfile_only, p)) + .collect(); + for pkg in packages { + let key = PurlKey::new(&pkg.purl); + if scanned_keys.contains(&key) { + continue; + } + if lock_only_pypi + .iter() + .any(|lock| socket_patch_core::utils::purl_key::pypi_same_release(lock, &pkg.purl)) + { + lockfile_only.insert(key); + } + } +} + fn effective_batch_size(requested: Option, use_public_proxy: bool) -> usize { requested .unwrap_or(if use_public_proxy { @@ -1829,7 +1876,10 @@ async fn run_scan( // that have NO installed copy (fresh clone, partial install). They join // discovery and are flagged "not yet installed". Scoped to the crawled // ecosystems. - let lockfile_only = lockfile_supplement(&ctx, &all_crawled, crawl_scope).await; + let mut lockfile_only = lockfile_supplement(&ctx, &all_crawled, crawl_scope).await; + // Counted once: #604 adds the API's spellings of lockfile-only PyPI pins + // to `lockfile_only.purls` after the batch query. + let lockfile_only_count = lockfile_only.purls.len(); // Unsupported layouts and malformed binary Bun locks, kept on empty // scans too: an unreadable graph is not evidence of no dependencies. let mut layout_refusals = unsupported_layout_warnings(&lockfile_only.unsupported); @@ -2266,8 +2316,8 @@ async fn run_scan( plural(package_count, "package", "packages") )); if human { - if !lockfile_only.purls.is_empty() { - eprintln!("{}", render::lockfile_only_note(lockfile_only.purls.len())); + if lockfile_only_count > 0 { + eprintln!("{}", render::lockfile_only_note(lockfile_only_count)); } print_layout_refusals(&layout_refusals, args.common.silent); policy.print_warnings(args.common.silent); @@ -2276,7 +2326,12 @@ async fn run_scan( // Query API in batches let mut all_packages_with_patches: Vec = Vec::new(); let mut can_access_paid_patches = false; - let chunks: Vec<&[String]> = batch_chunks(&all_purls, batch_size, BATCH_BODY_BYTE_CAP); + // #604: a lockfile-only PyPI pin is spelled as the user wrote it + // (`six==1.16` → `@1.16`) while the API keys the release as the + // registry published it (`@1.16.0`); pip treats both as one release + // (PEP 440). Ask for its equivalent spellings too. + let query_purls = with_pypi_equivalents(&all_purls, &lockfile_only.purls); + let chunks: Vec<&[String]> = batch_chunks(&query_purls, batch_size, BATCH_BODY_BYTE_CAP); let total_batches = chunks.len(); let mut batch_error_count = 0usize; let mut last_batch_error: Option = None; @@ -2387,6 +2442,13 @@ async fn run_scan( // drives the table, the `--json` `packages` array and the apply order, // which operators diff across runs. all_packages_with_patches.sort_by(|a, b| a.purl.cmp(&b.purl)); + // #604: a patch the API returned under an equivalent spelling of a + // lockfile-only PyPI pin is that lockfile-only package. + adopt_pypi_equivalents( + &all_packages_with_patches, + &all_purls, + &mut lockfile_only.purls, + ); // If every batch errored, surface a full scan failure rather than // silently reporting zero patches. @@ -2401,7 +2463,7 @@ async fn run_scan( "status": "error", "error": { "code": API_BATCH_FAILED, "message": err }, "scannedPackages": package_count, - "lockfileOnlyPackages": lockfile_only.purls.len(), + "lockfileOnlyPackages": lockfile_only_count, "packagesWithPatches": 0, "totalPatches": 0, "freePatches": 0, @@ -2481,7 +2543,7 @@ async fn run_scan( let mut result = serde_json::json!({ "status": "success", "scannedPackages": package_count, - "lockfileOnlyPackages": lockfile_only.purls.len(), + "lockfileOnlyPackages": lockfile_only_count, "packagesWithPatches": all_packages_with_patches.len(), "totalPatches": total_patches, "freePatches": free_patches, diff --git a/crates/socket-patch-cli/tests/mode_migration_pypi.rs b/crates/socket-patch-cli/tests/mode_migration_pypi.rs index 9296c6884..55d1d3c43 100644 --- a/crates/socket-patch-cli/tests/mode_migration_pypi.rs +++ b/crates/socket-patch-cli/tests/mode_migration_pypi.rs @@ -2002,3 +2002,56 @@ async fn pipenv_hosted_to_vendored_names_the_unpatched_requirements() { "the takeover names requirements.txt as an unpatched install source: {env:#}" ); } + +// ── #604: PEP 440-equivalent lock-only pins ────────────────────────────── + +/// #604: on a fresh checkout (no venv), `six==1.16` (or `==1.16.0.0`, +/// `==01.16.0`) is the release the patch API keys as `six@1.16.0`. Hosted +/// mode must find the patch and rewrite the pin, as it does when a venv +/// holds six 1.16.0, and report the package as not installed. +#[tokio::test] +async fn lock_only_pep440_equivalent_pin_is_patched() { + use wiremock::matchers::body_string_contains; + for pin in ["1.16", "1.16.0.0", "01.16.0"] { + let server = MockServer::start().await; + // The API matches purls exactly: only `@1.16.0` has the patch. + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/batch"))) + .and(body_string_contains(PURL)) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "packages": [{ "purl": PURL, "patches": [{ + "uuid": UUID, "purl": PURL, "tier": "free", "cveIds": [], "ghsaIds": [], + "severity": "high", "title": "pep440 fixture" + }]}], + "canAccessPaidPatches": false, + }))) + .with_priority(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/batch"))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "packages": [], "canAccessPaidPatches": false, + }))) + .with_priority(2) + .mount(&server) + .await; + let hosted_url = mount_hosted_api(&server, true).await; + let (_tmp, root) = project(); + std::fs::write( + root.join("requirements.txt"), + format!("idna==3.7\nsix=={pin}\n"), + ) + .unwrap(); + let (code, env) = hosted_scan(&root, &server); + assert_eq!(code, 0, "{pin}: {env:#}"); + assert_eq!(env["redirect"]["redirected"], 1, "{pin}: {env:#}"); + assert_eq!(env["packages"][0]["purl"], PURL, "{pin}: {env:#}"); + assert_eq!(env["packages"][0]["notInstalled"], true, "{pin}: {env:#}"); + let requirements = std::fs::read_to_string(root.join("requirements.txt")).unwrap(); + assert!( + requirements.contains(&format!("six @ {hosted_url}")), + "{pin}: the pin is rewritten:\n{requirements}" + ); + } +} diff --git a/crates/socket-patch-cli/tests/scan_requirements_lock_only.rs b/crates/socket-patch-cli/tests/scan_requirements_lock_only.rs index 47523f7a9..da8f60eaa 100644 --- a/crates/socket-patch-cli/tests/scan_requirements_lock_only.rs +++ b/crates/socket-patch-cli/tests/scan_requirements_lock_only.rs @@ -376,3 +376,40 @@ async fn lock_only_scan_discovers_pin_with_dangling_eof_continuation() { ) .await; } + +/// #604: pip resolves `six==1.16`, `six==1.16.0.0` and `six==01.16.0` to +/// the registry release `1.16.0` (PEP 440), but lock-only discovery queried +/// only the spelled version, so the patch keyed `@1.16.0` was never found. +/// Each spelling must also ask for the release's other spellings. +#[tokio::test] +async fn lock_only_scan_queries_pep440_equivalent_spellings() { + for (pin, spelled) in [ + ("1.16", "1.16"), + ("1.16.0.0", "1.16.0.0"), + ("01.16.0", "01.16.0"), + ] { + for mode in [&[][..], &["--mode", "vendored"][..]] { + let mock = MockServer::start().await; + mount_empty_batch(&mock).await; + let tmp = tempfile::tempdir().unwrap(); + std::fs::write( + tmp.path().join("requirements.txt"), + format!("sp-fixture-six=={pin}\n"), + ) + .unwrap(); + let (code, v) = run_scan(tmp.path(), &mock.uri(), mode, &[]); + assert_eq!(code, 0, "{pin} {mode:?}: {v}"); + assert_eq!(v["lockfileOnlyPackages"].as_u64(), Some(1), "{v}"); + let purls = batch_purls(&mock).await; + for want in [ + format!("pkg:pypi/sp-fixture-six@{spelled}"), + "pkg:pypi/sp-fixture-six@1.16.0".to_string(), + ] { + assert!( + purls.contains(&want), + "{pin} {mode:?}: {want} must reach the patch API; sent {purls:?}" + ); + } + } + } +} diff --git a/crates/socket-patch-core/src/utils/pep440.rs b/crates/socket-patch-core/src/utils/pep440.rs index 50543d5b4..a184827f5 100644 --- a/crates/socket-patch-core/src/utils/pep440.rs +++ b/crates/socket-patch-core/src/utils/pep440.rs @@ -130,6 +130,37 @@ pub(crate) fn is_exact_pin(specifier: &str) -> bool { .is_some_and(|pinned| parse(pinned).is_some()) } +/// The other spellings a registry may have published the pure release +/// `version` (digits and dots only) under, as PEP 440 equality reads them: +/// leading zeros dropped and the release padded with `.0` from its +/// trailing-zero-trimmed form up to at least three segments (`1.16` → +/// `1.16.0`; `1.16.0` → `1.16`; `01.16.0.0` → `1.16`, `1.16.0`). `version` +/// itself is not included; anything that is not a pure release has none. +pub(crate) fn equivalent_release_spellings(version: &str) -> Vec { + let valid = !version.is_empty() + && version.split('.').all(|segment| { + !segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_digit()) + }); + if !valid { + return Vec::new(); + } + let segments: Vec = version.split('.').map(number).collect(); + let mut trimmed = segments.clone(); + while trimmed.len() > 1 && trimmed.last().is_some_and(|segment| segment == "0") { + trimmed.pop(); + } + let mut out = Vec::new(); + for len in trimmed.len()..=segments.len().max(3) { + let mut release = trimmed.clone(); + release.resize(len, "0".to_string()); + let spelling = release.join("."); + if spelling != version && !out.contains(&spelling) { + out.push(spelling); + } + } + out +} + #[cfg(test)] mod tests { use super::*; @@ -182,6 +213,32 @@ mod tests { } } + #[test] + fn equivalent_release_spellings_pad_and_trim() { + assert_eq!(equivalent_release_spellings("1.16"), ["1.16.0"]); + assert_eq!(equivalent_release_spellings("1.16.0"), ["1.16"]); + assert_eq!(equivalent_release_spellings("1.16.0.0"), ["1.16", "1.16.0"]); + assert_eq!(equivalent_release_spellings("01.16.0"), ["1.16", "1.16.0"]); + assert_eq!(equivalent_release_spellings("2"), ["2.0", "2.0.0"]); + assert!(equivalent_release_spellings("2.8.2").is_empty()); + for other in [ + "1.0rc1", + "2.9.0.post0", + "1.0+local", + "", + "1..2", + "v1.0", + "1.*", + ] { + assert!(equivalent_release_spellings(other).is_empty(), "{other}"); + } + for version in ["1.16", "1.16.0.0", "01.16.0"] { + for spelling in equivalent_release_spellings(version) { + assert!(versions_equal(version, &spelling), "{version} {spelling}"); + } + } + } + #[test] fn exact_pin_spellings() { assert!(is_exact_pin_of("==1.16", "1.16.0")); diff --git a/crates/socket-patch-core/src/utils/purl_key.rs b/crates/socket-patch-core/src/utils/purl_key.rs index 5c0878380..ffaa1082a 100644 --- a/crates/socket-patch-core/src/utils/purl_key.rs +++ b/crates/socket-patch-core/src/utils/purl_key.rs @@ -173,6 +173,43 @@ fn composer_identity(canonical: &str) -> Option { )) } +/// `pkg:pypi/@` purls for every other spelling of `purl`'s pure +/// release (`@1.16` → `@1.16.0`, see +/// [`crate::utils::pep440::equivalent_release_spellings`]): the lockfile +/// spells a pin as the user wrote it (`six==1.16`), while the patch API +/// keys the release as the registry published it (#604). Empty for any +/// other purl. +pub fn pypi_equivalent_purls(purl: &str) -> Vec { + let canonical = canonical_base_purl(purl); + let Some((name, version)) = canonical + .strip_prefix("pkg:pypi/") + .and_then(|rest| rest.rsplit_once('@')) + else { + return Vec::new(); + }; + crate::utils::pep440::equivalent_release_spellings(version) + .into_iter() + .map(|spelling| format!("pkg:pypi/{name}@{spelling}")) + .collect() +} + +/// Whether two PyPI purls name the same release under PEP 440 equality +/// (`@1.16` and `@1.16.0`), the way pip resolves an `==` pin. `false` for +/// anything that is not two PyPI purls of one (PEP 503) name. +pub fn pypi_same_release(a: &str, b: &str) -> bool { + let (a, b) = (canonical_base_purl(a), canonical_base_purl(b)); + let split = |purl: &str| -> Option<(String, String)> { + let (name, version) = purl.strip_prefix("pkg:pypi/")?.rsplit_once('@')?; + Some((name.to_string(), version.to_string())) + }; + match (split(&a), split(&b)) { + (Some((name_a, version_a)), Some((name_b, version_b))) => { + name_a == name_b && crate::utils::pep440::versions_equal(&version_a, &version_b) + } + _ => false, + } +} + #[cfg(test)] #[path = "purl_key_nuget_vendor_tests.rs"] mod nuget_vendor_tests; diff --git a/docs/testing/uv-compatibility.md b/docs/testing/uv-compatibility.md index 5448481c0..de3591495 100644 --- a/docs/testing/uv-compatibility.md +++ b/docs/testing/uv-compatibility.md @@ -154,6 +154,11 @@ frozen, locked, and ordinary installation outcomes separately where supported. those cases requires marker-specific source mappings. Standalone PEP 751 rewriting selects the exact package version; duplicate entries for the same name and version are refused when source selection is ambiguous. +- Lock-only discovery (a fresh checkout, no venv) queries the patch API with + every PEP 440 spelling of an exact pure-release pin as well as the one + written (`six==1.16` asks for `@1.16` and `@1.16.0`; `==1.16.0` also asks for + `@1.16`), so it finds the patch the registry keys under its own spelling, as a + venv-backed run does, and reports the package as not installed (#604). - Hosted requirements select exact `==`/`===` pins or identifiable archive URLs. Other versions remain unchanged. A bare requirement is rewritten only when one row and one override version identify the selection. Ranges, wildcard