diff --git a/crates/socket-patch-cli/src/commands/scan/hosted/python.rs b/crates/socket-patch-cli/src/commands/scan/hosted/python.rs index f84734b24..70c097e46 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted/python.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted/python.rs @@ -57,11 +57,14 @@ pub(super) async fn stale_install_warnings( } else { socket_patch_core::crawlers::hatch_env::hatch_environments(&common.cwd).await }; - let paths = if common.is_global() { - crawler + // The base interpreter dirs a `--site-packages` venv inherits: a copy + // there survives Pipenv's usual remedies (#1450). + let (paths, inherited) = if common.is_global() { + let paths = crawler .get_site_packages_paths(&common.crawler_options()) .await - .unwrap_or_default() + .unwrap_or_default(); + (paths, Default::default()) } else { let sites = socket_patch_core::crawlers::python_crawler::find_local_venv_site_packages(&common.cwd) @@ -69,7 +72,21 @@ pub(super) async fn stale_install_warnings( // A `--system-site-packages` venv's own copy hides its base // interpreter's (#409). crawler.set_search_order(&sites).await; - sites + // Only Pipenv's remedy reads this, over the venvs Pipenv itself + // resolves (a `.venv` it shares with Hatch included): a base dir + // only a Hatch env reaches is not one `pipenv run` fixes. + let inherited = if pipenv_uuids.is_empty() { + Default::default() + } else { + let pipenv_sites = + socket_patch_core::crawlers::python_crawler::non_hatch_local_venv_site_packages( + &common.cwd, + ) + .await; + socket_patch_core::crawlers::python_crawler::inherited_base_site_dirs(&pipenv_sites) + .await + }; + (sites, inherited) }; #[derive(Default)] @@ -114,7 +131,7 @@ pub(super) async fn stale_install_warnings( } } } - for ((site, _, _), judgment) in judgments { + for ((site, _, version), judgment) in judgments { if judgment.patched || !judgment.stale { continue; } @@ -133,17 +150,47 @@ pub(super) async fn stale_install_warnings( .and_then(|rest| rest.split('@').next()) .unwrap_or("") .to_string(); - let remedy = socket_patch_core::vendor::pypi_pipenv::stale_install_remedy( - pipenv_lock.as_ref(), - &name, - ); - format!( - "Pipenv does not reinstall a release that is already present (`pipenv \ - install`, `pipenv install --deploy` and `pipenv sync` all keep those \ - bytes), so the rewritten Pipfile.lock only protects fresh installs. \ - {remedy}; then `socket-patch vex --product ` re-verifies the \ - installed files." - ) + if inherited.contains(&site) { + let remedy = + socket_patch_core::vendor::pypi_pipenv::base_site_stale_install_remedy( + pipenv_lock.as_ref(), + &name, + &site.display().to_string(), + ); + format!( + "{remedy}; then `socket-patch vex --product ` re-verifies the \ + installed files." + ) + } else if let Some(base) = + socket_patch_core::crawlers::python_crawler::inherited_same_release( + &site, &name, &version, + ) + .await + { + let remedy = + socket_patch_core::vendor::pypi_pipenv::shadowing_stale_install_remedy( + pipenv_lock.as_ref(), + &name, + &site.display().to_string(), + &base.display().to_string(), + ); + format!( + "{remedy}; then `socket-patch vex --product ` re-verifies the \ + installed files." + ) + } else { + let remedy = socket_patch_core::vendor::pypi_pipenv::stale_install_remedy( + pipenv_lock.as_ref(), + &name, + ); + format!( + "Pipenv does not reinstall a release that is already present (`pipenv \ + install`, `pipenv install --deploy` and `pipenv sync` all keep those \ + bytes), so the rewritten Pipfile.lock only protects fresh installs. \ + {remedy}; then `socket-patch vex --product ` re-verifies the \ + installed files." + ) + } } else { "Reinstall from the rewritten lock in this interpreter and verify with \ `socket-patch vex`. Poetry before 1.4 may keep same-version packages: \ @@ -370,4 +417,134 @@ mod tests { "{docs}" ); } + + /// #1450: a `--site-packages` venv imports the upstream release from its + /// base interpreter. pip refuses to uninstall that copy from the venv + /// and `pipenv sync` (also in a recreated venv) counts it as installed, + /// so the warning names an install over it inside the venv instead, and + /// does not claim the lock protects fresh installs. + #[tokio::test] + async fn pipenv_base_site_copy_gets_an_install_over_it() { + let tmp = tempfile::tempdir().unwrap(); + let project = tmp.path().join("app"); + let venv = project.join(".venv"); + let base = tmp.path().join("base"); + let (home, venv_site, base_site) = if cfg!(windows) { + std::fs::create_dir_all(base.join("Lib")).unwrap(); + std::fs::write(base.join("Lib").join("os.py"), "").unwrap(); + ( + base.clone(), + venv.join("Lib").join("site-packages"), + base.join("Lib").join("site-packages"), + ) + } else { + let lib = base.join("lib").join("python3.12"); + std::fs::create_dir_all(&lib).unwrap(); + std::fs::write(lib.join("os.py"), "").unwrap(); + std::fs::create_dir_all(base.join("bin")).unwrap(); + ( + base.join("bin"), + venv.join("lib").join("python3.12").join("site-packages"), + lib.join("site-packages"), + ) + }; + std::fs::create_dir_all(&venv_site).unwrap(); + std::fs::write( + venv.join("pyvenv.cfg"), + format!( + "home = {}\ninclude-system-site-packages = true\nversion = 3.12.4\n", + home.display() + ), + ) + .unwrap(); + let dist = base_site.join("six-1.16.0.dist-info"); + std::fs::create_dir_all(&dist).unwrap(); + std::fs::write(dist.join("METADATA"), "Name: six\nVersion: 1.16.0\n").unwrap(); + std::fs::write(base_site.join("six.py"), b"upstream").unwrap(); + + let common = crate::args::GlobalArgs { + cwd: project.clone(), + ..Default::default() + }; + let purl = "pkg:pypi/six@1.16.0"; + let confirmed = vec![(purl.to_string(), "six-uuid".to_string())]; + let ledger = BTreeMap::from([("k".into(), record("six-uuid", "six.py", b"patched"))]); + let pipenv = BTreeSet::from(["six-uuid".to_string()]); + let lock = r#"{"_meta": {"pipfile-spec": 6}, "default": {}, "develop": {"six": {}}}"#; + let out = stale_install_warnings(&common, &confirmed, &pipenv, Some(lock), &ledger).await; + assert_eq!(out.stale_purls, BTreeSet::from([purl.to_string()])); + assert_eq!(out.warnings.len(), 1); + let detail = out.warnings[0]["detail"].as_str().unwrap(); + assert!( + detail.contains(&base_site.display().to_string()), + "{detail}" + ); + assert!( + detail.contains( + "`pipenv requirements --dev > pipenv-locked.txt && pipenv run pip install \ + --no-deps --ignore-installed -r pipenv-locked.txt`" + ), + "{detail}" + ); + for broken in [ + "pip uninstall", + "pipenv --rm", + "only protects fresh installs", + ] { + assert!(!detail.contains(broken), "{broken}: {detail}"); + } + + // The venv's own stale copy hides the same release in the base dir: + // uninstalling it would only expose the base copy to `pipenv sync`, + // so it gets the install-over remedy too. + let dist = venv_site.join("six-1.16.0.dist-info"); + std::fs::create_dir_all(&dist).unwrap(); + std::fs::write(dist.join("METADATA"), "Name: six\nVersion: 1.16.0\n").unwrap(); + std::fs::write(venv_site.join("six.py"), b"upstream").unwrap(); + let out = stale_install_warnings(&common, &confirmed, &pipenv, Some(lock), &ledger).await; + assert_eq!(out.warnings.len(), 1); + let detail = out.warnings[0]["detail"].as_str().unwrap(); + assert!( + detail.contains(&format!( + "hides the same release in {}", + base_site.display() + )), + "{detail}" + ); + assert!( + detail.contains("pipenv run pip install --no-deps --ignore-installed"), + "{detail}" + ); + assert!(!detail.contains("pip uninstall -y six"), "{detail}"); + + // A different release in the base dir is not one `pipenv sync` + // counts as installed: the usual remedy replaces the venv's copy. + let base_dist = base_site.join("six-1.16.0.dist-info"); + std::fs::write(base_dist.join("METADATA"), "Name: six\nVersion: 1.15.0\n").unwrap(); + let out = stale_install_warnings(&common, &confirmed, &pipenv, Some(lock), &ledger).await; + assert_eq!(out.warnings.len(), 1); + let detail = out.warnings[0]["detail"].as_str().unwrap(); + assert!( + detail.contains("`pipenv run pip uninstall -y six && pipenv sync --dev`"), + "{detail}" + ); + std::fs::write(base_dist.join("METADATA"), "Name: six\nVersion: 1.16.0\n").unwrap(); + + // A `.venv` Pipenv shares with Hatch (`path = ".venv"`) is still + // Pipenv's: its inherited base copy keeps the install-over remedy. + std::fs::remove_file(venv_site.join("six.py")).unwrap(); + std::fs::remove_dir_all(venv_site.join("six-1.16.0.dist-info")).unwrap(); + std::fs::write( + project.join("pyproject.toml"), + "[project]\nname = \"app\"\nversion = \"0.1.0\"\n\n[tool.hatch.envs.default]\npath = \".venv\"\n", + ) + .unwrap(); + let out = stale_install_warnings(&common, &confirmed, &pipenv, Some(lock), &ledger).await; + assert_eq!(out.warnings.len(), 1); + let detail = out.warnings[0]["detail"].as_str().unwrap(); + assert!( + detail.contains("pipenv run pip install --no-deps --ignore-installed"), + "{detail}" + ); + } } diff --git a/crates/socket-patch-core/src/crawlers/python_crawler.rs b/crates/socket-patch-core/src/crawlers/python_crawler.rs index 998627d3b..02558cb2e 100644 --- a/crates/socket-patch-core/src/crawlers/python_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/python_crawler.rs @@ -617,6 +617,56 @@ fn shadowed_names_sync( shadowed } +/// The dirs among `sites` that a `--system-site-packages` venv also in +/// `sites` reaches in its base interpreter (see [`add_system_site_packages`]) +/// rather than a venv's own site-packages. A copy installed there is outside +/// every venv: pip refuses to uninstall it from one, and an install into the +/// venv counts it as already present (#1450). +pub async fn inherited_base_site_dirs(sites: &[PathBuf]) -> HashSet { + let var = |name: &str| std::env::var(name).ok(); + let sites = sites.to_vec(); + run_blocking(move || inherited_base_site_dirs_sync(&sites, &var)).await +} + +fn inherited_base_site_dirs_sync( + sites: &[PathBuf], + var: &impl Fn(&str) -> Option, +) -> HashSet { + let own: Vec<(PathBuf, &PathBuf)> = sites + .iter() + .filter_map(|site| venv_root_of_site(site).map(|root| (root, site))) + .collect(); + own.iter() + .flat_map(|(root, site)| system_site_dirs_of_venv(root, site, var)) + .filter(|dir| sites.contains(dir) && !own.iter().any(|(_, site)| *site == dir)) + .collect() +} + +/// The base interpreter site dir holding `name` at `version` that the +/// `--system-site-packages` venv owning `site` would fall back to once its +/// own copy is gone. pip and Pipenv count that copy as installed, so +/// removing the venv's copy exposes the base one instead of reinstalling. +pub async fn inherited_same_release(site: &Path, name: &str, version: &str) -> Option { + let var = |name: &str| std::env::var(name).ok(); + let (site, name, version) = ( + site.to_path_buf(), + canonicalize_pypi_name(name), + version.to_string(), + ); + run_blocking(move || { + let root = venv_root_of_site(&site)?; + system_site_dirs_of_venv(&root, &site, &var) + .into_iter() + .find_map(|base| { + list_dist_info_packages_sync(&base) + .into_iter() + .any(|(n, v)| n == name && v == version) + .then_some(base) + }) + }) + .await +} + /// Find local virtual environment `site-packages` directories. /// /// Checks (in order): @@ -3963,6 +4013,25 @@ mod tests { ); } + /// #1450: the base dirs a `--system-site-packages` venv inherits are + /// told apart from the venv's own site-packages. + #[test] + fn inherited_base_site_dirs_are_the_base_interpreters() { + let tmp = tempfile::tempdir().unwrap(); + let (_, venv_site, base_site) = system_site_layout(tmp.path(), "true"); + let no_env = env_of(&[]); + let sites = vec![venv_site.clone(), base_site.clone()]; + assert_eq!( + inherited_base_site_dirs_sync(&sites, &no_env), + HashSet::from([base_site.clone()]) + ); + // Only dirs actually in `sites` are reported. + assert!(inherited_base_site_dirs_sync(&[venv_site], &no_env).is_empty()); + let tmp = tempfile::tempdir().unwrap(); + let (_, venv_site, base_site) = system_site_layout(tmp.path(), "false"); + assert!(inherited_base_site_dirs_sync(&[venv_site, base_site], &no_env).is_empty()); + } + #[tokio::test] async fn isolated_venv_never_reaches_its_base_interpreter() { let tmp = tempfile::tempdir().unwrap(); diff --git a/crates/socket-patch-core/src/vendor/pypi.rs b/crates/socket-patch-core/src/vendor/pypi.rs index 510e82d4e..1dc99acdc 100644 --- a/crates/socket-patch-core/src/vendor/pypi.rs +++ b/crates/socket-patch-core/src/vendor/pypi.rs @@ -765,21 +765,65 @@ async fn pipenv_stale_install_warning( if stale_dirs.is_empty() { return None; } - let name = parse_pypi_purl(strip_purl_qualifiers(purl)) - .map(|(name, _)| name.to_string()) - .unwrap_or_else(|| purl.to_string()); - let listed = stale_dirs + let (name, version) = parse_pypi_purl(strip_purl_qualifiers(purl)) + .map(|(name, version)| (name.to_string(), version.to_string())) + .unwrap_or_else(|| (purl.to_string(), String::new())); + let join = |dirs: &[&std::path::PathBuf]| { + dirs.iter() + .map(|d| d.display().to_string()) + .collect::>() + .join(", ") + }; + let listed = join(&stale_dirs.iter().collect::>()); + // A base interpreter's copy a `--site-packages` venv inherits survives + // both the uninstall-and-sync and the recreate-and-sync remedies (#1450). + let inherited = crate::crawlers::python_crawler::inherited_base_site_dirs(&sites).await; + let (base, own): (Vec<_>, Vec<_>) = stale_dirs.iter().partition(|d| inherited.contains(*d)); + // A venv copy that hides the same release in its base interpreter: the + // uninstall-and-sync remedy would only expose that base copy. + let mut shadowing = Vec::new(); + let mut plain = Vec::new(); + for dir in own { + match crate::crawlers::python_crawler::inherited_same_release(dir, &name, &version).await { + Some(base_dir) => shadowing.push((dir, base_dir)), + None => plain.push(dir), + } + } + let own = plain; + let mut detail = + format!("{purl}: the UNPATCHED upstream release is still installed in {listed}."); + let several = [!own.is_empty(), !shadowing.is_empty(), !base.is_empty()] .iter() - .map(|d| d.display().to_string()) - .collect::>() - .join(", "); - let remedy = super::pypi_pipenv::stale_install_remedy(Some(lock), &name); - Some(VendorWarning::new( - "pypi_pipenv_stale_install", - format!( - "{purl}: the UNPATCHED upstream release is still installed in {listed}. Pipenv does not reinstall a release that is already present (`pipenv install`, `pipenv install --deploy` and `pipenv sync` all keep those bytes), so the wired Pipfile.lock only protects fresh installs. {remedy}; then `socket-patch vex` re-verifies the installed files." - ), - )) + .filter(|kind| **kind) + .count() + > 1; + if !own.is_empty() { + let remedy = super::pypi_pipenv::stale_install_remedy(Some(lock), &name); + let lead = if !several { + String::new() + } else { + format!("For {}: ", join(&own)) + }; + detail.push_str(&format!( + " {lead}Pipenv does not reinstall a release that is already present (`pipenv install`, `pipenv install --deploy` and `pipenv sync` all keep those bytes), so the wired Pipfile.lock only protects fresh installs. {remedy}." + )); + } + for (dir, base_dir) in &shadowing { + let remedy = super::pypi_pipenv::shadowing_stale_install_remedy( + Some(lock), + &name, + &dir.display().to_string(), + &base_dir.display().to_string(), + ); + detail.push_str(&format!(" {remedy}.")); + } + if !base.is_empty() { + let remedy = + super::pypi_pipenv::base_site_stale_install_remedy(Some(lock), &name, &join(&base)); + detail.push_str(&format!(" {remedy}.")); + } + detail.push_str(" Then `socket-patch vex` re-verifies the installed files."); + Some(VendorWarning::new("pypi_pipenv_stale_install", detail)) } /// The Hatch twin of [`pipenv_stale_install_warning`]: Hatch keeps the @@ -4190,6 +4234,186 @@ mod tests { ); } + /// #1450: when the stale upstream copy is the one a `--site-packages` + /// venv inherits from its base interpreter, neither uninstall-and-sync + /// nor recreate-and-sync can replace it, so the warning names an + /// install over it inside the venv instead. + #[tokio::test] + async fn pipenv_stale_install_in_a_base_site_dir_installs_over_it() { + let fx = e2e_fixture().await; + let venv = fx.root.join(".venv"); + let base = fx.root.join("base"); + let (home, venv_site, base_site) = if cfg!(windows) { + tokio::fs::create_dir_all(base.join("Lib")).await.unwrap(); + touch(&base.join("Lib"), "os.py", "").await; + ( + base.clone(), + venv.join("Lib").join("site-packages"), + base.join("Lib").join("site-packages"), + ) + } else { + tokio::fs::create_dir_all(base.join("lib").join("python3.12")) + .await + .unwrap(); + touch(&base.join("lib").join("python3.12"), "os.py", "").await; + tokio::fs::create_dir_all(base.join("bin")).await.unwrap(); + ( + base.join("bin"), + fx.site_packages.clone(), + base.join("lib").join("python3.12").join("site-packages"), + ) + }; + // The venv holds no six of its own; the base interpreter's upstream + // copy is the one it imports. + let _ = tokio::fs::remove_dir_all(&fx.site_packages).await; + tokio::fs::create_dir_all(&venv_site).await.unwrap(); + let di = base_site.join("six-1.16.0.dist-info"); + tokio::fs::create_dir_all(&di).await.unwrap(); + tokio::fs::write(base_site.join("six.py"), ORIG) + .await + .unwrap(); + touch( + &di, + "METADATA", + "Metadata-Version: 2.1\nName: six\nVersion: 1.16.0\n", + ) + .await; + touch( + &venv, + "pyvenv.cfg", + &format!( + "home = {}\ninclude-system-site-packages = true\nversion = 3.12.4\n", + home.display() + ), + ) + .await; + let lock: serde_json::Value = serde_json::from_str( + r#"{"_meta": {"pipfile-spec": 6}, "default": {"six": {"version": "==1.16.0"}}, "develop": {}}"#, + ) + .unwrap(); + let warning = pipenv_stale_install_warning( + &fx.root, + "pkg:pypi/six@1.16.0", + &fx.record, + &InstalledSiteListings::default(), + &lock, + ) + .await + .expect("the inherited upstream six.py is reported"); + assert_eq!(warning.code, "pypi_pipenv_stale_install"); + assert!( + warning.detail.contains(&base_site.display().to_string()), + "{}", + warning.detail + ); + assert!( + warning + .detail + .contains("pipenv run pip install --no-deps --ignore-installed"), + "{}", + warning.detail + ); + for broken in [ + "pip uninstall", + "pipenv --rm", + "only protects fresh installs", + ] { + assert!( + !warning.detail.contains(broken), + "{broken}: {}", + warning.detail + ); + } + + // A second venv with its own stale copy keeps the usual remedy for + // that copy, beside the base-copy one. + let other = if cfg!(windows) { + fx.root.join("venv").join("Lib").join("site-packages") + } else { + fx.root + .join("venv") + .join("lib") + .join("python3.12") + .join("site-packages") + }; + let di = other.join("six-1.16.0.dist-info"); + tokio::fs::create_dir_all(&di).await.unwrap(); + tokio::fs::write(other.join("six.py"), ORIG).await.unwrap(); + touch( + &di, + "METADATA", + "Metadata-Version: 2.1\nName: six\nVersion: 1.16.0\n", + ) + .await; + // Without system site-packages, so the host's own base dirs stay out. + touch( + &fx.root.join("venv"), + "pyvenv.cfg", + "home = /usr/bin\ninclude-system-site-packages = false\n", + ) + .await; + let warning = pipenv_stale_install_warning( + &fx.root, + "pkg:pypi/six@1.16.0", + &fx.record, + &InstalledSiteListings::default(), + &lock, + ) + .await + .expect("both stale copies are reported"); + let detail = &warning.detail; + assert!( + detail.contains(&format!("For {}: ", other.display())), + "{detail}" + ); + assert!( + detail.contains("`pipenv run pip uninstall -y six && pipenv sync`"), + "{detail}" + ); + assert!( + detail.contains("pipenv run pip install --no-deps --ignore-installed"), + "{detail}" + ); + + // The inheriting venv's own stale copy of the same release hides the + // base one: uninstalling it would only expose the base copy to + // `pipenv sync`, so that copy gets the install-over remedy too. + let di = venv_site.join("six-1.16.0.dist-info"); + tokio::fs::create_dir_all(&di).await.unwrap(); + tokio::fs::write(venv_site.join("six.py"), ORIG) + .await + .unwrap(); + touch( + &di, + "METADATA", + "Metadata-Version: 2.1\nName: six\nVersion: 1.16.0\n", + ) + .await; + let warning = pipenv_stale_install_warning( + &fx.root, + "pkg:pypi/six@1.16.0", + &fx.record, + &InstalledSiteListings::default(), + &lock, + ) + .await + .expect("both venv copies are reported"); + let detail = &warning.detail; + assert!( + detail.contains(&format!( + "The upstream six in {} hides the same release in {}", + venv_site.display(), + base_site.display() + )), + "{detail}" + ); + // The non-inheriting venv keeps the usual remedy. + assert!( + detail.contains(&format!("For {}: ", other.display())), + "{detail}" + ); + } + #[tokio::test] async fn end_to_end_requirements_vendor_and_revert() { let fx = e2e_fixture().await; diff --git a/crates/socket-patch-core/src/vendor/pypi_pipenv.rs b/crates/socket-patch-core/src/vendor/pypi_pipenv.rs index 4eac7375b..9aa8b35d7 100644 --- a/crates/socket-patch-core/src/vendor/pypi_pipenv.rs +++ b/crates/socket-patch-core/src/vendor/pypi_pipenv.rs @@ -784,29 +784,7 @@ fn ordered_sections(mut sections: Vec<&str>) -> Vec<&str> { /// category, so `--rm` drops nothing the project had installed. With no /// readable lock, falls back to the `default`-only commands. pub fn stale_install_remedy(lock: Option<&Value>, name: &str) -> String { - let canon = canonicalize_pypi_name(name); - let (package, all): (Vec<&str>, Vec<&str>) = match lock { - Some(lock) => ( - ordered_sections( - find_entries(lock, &canon) - .into_iter() - .map(|(section, _, _)| section) - .collect(), - ), - ordered_sections( - lock.as_object() - .into_iter() - .flat_map(|map| map.iter()) - .filter(|(section, value)| { - section.as_str() != "_meta" - && value.as_object().is_some_and(|m| !m.is_empty()) - }) - .map(|(section, _)| section.as_str()) - .collect(), - ), - ), - None => (Vec::new(), Vec::new()), - }; + let (package, all) = lock_sections(lock, name); let reinstall = sync_args(&package); let clean = sync_args(&all); // Named categories arrived in Pipenv 2022, so the pre-2018 spelling only @@ -824,6 +802,93 @@ pub fn stale_install_remedy(lock: Option<&Value>, name: &str) -> String { ) } +/// The lock sections that pin `name`, and every non-empty section, each in +/// [`ordered_sections`] order. Both are empty with no readable lock. +fn lock_sections<'a>(lock: Option<&'a Value>, name: &str) -> (Vec<&'a str>, Vec<&'a str>) { + let canon = canonicalize_pypi_name(name); + let Some(lock) = lock else { + return (Vec::new(), Vec::new()); + }; + ( + ordered_sections( + find_entries(lock, &canon) + .into_iter() + .map(|(section, _, _)| section) + .collect(), + ), + ordered_sections( + lock.as_object() + .into_iter() + .flat_map(|map| map.iter()) + .filter(|(section, value)| { + section.as_str() != "_meta" && value.as_object().is_some_and(|m| !m.is_empty()) + }) + .map(|(section, _)| section.as_str()) + .collect(), + ), + ) +} + +/// The remedy for a stale `name` that a `--site-packages` virtualenv +/// imports from its base interpreter's site dir `dir` (#1450), in place of +/// [`stale_install_remedy`], whose commands both keep that copy: pip +/// refuses to uninstall a release outside the virtualenv, and `pipenv +/// sync` (in the same or a recreated `--site-packages` virtualenv) counts +/// it as installed. Installing the lock with `--ignore-installed` puts the +/// locked release in the virtualenv itself, ahead of the base copy on +/// `sys.path`, and leaves the base interpreter untouched. The +/// `pipenv requirements` arguments follow the categories that pin `name`, +/// as [`stale_install_remedy`]'s `sync` ones do. +pub fn base_site_stale_install_remedy(lock: Option<&Value>, name: &str, dir: &str) -> String { + format!( + "The upstream {name} in {dir} belongs to the base interpreter this `--site-packages` \ + virtualenv inherits: pip refuses to uninstall it from the virtualenv, and `pipenv \ + sync` (also in a recreated `--site-packages` virtualenv) counts it as installed, so \ + Pipfile.lock does not protect installs on this host. Install the locked release \ + over it inside the virtualenv, leaving the base interpreter untouched: {}", + install_over_command(lock, name) + ) +} + +/// The remedy for a venv's own stale copy of a release its base interpreter +/// also holds (see [`base_site_stale_install_remedy`]): uninstalling the +/// venv copy would leave `pipenv sync` counting the base copy as installed. +pub fn shadowing_stale_install_remedy( + lock: Option<&Value>, + name: &str, + dir: &str, + base: &str, +) -> String { + format!( + "The upstream {name} in {dir} hides the same release in {base}, which belongs to the \ + base interpreter this `--site-packages` virtualenv inherits: removing the virtualenv's \ + copy leaves `pipenv sync` counting the base one as installed, so Pipfile.lock does not \ + protect installs on this host. Install the locked release over the virtualenv's copy, \ + leaving the base interpreter untouched: {}", + install_over_command(lock, name) + ) +} + +/// The install-over command both base-interpreter remedies share. +fn install_over_command(lock: Option<&Value>, name: &str) -> String { + let (package, _) = lock_sections(lock, name); + let categories = sync_args(&package); + // `pipenv requirements` arrived in 2022.4; its `--categories` later + // (present in 2022.12). + let since = if categories.starts_with(" --categories") { + "2022.12" + } else { + "2022.4" + }; + format!( + "`pipenv requirements{categories} > pipenv-locked.txt && pipenv run pip install \ + --no-deps --ignore-installed -r pipenv-locked.txt` (Pipenv {since}+; it reinstalls \ + every package of those lock categories into the virtualenv at its locked version), \ + then delete pipenv-locked.txt — NOT `pipenv uninstall`, which rewrites the Pipfile \ + and re-locks the patch away" + ) +} + /// Splice one entry into the lock text in Pipenv's layout: every object /// key-sorted, as `json.dumps(sort_keys=True)` writes it (see /// [`splice_entry`]). `None` removes the entry. @@ -2277,4 +2342,58 @@ mod tests { assert!(remedy.contains("-y six && pipenv sync` ("), "{remedy}"); assert!(remedy.contains("NOT `pipenv uninstall`"), "{remedy}"); } + + /// #1450: a base-interpreter copy a `--site-packages` virtualenv + /// inherits gets an install over it inside the virtualenv, never the + /// uninstall-and-sync or recreate-and-sync commands that both keep it. + #[test] + fn base_site_stale_install_remedy_installs_over_the_base_copy() { + let lock = |body: &str| -> Value { + serde_json::from_str(&format!(r#"{{"_meta": {{"pipfile-spec": 6}}, {body}}}"#)).unwrap() + }; + let six = r#"{"version": "==1.16.0"}"#; + let dir = "/usr/lib/python3/dist-packages"; + let only_default = lock(&format!(r#""default": {{"six": {six}}}, "develop": {{}}"#)); + let remedy = base_site_stale_install_remedy(Some(&only_default), "six", dir); + assert!( + remedy.contains( + "`pipenv requirements > pipenv-locked.txt && pipenv run pip install --no-deps \ + --ignore-installed -r pipenv-locked.txt`" + ), + "{remedy}" + ); + assert!(remedy.contains(dir), "{remedy}"); + assert!(remedy.contains("NOT `pipenv uninstall`"), "{remedy}"); + assert!( + remedy.contains("reinstalls every package of those lock categories"), + "{remedy}" + ); + for broken in [ + "pip uninstall", + "pipenv --rm", + "only protects fresh installs", + ] { + assert!(!remedy.contains(broken), "{broken}: {remedy}"); + } + // The requirements export follows the categories that pin it. + let develop = lock(&format!( + r#""default": {{"requests": {six}}}, "develop": {{"six": {six}}}"# + )); + let remedy = base_site_stale_install_remedy(Some(&develop), "six", dir); + assert!( + remedy.contains("`pipenv requirements --dev > pipenv-locked.txt"), + "{remedy}" + ); + let named = lock(&format!( + r#""default": {{"six": {six}}}, "docs": {{"six": {six}}}"# + )); + let remedy = base_site_stale_install_remedy(Some(&named), "six", dir); + assert!( + remedy.contains("`pipenv requirements --categories \"packages docs\" > "), + "{remedy}" + ); + assert!(remedy.contains("(Pipenv 2022.12+;"), "{remedy}"); + let remedy = base_site_stale_install_remedy(None, "six", dir); + assert!(remedy.contains("`pipenv requirements > "), "{remedy}"); + } } diff --git a/docs/testing/pipenv-compatibility.md b/docs/testing/pipenv-compatibility.md index e48a1c1c6..28e9c89fb 100644 --- a/docs/testing/pipenv-compatibility.md +++ b/docs/testing/pipenv-compatibility.md @@ -56,6 +56,17 @@ import on modern Pythons); 2018–2022 on Python 3.8; 2023+ on Python 3.12. re-syncs every non-empty category (#790). `pipenv uninstall ` is **not** a remedy: it rewrites the Pipfile and re-locks the patch away. `PIP_FORCE_REINSTALL=1 pipenv sync` works on 2018 but is ignored by 2026. +- **Base-interpreter copies survive both remedies.** A `--site-packages` + (`PIPENV_SITE_PACKAGES=1`) venv with no copy of its own imports the + release from its base interpreter (e.g. Debian's + `/usr/lib/python3/dist-packages`). pip refuses to uninstall it from the + venv, and `pipenv sync`, in the same or a recreated `--site-packages` + venv, counts it as installed, so the lock protects no install on that + host. For a stale copy there the warning instead prints `pipenv + requirements > pipenv-locked.txt && pipenv run pip install --no-deps + --ignore-installed -r pipenv-locked.txt` (Pipenv 2022.4+, with the same + category arguments), which installs the locked release inside the venv + and leaves the base copy untouched (#1450). - **Relocking drops the reference.** `pipenv lock` (and `update`, and `install ` on releases before 2024, where it is a full relock) regenerates the redirected entry to its registry reference on every major,