diff --git a/Cargo.lock b/Cargo.lock index 7f4a067cd2..e5b59b2fe3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -233,6 +233,7 @@ dependencies = [ "chrono", "dirs", "fs2", + "libc", "serde", "serde_json", "sha2 0.10.9", @@ -745,6 +746,7 @@ dependencies = [ name = "astra-tools" version = "0.2.13" dependencies = [ + "astra-config", "astra-core", "astra-memoria", "astra-prompts", @@ -760,6 +762,8 @@ dependencies = [ "futures", "gix", "hex", + "hickory-proto", + "ipnet", "libc", "nix 0.31.3", "regex", diff --git a/SECURITY.md b/SECURITY.md index 65ba705258..d0c82d3216 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -84,8 +84,16 @@ beyond a development machine, review the `ASTRA_RUNTIME_ROOT_SECRET`, and `MEMORIA_MASTER_KEY`. Never ship the template values. - Keep `.env`, `.env.production`, and `.models.yaml` out of version control. + The CLI loads `.env` from the working directory but refuses one that sets + `ASTRA_LOCAL_STATE_ROOT` or `MOI_AUTH_DIR`; set those in the shell. - Bind development services to loopback. `ASTRA_BIND_ADDRESS` defaults to `127.0.0.1` for a reason. +- Leave Edge/CLI `[web_fetch] fake_ip` at `block` unless a fake-IP VPN + blocks web access. `allow_doh_screened` relaxes the private-network SSRF + boundary for fake-IP answers: the VPN picks the real destination, and the + DoH screen cannot detect a host that answers the VPN with a private address; + see the + [configuration reference](docs/reference/configuration.md#local-web_fetch-behind-fake-ip-dns). - Remember that Runner placement controls where execution happens, not where data goes. Model endpoints, context disclosure, and tool-result handling remain explicit policy choices. diff --git a/crates/astra-cli/src/edge_tools.rs b/crates/astra-cli/src/edge_tools.rs index 861c6792dd..d68f6da16d 100644 --- a/crates/astra-cli/src/edge_tools.rs +++ b/crates/astra-cli/src/edge_tools.rs @@ -1624,7 +1624,7 @@ impl ToolExecutor { cancel_token: None, }, ) - .with_local_network(), + .with_local_network(astra_tools::web_fetch::configured_fake_ip_policy()), plan_mode_authoring_cache: std::sync::Arc::new(tokio::sync::RwLock::new(None)), ask_user_request_tx: std::sync::Mutex::new(None), plan_review_request_tx: std::sync::Mutex::new(None), @@ -5201,7 +5201,7 @@ impl ToolExecutor { let result = astra_tools::web_fetch::fetch_with_cache_scope( args, &cache_scope, - astra_tools::web_fetch::FetchTransport::LocalEnvironment, + self.default_executor.fetch_network(), ) .await; *source_is_error = Some(result.is_error); @@ -5491,7 +5491,7 @@ impl ToolExecutor { let result = astra_tools::web_search::perform_web_search( args, &cache_scope, - astra_tools::web_fetch::FetchTransport::LocalEnvironment, + self.default_executor.fetch_network(), ) .await; *source_is_error = Some(result.is_error); diff --git a/crates/astra-cli/src/entrypoint.rs b/crates/astra-cli/src/entrypoint.rs index 1c244fcc97..2707466548 100644 --- a/crates/astra-cli/src/entrypoint.rs +++ b/crates/astra-cli/src/entrypoint.rs @@ -55,26 +55,42 @@ use cli::project_instructions::{ // ════════════════════════════════════════════════════════════════ main ════ -fn validate_native_directory_source( +/// Variables that relocate trusted user state (credentials, user-level +/// configuration, permissions and workspace trust). A workspace `.env` must +/// not supply them; only the launching shell may. `HOME` is the fallback root +/// for all of these when the others are unset, so a sanitized environment +/// without it must not let the workspace choose it. +const SHELL_ONLY_ENV: [&str; 3] = [ + "MOI_AUTH_DIR", + astra_core::local_state::ASTRA_LOCAL_STATE_ROOT_ENV, + "HOME", +]; + +fn validate_shell_only_source( + name: &str, before: Option, after: Option, -) -> Result<(), &'static str> { +) -> Result<(), String> { if before != after { - return Err( - "MOI_AUTH_DIR cannot be supplied by a workspace .env; set it explicitly in your shell", - ); + return Err(format!( + "{name} cannot be supplied by a workspace .env; set it explicitly in your shell" + )); } Ok(()) } #[test] -fn workspace_dotenv_cannot_select_native_auth_directory() { - assert!(validate_native_directory_source(None, Some("/untrusted".into())).is_err()); - assert!(validate_native_directory_source(None, Some("".into())).is_err()); - assert!(validate_native_directory_source(None, None).is_ok()); - assert!( - validate_native_directory_source(Some("/trusted".into()), Some("/trusted".into())).is_ok() - ); +fn workspace_dotenv_cannot_select_trusted_state_roots() { + for name in SHELL_ONLY_ENV { + let rejected = validate_shell_only_source(name, None, Some("/untrusted".into())); + assert!(rejected.unwrap_err().contains(name)); + assert!(validate_shell_only_source(name, None, Some("".into())).is_err()); + assert!(validate_shell_only_source(name, None, None).is_ok()); + assert!( + validate_shell_only_source(name, Some("/trusted".into()), Some("/trusted".into())) + .is_ok() + ); + } } pub fn run() -> i32 { @@ -155,13 +171,13 @@ async fn run_async() -> i32 { // Chat/login can launch credential-consuming children too. Inspect the // actual loaded environment, not a second parse of a mutable .env file. if !explicit_env_config { - let native_directory = std::env::var_os("MOI_AUTH_DIR"); + let shell_values = SHELL_ONLY_ENV.map(std::env::var_os); dotenvy::dotenv().ok(); - if let Err(error) = - validate_native_directory_source(native_directory, std::env::var_os("MOI_AUTH_DIR")) - { - eprintln!("Error: {error}"); - return 2; + for (name, before) in SHELL_ONLY_ENV.into_iter().zip(shell_values) { + if let Err(error) = validate_shell_only_source(name, before, std::env::var_os(name)) { + eprintln!("Error: {error}"); + return 2; + } } } cli::diagnostic_log::init_cli_observability(&cli); diff --git a/crates/astra-cli/src/tui/config_edit_router.rs b/crates/astra-cli/src/tui/config_edit_router.rs index afa8aa5e5f..f09b72dba7 100644 --- a/crates/astra-cli/src/tui/config_edit_router.rs +++ b/crates/astra-cli/src/tui/config_edit_router.rs @@ -9,7 +9,7 @@ use crate::cli::session::{session_startup, session_state::SessionState}; use crate::tui::bottom_pane::view::ConfigEditDisposition; use astra_config::config_versions::{ConfigVersionStore, LocalFileStore, PutMetadata, VersionId}; -use astra_config::runtime_config::{RuntimeConfig, user_runtime_config_path}; +use astra_config::runtime_config::{RuntimeConfig, USER_ONLY_SECTIONS, user_runtime_config_path}; use std::path::PathBuf; /// Result of resolving a config-editor completion. @@ -149,9 +149,52 @@ fn save_and_report( std::fs::create_dir_all(parent) .map_err(|e| format!("Could not create {}: {e}", parent.display()))?; } - let pretty = toml::to_string_pretty(&parsed) + if scope == "project" + && let Some(user_path) = user_runtime_config_path() + && same_config_target(&path, &user_path) + { + return Err("Project and user configuration refer to the same file; save as user configuration instead".into()); + } + let complete = toml::to_string_pretty(&parsed) .map_err(|e| format!("Could not serialize config back to TOML: {e}"))?; - std::fs::write(&path, &pretty).map_err(|e| format!("Write failed {}: {e}", path.display()))?; + let pretty = if scope == "project" { + let mut value: toml::Value = toml::from_str(&complete) + .map_err(|e| format!("Could not prepare project configuration: {e}"))?; + if let Some(table) = value.as_table_mut() { + for section in USER_ONLY_SECTIONS { + table.remove(*section); + } + } + toml::to_string_pretty(&value) + .map_err(|e| format!("Could not serialize project configuration: {e}"))? + } else { + complete.clone() + }; + if scope == "user" { + use std::io::Write; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options + .open(&path) + .map_err(|e| format!("Write failed {}: {e}", path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(std::fs::Permissions::from_mode(0o600)) + .map_err(|e| format!("Could not protect {}: {e}", path.display()))?; + } + file.set_len(0) + .and_then(|_| file.write_all(pretty.as_bytes())) + .map_err(|e| format!("Write failed {}: {e}", path.display()))?; + } else { + std::fs::write(&path, &pretty) + .map_err(|e| format!("Write failed {}: {e}", path.display()))?; + } // Content-addressed put: computes the new id and dedups on repeat // saves. Best-effort — if the store is unavailable (no HOME, disk @@ -167,12 +210,12 @@ fn save_and_report( .put(&parsed, meta) .map(|id| id.as_str().to_string()) .unwrap_or_else(|_| { - VersionId::from_toml_bytes(pretty.as_bytes()) + VersionId::from_toml_bytes(complete.as_bytes()) .as_str() .to_string() }) } - None => VersionId::from_toml_bytes(pretty.as_bytes()) + None => VersionId::from_toml_bytes(complete.as_bytes()) .as_str() .to_string(), }; @@ -186,6 +229,29 @@ fn save_and_report( }) } +fn same_config_target(left: &std::path::Path, right: &std::path::Path) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if let (Ok(left), Ok(right)) = (std::fs::metadata(left), std::fs::metadata(right)) + && left.dev() == right.dev() + && left.ino() == right.ino() + { + return true; + } + } + fn resolve(path: &std::path::Path) -> Option { + std::fs::canonicalize(path).ok().or_else(|| { + Some( + std::fs::canonicalize(path.parent()?) + .ok()? + .join(path.file_name()?), + ) + }) + } + matches!((resolve(left), resolve(right)), (Some(left), Some(right)) if left == right) +} + fn scope_path(scope: &str) -> Result { match scope { "user" => user_runtime_config_path() @@ -203,6 +269,100 @@ mod tests { use super::finalize; use crate::tui::bottom_pane::view::ConfigEditDisposition; + #[test] + fn project_save_omits_user_only_policy_and_rejects_shared_target() { + const CHILD: &str = "ASTRA_TEST_PROJECT_CONFIG_SAVE"; + if std::env::var_os(CHILD).is_some() { + let trusted_root = std::env::var_os("ASTRA_LOCAL_STATE_ROOT").unwrap(); + let mut config = astra_config::RuntimeConfig::default(); + config.memory.retrieval_top_k = 7; + config.web_fetch.doh_resolvers = + vec!["https://dns.example/private-token?key=secret".into()]; + let body = toml::to_string_pretty(&config).unwrap(); + finalize(ConfigEditDisposition::SaveProject, &body).unwrap(); + let project = std::fs::read_to_string(".astra/config/runtime.toml").unwrap(); + assert!(!project.contains("web_fetch")); + assert!(!project.contains("private-token")); + assert_eq!( + toml::from_str::(&project) + .unwrap() + .memory + .retrieval_top_k, + 7 + ); + unsafe { + std::env::set_var("ASTRA_LOCAL_STATE_ROOT", ".astra"); + } + assert!( + finalize(ConfigEditDisposition::SaveProject, &body) + .unwrap_err() + .contains("same file") + ); + assert_eq!( + std::fs::read_to_string(".astra/config/runtime.toml").unwrap(), + project + ); + #[cfg(unix)] + { + let user_file = std::path::PathBuf::from(&trusted_root).join("config/runtime.toml"); + std::fs::create_dir_all(user_file.parent().unwrap()).unwrap(); + std::fs::write(&user_file, "user policy").unwrap(); + std::fs::remove_file(".astra/config/runtime.toml").unwrap(); + std::fs::hard_link(&user_file, ".astra/config/runtime.toml").unwrap(); + unsafe { + std::env::set_var("ASTRA_LOCAL_STATE_ROOT", trusted_root); + } + assert!( + finalize(ConfigEditDisposition::SaveProject, &body) + .unwrap_err() + .contains("same file") + ); + assert_eq!(std::fs::read_to_string(user_file).unwrap(), "user policy"); + } + return; + } + let workspace = tempfile::tempdir().unwrap(); + let state = tempfile::tempdir().unwrap(); + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "tui::config_edit_router::tests::project_save_omits_user_only_policy_and_rejects_shared_target", "--nocapture"]) + .env(CHILD, "1") + .env("ASTRA_LOCAL_STATE_ROOT", state.path()) + .current_dir(workspace.path()) + .output().unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + } + + #[cfg(unix)] + #[test] + #[serial_test::serial] + fn user_save_protects_new_and_existing_configuration_files() { + use std::os::unix::fs::PermissionsExt; + let root = tempfile::tempdir().unwrap(); + let _env = crate::test_utils::ProcessEnvGuard::remove("ASTRA_LOCAL_STATE_ROOT"); + unsafe { + std::env::set_var("ASTRA_LOCAL_STATE_ROOT", root.path()); + } + let mut config = astra_config::RuntimeConfig::default(); + config.web_fetch.doh_resolvers = vec!["https://dns.example/private-token".into()]; + let body = toml::to_string_pretty(&config).unwrap(); + finalize(ConfigEditDisposition::SaveUser, &body).unwrap(); + let file = root.path().join("config/runtime.toml"); + assert_eq!( + std::fs::metadata(&file).unwrap().permissions().mode() & 0o777, + 0o600 + ); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o644)).unwrap(); + finalize(ConfigEditDisposition::SaveUser, &body).unwrap(); + assert_eq!( + std::fs::metadata(&file).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + #[test] fn discard_and_cancel_produce_friendly_messages() { let a = finalize(ConfigEditDisposition::Discard, "").unwrap(); diff --git a/crates/astra-cli/tests/config_version_cli.rs b/crates/astra-cli/tests/config_version_cli.rs index a99c8effe0..120f5700cc 100644 --- a/crates/astra-cli/tests/config_version_cli.rs +++ b/crates/astra-cli/tests/config_version_cli.rs @@ -392,3 +392,121 @@ async fn configuration_layers_preserve_explicit_values_at_the_cli_entrypoint() { ); } } + +#[tokio::test] +async fn workspace_dotenv_cannot_relocate_the_user_configuration_root() { + use astra_config::config_versions::VersionId; + use astra_config::runtime_config::FakeIpMode; + + // A cloned repository ships a `.env` pointing the user root at its own + // `.astra`, whose runtime.toml would otherwise load as user-level policy. + let workspace = tempfile::tempdir().unwrap(); + let home = workspace.path().join("home"); + let trusted = workspace.path().join("trusted"); + std::fs::create_dir_all(&home).unwrap(); + std::fs::create_dir_all(workspace.path().join(".astra/config")).unwrap(); + std::fs::write( + workspace.path().join(".env"), + "ASTRA_LOCAL_STATE_ROOT=.astra\n", + ) + .unwrap(); + std::fs::write( + workspace.path().join(".astra/config/runtime.toml"), + r#" + [web_fetch] + fake_ip = "allow_doh_screened" + doh_resolvers = ["https://doh.attacker.example/dns-query"] + "#, + ) + .unwrap(); + let run_with = |shell_home: Option<&std::path::Path>, shell_root: Option<&std::path::Path>| { + let mut command = tokio::process::Command::new(env!("CARGO_BIN_EXE_astra")); + command + .env_clear() + .current_dir(workspace.path()) + .args(["config", "version", "current"]) + .stdin(std::process::Stdio::null()) + .kill_on_drop(true); + if let Some(home) = shell_home { + command.env("HOME", home); + } + if let Some(root) = shell_root { + command.env("ASTRA_LOCAL_STATE_ROOT", root); + } + async move { + tokio::time::timeout(std::time::Duration::from_secs(10), command.output()) + .await + .expect("local configuration command must finish without network or input") + .unwrap() + } + }; + let run = |shell_root: Option<&std::path::Path>| run_with(Some(&home), shell_root); + let hash = |config: &RuntimeConfig| { + VersionId::from_toml_bytes(toml::to_string_pretty(config).unwrap().as_bytes()) + }; + + let output = run(None).await; + assert_eq!(output.status.code(), Some(2)); + let diagnostic = String::from_utf8_lossy(&output.stderr); + assert!( + diagnostic.contains("ASTRA_LOCAL_STATE_ROOT cannot be supplied by a workspace .env"), + "{diagnostic}" + ); + assert!(output.stdout.is_empty()); + + // A shell-selected root wins over the `.env` value, and the same file read + // as the project layer cannot enable the user-only policy. + let output = run(Some(&trusted)).await; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8(output.stdout).unwrap().trim(), + hash(&RuntimeConfig::default()).as_str() + ); + + // The user may still deliberately select that directory from the shell. + let output = run(Some(&workspace.path().join(".astra"))).await; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let mut opted_in = RuntimeConfig::default(); + opted_in.web_fetch.fake_ip = FakeIpMode::AllowDohScreened; + opted_in.web_fetch.doh_resolvers = vec!["https://doh.attacker.example/dns-query".into()]; + assert_eq!( + String::from_utf8(output.stdout).unwrap().trim(), + hash(&opted_in).as_str() + ); + + // `HOME` is the fallback root. A sanitized environment without it must + // not let the `.env` point it at the workspace, whose `.astra` would then + // load unfiltered as the user layer. + std::fs::write( + workspace.path().join(".env"), + format!("HOME={}\n", workspace.path().display()), + ) + .unwrap(); + let output = run_with(None, None).await; + assert_eq!(output.status.code(), Some(2)); + let diagnostic = String::from_utf8_lossy(&output.stderr); + assert!( + diagnostic.contains("HOME cannot be supplied by a workspace .env"), + "{diagnostic}" + ); + assert!(output.stdout.is_empty()); + // A shell HOME is kept; the `.env` value does not override it. + let output = run(None).await; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8(output.stdout).unwrap().trim(), + hash(&RuntimeConfig::default()).as_str() + ); +} diff --git a/crates/astra-config/Cargo.toml b/crates/astra-config/Cargo.toml index 93455d6ce7..51b04e9cf1 100644 --- a/crates/astra-config/Cargo.toml +++ b/crates/astra-config/Cargo.toml @@ -11,12 +11,13 @@ astra-turn-types.workspace = true chrono = { workspace = true, features = ["serde"] } dirs = "6.0.0" fs2 = "0.4.3" +libc.workspace = true serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } sha2.workspace = true thiserror.workspace = true toml = { version = "1.1.2", features = ["preserve_order"] } tracing = { workspace = true } +tempfile = "3" [dev-dependencies] -tempfile = "3" diff --git a/crates/astra-config/src/config_overlay.rs b/crates/astra-config/src/config_overlay.rs index e5e6678b51..4a43e4418e 100644 --- a/crates/astra-config/src/config_overlay.rs +++ b/crates/astra-config/src/config_overlay.rs @@ -97,8 +97,30 @@ impl RuntimeConfigLayer { } pub fn from_toml(source: &str) -> Result { - let config: RuntimeConfig = toml::from_str(source)?; + Self::from_toml_value(toml::from_str(source)?) + } + + /// Parse a TOML layer that may not set `sections`. They are dropped before + /// typed validation, so a malformed disallowed section cannot reject the + /// rest of the layer. Returns the sections the source actually carried. + pub fn from_toml_without_sections( + source: &str, + sections: &[&'static str], + ) -> Result<(Self, Vec<&'static str>), OverlayError> { let mut value: toml::Value = toml::from_str(source)?; + let removed = match value.as_table_mut() { + Some(table) => sections + .iter() + .copied() + .filter(|section| table.remove(*section).is_some()) + .collect(), + None => Vec::new(), + }; + Ok((Self::from_toml_value(value)?, removed)) + } + + fn from_toml_value(mut value: toml::Value) -> Result { + let config: RuntimeConfig = value.clone().try_into()?; // TOML supports non-finite floats, JSON does not. Preserve the existing // trace normalization before conversion instead of turning them into null. if let Some(rate) = value diff --git a/crates/astra-config/src/config_versions.rs b/crates/astra-config/src/config_versions.rs index 3887ab0b1d..1742927437 100644 --- a/crates/astra-config/src/config_versions.rs +++ b/crates/astra-config/src/config_versions.rs @@ -33,7 +33,7 @@ use fs2::FileExt; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::fs::{self, OpenOptions}; -use std::io::{BufRead, BufReader, Write}; +use std::io::{BufRead, BufReader, Read, Write}; use std::path::PathBuf; // ─── Public types ──────────────────────────────────────────────────────── @@ -121,6 +121,8 @@ pub enum StoreError { }, #[error("corrupt index row: {0}")] CorruptIndex(String), + #[error("corrupt config version blob: {0}")] + CorruptBlob(String), } // ─── Trait ─────────────────────────────────────────────────────────────── @@ -176,10 +178,10 @@ impl LocalFileStore { }) } - fn append_index(&self, entry: &IndexEntry) -> Result<(), StoreError> { + fn lock_index(&self) -> Result { self.ensure_root()?; let path = self.index_path(); - let mut f = OpenOptions::new() + let f = OpenOptions::new() .create(true) .append(true) .open(&path) @@ -192,10 +194,13 @@ impl LocalFileStore { source, })?; - // Hold an advisory file lock for the complete append. `write_all` - // may loop internally, so O_APPEND alone cannot guarantee a - // one-row-per-line record when multiple CLI processes save config - // versions at the same time. + Ok(f) + } + + fn append_index(&self, mut f: fs::File, entry: &IndexEntry) -> Result<(), StoreError> { + let path = self.index_path(); + // The caller holds the index lock across blob publication and append. + // `write_all` may loop internally, so O_APPEND alone is insufficient. let mut line = serde_json::to_string(entry).map_err(|e| StoreError::CorruptIndex(e.to_string()))?; line.push('\n'); @@ -248,22 +253,89 @@ impl ConfigVersionStore for LocalFileStore { // round-trip is the bytes, not the struct. let toml_bytes = toml::to_string_pretty(config)?; let id = VersionId::from_toml_bytes(toml_bytes.as_bytes()); + // Reuse the index lock to serialize publication and validation. This + // also protects the hard-link based no-clobber fallback used on some + // filesystems from another put observing its temporary second link. + let index = self.lock_index()?; - // 2. Write blob iff absent. `create_new` would race two - // concurrent puts; we check-then-write on the assumption - // that collisions on the same content are fine. + // 2. Publish a fully written private blob without following an existing + // destination. Concurrent writers may produce the same version. let blob = self.blob_path(&id); if !blob.exists() { - let mut f = fs::File::create(&blob).map_err(|source| StoreError::Io { + let mut staged = + tempfile::NamedTempFile::new_in(&self.root).map_err(|source| StoreError::Io { + path: self.root.clone(), + source, + })?; + staged + .write_all(toml_bytes.as_bytes()) + .map_err(|source| StoreError::Io { + path: blob.clone(), + source, + })?; + if let Err(error) = staged.persist_noclobber(&blob) + && error.error.kind() != std::io::ErrorKind::AlreadyExists + { + return Err(StoreError::Io { + path: blob.clone(), + source: error.error, + }); + } + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + #[cfg(not(unix))] + if fs::symlink_metadata(&blob).is_ok_and(|m| m.file_type().is_symlink()) { + return Err(StoreError::Io { path: blob.clone(), - source, - })?; - f.write_all(toml_bytes.as_bytes()) + source: std::io::Error::other("config version blob is a symlink"), + }); + } + let mut existing = options.open(&blob).map_err(|source| StoreError::Io { + path: blob.clone(), + source, + })?; + let metadata = existing.metadata().map_err(|source| StoreError::Io { + path: blob.clone(), + source, + })?; + if !metadata.is_file() { + return Err(StoreError::CorruptBlob( + "version is not a regular file".into(), + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + if metadata.nlink() != 1 { + return Err(StoreError::CorruptBlob( + "version has multiple hard links".into(), + )); + } + existing + .set_permissions(fs::Permissions::from_mode(0o600)) .map_err(|source| StoreError::Io { path: blob.clone(), source, })?; } + let mut contents = String::new(); + existing + .read_to_string(&mut contents) + .map_err(|source| StoreError::Io { + path: blob.clone(), + source, + })?; + if contents != toml_bytes { + return Err(StoreError::CorruptBlob( + "content does not match its version".into(), + )); + } // 3. Append index row — always, even on duplicate put. The // second session that arrives at the same config IS a @@ -274,7 +346,7 @@ impl ConfigVersionStore for LocalFileStore { source_session: meta.source_session, parent: meta.parent, }; - self.append_index(&entry)?; + self.append_index(index, &entry)?; Ok(id) } diff --git a/crates/astra-config/src/runtime_config.rs b/crates/astra-config/src/runtime_config.rs index 15f893f31f..461b946f4a 100644 --- a/crates/astra-config/src/runtime_config.rs +++ b/crates/astra-config/src/runtime_config.rs @@ -11,7 +11,7 @@ //! 4. Environment variables (ASTRA_*) //! 5. Explicit CLI settings and trace flags -use crate::config_overlay::RuntimeConfigLayer; +use crate::config_overlay::{OverlayError, RuntimeConfigLayer}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -70,6 +70,45 @@ pub struct RuntimeConfig { /// Explain Analyze presentation and capture preferences. #[serde(default)] pub explain: ExplainConfig, + + /// Local `web_fetch` network policy. User-level only: the project layer + /// cannot set it (see [`USER_ONLY_SECTIONS`]). + #[serde(default)] + pub web_fetch: WebFetchConfig, +} + +/// Sections a project `.astra/config/runtime.toml` must not set. They relax a +/// safety boundary, and a project file arrives with untrusted repositories. +pub const USER_ONLY_SECTIONS: &[&str] = &["web_fetch"]; + +/// How Edge/CLI `web_fetch` treats answers from a fake-IP DNS (Clash/mihomo, +/// Surge, sing-box). Server transports never read this. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct WebFetchConfig { + /// `block` (default) treats fake-IP answers like any reserved address. + /// `allow_doh_screened` relaxes the private-network boundary for them: + /// the fetch goes to the fake address, and the VPN chooses the real + /// destination. `doh_resolvers` only screen out hosts whose public DNS + /// answer is private; they cannot confirm where the VPN connects. + #[serde(default)] + pub fake_ip: FakeIpMode, + /// IPv4 CIDRs the local DNS uses for fake answers. Empty means + /// `198.18.0.0/15`; entries must lie inside that range. + #[serde(default)] + pub fake_ip_ranges: Vec, + /// HTTPS DNS-over-HTTPS endpoints (RFC 8484), tried in order. Required + /// when `fake_ip = "allow_doh_screened"`; there is no built-in provider. + #[serde(default)] + pub doh_resolvers: Vec, +} + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum FakeIpMode { + #[default] + Block, + AllowDohScreened, } /// Human-readable Explain Analyze report formats. @@ -305,6 +344,7 @@ impl Default for RuntimeConfig { runtime_limits: RuntimeLimitsConfig::default(), agent_binding_registry: AgentBindingRegistryConfig::default(), explain: ExplainConfig::default(), + web_fetch: WebFetchConfig::default(), } } } @@ -1192,6 +1232,20 @@ pub fn user_runtime_config_path() -> Option { // ─── Configuration Loading ─────────────────────────────────────────────────── +/// Parse a project `runtime.toml` without the sections only the user may set. +fn project_layer_from_toml(content: &str) -> Result { + let (layer, ignored) = + RuntimeConfigLayer::from_toml_without_sections(content, USER_ONLY_SECTIONS)?; + for section in ignored { + tracing::warn!( + section, + "ignoring user-only section in project runtime.toml; \ + set it in the user-level runtime.toml instead" + ); + } + Ok(layer) +} + impl RuntimeConfig { /// Return a process-wide cached `RuntimeConfig`. /// @@ -1238,8 +1292,7 @@ impl RuntimeConfig { // Project-level config let project_config = PathBuf::from(".astra/config/runtime.toml"); if let Ok(content) = std::fs::read_to_string(&project_config) { - match RuntimeConfigLayer::from_toml(&content).and_then(|layer| layer.apply_to(&config)) - { + match project_layer_from_toml(&content).and_then(|layer| layer.apply_to(&config)) { Ok(project) => config = project, Err(err) => { tracing::warn!( @@ -2210,6 +2263,59 @@ mod tests { assert!(result.is_err(), "unknown trust_mode should fail to parse"); } + #[test] + fn project_layer_cannot_set_user_only_web_fetch_policy() { + let user = RuntimeConfigLayer::from_toml( + "[web_fetch]\nfake_ip = \"allow_doh_screened\"\ndoh_resolvers = [\"https://1.12.12.12/dns-query\"]\n", + ) + .unwrap() + .apply_to(&RuntimeConfig::default()) + .unwrap(); + assert_eq!(user.web_fetch.fake_ip, FakeIpMode::AllowDohScreened); + + // A cloned repository can neither enable nor redirect the opt-in, and + // the rest of its project file still applies. + let project = project_layer_from_toml( + "[web_fetch]\nfake_ip = \"allow_doh_screened\"\ndoh_resolvers = [\"https://evil.example/dns-query\"]\n\ + [safety]\ntrust_mode = \"strict\"\n", + ) + .unwrap(); + let merged = project.apply_to(&RuntimeConfig::default()).unwrap(); + assert_eq!(merged.web_fetch, WebFetchConfig::default()); + assert_eq!(merged.safety.trust_mode, Some(TrustModeSerde::Strict)); + let merged = project.apply_to(&user).unwrap(); + assert_eq!(merged.web_fetch, user.web_fetch); + + // The ignored section is not validated, so a malformed one cannot + // discard the project's other settings. + for ignored in [ + "[web_fetch]\nfake_ip = \"typo\"\n", + "[web_fetch]\nunknown_field = true\n", + "web_fetch = 3\n", + ] { + let project = + project_layer_from_toml(&format!("{ignored}[safety]\ntrust_mode = \"strict\"\n")) + .unwrap_or_else(|error| panic!("{ignored}: {error}")); + let merged = project.apply_to(&RuntimeConfig::default()).unwrap(); + assert_eq!(merged.web_fetch, WebFetchConfig::default()); + assert_eq!(merged.safety.trust_mode, Some(TrustModeSerde::Strict)); + } + // Other sections are still validated. + assert!(project_layer_from_toml("[safety]\ntrust_mode = \"typo\"\n").is_err()); + } + + #[test] + fn web_fetch_policy_defaults_off_and_rejects_unknown_fields() { + assert_eq!( + RuntimeConfig::default().web_fetch.fake_ip, + FakeIpMode::Block + ); + assert!( + RuntimeConfigLayer::from_toml("[web_fetch]\nfake_ip_validation = \"doh\"\n").is_err() + ); + assert!(RuntimeConfigLayer::from_toml("[web_fetch]\nfake_ip = \"on\"\n").is_err()); + } + #[test] fn safety_layers_preserve_absence_and_allow_explicit_strict() { let strict = RuntimeConfig::default(); diff --git a/crates/astra-config/tests/config_versions.rs b/crates/astra-config/tests/config_versions.rs index 5413fa7836..ec90eef2a9 100644 --- a/crates/astra-config/tests/config_versions.rs +++ b/crates/astra-config/tests/config_versions.rs @@ -36,6 +36,46 @@ //! 7. **Missing id is a structured error**, not a panic. use astra_config::config_versions::{ConfigVersionStore, LocalFileStore, PutMetadata, VersionId}; + +#[cfg(unix)] +#[test] +fn version_blobs_with_resolver_tokens_are_private_and_never_follow_symlinks() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let (root, store) = tmp_store(); + let mut config = astra_config::RuntimeConfig::default(); + config.web_fetch.doh_resolvers = vec!["https://dns.example/private-token?key=secret".into()]; + let id = store.put(&config, PutMetadata::default()).unwrap(); + let blob = root.path().join(format!("{}.toml", id.as_str())); + assert_eq!( + std::fs::metadata(&blob).unwrap().permissions().mode() & 0o777, + 0o600 + ); + std::fs::set_permissions(&blob, std::fs::Permissions::from_mode(0o644)).unwrap(); + store.put(&config, PutMetadata::default()).unwrap(); + assert_eq!( + std::fs::metadata(&blob).unwrap().permissions().mode() & 0o777, + 0o600 + ); + std::fs::remove_file(&blob).unwrap(); + let target = root.path().join("other-file"); + std::fs::write(&target, "untouched").unwrap(); + symlink(&target, &blob).unwrap(); + assert!(store.put(&config, PutMetadata::default()).is_err()); + assert_eq!(std::fs::read_to_string(&target).unwrap(), "untouched"); + std::fs::remove_file(&blob).unwrap(); + std::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o644)).unwrap(); + std::fs::hard_link(&target, &blob).unwrap(); + assert!(store.put(&config, PutMetadata::default()).is_err()); + assert_eq!( + std::fs::metadata(&target).unwrap().permissions().mode() & 0o777, + 0o644 + ); + std::fs::remove_file(&blob).unwrap(); + use std::os::unix::ffi::OsStrExt; + let fifo = std::ffi::CString::new(blob.as_os_str().as_bytes()).unwrap(); + assert_eq!(unsafe { libc::mkfifo(fifo.as_ptr(), 0o600) }, 0); + assert!(store.put(&config, PutMetadata::default()).is_err()); +} use astra_config::runtime_config::RuntimeConfig; use fs2::FileExt; use std::fs::OpenOptions; @@ -263,6 +303,31 @@ fn duplicate_put_with_different_metadata_keeps_first_blob_records_second_index_r ); } +#[test] +fn concurrent_puts_publish_one_complete_blob_and_all_index_rows() { + let (dir, store) = tmp_store(); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(6)); + let mut threads = Vec::new(); + for _ in 0..6 { + let barrier = barrier.clone(); + let root = dir.path().to_path_buf(); + threads.push(std::thread::spawn(move || { + barrier.wait(); + LocalFileStore::new(root) + .put(&RuntimeConfig::default(), PutMetadata::default()) + .unwrap() + })); + } + let ids: Vec<_> = threads + .into_iter() + .map(|thread| thread.join().unwrap()) + .collect(); + assert!(ids.iter().all(|id| id == &ids[0])); + assert_eq!(store.list().unwrap().len(), 6); + let body = store.get_toml(&ids[0]).unwrap().unwrap(); + assert!(toml::from_str::(&body).is_ok()); +} + #[test] fn put_waits_for_exclusive_index_lock_before_appending() { let (dir, store) = tmp_store(); diff --git a/crates/astra-edge/src/main.rs b/crates/astra-edge/src/main.rs index 4442562bfb..8efc245933 100644 --- a/crates/astra-edge/src/main.rs +++ b/crates/astra-edge/src/main.rs @@ -812,7 +812,7 @@ async fn run_edge_connection(config: &EdgeConfig) -> Result<(), Box>, progress_callback: Option>, @@ -108,7 +108,7 @@ impl DefaultToolExecutor { pub fn new(ctx: ToolContext) -> Self { Self { ctx, - fetch_transport: Default::default(), + fetch_network: Default::default(), approval_gate: None, progress_callback: None, @@ -143,13 +143,23 @@ impl DefaultToolExecutor { self } - /// Install the network authority of a user-owned execution boundary. - /// Server constructors intentionally retain the pinned-direct default. - pub fn with_local_network(mut self) -> Self { - self.fetch_transport = crate::web_fetch::FetchTransport::LocalEnvironment; + /// Install the network authority of a user-owned execution boundary, + /// with the user's fake-IP opt-in when configured (see + /// [`crate::web_fetch::configured_fake_ip_policy`]). Server constructors + /// intentionally retain the pinned-direct default. + pub fn with_local_network( + mut self, + fake_ip: Result>, String>, + ) -> Self { + self.fetch_network = crate::web_fetch::FetchNetwork::local_environment(fake_ip); self } + /// Network authority used by this executor's web tools. + pub fn fetch_network(&self) -> &crate::web_fetch::FetchNetwork { + &self.fetch_network + } + /// Access the underlying context. pub fn context(&self) -> &ToolContext { &self.ctx @@ -631,7 +641,7 @@ impl DefaultToolExecutor { // ── Web search ─────────────────────────────────────────── "web_search" => { let cache_scope = format!("{}:{}", self.ctx.user_id, self.ctx.session_id); - crate::web_search::perform_web_search(args, &cache_scope, self.fetch_transport).await + crate::web_search::perform_web_search(args, &cache_scope, &self.fetch_network).await } // ── Utility tools ──────────────────────────────────────── @@ -664,7 +674,7 @@ impl DefaultToolExecutor { // ── Web fetch (HTTP GET) ───────────────────────────────── "web_fetch" => { let cache_scope = format!("{}:{}", self.ctx.user_id, self.ctx.session_id); - crate::web_fetch::fetch_with_cache_scope(args, &cache_scope, self.fetch_transport).await + crate::web_fetch::fetch_with_cache_scope(args, &cache_scope, &self.fetch_network).await } // ── Display sixel (terminal image rendering) ────────────── diff --git a/crates/astra-tools/src/web_fetch.rs b/crates/astra-tools/src/web_fetch.rs index 0fc5bdf03f..13a03a78b5 100644 --- a/crates/astra-tools/src/web_fetch.rs +++ b/crates/astra-tools/src/web_fetch.rs @@ -3,10 +3,11 @@ //! Fetches URLs and transforms raw HTML into model-friendly Markdown with //! metadata extraction, link discovery, and content-aware routing. -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; +use ipnet::Ipv6Net; use reqwest::header; use serde::Serialize; use serde_json::Value; @@ -14,6 +15,10 @@ use tokio::sync::Mutex; use crate::ToolResult; +mod fake_ip; + +pub use fake_ip::{FakeIpPolicy, configured_fake_ip_policy}; + /// Selected by the execution owner, never by tool arguments. A user-owned /// proxy is a trusted network boundary and may resolve destinations itself; /// Server traffic must retain direct, pinned connections instead. @@ -25,6 +30,41 @@ pub enum FetchTransport { LocalEnvironment, } +/// The network authority a fetch runs under. Only the user-owned local +/// environment can carry the user's fake-IP opt-in; Server transports are +/// constructed without one and cannot acquire it. +#[derive(Debug, Clone)] +pub struct FetchNetwork { + transport: FetchTransport, + fake_ip: Result>, String>, +} + +impl Default for FetchNetwork { + fn default() -> Self { + Self { + transport: FetchTransport::DirectPinned, + fake_ip: Ok(None), + } + } +} + +impl FetchNetwork { + pub fn direct_pinned() -> Self { + Self::default() + } + + pub fn local_environment(fake_ip: Result>, String>) -> Self { + Self { + transport: FetchTransport::LocalEnvironment, + fake_ip, + } + } + + pub fn transport(&self) -> FetchTransport { + self.transport + } +} + pub use convert::{to_markdown, to_text}; pub use extract::{ExtractedLink, PageMetadata, extract_links, extract_metadata}; @@ -71,8 +111,112 @@ pub enum FetchError { Network(String), #[error("{0}")] Timeout(String), - #[error("SSRF blocked: {0} resolves to a private/internal address")] - SsrfBlocked(String), + #[error("{0}")] + SsrfBlocked(Box), +} + +impl FetchError { + fn ssrf(host: &str, addrs: Vec, reason: SsrfBlockReason) -> Self { + Self::SsrfBlocked(Box::new(SsrfBlock { + host: host.to_string(), + addrs, + reason, + })) + } +} + +#[derive(Debug)] +pub struct SsrfBlock { + host: String, + addrs: Vec, + reason: SsrfBlockReason, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum SsrfBlockReason { + PrivateAddress, + /// Every address is in the fake-IP range and the user has not opted in. + FakeIpNotEnabled, + FakeIpInvalidConfiguration(String), + /// The answer looks like fake-IP DNS but is not purely IPv4 fake + /// addresses (an IPv6 fake range, or fake mixed with real addresses). + FakeIpUnsupportedAnswer, + FakeIpDohUnavailable(String), + FakeIpNoDohAnswer, + FakeIpRealAddressPrivate(Vec), +} + +/// The message is read by the model first, so each case says what is known +/// and who may act. The opt-in is the user's; the model must not enable it. +impl std::fmt::Display for SsrfBlock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let host = &self.host; + let addrs = join_ips(&self.addrs); + match &self.reason { + SsrfBlockReason::PrivateAddress if self.addrs.is_empty() => { + write!(f, "SSRF blocked: {host} is a private/internal address") + } + SsrfBlockReason::PrivateAddress => write!( + f, + "SSRF blocked: {host} resolves to a private/internal address ({addrs})" + ), + SsrfBlockReason::FakeIpNotEnabled => write!( + f, + "SSRF blocked: {host} resolves to {addrs}, inside {range}. Local VPN/proxy \ + tools with fake-IP DNS (Clash/mihomo, Surge, sing-box) answer from this \ + range and choose the real destination themselves, so Astra cannot check \ + it. The user can choose to relax private-network protection for such \ + answers in their user-level runtime.toml (~/.astra/config/runtime.toml); \ + a DNS-over-HTTPS lookup then screens out hosts whose public address is \ + private, but cannot confirm where the VPN connects:\n\n[web_fetch]\n\ + fake_ip = \"allow_doh_screened\"\n\ + doh_resolvers = [\"https:///dns-query\"]\n\n\ + See docs/reference/configuration.md. Tell the user about this option; \ + do not change their configuration on their behalf.", + range = fake_ip::FAKE_IP_RANGE, + ), + SsrfBlockReason::FakeIpInvalidConfiguration(detail) => write!( + f, + "SSRF blocked: {host} resolves to fake-IP address {addrs}, but the local \ + [web_fetch] configuration is invalid: {detail}" + ), + SsrfBlockReason::FakeIpUnsupportedAnswer => write!( + f, + "SSRF blocked: {host} resolves to {addrs}, which looks like fake-IP DNS from \ + a local VPN/proxy. Only answers made entirely of IPv4 fake addresses in {v4} \ + can be allowed; IPv6 fake-IP ranges such as {v6} and answers mixing fake \ + and real addresses are not supported. Tell the user; do not change their \ + configuration on their behalf.", + v4 = fake_ip::FAKE_IP_RANGE, + v6 = fake_ip::FAKE_IPV6_RANGE, + ), + SsrfBlockReason::FakeIpDohUnavailable(detail) => write!( + f, + "SSRF blocked: {host} resolves to fake-IP address {addrs}, and no configured \ + DoH resolver returned a usable response ({detail}), so the host could not be screened. The user can configure a reachable endpoint in [web_fetch] \ + doh_resolvers of their user-level runtime.toml." + ), + SsrfBlockReason::FakeIpNoDohAnswer => write!( + f, + "SSRF blocked: {host} resolves to fake-IP address {addrs}, but DoH returned \ + no address for it, so the host could not be screened." + ), + SsrfBlockReason::FakeIpRealAddressPrivate(real) => write!( + f, + "SSRF blocked: {host} resolves to fake-IP address {addrs}, and the DoH response \ + contains a private/internal address ({real})", + real = join_ips(real), + ), + } + } +} + +fn join_ips(addrs: &[IpAddr]) -> String { + addrs + .iter() + .map(ToString::to_string) + .collect::>() + .join(", ") } // ─── Configuration ─────────────────────────────────────────────────────────── @@ -239,19 +383,34 @@ const MAX_WALK_DEPTH: usize = 256; pub async fn fetch_with_cache_scope( args: &Value, cache_scope: &str, - transport: FetchTransport, + network: &FetchNetwork, ) -> ToolResult { let timeout = Duration::from_secs(args.get("timeout").and_then(Value::as_u64).unwrap_or(30)); - let result = tokio::time::timeout(timeout, fetch_inner(args, cache_scope, transport)) + let result = tokio::time::timeout(timeout, fetch_inner(args, cache_scope, network)) .await .unwrap_or_else(|_| Err(FetchError::Timeout("Total fetch deadline exceeded".into()))); - render_fetch_result(result, transport) + render_fetch_result(result, network) } fn render_fetch_result( - result: Result, FetchError>, - transport: FetchTransport, + mut result: Result, FetchError>, + network: &FetchNetwork, ) -> ToolResult { + if let Err(FetchError::SsrfBlocked(block)) = &mut result { + if network.transport == FetchTransport::DirectPinned + && matches!( + block.reason, + SsrfBlockReason::FakeIpNotEnabled | SsrfBlockReason::FakeIpUnsupportedAnswer + ) + { + block.reason = SsrfBlockReason::PrivateAddress; + } else if block.reason == SsrfBlockReason::FakeIpNotEnabled + && let Err(detail) = &network.fake_ip + { + block.reason = SsrfBlockReason::FakeIpInvalidConfiguration(detail.clone()); + } + } + let transport = network.transport(); use astra_core::{ErrorKind, ToolFailureEvidence}; let (kind, detail, response) = match result { Ok(result) if result.status < 400 => { @@ -296,8 +455,10 @@ fn render_fetch_result( async fn fetch_inner( args: &Value, cache_scope: &str, - transport: FetchTransport, + network: &FetchNetwork, ) -> Result, FetchError> { + let transport = network.transport(); + let fake_ip = network.fake_ip.as_ref().ok().and_then(Option::as_deref); let (raw_url, config) = FetchConfig::from_args(args)?; let url = if config.allow_http { raw_url.clone() @@ -305,10 +466,10 @@ async fn fetch_inner( upgrade_scheme(&raw_url) }; validate_url(&url)?; - // Resolve once for validation; the actual fetch paths re-resolve per hop - // during redirect following (different host), so this is intentionally - // consumed only for early fail on private-IP targets. - let _early_validate = resolve_and_validate_host(&url).await?; + // Validate before the cache lookup so a cached body is never served for a + // host that is now blocked. A cache miss pins the first hop to these + // addresses; redirects are resolved and validated per hop. + let first_hop = resolve_and_validate_host(&url, fake_ip).await?; let cache_key = CacheKey { scope: cache_scope.to_string(), @@ -330,7 +491,7 @@ async fn fetch_inner( let start = Instant::now(); let (status, final_url, content_type, body) = - fetch_reqwest(&url, config.timeout, transport).await?; + fetch_reqwest(&url, first_hop, config.timeout, transport, fake_ip).await?; // Store one bounded representation independent of the caller's requested // slice. A later model round asking for 8k after an initial 3k read can // reuse the same network response; only the returned projection changes. @@ -448,7 +609,11 @@ fn validate_url(url: &str) -> Result<(), FetchError> { .ok_or_else(|| FetchError::Validation("URL has no host".into()))?; if is_private_host(host) { - return Err(FetchError::SsrfBlocked(host.to_string())); + return Err(FetchError::ssrf( + host, + Vec::new(), + SsrfBlockReason::PrivateAddress, + )); } Ok(()) @@ -457,7 +622,10 @@ fn validate_url(url: &str) -> Result<(), FetchError> { /// Resolve DNS and validate ALL returned addresses are public. /// Returns the validated addresses so callers can pin them for the connection, /// preventing DNS rebinding (where a second resolution returns a different IP). -async fn resolve_and_validate_host(url: &str) -> Result, FetchError> { +async fn resolve_and_validate_host( + url: &str, + fake_ip: Option<&FakeIpPolicy>, +) -> Result, FetchError> { let parsed = url::Url::parse(url).map_err(|e| FetchError::Validation(format!("Invalid URL: {e}")))?; let host = parsed @@ -468,10 +636,15 @@ async fn resolve_and_validate_host(url: &str) -> Result, FetchEr .port_or_known_default() .ok_or_else(|| FetchError::Validation("URL has no port for scheme".into()))?; - // If host is already an IP literal, validate directly. + // If host is already an IP literal, validate directly. A literal fake-IP + // address has no hostname to verify, so it is never admitted. if let Ok(ip) = host.parse::() { if is_private_ip(ip) { - return Err(FetchError::SsrfBlocked(host.to_string())); + return Err(FetchError::ssrf( + host, + Vec::new(), + SsrfBlockReason::PrivateAddress, + )); } return Ok(vec![SocketAddr::new(ip, port)]); } @@ -487,12 +660,44 @@ async fn resolve_and_validate_host(url: &str) -> Result, FetchEr ))); } - for addr in &addrs { - if is_private_ip(addr.ip()) { - return Err(FetchError::SsrfBlocked(host.to_string())); + validate_resolved_addrs(host, addrs, fake_ip).await +} + +/// Admit public answers. Private answers are blocked unless every one lies in +/// the user's IPv4 fake-IP range, the user relaxed the boundary for such +/// answers, and DoH screening passes. The pinned connection uses the fake +/// addresses, so the local VPN, not this check, picks the real destination. +async fn validate_resolved_addrs( + host: &str, + addrs: Vec, + fake_ip: Option<&FakeIpPolicy>, +) -> Result, FetchError> { + let ips: Vec = addrs.iter().map(SocketAddr::ip).collect(); + if !ips.iter().copied().any(is_private_ip) { + return Ok(addrs); + } + let ipv4_fake = |ip: &IpAddr| match (ip, fake_ip) { + (IpAddr::V4(_), Some(policy)) => policy.contains(*ip), + (IpAddr::V4(v4), None) => fake_ip::FAKE_IP_RANGE.contains(v4), + (IpAddr::V6(_), _) => false, + }; + let reason = if ips.iter().all(ipv4_fake) { + match fake_ip { + Some(policy) => match policy.screen(host).await { + Ok(()) => return Ok(addrs), + Err(reason) => reason, + }, + None => SsrfBlockReason::FakeIpNotEnabled, } - } - Ok(addrs) + } else if ips + .iter() + .all(|ip| ipv4_fake(ip) || fake_ip::is_ipv6_fake_candidate(*ip) || !is_private_ip(*ip)) + { + SsrfBlockReason::FakeIpUnsupportedAnswer + } else { + SsrfBlockReason::PrivateAddress + }; + Err(FetchError::ssrf(host, ips, reason)) } fn is_private_host(host: &str) -> bool { @@ -547,16 +752,33 @@ fn is_private_ip(ip: IpAddr) -> bool { (v6.segments()[7] >> 8) as u8, v6.segments()[7] as u8, ))) + || NOT_GLOBALLY_REACHABLE_IPV6 + .iter() + .any(|range| range.contains(&v6)) } } } +/// IANA IPv6 special-purpose blocks marked not globally reachable that the +/// checks above do not already cover. `2001:2::/48` is also the IPv6 fake-IP +/// range in sing-box's documented configuration. +const NOT_GLOBALLY_REACHABLE_IPV6: [Ipv6Net; 8] = [ + Ipv6Net::new_assert(Ipv6Addr::new(0x0064, 0xff9b, 1, 0, 0, 0, 0, 0), 48), // local-use NAT64 (RFC 8215) + Ipv6Net::new_assert(Ipv6Addr::new(0x0100, 0, 0, 0, 0, 0, 0, 0), 64), // discard-only (RFC 6666) + Ipv6Net::new_assert(Ipv6Addr::new(0x0100, 0, 0, 1, 0, 0, 0, 0), 64), // dummy prefix (RFC 9780) + fake_ip::FAKE_IPV6_RANGE, // benchmarking (RFC 5180) + Ipv6Net::new_assert(Ipv6Addr::new(0x2001, 0x0db8, 0, 0, 0, 0, 0, 0), 32), // documentation (RFC 3849) + Ipv6Net::new_assert(Ipv6Addr::new(0x3fff, 0, 0, 0, 0, 0, 0, 0), 20), // documentation (RFC 9637) + Ipv6Net::new_assert(Ipv6Addr::new(0x5f00, 0, 0, 0, 0, 0, 0, 0), 16), // SRv6 SIDs (RFC 9602) + Ipv6Net::new_assert(Ipv6Addr::new(0xfec0, 0, 0, 0, 0, 0, 0, 0), 10), // deprecated site-local (RFC 3879) +]; + fn is_private_ipv4(v4: Ipv4Addr) -> bool { let octets = v4.octets(); v4.is_loopback() || v4.is_private() || v4.is_link_local() - || v4.is_unspecified() + || octets[0] == 0 // This network (RFC 1122), including 0.0.0.0. || v4.is_broadcast() || v4.is_multicast() // CGNAT / Shared Address Space (RFC 6598) @@ -577,18 +799,26 @@ fn is_private_ipv4(v4: Ipv4Addr) -> bool { // ─── HTTP Fetch ────────────────────────────────────────────────────────────── +/// `first_hop` is the validated address set for `url`; every redirect hop is +/// resolved and validated again. async fn fetch_reqwest( url: &str, + first_hop: Vec, timeout: Duration, transport: FetchTransport, + fake_ip: Option<&FakeIpPolicy>, ) -> Result<(u16, Option, String, String), FetchError> { let mut current = url.to_string(); let mut final_url = None; + let mut first_hop = Some(first_hop); for redirect_count in 0..=MAX_REDIRECTS { validate_url(¤t)?; // Resolve DNS and validate ALL IPs are public; pin them for the connection. - let resolved = resolve_and_validate_host(¤t).await?; + let resolved = match first_hop.take() { + Some(resolved) => resolved, + None => resolve_and_validate_host(¤t, fake_ip).await?, + }; let parsed = url::Url::parse(¤t) .map_err(|e| FetchError::Validation(format!("Invalid URL: {e}")))?; let host = parsed.host_str().unwrap_or_default().to_string(); @@ -1396,6 +1626,22 @@ fn resolve_url(base: &str, href: &str) -> Option { mod tests { use super::*; + #[test] + fn server_fake_ip_denial_does_not_suggest_a_local_opt_in() { + for reason in [ + SsrfBlockReason::FakeIpNotEnabled, + SsrfBlockReason::FakeIpUnsupportedAnswer, + ] { + let blocked = + FetchError::ssrf("example.com", vec!["198.18.0.1".parse().unwrap()], reason); + let result = render_fetch_result(Err(blocked), &FetchNetwork::direct_pinned()); + assert!(result.is_error); + assert!(result.output.contains("private/internal")); + assert!(!result.output.contains("allow_doh_screened")); + assert!(!result.output.contains("can be allowed")); + } + } + // ── SSRF Protection ─────────────────────────────────────────────── #[test] @@ -1474,6 +1720,8 @@ mod tests { #[test] fn blocks_cgnat_and_reserved_ranges() { + assert!(validate_url("http://0.0.0.1/").is_err()); + assert!(validate_url("http://0.255.255.255/").is_err()); // CGNAT (RFC 6598) assert!(validate_url("http://100.64.0.1/").is_err()); assert!(validate_url("http://100.127.255.254/").is_err()); @@ -1494,10 +1742,82 @@ mod tests { // NAT64 well-known prefix wrapping a private IPv4 assert!(validate_url("http://[64:ff9b::127.0.0.1]/").is_err()); assert!(validate_url("http://[64:ff9b::10.0.0.1]/").is_err()); + assert!(validate_url("http://[64:ff9b::0.0.0.1]/").is_err()); + assert!(validate_url("http://[::ffff:0.0.0.1]/").is_err()); // NAT64 wrapping a public IP should pass assert!(validate_url("http://[64:ff9b::93.184.216.34]/").is_ok()); } + #[test] + fn blocks_ipv6_special_purpose_ranges_that_are_not_globally_reachable() { + for literal in [ + "64:ff9b:1::1", + "100::1", + "100:0:0:1::1", + "2001:2::1", + "2001:db8::1", + "3fff::1", + "5f00::1", + "fec0::1", + ] { + assert!( + validate_url(&format!("http://[{literal}]/")).is_err(), + "{literal}" + ); + } + // Neighbouring globally reachable blocks stay allowed. + for literal in ["2001:3::1", "2001:4860::8888", "2606:4700::1111"] { + assert!( + validate_url(&format!("http://[{literal}]/")).is_ok(), + "{literal}" + ); + } + } + + #[tokio::test] + async fn first_hop_reuses_validated_addresses_and_redirects_are_revalidated() { + use wiremock::matchers::path; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(path("/page")) + .respond_with(ResponseTemplate::new(200).set_body_string("first hop")) + .mount(&server) + .await; + Mock::given(path("/redirect")) + .respond_with( + ResponseTemplate::new(302) + .insert_header("location", format!("{}/page", server.uri())), + ) + .mount(&server) + .await; + // `.example` never resolves to this server: the request can only reach + // it through the pinned first-hop addresses, without another lookup. + let pinned = vec![*server.address()]; + let (status, _, _, body) = fetch_reqwest( + "http://first-hop.example/page", + pinned.clone(), + Duration::from_secs(5), + FetchTransport::DirectPinned, + None, + ) + .await + .unwrap(); + assert_eq!((status, body.as_str()), (200, "first hop")); + + // The redirect target is a new hop and goes through validation again. + let error = fetch_reqwest( + "http://first-hop.example/redirect", + pinned, + Duration::from_secs(5), + FetchTransport::DirectPinned, + None, + ) + .await + .unwrap_err(); + assert!(matches!(error, FetchError::SsrfBlocked(_)), "{error}"); + } + #[test] fn blocks_userinfo_in_url() { assert!(validate_url("https://evil.com@internal.example.com/").is_err()); @@ -2013,8 +2333,8 @@ mod tests { #[tokio::test] async fn error_on_missing_url() { - let r = - fetch_with_cache_scope(&serde_json::json!({}), "", FetchTransport::DirectPinned).await; + let r = fetch_with_cache_scope(&serde_json::json!({}), "", &FetchNetwork::direct_pinned()) + .await; assert!(r.is_error); assert!(r.output.contains("Missing 'url'")); assert_eq!(r.metadata.unwrap()["error_kind"], "tool_invalid_args"); @@ -2022,14 +2342,14 @@ mod tests { #[tokio::test] async fn error_on_private_ip() { - for transport in [ - FetchTransport::DirectPinned, - FetchTransport::LocalEnvironment, + for network in [ + FetchNetwork::direct_pinned(), + FetchNetwork::local_environment(Ok(None)), ] { let r = fetch_with_cache_scope( &serde_json::json!({"url": "http://127.0.0.1/admin"}), "", - transport, + &network, ) .await; assert!(r.is_error); @@ -2043,7 +2363,7 @@ mod tests { let r = fetch_with_cache_scope( &serde_json::json!({"url": "https://x.com", "format": "raw"}), "", - FetchTransport::DirectPinned, + &FetchNetwork::direct_pinned(), ) .await; assert!(r.is_error); @@ -2096,7 +2416,7 @@ mod tests { Duration::ZERO, ); assert_eq!(result.status, 410); - let result = render_fetch_result(Ok(Arc::new(result)), FetchTransport::DirectPinned); + let result = render_fetch_result(Ok(Arc::new(result)), &FetchNetwork::direct_pinned()); assert!(result.is_error); assert_eq!(result.metadata.unwrap()["error_kind"], "tool_not_found"); let output: Value = serde_json::from_str(&result.output).unwrap(); diff --git a/crates/astra-tools/src/web_fetch/fake_ip.rs b/crates/astra-tools/src/web_fetch/fake_ip.rs new file mode 100644 index 0000000000..48b1568bf6 --- /dev/null +++ b/crates/astra-tools/src/web_fetch/fake_ip.rs @@ -0,0 +1,1177 @@ +//! User opt-in that relaxes the private-network SSRF boundary for fake-IP +//! DNS answers. +//! +//! Fake-IP DNS (Clash/mihomo, Surge, sing-box) answers every lookup with an +//! address from `198.18.0.0/15` and routes the connection by hostname. The +//! SSRF guard cannot judge such an answer, so it blocks it by default. When +//! the user opts in through the user-level `[web_fetch]` configuration, the +//! fetch goes to the fake address and the VPN resolves and picks the real +//! destination. Astra cannot observe that destination. A DNS-over-HTTPS lookup +//! screens out hosts whose public answer is private; it is not proof of where +//! the VPN connects, because a host may answer the resolver and the VPN +//! differently. +//! +//! Only `FetchTransport::LocalEnvironment` carries a policy; Server transports +//! never consult it. + +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::sync::Arc; +use std::time::Duration; + +use astra_config::runtime_config::{FakeIpMode, RuntimeConfig, WebFetchConfig}; +use hickory_proto::op::{Message, MessageType, OpCode, Query, ResponseCode}; +use hickory_proto::rr::{DNSClass, Name, RData, RecordType}; +use ipnet::{Ipv4Net, Ipv6Net}; +use reqwest::header; + +use super::{SsrfBlockReason, is_private_ip}; + +const DNS_MESSAGE: &str = "application/dns-message"; +const DOH_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_DNS_MESSAGE_BYTES: usize = 64 * 1024; +const MAX_CNAME_HOPS: usize = 16; + +/// RFC 2544 benchmarking range: never assigned to real hosts, and the range +/// fake-IP implementations draw from. +pub(super) const FAKE_IP_RANGE: Ipv4Net = Ipv4Net::new_assert(Ipv4Addr::new(198, 18, 0, 0), 15); + +/// RFC 5180 IPv6 benchmarking range, used by sing-box's documented IPv6 +/// fake-IP configuration. Only IPv4 fake answers can be allowed; an answer +/// from this range is reported as unsupported rather than as a LAN address. +pub(super) const FAKE_IPV6_RANGE: Ipv6Net = + Ipv6Net::new_assert(Ipv6Addr::new(0x2001, 0x0002, 0, 0, 0, 0, 0, 0), 48); + +pub(super) fn is_ipv6_fake_candidate(ip: IpAddr) -> bool { + matches!(ip, IpAddr::V6(v6) if FAKE_IPV6_RANGE.contains(&v6)) +} + +#[derive(Debug)] +pub struct FakeIpPolicy { + ranges: Vec, + resolvers: Vec, +} + +impl FakeIpPolicy { + /// Build the policy selected by configuration. `Ok(None)` means the user + /// has not opted in. + pub fn from_config(config: &WebFetchConfig) -> Result, String> { + if config.fake_ip == FakeIpMode::Block { + return Ok(None); + } + let ranges = if config.fake_ip_ranges.is_empty() { + vec![FAKE_IP_RANGE] + } else { + config + .fake_ip_ranges + .iter() + .map(|raw| parse_fake_ip_range(raw)) + .collect::>()? + }; + if config.doh_resolvers.is_empty() { + return Err( + "web_fetch.fake_ip = \"allow_doh_screened\" requires at least one doh_resolvers entry" + .into(), + ); + } + let resolvers = config + .doh_resolvers + .iter() + .enumerate() + .map(|(index, raw)| parse_doh_resolver(index, raw)) + .collect::>()?; + Ok(Some(Self { ranges, resolvers })) + } + + #[cfg(test)] + pub(super) fn for_test(resolvers: Vec) -> Self { + Self { + ranges: vec![FAKE_IP_RANGE], + resolvers, + } + } + + pub(super) fn contains(&self, ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => self.ranges.iter().any(|range| range.contains(&v4)), + IpAddr::V6(_) => false, + } + } + + /// Pass `host` only when a configured resolver gives complete, consistent + /// answers for both address families and no reported address is in an + /// Astra-blocked range. Fails closed when no resolver does. Passing does not bind the + /// VPN's destination; see the module documentation. + /// + /// A private address blocks the host even when it came from a truncated + /// response or the other family's query failed: a later resolver must not + /// be able to outvote evidence that has already been seen. + pub(super) async fn screen(&self, host: &str) -> Result<(), SsrfBlockReason> { + let mut name = Name::from_ascii(host).map_err(|_| SsrfBlockReason::FakeIpNoDohAnswer)?; + name.set_fqdn(true); + let mut failures = Vec::new(); + let mut empty_answers = 0; + for (index, resolver) in self.resolvers.iter().enumerate() { + let (v4, v6) = resolve_via_doh(resolver, &name).await; + let real: Vec = v4.addrs.iter().chain(&v6.addrs).copied().collect(); + if real.iter().any(|ip| is_private_ip(*ip)) { + return Err(SsrfBlockReason::FakeIpRealAddressPrivate(real)); + } + let failure = match (&v4.status, &v6.status) { + (AnswerStatus::Failed(error), _) | (_, AnswerStatus::Failed(error)) => { + error.clone() + } + // NXDOMAIN says the name has no records of any type. + (AnswerStatus::NxDomain, _) | (_, AnswerStatus::NxDomain) if !real.is_empty() => { + "contradictory answers: NXDOMAIN for one address family, records for the other" + .to_string() + } + _ if real.is_empty() => { + empty_answers += 1; + "no address records".to_string() + } + _ => return Ok(()), + }; + failures.push(format!("{}: {failure}", resolver_label(index, resolver))); + } + if empty_answers == self.resolvers.len() { + Err(SsrfBlockReason::FakeIpNoDohAnswer) + } else { + Err(SsrfBlockReason::FakeIpDohUnavailable(failures.join("; "))) + } + } +} + +/// The policy from the user's runtime configuration, for Edge/CLI owners. +/// Invalid configuration is reported and leaves the default block in place. +pub fn configured_fake_ip_policy() -> Result>, String> { + FakeIpPolicy::from_config(&RuntimeConfig::cached().web_fetch).map(|policy| policy.map(Arc::new)) +} + +fn parse_fake_ip_range(raw: &str) -> Result { + let range = raw + .trim() + .parse::() + .map_err(|_| "invalid web_fetch.fake_ip_ranges entry".to_string())? + .trunc(); + if !FAKE_IP_RANGE.contains(&range) { + return Err(format!( + "web_fetch.fake_ip_ranges entry must lie within {FAKE_IP_RANGE}" + )); + } + Ok(range) +} + +/// Resolver URLs may carry account identifiers in their path or query, so +/// diagnostics name a resolver only by position and host. +fn resolver_label(index: usize, resolver: &url::Url) -> String { + format!( + "doh_resolvers[{index}] ({})", + resolver.host_str().unwrap_or("?") + ) +} + +fn parse_doh_resolver(index: usize, raw: &str) -> Result { + let entry = format!("web_fetch.doh_resolvers[{index}]"); + let url = url::Url::parse(raw.trim()).map_err(|e| format!("invalid {entry}: {e}"))?; + if url.scheme() != "https" || url.host_str().is_none() { + return Err(format!("{entry} must be an https:// URL")); + } + if !url.username().is_empty() || url.password().is_some() { + return Err(format!("{entry} must not embed credentials")); + } + Ok(url) +} + +/// One address family's answer from one resolver. +struct FamilyAnswer { + /// Addresses the response carried, even when it is unusable as a whole: a + /// private address in a truncated response is still evidence. + addrs: Vec, + status: AnswerStatus, +} + +enum AnswerStatus { + Complete, + NxDomain, + Failed(String), +} + +impl FamilyAnswer { + fn failed(error: String) -> Self { + Self { + addrs: Vec::new(), + status: AnswerStatus::Failed(error), + } + } +} + +/// A and AAAA answers from one resolver, kept apart so a failure in one +/// family does not discard what the other reported. +async fn resolve_via_doh(resolver: &url::Url, name: &Name) -> (FamilyAnswer, FamilyAnswer) { + let client = match astra_core::net::client_builder_for_target(resolver.as_str()) + .timeout(DOH_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) + .build() + { + Ok(client) => client, + Err(error) => { + let error = format!("client init failed: {}", transport_failure(error)); + return ( + FamilyAnswer::failed(error.clone()), + FamilyAnswer::failed(error), + ); + } + }; + let (v4, v6) = futures::join!( + bounded_query(&client, resolver, name, RecordType::A), + bounded_query(&client, resolver, name, RecordType::AAAA) + ); + (v4, v6) +} + +async fn bounded_query( + client: &reqwest::Client, + resolver: &url::Url, + name: &Name, + record_type: RecordType, +) -> FamilyAnswer { + match tokio::time::timeout(DOH_TIMEOUT, query(client, resolver, name, record_type)).await { + Ok(answer) => answer, + Err(_) => FamilyAnswer::failed(format!("{record_type} query timed out")), + } +} + +async fn query( + client: &reqwest::Client, + resolver: &url::Url, + name: &Name, + record_type: RecordType, +) -> FamilyAnswer { + let mut current = name.clone(); + let mut visited = vec![current.clone()]; + loop { + let answer = match query_message(client, resolver, ¤t, record_type).await { + Ok(answer) => answer, + Err(error) => return FamilyAnswer::failed(error), + }; + // Retain private evidence even if this response is otherwise invalid. + let addrs: Vec = answer + .answers + .iter() + .filter_map(|record| match &record.data { + RData::A(a) => Some(IpAddr::V4(a.0)), + RData::AAAA(a) => Some(IpAddr::V6(a.0)), + _ => None, + }) + .collect(); + let failed = |error: String| FamilyAnswer { + addrs: addrs.clone(), + status: AnswerStatus::Failed(error), + }; + if answer.metadata.message_type != MessageType::Response + || answer.metadata.op_code != OpCode::Query + || answer.metadata.id != 0 + || answer.queries.len() != 1 + || answer.queries[0].name() != ¤t + || answer.queries[0].query_type() != record_type + || answer.queries[0].query_class() != DNSClass::IN + { + return failed("DNS response does not match the query".into()); + } + if addrs.iter().any(|ip| is_private_ip(*ip)) { + return FamilyAnswer { + addrs, + status: AnswerStatus::Complete, + }; + } + if answer.metadata.truncation { + return failed(format!("truncated {record_type} response")); + } + match answer.metadata.response_code { + ResponseCode::NoError => {} + ResponseCode::NXDomain => { + return FamilyAnswer { + addrs, + status: AnswerStatus::NxDomain, + }; + } + code => return failed(format!("DNS response code {code}")), + } + + let mut terminal = current.clone(); + let mut chain = vec![terminal.clone()]; + loop { + let mut targets = answer.answers.iter().filter_map(|record| { + if record.name == terminal + && record.dns_class == DNSClass::IN + && let RData::CNAME(target) = &record.data + { + return Some(target.0.clone()); + } + None + }); + let Some(target) = targets.next() else { break }; + if targets.any(|other| other != target) + || visited.contains(&target) + || visited.len() >= MAX_CNAME_HOPS + { + return failed("invalid or cyclic CNAME chain".into()); + } + visited.push(target.clone()); + chain.push(target.clone()); + terminal = target; + } + for record in &answer.answers { + match &record.data { + RData::A(_) | RData::AAAA(_) => { + if record.name != terminal + || record.dns_class != DNSClass::IN + || record.record_type() != record_type + { + return failed("unrelated DNS address record".into()); + } + } + RData::CNAME(_) + if record.dns_class != DNSClass::IN + || !chain[..chain.len() - 1].contains(&record.name) => + { + return failed("unrelated DNS CNAME record".into()); + } + _ => {} + } + } + if !addrs.is_empty() { + return FamilyAnswer { + addrs, + status: AnswerStatus::Complete, + }; + } + let has_soa = answer.authorities.iter().any(|r| { + r.dns_class == DNSClass::IN + && matches!(r.data, RData::SOA(_)) + && r.name.zone_of(&terminal) + }); + let referral = answer + .authorities + .iter() + .any(|r| matches!(r.data, RData::NS(_))); + if referral && !has_soa { + return failed("DNS response is a referral".into()); + } + if chain.len() == 1 || has_soa { + return FamilyAnswer { + addrs, + status: AnswerStatus::Complete, + }; + } + current = terminal; + } +} + +async fn query_message( + client: &reqwest::Client, + resolver: &url::Url, + name: &Name, + record_type: RecordType, +) -> Result { + // RFC 8484 recommends ID 0 so responses stay cacheable. + let mut request = Message::new(0, MessageType::Query, OpCode::Query); + request.metadata.recursion_desired = true; + request.add_query(Query::query(name.clone(), record_type)); + let body = request.to_vec().map_err(|e| e.to_string())?; + let mut response = client + .post(resolver.clone()) + .header(header::CONTENT_TYPE, DNS_MESSAGE) + .header(header::ACCEPT, DNS_MESSAGE) + .body(body) + .send() + .await + .map_err(transport_failure)?; + if !response.status().is_success() { + return Err(format!("HTTP {}", response.status())); + } + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(transport_failure)? { + if bytes.len() + chunk.len() > MAX_DNS_MESSAGE_BYTES { + return Err("DNS response too large".into()); + } + bytes.extend_from_slice(&chunk); + } + Message::from_vec(&bytes).map_err(|e| format!("malformed DNS response: {e}")) +} + +/// Keep the underlying cause (for example "connection reset by peer"): +/// reqwest's own message alone does not tell the user why DoH failed. +fn transport_failure(error: reqwest::Error) -> String { + let error = error.without_url(); + let mut message = error.to_string(); + let mut source = std::error::Error::source(&error); + while let Some(cause) = source { + message.push_str(": "); + message.push_str(&cause.to_string()); + source = cause.source(); + } + message +} + +#[cfg(test)] +mod tests { + use std::net::SocketAddr; + + use hickory_proto::rr::Record; + use hickory_proto::rr::rdata::{A, AAAA}; + use wiremock::matchers::{header, method}; + use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; + + use super::*; + use crate::web_fetch::{ + FetchError, FetchNetwork, resolve_and_validate_host, validate_resolved_addrs, + }; + + /// How a DoH fixture misanswers queries for one record type. + #[derive(Clone, Copy, Debug)] + enum Degrade { + Http503, + /// TC=1, still carrying the configured records. + Truncated, + /// NXDOMAIN with no records, while the other type answers normally. + NxDomain, + } + + /// A DoH endpoint answering A and AAAA queries from fixed record sets. + struct DohAnswers { + code: ResponseCode, + v4: Vec, + v6: Vec, + degraded: Option<(RecordType, Degrade)>, + } + + impl Respond for DohAnswers { + fn respond(&self, request: &Request) -> ResponseTemplate { + let query = Message::from_vec(&request.body).expect("DNS query body"); + let question = query.queries[0].clone(); + let degrade = self + .degraded + .filter(|(record_type, _)| *record_type == question.query_type()) + .map(|(_, degrade)| degrade); + let mut response = Message::response(query.metadata.id, OpCode::Query); + response.add_query(question.clone()); + response.metadata.response_code = self.code; + match degrade { + Some(Degrade::Http503) => return ResponseTemplate::new(503), + Some(Degrade::Truncated) => response.metadata.truncation = true, + Some(Degrade::NxDomain) => response.metadata.response_code = ResponseCode::NXDomain, + None => {} + } + let name = question.name().clone(); + let records: Vec = match (degrade, question.query_type()) { + (Some(Degrade::NxDomain), _) => Vec::new(), + (_, RecordType::A) => self + .v4 + .iter() + .map(|ip| Record::from_rdata(name.clone(), 60, RData::A(A(*ip)))) + .collect(), + (_, RecordType::AAAA) => self + .v6 + .iter() + .map(|ip| Record::from_rdata(name.clone(), 60, RData::AAAA(AAAA(*ip)))) + .collect(), + _ => Vec::new(), + }; + response.add_answers(records); + ResponseTemplate::new(200) + .insert_header("content-type", DNS_MESSAGE) + .set_body_bytes(response.to_vec().expect("encode DNS response")) + } + } + + async fn doh(code: ResponseCode, v4: &[&str], v6: &[&str]) -> MockServer { + doh_degraded(code, v4, v6, None).await + } + + async fn doh_degraded( + code: ResponseCode, + v4: &[&str], + v6: &[&str], + degraded: Option<(RecordType, Degrade)>, + ) -> MockServer { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(header("content-type", DNS_MESSAGE)) + .respond_with(DohAnswers { + code, + v4: v4.iter().map(|ip| ip.parse().unwrap()).collect(), + v6: v6.iter().map(|ip| ip.parse().unwrap()).collect(), + degraded, + }) + .mount(&server) + .await; + server + } + + fn policy(servers: &[&MockServer]) -> FakeIpPolicy { + FakeIpPolicy::for_test( + servers + .iter() + .map(|server| url::Url::parse(&format!("{}/dns-query", server.uri())).unwrap()) + .collect(), + ) + } + + fn addrs(ips: &[&str]) -> Vec { + ips.iter() + .map(|ip| SocketAddr::new(ip.parse().unwrap(), 443)) + .collect() + } + + fn reason(error: FetchError) -> SsrfBlockReason { + match error { + FetchError::SsrfBlocked(block) => block.reason, + other => panic!("expected an SSRF block, got {other:?}"), + } + } + + fn config(mode: FakeIpMode, ranges: &[&str], resolvers: &[&str]) -> WebFetchConfig { + WebFetchConfig { + fake_ip: mode, + fake_ip_ranges: ranges.iter().map(ToString::to_string).collect(), + doh_resolvers: resolvers.iter().map(ToString::to_string).collect(), + } + } + + #[derive(Clone, Copy)] + enum ChallengedAnswer { + WrongQuestion, + WrongId, + WrongType, + WrongClass, + WrongRecordClass, + WrongQuestionPrivate, + UnrelatedAddress, + Alias, + AliasPrivate, + AliasPrivateV6, + BundledAlias, + AliasSoa, + UnrelatedSoaReferral, + AliasLoop, + Referral, + } + + impl Respond for ChallengedAnswer { + fn respond(&self, request: &Request) -> ResponseTemplate { + use hickory_proto::rr::rdata::{CNAME, NS, SOA}; + let request = Message::from_vec(&request.body).unwrap(); + let question = &request.queries[0]; + let mut response = Message::response( + if matches!(self, Self::WrongId) { + 42 + } else { + request.metadata.id + }, + OpCode::Query, + ); + let unrelated = Name::from_ascii("unrelated.example.").unwrap(); + let alias = Name::from_ascii("alias.example.").unwrap(); + let mut echoed = question.clone(); + if matches!(self, Self::WrongQuestion | Self::WrongQuestionPrivate) { + echoed = Query::query(unrelated.clone(), question.query_type()); + } + if matches!(self, Self::WrongType) { + echoed = Query::query(question.name().clone(), RecordType::MX); + } + if matches!(self, Self::WrongClass) { + echoed.set_query_class(DNSClass::CH); + } + response.add_query(echoed); + match self { + Self::Alias | Self::AliasPrivate | Self::AliasPrivateV6 | Self::AliasLoop + if question.name() != &alias => + { + response.add_answer(Record::from_rdata( + question.name().clone(), + 60, + RData::CNAME(CNAME(alias)), + )); + } + Self::AliasLoop => { + response.add_answer(Record::from_rdata( + alias, + 60, + RData::CNAME(CNAME(Name::from_ascii("target.example.").unwrap())), + )); + } + Self::Referral => { + response.add_authority(Record::from_rdata( + question.name().clone(), + 60, + RData::NS(NS(alias)), + )); + } + Self::BundledAlias if question.query_type() == RecordType::A => { + let terminal = Name::from_ascii("END.example.").unwrap(); + response.add_answer(Record::from_rdata( + terminal.clone(), + 60, + RData::A(A("8.8.8.8".parse().unwrap())), + )); + response.add_answer(Record::from_rdata( + alias.clone(), + 60, + RData::CNAME(CNAME(terminal)), + )); + response.add_answer(Record::from_rdata( + question.name().clone(), + 60, + RData::CNAME(CNAME(Name::from_ascii("ALIAS.example.").unwrap())), + )); + } + Self::AliasSoa | Self::UnrelatedSoaReferral + if question.query_type() == RecordType::AAAA => + { + response.add_answer(Record::from_rdata( + question.name().clone(), + 60, + RData::CNAME(CNAME(alias.clone())), + )); + let zone = if matches!(self, Self::AliasSoa) { + Name::from_ascii("example.").unwrap() + } else { + unrelated.clone() + }; + response.add_authority(Record::from_rdata( + zone, + 60, + RData::SOA(SOA::new(alias.clone(), alias.clone(), 1, 1, 1, 1, 1)), + )); + if matches!(self, Self::UnrelatedSoaReferral) { + response.add_authority(Record::from_rdata( + unrelated.clone(), + 60, + RData::NS(NS(alias)), + )); + } + } + Self::AliasPrivateV6 if question.query_type() == RecordType::AAAA => { + response.add_answer(Record::from_rdata( + question.name().clone(), + 60, + RData::AAAA(AAAA("::1".parse().unwrap())), + )); + } + _ if question.query_type() == RecordType::A => { + let owner = if matches!(self, Self::UnrelatedAddress) { + unrelated + } else { + question.name().clone() + }; + let ip = if matches!(self, Self::AliasPrivate | Self::WrongQuestionPrivate) { + "127.0.0.1" + } else { + "8.8.8.8" + }; + let mut record = + Record::from_rdata(owner, 60, RData::A(A(ip.parse().unwrap()))); + if matches!(self, Self::WrongRecordClass) { + record.dns_class = DNSClass::CH; + } + response.add_answer(record); + } + _ => {} + } + ResponseTemplate::new(200).set_body_bytes(response.to_vec().unwrap()) + } + } + + #[tokio::test] + async fn screening_checks_question_and_address_ownership() { + for answer in [ + ChallengedAnswer::WrongQuestion, + ChallengedAnswer::WrongId, + ChallengedAnswer::WrongType, + ChallengedAnswer::WrongClass, + ChallengedAnswer::WrongRecordClass, + ChallengedAnswer::UnrelatedAddress, + ChallengedAnswer::UnrelatedSoaReferral, + ChallengedAnswer::AliasLoop, + ChallengedAnswer::Referral, + ] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(answer) + .mount(&server) + .await; + assert!(policy(&[&server]).screen("target.example").await.is_err()); + } + } + + #[tokio::test] + async fn screening_follows_aliases_and_blocks_private_terminal_addresses() { + for (answer, accepted) in [ + (ChallengedAnswer::Alias, true), + (ChallengedAnswer::AliasPrivate, false), + (ChallengedAnswer::AliasPrivateV6, false), + (ChallengedAnswer::BundledAlias, true), + (ChallengedAnswer::AliasSoa, true), + ] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(answer) + .mount(&server) + .await; + let result = policy(&[&server]).screen("target.example").await; + assert_eq!(result.is_ok(), accepted, "{result:?}"); + } + } + + #[tokio::test] + async fn invalid_policy_reason_reaches_the_local_tool_result() { + let selection = FakeIpPolicy::from_config(&config(FakeIpMode::AllowDohScreened, &[], &[])) + .map(|policy| policy.map(Arc::new)); + let network = FetchNetwork::local_environment(selection); + let blocked = validate_resolved_addrs("target.example", addrs(&["198.18.0.1"]), None) + .await + .unwrap_err(); + let result = crate::web_fetch::render_fetch_result(Err(blocked), &network); + assert!(result.is_error); + assert!( + result + .output + .contains("requires at least one doh_resolvers"), + "{}", + result.output + ); + assert!(!result.output.contains("The user can choose to relax")); + } + + #[tokio::test] + async fn private_evidence_in_a_mismatched_answer_cannot_be_overruled() { + let suspect = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ChallengedAnswer::WrongQuestionPrivate) + .mount(&suspect) + .await; + let public = doh(ResponseCode::NoError, &["8.8.8.8"], &[]).await; + let result = policy(&[&suspect, &public]).screen("target.example").await; + assert!(matches!( + result, + Err(SsrfBlockReason::FakeIpRealAddressPrivate(_)) + )); + assert!(public.received_requests().await.unwrap().is_empty()); + } + + #[test] + fn configuration_is_opt_in_and_validated() { + let doh = FakeIpMode::AllowDohScreened; + let resolver = ["https://1.12.12.12/dns-query"]; + assert!( + FakeIpPolicy::from_config(&config(FakeIpMode::Block, &[], &resolver)) + .unwrap() + .is_none() + ); + let default = FakeIpPolicy::from_config(&config(doh, &[], &resolver)) + .unwrap() + .unwrap(); + assert!(default.contains("198.19.255.1".parse().unwrap())); + let narrow = FakeIpPolicy::from_config(&config(doh, &["198.18.0.1/16"], &resolver)) + .unwrap() + .unwrap(); + assert!(narrow.contains("198.18.3.4".parse().unwrap())); + assert!(!narrow.contains("198.19.0.1".parse().unwrap())); + + for (ranges, resolvers, expected) in [ + (&[][..], &[][..], "requires at least one doh_resolvers"), + ( + &[][..], + &["http://1.12.12.12/dns-query"][..], + "must be an https:// URL", + ), + ( + &[][..], + &["https://user:secret@doh.example/dns-query"][..], + "doh_resolvers[0] must not embed credentials", + ), + ( + &["10.0.0.0/8"][..], + &resolver[..], + "must lie within 198.18.0.0/15", + ), + ( + &["198.0.0.0/8"][..], + &resolver[..], + "must lie within 198.18.0.0/15", + ), + ( + &["fc00::/7"][..], + &resolver[..], + "invalid web_fetch.fake_ip_ranges", + ), + ] { + let error = FakeIpPolicy::from_config(&config(doh, ranges, resolvers)).unwrap_err(); + assert!(error.contains(expected), "{error}"); + assert!(!error.contains("secret"), "{error}"); + } + } + + #[tokio::test] + async fn fake_ip_answer_is_blocked_with_user_opt_in_hint_by_default() { + let error = validate_resolved_addrs("www.bing.com", addrs(&["198.18.5.51"]), None) + .await + .unwrap_err(); + let message = error.to_string(); + assert!(message.contains("198.18.5.51"), "{message}"); + assert!( + message.contains("fake_ip = \"allow_doh_screened\""), + "{message}" + ); + assert!( + message.contains("do not change their configuration"), + "{message}" + ); + assert_eq!(reason(error), SsrfBlockReason::FakeIpNotEnabled); + } + + #[tokio::test] + async fn real_private_answer_never_suggests_the_opt_in() { + for answer in [&["10.0.0.5"][..], &["198.18.0.9", "127.0.0.1"][..]] { + let error = validate_resolved_addrs("intranet.example", addrs(answer), None) + .await + .unwrap_err(); + assert!(!error.to_string().contains("fake_ip")); + assert_eq!(reason(error), SsrfBlockReason::PrivateAddress); + } + } + + #[tokio::test] + async fn screened_host_connects_through_the_fake_address() { + let server = doh(ResponseCode::NoError, &["140.82.113.4"], &[]).await; + let fake = addrs(&["198.18.0.23"]); + let admitted = + validate_resolved_addrs("github.com", fake.clone(), Some(&policy(&[&server]))) + .await + .unwrap(); + assert_eq!( + admitted, fake, + "the pinned connection must use the VPN's fake address" + ); + } + + /// Documents the accepted limitation: screening sees only the DoH answer. + /// A host that gives DoH a public address and the VPN's DNS a private one + /// is admitted, and the connection goes to the fake address, which the VPN + /// then routes to wherever its own lookup pointed. + #[tokio::test] + async fn screening_cannot_bind_the_destination_the_vpn_picks() { + let doh_view = doh(ResponseCode::NoError, &["93.184.216.34"], &[]).await; + let vpn_view = "127.0.0.1".parse::().unwrap(); + let fake = addrs(&["198.18.0.7"]); + let admitted = + validate_resolved_addrs("rebind.example", fake.clone(), Some(&policy(&[&doh_view]))) + .await + .expect("screening passes on the DoH view alone"); + assert_eq!(admitted, fake); + assert!( + admitted.iter().all(|addr| addr.ip() != vpn_view), + "Astra never sees the VPN's destination, so it cannot reject it" + ); + } + + #[tokio::test] + async fn doh_reporting_any_private_real_address_is_blocked() { + for (v4, v6) in [ + (&["0.0.0.1"][..], &[][..]), + (&["127.0.0.1"][..], &[][..]), + (&["93.184.216.34", "10.1.2.3"][..], &[][..]), + (&["93.184.216.34"][..], &["::1"][..]), + // A fake-range answer from DoH is not a real address either. + (&["198.18.0.1"][..], &[][..]), + (&["93.184.216.34"][..], &["2001:2::1"][..]), + ] { + let server = doh(ResponseCode::NoError, v4, v6).await; + let error = validate_resolved_addrs( + "evil.example", + addrs(&["198.18.0.7"]), + Some(&policy(&[&server])), + ) + .await + .unwrap_err(); + assert!( + matches!(reason(error), SsrfBlockReason::FakeIpRealAddressPrivate(_)), + "{v4:?} {v6:?}" + ); + } + } + + #[tokio::test] + async fn missing_doh_answer_is_blocked() { + for code in [ResponseCode::NoError, ResponseCode::NXDomain] { + let server = doh(code, &[], &[]).await; + let error = validate_resolved_addrs( + "gone.example", + addrs(&["198.18.0.7"]), + Some(&policy(&[&server])), + ) + .await + .unwrap_err(); + assert_eq!(reason(error), SsrfBlockReason::FakeIpNoDohAnswer); + } + } + + #[tokio::test] + async fn unreachable_doh_fails_closed_and_later_resolvers_are_tried() { + let broken = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(503)) + .mount(&broken) + .await; + let error = validate_resolved_addrs( + "github.com", + addrs(&["198.18.0.23"]), + Some(&policy(&[&broken])), + ) + .await + .unwrap_err(); + let SsrfBlockReason::FakeIpDohUnavailable(detail) = reason(error) else { + panic!("an unreachable resolver must fail closed"); + }; + assert!(detail.contains("HTTP 503"), "{detail}"); + assert!(detail.contains("doh_resolvers[0] (127.0.0.1)"), "{detail}"); + + let working = doh(ResponseCode::NoError, &["140.82.113.4"], &[]).await; + validate_resolved_addrs( + "github.com", + addrs(&["198.18.0.23"]), + Some(&policy(&[&broken, &working])), + ) + .await + .expect("the next configured resolver answers"); + + let empty = doh(ResponseCode::NoError, &[], &[]).await; + validate_resolved_addrs( + "github.com", + addrs(&["198.18.0.23"]), + Some(&policy(&[&empty, &working])), + ) + .await + .expect("an empty answer does not prevent trying the next resolver"); + } + + #[tokio::test] + async fn private_answer_in_one_family_survives_a_failure_in_the_other() { + let public = doh(ResponseCode::NoError, &["140.82.113.4"], &[]).await; + for (failing, v4, v6) in [ + (RecordType::AAAA, &["127.0.0.1"][..], &[][..]), + (RecordType::A, &[][..], &["::1"][..]), + ] { + let partial = doh_degraded( + ResponseCode::NoError, + v4, + v6, + Some((failing, Degrade::Http503)), + ) + .await; + let error = validate_resolved_addrs( + "evil.example", + addrs(&["198.18.0.7"]), + Some(&policy(&[&partial, &public])), + ) + .await + .unwrap_err(); + assert!( + matches!(reason(error), SsrfBlockReason::FakeIpRealAddressPrivate(_)), + "{failing:?} failure must not let the next resolver admit the host" + ); + } + // A partial public answer is not proof either: try the next resolver. + let partial = doh_degraded( + ResponseCode::NoError, + &["140.82.113.4"], + &[], + Some((RecordType::AAAA, Degrade::Http503)), + ) + .await; + let error = validate_resolved_addrs( + "github.com", + addrs(&["198.18.0.23"]), + Some(&policy(&[&partial])), + ) + .await + .unwrap_err(); + assert!(matches!( + reason(error), + SsrfBlockReason::FakeIpDohUnavailable(_) + )); + } + + #[tokio::test] + async fn incomplete_or_contradictory_doh_answers_are_not_trusted() { + let public = doh(ResponseCode::NoError, &["140.82.113.4"], &[]).await; + for (degraded, v4, v6, expected) in [ + ( + (RecordType::A, Degrade::Truncated), + &["140.82.113.4"][..], + &[][..], + "truncated A response", + ), + ( + (RecordType::AAAA, Degrade::Truncated), + &["140.82.113.4"][..], + &["2606:50c0:8000::154"][..], + "truncated AAAA response", + ), + ( + (RecordType::A, Degrade::NxDomain), + &[][..], + &["2606:50c0:8000::154"][..], + "contradictory answers", + ), + ( + (RecordType::AAAA, Degrade::NxDomain), + &["140.82.113.4"][..], + &[][..], + "contradictory answers", + ), + ] { + let suspect = doh_degraded(ResponseCode::NoError, v4, v6, Some(degraded)).await; + let error = validate_resolved_addrs( + "github.com", + addrs(&["198.18.0.23"]), + Some(&policy(&[&suspect])), + ) + .await + .unwrap_err(); + let SsrfBlockReason::FakeIpDohUnavailable(detail) = reason(error) else { + panic!("{degraded:?} must fail closed"); + }; + assert!(detail.contains(expected), "{degraded:?}: {detail}"); + validate_resolved_addrs( + "github.com", + addrs(&["198.18.0.23"]), + Some(&policy(&[&suspect, &public])), + ) + .await + .unwrap_or_else(|error| panic!("{degraded:?} must fall back: {error:?}")); + } + + // What a truncated response did carry still counts as evidence. + let truncated = doh_degraded( + ResponseCode::NoError, + &["140.82.113.4", "127.0.0.1"], + &[], + Some((RecordType::A, Degrade::Truncated)), + ) + .await; + let error = validate_resolved_addrs( + "evil.example", + addrs(&["198.18.0.7"]), + Some(&policy(&[&truncated, &public])), + ) + .await + .unwrap_err(); + assert!(matches!( + reason(error), + SsrfBlockReason::FakeIpRealAddressPrivate(_) + )); + } + + #[tokio::test] + async fn resolver_diagnostics_omit_url_path_and_query() { + let broken = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(503)) + .mount(&broken) + .await; + let resolver = + url::Url::parse(&format!("{}/profile-secret?token=secret", broken.uri())).unwrap(); + let error = validate_resolved_addrs( + "github.com", + addrs(&["198.18.0.23"]), + Some(&FakeIpPolicy::for_test(vec![resolver])), + ) + .await + .unwrap_err(); + let message = error.to_string(); + assert!(message.contains("doh_resolvers[0]"), "{message}"); + assert!(!message.contains("secret"), "{message}"); + } + + #[tokio::test] + async fn unsupported_fake_ip_answers_are_rejected_without_consulting_doh() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(500)) + .expect(0) + .mount(&server) + .await; + let opted_in = policy(&[&server]); + for answer in [ + // sing-box's documented IPv6 fake-IP range + &["2001:2::7"][..], + &["198.18.0.7", "2001:2::7"][..], + // IPv4 fake address next to a real public IPv6 address + &["198.18.0.7", "2606:4700::1111"][..], + ] { + for fake_ip in [None, Some(&opted_in)] { + let error = validate_resolved_addrs("dual.example", addrs(answer), fake_ip) + .await + .unwrap_err(); + let message = error.to_string(); + assert!(message.contains("not supported"), "{message}"); + assert!( + message.contains("do not change their configuration"), + "{message}" + ); + assert_eq!( + reason(error), + SsrfBlockReason::FakeIpUnsupportedAnswer, + "{answer:?}" + ); + } + } + // A real LAN address is never described as fake-IP. + let error = validate_resolved_addrs("lan.example", addrs(&["fd00::5"]), Some(&opted_in)) + .await + .unwrap_err(); + assert_eq!(reason(error), SsrfBlockReason::PrivateAddress); + } + + #[tokio::test] + async fn doh_is_consulted_only_when_every_answer_is_fake() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(500)) + .expect(0) + .mount(&server) + .await; + let policy = policy(&[&server]); + let public = addrs(&["140.82.113.4"]); + assert_eq!( + validate_resolved_addrs("github.com", public.clone(), Some(&policy)) + .await + .unwrap(), + public + ); + let error = validate_resolved_addrs( + "mixed.example", + addrs(&["198.18.0.7", "10.0.0.1"]), + Some(&policy), + ) + .await + .unwrap_err(); + assert_eq!(reason(error), SsrfBlockReason::PrivateAddress); + // A literal fake address has no hostname to screen. + let error = resolve_and_validate_host("http://198.18.0.5/", Some(&policy)) + .await + .unwrap_err(); + assert_eq!(reason(error), SsrfBlockReason::PrivateAddress); + } + + #[test] + fn only_the_local_environment_can_carry_the_opt_in() { + assert!(matches!(FetchNetwork::direct_pinned().fake_ip, Ok(None))); + assert!(matches!(FetchNetwork::default().fake_ip, Ok(None))); + let local = + FetchNetwork::local_environment(Ok(Some(Arc::new(FakeIpPolicy::for_test(Vec::new()))))); + assert!(matches!(local.fake_ip, Ok(Some(_)))); + } +} diff --git a/crates/astra-tools/src/web_search.rs b/crates/astra-tools/src/web_search.rs index bdf061b9c1..653578fde1 100644 --- a/crates/astra-tools/src/web_search.rs +++ b/crates/astra-tools/src/web_search.rs @@ -113,10 +113,10 @@ pub fn web_search(args: &Value) -> String { pub async fn perform_web_search( args: &Value, cache_scope: &str, - transport: crate::web_fetch::FetchTransport, + network: &crate::web_fetch::FetchNetwork, ) -> ToolResult { perform_web_search_with(args, |fetch_args| async move { - crate::web_fetch::fetch_with_cache_scope(&fetch_args, cache_scope, transport).await + crate::web_fetch::fetch_with_cache_scope(&fetch_args, cache_scope, network).await }) .await } diff --git a/crates/astra-tools/tests/web_transport.rs b/crates/astra-tools/tests/web_transport.rs index dfac9d8547..09882e6798 100644 --- a/crates/astra-tools/tests/web_transport.rs +++ b/crates/astra-tools/tests/web_transport.rs @@ -1,6 +1,6 @@ //! Exercise the real HTTP client in isolated subprocess environments: never //! mutate proxy variables in the shared test process or need external servers. -use astra_tools::web_fetch::{FetchTransport, fetch_with_cache_scope}; +use astra_tools::web_fetch::{FetchNetwork, fetch_with_cache_scope}; use astra_tools::{ToolExecutor, executor::DefaultToolExecutor}; use serde_json::{Value, json}; use std::io::{Read, Write}; @@ -15,7 +15,7 @@ async fn live_owner_network() { let workspace = tempfile::tempdir().unwrap(); let direct = DefaultToolExecutor::for_workspace(workspace.path(), "network-test", "direct"); let local = DefaultToolExecutor::for_workspace(workspace.path(), "network-test", "local") - .with_local_network(); + .with_local_network(Ok(None)); let url = std::env::var("ASTRA_TEST_PUBLIC_URL").expect("explicit public URL required"); let args = json!({"url": url, "timeout": 5, "max_content": 2048}); let result = local.execute("web_fetch", &args).await; @@ -48,26 +48,53 @@ async fn live_owner_network() { ); } +#[tokio::test] +#[ignore = "requires a fake-IP DNS, a reachable DoH resolver and public network"] +async fn live_fake_ip_screening() { + use astra_config::runtime_config::{FakeIpMode, WebFetchConfig}; + use astra_tools::web_fetch::FakeIpPolicy; + let url = std::env::var("ASTRA_TEST_PUBLIC_URL").expect("explicit public URL required"); + let resolver = std::env::var("ASTRA_TEST_DOH_RESOLVER").expect("explicit DoH URL required"); + let args = json!({"url": url, "timeout": 15, "max_content": 2048}); + let blocked = + fetch_with_cache_scope(&args, "", &FetchNetwork::local_environment(Ok(None))).await; + assert!( + blocked.is_error, + "requires a host the local DNS maps to 198.18.0.0/15" + ); + println!("without opt-in: {}", blocked.output); + let policy = FakeIpPolicy::from_config(&WebFetchConfig { + fake_ip: FakeIpMode::AllowDohScreened, + fake_ip_ranges: Vec::new(), + doh_resolvers: vec![resolver], + }) + .unwrap() + .map(std::sync::Arc::new); + let admitted = + fetch_with_cache_scope(&args, "", &FetchNetwork::local_environment(Ok(policy))).await; + assert!(!admitted.is_error, "with opt-in: {admitted:?}"); + let data: Value = serde_json::from_str(&admitted.output).unwrap(); + println!( + "with opt-in: status={} content_bytes={}", + data["status"], data["content_length"] + ); +} + #[tokio::test] async fn proxy_worker() { let Ok(mode) = std::env::var("ASTRA_TEST_WEB_TRANSPORT") else { return; }; - let policy = if mode == "direct" || mode == "no_proxy" { - FetchTransport::DirectPinned - } else { - FetchTransport::LocalEnvironment - }; // no_proxy must exercise LocalEnvironment while bypassing the proxy. - let policy = if mode == "no_proxy" { - FetchTransport::LocalEnvironment + let network = if mode == "direct" { + FetchNetwork::direct_pinned() } else { - policy + FetchNetwork::local_environment(Ok(None)) }; let result = fetch_with_cache_scope( &json!({"url": TARGET, "allow_http": true, "timeout": 1}), "transport-test", - policy, + &network, ) .await; let output: Value = serde_json::from_str(&result.output).unwrap(); @@ -78,7 +105,7 @@ async fn proxy_worker() { let direct = fetch_with_cache_scope( &json!({"url": TARGET, "allow_http": true, "timeout": 1}), "transport-test", - FetchTransport::DirectPinned, + &FetchNetwork::direct_pinned(), ) .await; assert!( diff --git a/docs/design/edge-cloud-execution.md b/docs/design/edge-cloud-execution.md index 2f181eda90..244ecbe582 100644 --- a/docs/design/edge-cloud-execution.md +++ b/docs/design/edge-cloud-execution.md @@ -81,6 +81,18 @@ guarantee. Both modes validate public destinations and every redirect locally; proxy-only DNS names are not supported. A failed proxy request never silently falls back to direct or Server execution. Cache identity includes transport mode. +Fake-IP DNS (a local VPN answering from `198.18.0.0/15`) is blocked like any +reserved address. The user may relax the private-network boundary for the local +transport through the user-level `[web_fetch]` configuration: a fetch whose +answers all lie in the IPv4 fake range goes to the fake address, and the VPN +picks the real destination. A DNS-over-HTTPS screen blocks hosts whose public +answer is private, but it cannot bind the VPN's destination, so the opt-in is an +explicit relaxation rather than a verified boundary. IPv6 fake-IP answers and +answers mixing fake and real addresses are rejected as unsupported. Only the +local transport can carry this opt-in; Server transports are constructed without +it. See the +[configuration reference](../reference/configuration.md#local-web_fetch-behind-fake-ip-dns). + One total fetch deadline covers DNS, redirects and body reads. Network, timeout, HTTP and policy failures retain typed failure metadata through search and local tool callbacks; a failed connection is not reported as an unreadable search page. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 9fc7f461f9..0d4efb7be8 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -203,6 +203,12 @@ retain the canonical identity files alongside any isolated cache. A writable existing identity preflight reports this prerequisite before turn admission; do not delete identity files or login credentials as a repair step. +`ASTRA_LOCAL_STATE_ROOT` relocates user-level state, including the user +configuration, permissions and workspace trust, so it must be set in the +launching shell. The CLI exits with an error when a workspace `.env` supplies it, +as it does for `MOI_AUTH_DIR`. The same applies to `HOME`, the fallback root for +all of these state locations, when the launching environment does not set it. + The configuration editor saves user defaults under `ASTRA_LOCAL_STATE_ROOT/config/runtime.toml` when that root is set, otherwise under `~/.astra/config/runtime.toml`; local versions live in the adjacent `versions` directory. Saving defaults retains invocation CLI settings @@ -286,6 +292,71 @@ Provider admission is intentionally configured by capacity inputs only. Scope is Server-loop Memoria observer and post-loop memory cleanup are fixed internal async best-effort side effects with bounded in-process concurrency. They are intentionally not environment-configurable; they must not hold run admission slots or become deployment-specific tuning surfaces. +### Local `web_fetch` behind fake-IP DNS + +Edge/CLI `web_fetch` and `web_search` block destinations that resolve to a +known private or reserved address, including `198.18.0.0/15`. Local VPN/proxy tools +in fake-IP mode (Clash/mihomo, Surge, sing-box) answer every lookup from that +range, so every fetch is blocked. Users on such a network can relax the +private-network boundary for fake-IP answers in the user-level +`~/.astra/config/runtime.toml`: + +```toml +[web_fetch] +fake_ip = "allow_doh_screened" # block (default) or allow_doh_screened +doh_resolvers = ["https://1.12.12.12/dns-query"] # required, tried in order +# fake_ip_ranges = ["198.18.0.0/16"] # optional, within 198.18.0.0/15 +``` + +This is a relaxation, not a verification. When every address from the local +DNS lies in an IPv4 fake range, the fetch goes to the fake address and the VPN +resolves the hostname and picks the real destination; Astra cannot see or +check it. Before allowing the fetch, Astra screens the hostname by asking the +resolvers (RFC 8484) for A and AAAA records and blocks it if any falls in +Astra's blocked address ranges. The list is not an exhaustive registry of all +special-purpose addresses; broader classification is tracked in #1140. +Resolvers are tried in order when one is unreachable, gives +an unusable response, or returns no address. Literal fake addresses and hosts +with no usable resolver answer stay blocked, and a private address reported for either +record type blocks the fetch even if the other query failed. There is no +built-in resolver: pick one your network does not block, because many networks +that use fake-IP DNS also block well-known DoH endpoints. Resolver URLs must +not contain userinfo; paths and query parameters may contain sensitive values, +and errors name a resolver only by its position and host. +The `0.0.0.0/8` this-network range is also blocked, including when embedded in +IPv4-mapped or NAT64 IPv6 addresses. + +Only IPv4 fake answers are supported. IPv6 fake-IP ranges (for example +`2001:2::/48`, used in sing-box's documented configuration) and answers that mix +fake addresses with real ones, such as an IPv4 fake address next to a public +IPv6 address, are rejected with an "unsupported" error whether or not the option +is enabled. Configure the VPN to answer with IPv4 fake addresses only if you +need this option. + +Only the user-level file and an explicit `--settings` overlay can set this +section. A project `.astra/config/runtime.toml` cannot; the section is ignored +with a warning, and the config editor omits it when saving project defaults. +If the project and user file are the same file, save as user defaults instead. +User defaults and local config-version blobs are written with private file +permissions on Unix. There is intentionally no environment variable, because the CLI +loads `.env` from the working directory. For the same reason a workspace `.env` +cannot set `ASTRA_LOCAL_STATE_ROOT` or a missing `HOME`, which locate the +user-level file. Server +`web_fetch` never reads the section. An invalid resolver or fake-IP range +keeps the default block and reports the configuration error when a fake-IP +host is fetched. A syntactically invalid user file is rejected as a whole; +use `--diagnostic-log` to see the configuration-loader warning. + +Screening does not bind the VPN's destination. A hostname that answers the +resolver with a public address and the VPN's DNS with a private one (DNS +rebinding by an attacker-controlled domain, or split-horizon DNS) passes the +screen, and the VPN then connects to the private address. With this option on, +Edge/CLI `web_fetch` therefore no longer guarantees that requests stay off +private networks for hosts answered with fake addresses; it relies on the VPN's +routing instead. Keep it at `block` unless fake-IP DNS blocks web access. Like other Astra +configuration, the file is protected from agent writes only by the permission +mode: Bypass, or Auto with `allow_sensitive_path_writes`, lets the agent edit it. + ### Observability - `ASTRA_LOG_FORMAT`, `ASTRA_SERVICE_NAME`, `ASTRA_OTEL_ENABLED`