diff --git a/AGENTS.md b/AGENTS.md index c6680c3c..3d77ae4e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,6 +10,10 @@ Use `make` to see available commands for building, testing, and formatting. `make check-rust` requires `cargo-hack` 0.6.45. The Rust check and test targets require the Hyperscan development library so they can check every Cargo feature. +Optional heavy dependencies are behind default-on Cargo features. Build with `--no-default-features` +to drop them: `third-party-active-checkers` (reqwest, aws-sign-v4) and `token-efficiency-checker` +(tiktoken-rs and its BPE vocabulary; gates `TokenEfficiencyCheck` and `NonHexPlusTokenEfficiencyChecker`). + ## Publishing - `.github/workflows/publish-crate.yml` publishes the crate on pushes to `main` and supports manual publication or a dry run. diff --git a/scripts/generate_license_3rdparty.sh b/scripts/generate_license_3rdparty.sh index f8b910cf..4107b626 100755 --- a/scripts/generate_license_3rdparty.sh +++ b/scripts/generate_license_3rdparty.sh @@ -30,8 +30,8 @@ import sys path = pathlib.Path(sys.argv[1]) contents = path.read_text() -default_features = 'default = ["dd-sds", "third-party-active-checkers"]' -license_features = 'default = ["dd-sds", "third-party-active-checkers", "sds-bindings-utils", "dd_sds_go", "sds-fuzz", "hyperscan", "manual_test"]' +default_features = 'default = ["dd-sds", "third-party-active-checkers", "token-efficiency-checker"]' +license_features = 'default = ["dd-sds", "third-party-active-checkers", "token-efficiency-checker", "sds-bindings-utils", "dd_sds_go", "sds-fuzz", "hyperscan", "manual_test"]' if default_features not in contents: raise SystemExit(f"Unable to find expected default feature line in {path}") diff --git a/sds/Cargo.toml b/sds/Cargo.toml index 4ae5bb53..d11ea289 100644 --- a/sds/Cargo.toml +++ b/sds/Cargo.toml @@ -12,13 +12,17 @@ name = "dd_sds" crate-type = ["staticlib", "cdylib", "lib"] [features] -default = ["dd-sds", "third-party-active-checkers"] +default = ["dd-sds", "third-party-active-checkers", "token-efficiency-checker"] dd-sds = [] # Enables third-party active checkers (HTTP and AWS match validation) and their # associated network dependencies (reqwest, aws-sign-v4). Enabled by default. # Build with `--no-default-features` (plus the features you need) to produce a # scanner without these dependencies. third-party-active-checkers = ["dep:reqwest", "dep:aws-sign-v4", "dep:rayon"] +# Enables the TokenEfficiencyCheck and NonHexPlusTokenEfficiencyChecker validators, which pull in +# tiktoken-rs and its embedded ~100k-entry BPE vocabulary. Enabled by default. Build with +# `--no-default-features` (plus the features you need) to produce a smaller scanner without them. +token-efficiency-checker = ["dep:tiktoken-rs"] sds-bindings-utils = ["dd-sds"] dd_sds_go = ["dd-sds", "sds-bindings-utils", "dep:serde_path_to_error"] sds-fuzz = ["dd-sds", "dep:afl", "dep:rand"] @@ -72,7 +76,7 @@ rand = { version = "0.8.5", optional = true } slab = "0.4.11" farmhash2 = "1.1.5" regex-pool = "0.1.1" -tiktoken-rs = "0.11.0" +tiktoken-rs = { version = "0.11.0", optional = true } # Optional fuzzing dependency. hyperscan = { version = "0.3.2", features = ["static"], optional = true } diff --git a/sds/benches/bench.rs b/sds/benches/bench.rs index 3c6f4227..360b6652 100644 --- a/sds/benches/bench.rs +++ b/sds/benches/bench.rs @@ -1,9 +1,11 @@ use criterion::Criterion; +#[cfg(feature = "token-efficiency-checker")] +use dd_sds::TokenEfficiencyCheck; use dd_sds::{ ContentVisitor, ExclusionCheck, Path, PathSegment, ProximityKeywordsConfig, RegexRuleConfig, RootRuleConfig, RuleIndexVisitor, ScannerError, Scope, ScopedRuleSet, }; -use dd_sds::{EntropyCheck, LuhnChecksum, TokenEfficiencyCheck, Validator}; +use dd_sds::{EntropyCheck, LuhnChecksum, Validator}; use dd_sds::{Scanner, SimpleEvent}; use std::collections::BTreeMap; @@ -119,7 +121,7 @@ pub fn luhn_checksum(c: &mut Criterion) { /// `TokenEfficiencyCheck` runs a full BPE tokenization, so it is far more expensive than the /// other statistical validator. Benchmark it against `EntropyCheck` on identical inputs to keep -/// that gap visible. +/// that gap visible. Without the `token-efficiency-checker` feature only the entropy baseline runs. pub fn token_efficiency_vs_entropy(c: &mut Criterion) { let values = vec![ "SOKXxs00k30PUuH4KLoDPNmwlQ4EwXKw", @@ -136,14 +138,17 @@ pub fn token_efficiency_vs_entropy(c: &mut Criterion) { }) }); - let token_efficiency = TokenEfficiencyCheck::new(); - c.bench_function("token-efficiency-check", |b| { - b.iter(|| { - for value in &values { - token_efficiency.is_valid_match(value); - } - }) - }); + #[cfg(feature = "token-efficiency-checker")] + { + let token_efficiency = TokenEfficiencyCheck::new(); + c.bench_function("token-efficiency-check", |b| { + b.iter(|| { + for value in &values { + token_efficiency.is_valid_match(value); + } + }) + }); + } } pub fn included_keywords(c: &mut Criterion) { diff --git a/sds/src/lib.rs b/sds/src/lib.rs index fb45c512..c4ad6076 100644 --- a/sds/src/lib.rs +++ b/sds/src/lib.rs @@ -131,5 +131,12 @@ pub use scanner::debug_scan::debug_scan; #[cfg(all(feature = "dd-sds", any(feature = "testing", feature = "bench")))] pub use crate::{ scoped_ruleset::{ContentVisitor, RuleIndexVisitor, ScopedRuleSet}, - secondary_validation::{EntropyCheck, LuhnChecksum, TokenEfficiencyCheck, Validator}, + secondary_validation::{EntropyCheck, LuhnChecksum, Validator}, }; + +#[cfg(all( + feature = "dd-sds", + feature = "token-efficiency-checker", + any(feature = "testing", feature = "bench") +))] +pub use crate::secondary_validation::TokenEfficiencyCheck; diff --git a/sds/src/scanner/regex_rule/config.rs b/sds/src/scanner/regex_rule/config.rs index 6a3879a2..d4600ceb 100644 --- a/sds/src/scanner/regex_rule/config.rs +++ b/sds/src/scanner/regex_rule/config.rs @@ -248,7 +248,9 @@ pub enum SecondaryValidator { IbanChecker, IrishPpsChecksum, ItalianNationalIdChecksum, - JwtClaimsValidator { config: JwtClaimsValidatorConfig }, + JwtClaimsValidator { + config: JwtClaimsValidatorConfig, + }, JwtExpirationChecker, LatviaNationalIdChecksum, LithuanianPersonalIdentificationNumberChecksum, @@ -266,6 +268,7 @@ pub enum SecondaryValidator { NhsCheckDigit, NirChecksum, NonHexChecker, + #[cfg(feature = "token-efficiency-checker")] NonHexPlusTokenEfficiencyChecker, PolishNationalIdChecksum, PolishNipChecksum, @@ -279,6 +282,7 @@ pub enum SecondaryValidator { SpanishNussChecksum, SwedenPINChecksum, ThaiNationalIdChecksum, + #[cfg(feature = "token-efficiency-checker")] TokenEfficiencyCheck, UkNinoFormatCheck, UkTrnChecksum, diff --git a/sds/src/scanner/test/validators.rs b/sds/src/scanner/test/validators.rs index ec6b95cb..c90cbb59 100644 --- a/sds/src/scanner/test/validators.rs +++ b/sds/src/scanner/test/validators.rs @@ -1,7 +1,9 @@ use crate::SecondaryValidator::{ ChineseIdChecksum, GithubTokenChecksum, IbanChecker, JwtExpirationChecker, NhsCheckDigit, - NonHexChecker, NonHexPlusTokenEfficiencyChecker, TokenEfficiencyCheck, + NonHexChecker, }; +#[cfg(feature = "token-efficiency-checker")] +use crate::SecondaryValidator::{NonHexPlusTokenEfficiencyChecker, TokenEfficiencyCheck}; use crate::scanner::RootRuleConfig; use crate::{MatchAction, RegexRuleConfig, ScannerBuilder, SecondaryValidator}; use chrono::Utc; @@ -201,6 +203,7 @@ fn test_non_hex_checker_filters_pure_hex() { assert_eq!(with_prefix, "[token]"); } +#[cfg(feature = "token-efficiency-checker")] #[test] fn test_token_efficiency_check_filters_identifiers() { let rule = RegexRuleConfig::new("[A-Za-z0-9]{16,}"); @@ -232,6 +235,7 @@ fn test_token_efficiency_check_filters_identifiers() { assert_eq!(content, "[secret] LibraryWebpageUploadUr1RowStatus"); } +#[cfg(feature = "token-efficiency-checker")] #[test] fn test_non_hex_plus_token_efficiency_checker_filters_hex_and_identifiers() { let rule = RegexRuleConfig::new("[A-Za-z0-9]{16,}"); diff --git a/sds/src/secondary_validation/mod.rs b/sds/src/secondary_validation/mod.rs index 99d210b2..23995509 100644 --- a/sds/src/secondary_validation/mod.rs +++ b/sds/src/secondary_validation/mod.rs @@ -39,6 +39,7 @@ mod monero_address; mod nhs_check_digit; mod nir_checksum; mod non_hex_checker; +#[cfg(feature = "token-efficiency-checker")] mod non_hex_plus_token_efficiency_checker; mod polish_national_id_checksum; mod polish_nip_checksum; @@ -52,6 +53,7 @@ mod spain_dni_checksum; mod spanish_nuss_checksum; mod sweden_pin_checksum; mod thai_national_id_checksum; +#[cfg(feature = "token-efficiency-checker")] mod token_efficiency; mod uk_nino_format_check; mod uk_trn_checksum; @@ -108,6 +110,7 @@ pub use crate::secondary_validation::monero_address::MoneroAddress; pub use crate::secondary_validation::nhs_check_digit::NhsCheckDigit; pub use crate::secondary_validation::nir_checksum::NirChecksum; pub use crate::secondary_validation::non_hex_checker::NonHexChecker; +#[cfg(feature = "token-efficiency-checker")] pub use crate::secondary_validation::non_hex_plus_token_efficiency_checker::NonHexPlusTokenEfficiencyChecker; pub use crate::secondary_validation::polish_national_id_checksum::PolishNationalIdChecksum; pub use crate::secondary_validation::polish_nip_checksum::PolishNipChecksum; @@ -121,6 +124,7 @@ pub use crate::secondary_validation::spain_dni_checksum::SpanishDniChecksum; pub use crate::secondary_validation::spanish_nuss_checksum::SpanishNussChecksum; pub use crate::secondary_validation::sweden_pin_checksum::SwedenPINChecksum; pub use crate::secondary_validation::thai_national_id_checksum::ThaiNationalIdChecksum; +#[cfg(feature = "token-efficiency-checker")] pub use crate::secondary_validation::token_efficiency::TokenEfficiencyCheck; pub use crate::secondary_validation::uk_nino_format_check::UkNinoFormatCheck; pub use crate::secondary_validation::uk_trn_checksum::UkTrnChecksum; @@ -267,6 +271,7 @@ impl SecondaryValidator { SecondaryValidator::NhsCheckDigit => Arc::new(NhsCheckDigit), SecondaryValidator::NirChecksum => Arc::new(NirChecksum), SecondaryValidator::NonHexChecker => Arc::new(NonHexChecker), + #[cfg(feature = "token-efficiency-checker")] SecondaryValidator::NonHexPlusTokenEfficiencyChecker => { Arc::new(NonHexPlusTokenEfficiencyChecker::new()) } @@ -284,6 +289,7 @@ impl SecondaryValidator { SecondaryValidator::SpanishNussChecksum => Arc::new(SpanishNussChecksum), SecondaryValidator::SwedenPINChecksum => Arc::new(SwedenPINChecksum), SecondaryValidator::ThaiNationalIdChecksum => Arc::new(ThaiNationalIdChecksum), + #[cfg(feature = "token-efficiency-checker")] SecondaryValidator::TokenEfficiencyCheck => Arc::new(TokenEfficiencyCheck::new()), SecondaryValidator::UkNinoFormatCheck => Arc::new(UkNinoFormatCheck), SecondaryValidator::UkTrnChecksum => Arc::new(UkTrnChecksum),