[DATASEC-498] Add token-efficiency-checker feature to drop tiktoken-rs

AGENTS.md

@@ -10,6 +10,10 @@ Use `make` to see available commands for building, testing, and formatting.
 `make check-rust` requires `cargo-hack` 0.6.45. The Rust check and test targets require the
 Hyperscan development library so they can check every Cargo feature.
 
+Optional heavy dependencies are behind default-on Cargo features. Build with `--no-default-features`
+to drop them: `third-party-active-checkers` (reqwest, aws-sign-v4) and `token-efficiency-checker`
+(tiktoken-rs and its BPE vocabulary; gates `TokenEfficiencyCheck` and `NonHexPlusTokenEfficiencyChecker`).
+
 ## Publishing
 
 - `.github/workflows/publish-crate.yml` publishes the crate on pushes to `main` and supports manual publication or a dry run.

scripts/generate_license_3rdparty.sh

@@ -30,8 +30,8 @@ import sys
 
 path = pathlib.Path(sys.argv[1])
 contents = path.read_text()
-default_features = 'default = ["dd-sds", "third-party-active-checkers"]'
-license_features = 'default = ["dd-sds", "third-party-active-checkers", "sds-bindings-utils", "dd_sds_go", "sds-fuzz", "hyperscan", "manual_test"]'
+default_features = 'default = ["dd-sds", "third-party-active-checkers", "token-efficiency-checker"]'
+license_features = 'default = ["dd-sds", "third-party-active-checkers", "token-efficiency-checker", "sds-bindings-utils", "dd_sds_go", "sds-fuzz", "hyperscan", "manual_test"]'
 
 if default_features not in contents:
     raise SystemExit(f"Unable to find expected default feature line in {path}")

sds/Cargo.toml

@@ -12,13 +12,17 @@ name = "dd_sds"
 crate-type = ["staticlib", "cdylib", "lib"]
 
 [features]
-default = ["dd-sds", "third-party-active-checkers"]
+default = ["dd-sds", "third-party-active-checkers", "token-efficiency-checker"]
 dd-sds = []
 # Enables third-party active checkers (HTTP and AWS match validation) and their
 # associated network dependencies (reqwest, aws-sign-v4). Enabled by default.
 # Build with `--no-default-features` (plus the features you need) to produce a
 # scanner without these dependencies.
 third-party-active-checkers = ["dep:reqwest", "dep:aws-sign-v4", "dep:rayon"]
+# Enables the TokenEfficiencyCheck and NonHexPlusTokenEfficiencyChecker validators, which pull in
+# tiktoken-rs and its embedded ~100k-entry BPE vocabulary. Enabled by default. Build with
+# `--no-default-features` (plus the features you need) to produce a smaller scanner without them.
+token-efficiency-checker = ["dep:tiktoken-rs"]
 sds-bindings-utils = ["dd-sds"]
 dd_sds_go = ["dd-sds", "sds-bindings-utils", "dep:serde_path_to_error"]
 sds-fuzz = ["dd-sds", "dep:afl", "dep:rand"]
@@ -72,7 +76,7 @@ rand = { version = "0.8.5", optional = true }
 slab = "0.4.11"
 farmhash2 = "1.1.5"
 regex-pool = "0.1.1"
-tiktoken-rs = "0.11.0"
+tiktoken-rs = { version = "0.11.0", optional = true }
 
 # Optional fuzzing dependency.
 hyperscan = { version = "0.3.2", features = ["static"], optional = true }

sds/benches/bench.rs

@@ -1,9 +1,11 @@
 use criterion::Criterion;
+#[cfg(feature = "token-efficiency-checker")]
+use dd_sds::TokenEfficiencyCheck;
 use dd_sds::{
     ContentVisitor, ExclusionCheck, Path, PathSegment, ProximityKeywordsConfig, RegexRuleConfig,
     RootRuleConfig, RuleIndexVisitor, ScannerError, Scope, ScopedRuleSet,
 };
-use dd_sds::{EntropyCheck, LuhnChecksum, TokenEfficiencyCheck, Validator};
+use dd_sds::{EntropyCheck, LuhnChecksum, Validator};
 use dd_sds::{Scanner, SimpleEvent};
 use std::collections::BTreeMap;
 
@@ -119,7 +121,7 @@ pub fn luhn_checksum(c: &mut Criterion) {
 
 /// `TokenEfficiencyCheck` runs a full BPE tokenization, so it is far more expensive than the
 /// other statistical validator. Benchmark it against `EntropyCheck` on identical inputs to keep
-/// that gap visible.
+/// that gap visible. Without the `token-efficiency-checker` feature only the entropy baseline runs.
 pub fn token_efficiency_vs_entropy(c: &mut Criterion) {
     let values = vec![
         "SOKXxs00k30PUuH4KLoDPNmwlQ4EwXKw",
@@ -136,14 +138,17 @@ pub fn token_efficiency_vs_entropy(c: &mut Criterion) {
         })
     });
 
-    let token_efficiency = TokenEfficiencyCheck::new();
-    c.bench_function("token-efficiency-check", |b| {
-        b.iter(|| {
-            for value in &values {
-                token_efficiency.is_valid_match(value);
-            }
-        })
-    });
+    #[cfg(feature = "token-efficiency-checker")]
+    {
+        let token_efficiency = TokenEfficiencyCheck::new();
+        c.bench_function("token-efficiency-check", |b| {
+            b.iter(|| {
+                for value in &values {
+                    token_efficiency.is_valid_match(value);
+                }
+            })
+        });
+    }
 }
 
 pub fn included_keywords(c: &mut Criterion) {

sds/src/lib.rs

@@ -131,5 +131,12 @@ pub use scanner::debug_scan::debug_scan;
 #[cfg(all(feature = "dd-sds", any(feature = "testing", feature = "bench")))]
 pub use crate::{
     scoped_ruleset::{ContentVisitor, RuleIndexVisitor, ScopedRuleSet},
-    secondary_validation::{EntropyCheck, LuhnChecksum, TokenEfficiencyCheck, Validator},
+    secondary_validation::{EntropyCheck, LuhnChecksum, Validator},
 };
+
+#[cfg(all(
+    feature = "dd-sds",
+    feature = "token-efficiency-checker",
+    any(feature = "testing", feature = "bench")
+))]
+pub use crate::secondary_validation::TokenEfficiencyCheck;

sds/src/scanner/regex_rule/config.rs

@@ -248,7 +248,9 @@ pub enum SecondaryValidator {
     IbanChecker,
     IrishPpsChecksum,
     ItalianNationalIdChecksum,
-    JwtClaimsValidator { config: JwtClaimsValidatorConfig },
+    JwtClaimsValidator {
+        config: JwtClaimsValidatorConfig,
+    },
     JwtExpirationChecker,
     LatviaNationalIdChecksum,
     LithuanianPersonalIdentificationNumberChecksum,
@@ -266,6 +268,7 @@ pub enum SecondaryValidator {
     NhsCheckDigit,
     NirChecksum,
     NonHexChecker,
+    #[cfg(feature = "token-efficiency-checker")]
     NonHexPlusTokenEfficiencyChecker,
     PolishNationalIdChecksum,
     PolishNipChecksum,
@@ -279,6 +282,7 @@ pub enum SecondaryValidator {
     SpanishNussChecksum,
     SwedenPINChecksum,
     ThaiNationalIdChecksum,
+    #[cfg(feature = "token-efficiency-checker")]
     TokenEfficiencyCheck,
     UkNinoFormatCheck,
     UkTrnChecksum,

sds/src/scanner/test/validators.rs

@@ -1,7 +1,9 @@
 use crate::SecondaryValidator::{
     ChineseIdChecksum, GithubTokenChecksum, IbanChecker, JwtExpirationChecker, NhsCheckDigit,
-    NonHexChecker, NonHexPlusTokenEfficiencyChecker, TokenEfficiencyCheck,
+    NonHexChecker,
 };
+#[cfg(feature = "token-efficiency-checker")]
+use crate::SecondaryValidator::{NonHexPlusTokenEfficiencyChecker, TokenEfficiencyCheck};
 use crate::scanner::RootRuleConfig;
 use crate::{MatchAction, RegexRuleConfig, ScannerBuilder, SecondaryValidator};
 use chrono::Utc;
@@ -201,6 +203,7 @@ fn test_non_hex_checker_filters_pure_hex() {
     assert_eq!(with_prefix, "[token]");
 }
 
+#[cfg(feature = "token-efficiency-checker")]
 #[test]
 fn test_token_efficiency_check_filters_identifiers() {
     let rule = RegexRuleConfig::new("[A-Za-z0-9]{16,}");
@@ -232,6 +235,7 @@ fn test_token_efficiency_check_filters_identifiers() {
     assert_eq!(content, "[secret] LibraryWebpageUploadUr1RowStatus");
 }
 
+#[cfg(feature = "token-efficiency-checker")]
 #[test]
 fn test_non_hex_plus_token_efficiency_checker_filters_hex_and_identifiers() {
     let rule = RegexRuleConfig::new("[A-Za-z0-9]{16,}");

sds/src/secondary_validation/mod.rs

@@ -39,6 +39,7 @@ mod monero_address;
 mod nhs_check_digit;
 mod nir_checksum;
 mod non_hex_checker;
+#[cfg(feature = "token-efficiency-checker")]
 mod non_hex_plus_token_efficiency_checker;
 mod polish_national_id_checksum;
 mod polish_nip_checksum;
@@ -52,6 +53,7 @@ mod spain_dni_checksum;
 mod spanish_nuss_checksum;
 mod sweden_pin_checksum;
 mod thai_national_id_checksum;
+#[cfg(feature = "token-efficiency-checker")]
 mod token_efficiency;
 mod uk_nino_format_check;
 mod uk_trn_checksum;
@@ -108,6 +110,7 @@ pub use crate::secondary_validation::monero_address::MoneroAddress;
 pub use crate::secondary_validation::nhs_check_digit::NhsCheckDigit;
 pub use crate::secondary_validation::nir_checksum::NirChecksum;
 pub use crate::secondary_validation::non_hex_checker::NonHexChecker;
+#[cfg(feature = "token-efficiency-checker")]
 pub use crate::secondary_validation::non_hex_plus_token_efficiency_checker::NonHexPlusTokenEfficiencyChecker;
 pub use crate::secondary_validation::polish_national_id_checksum::PolishNationalIdChecksum;
 pub use crate::secondary_validation::polish_nip_checksum::PolishNipChecksum;
@@ -121,6 +124,7 @@ pub use crate::secondary_validation::spain_dni_checksum::SpanishDniChecksum;
 pub use crate::secondary_validation::spanish_nuss_checksum::SpanishNussChecksum;
 pub use crate::secondary_validation::sweden_pin_checksum::SwedenPINChecksum;
 pub use crate::secondary_validation::thai_national_id_checksum::ThaiNationalIdChecksum;
+#[cfg(feature = "token-efficiency-checker")]
 pub use crate::secondary_validation::token_efficiency::TokenEfficiencyCheck;
 pub use crate::secondary_validation::uk_nino_format_check::UkNinoFormatCheck;
 pub use crate::secondary_validation::uk_trn_checksum::UkTrnChecksum;
@@ -267,6 +271,7 @@ impl SecondaryValidator {
             SecondaryValidator::NhsCheckDigit => Arc::new(NhsCheckDigit),
             SecondaryValidator::NirChecksum => Arc::new(NirChecksum),
             SecondaryValidator::NonHexChecker => Arc::new(NonHexChecker),
+            #[cfg(feature = "token-efficiency-checker")]
             SecondaryValidator::NonHexPlusTokenEfficiencyChecker => {
                 Arc::new(NonHexPlusTokenEfficiencyChecker::new())
             }
@@ -284,6 +289,7 @@ impl SecondaryValidator {
             SecondaryValidator::SpanishNussChecksum => Arc::new(SpanishNussChecksum),
             SecondaryValidator::SwedenPINChecksum => Arc::new(SwedenPINChecksum),
             SecondaryValidator::ThaiNationalIdChecksum => Arc::new(ThaiNationalIdChecksum),
+            #[cfg(feature = "token-efficiency-checker")]
             SecondaryValidator::TokenEfficiencyCheck => Arc::new(TokenEfficiencyCheck::new()),
             SecondaryValidator::UkNinoFormatCheck => Arc::new(UkNinoFormatCheck),
             SecondaryValidator::UkTrnChecksum => Arc::new(UkTrnChecksum),