diff --git a/CHANGELOG.md b/CHANGELOG.md index 6103a6e..ae40563 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v2.0.0 - 2026-09-08 + +- Cut the Rust source API over to explicit borrowed execution options for CSV analysis, preview, output, paste transformation, and quick generation; the public wire contracts and CLI/Tauri command names remain unchanged. +- Centralize frontend protection readiness so disabled controls and direct CSV, paste, and quick actions agree, including selected Local AI download blocking and the specific malformed tokenization-key error. +- Isolate anonymization job transport tracking and split internal core type ownership behind the unchanged root type façade. + ## v1.0.93 - 2026-09-08 - Close anonymization disclosure paths in pasted text and XML, including complete PEM redaction, character-reference transformation, and honest residual-audit evidence. diff --git a/Cargo.lock b/Cargo.lock index a8db20d..0a13deb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -681,7 +681,7 @@ dependencies = [ [[package]] name = "csv-anonymizer-app" -version = "1.0.93" +version = "2.0.0" dependencies = [ "csv-anonymizer-core", "tempfile", @@ -689,7 +689,7 @@ dependencies = [ [[package]] name = "csv-anonymizer-core" -version = "1.0.93" +version = "2.0.0" dependencies = [ "blake3", "card-validate", @@ -720,7 +720,7 @@ dependencies = [ [[package]] name = "csv-anonymizer-tauri" -version = "1.0.93" +version = "2.0.0" dependencies = [ "csv-anonymizer-core", "directories", diff --git a/Cargo.toml b/Cargo.toml index dcd0582..3460d5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ resolver = "3" [workspace.package] -version = "1.0.93" +version = "2.0.0" edition = "2024" license = "MIT" authors = ["Douwe de Vries "] diff --git a/build/linux/io.github.ddv1982.csv-data-anonymizer.metainfo.xml b/build/linux/io.github.ddv1982.csv-data-anonymizer.metainfo.xml index e56d2f2..727cfa0 100644 --- a/build/linux/io.github.ddv1982.csv-data-anonymizer.metainfo.xml +++ b/build/linux/io.github.ddv1982.csv-data-anonymizer.metainfo.xml @@ -17,6 +17,7 @@ Utility + diff --git a/crates/csv-anonymizer-app/src/cli.rs b/crates/csv-anonymizer-app/src/cli.rs index 8222152..b87a615 100644 --- a/crates/csv-anonymizer-app/src/cli.rs +++ b/crates/csv-anonymizer-app/src/cli.rs @@ -127,7 +127,7 @@ pub(crate) fn run_cli(action: CliAction) -> Result<(), String> { match action { CliAction::Analyze { input } => { let headers = service - .analyze_csv(&input) + .analyze_csv(&input, csv_anonymizer_core::CsvAnalysisOptions::default()) .map_err(|error| error.to_string())?; println!( "CSV Anonymizer {} inspected {} rows in {}", @@ -145,7 +145,7 @@ pub(crate) fn run_cli(action: CliAction) -> Result<(), String> { } CliAction::SmokeAnonymize { input, output } => { let headers = service - .analyze_csv(&input) + .analyze_csv(&input, csv_anonymizer_core::CsvAnalysisOptions::default()) .map_err(|error| error.to_string())?; let columns = headers .columns @@ -158,29 +158,35 @@ pub(crate) fn run_cli(action: CliAction) -> Result<(), String> { } let preview = service - .preview_anonymization(PreviewParams { - file_path: input.clone(), - columns: columns.clone(), - controls: vec![], - sample_count: 2, - // The analyze and anonymize calls either side of this one take - // the service default, so the preview has to ask for it too. - sample_row_count: 100, - }) + .preview_anonymization( + PreviewParams { + file_path: input.clone(), + columns: columns.clone(), + controls: vec![], + sample_count: 2, + // The analyze and anonymize calls either side of this one take + // the service default, so the preview has to ask for it too. + sample_row_count: 100, + }, + csv_anonymizer_core::TransformRuntime::default(), + ) .map_err(|error| error.to_string())?; if preview.previews.is_empty() { return Err("smoke preview did not produce any column samples".to_string()); } let result = service - .anonymize_csv(AnonymizeParams { - file_path: input, - output_path: output, - columns, - controls: vec![], - force: true, - preview_smart_replacements: preview.smart_replacements, - }) + .anonymize_csv( + AnonymizeParams { + file_path: input, + output_path: output, + columns, + controls: vec![], + force: true, + preview_smart_replacements: preview.smart_replacements, + }, + csv_anonymizer_core::CsvRunOptions::default(), + ) .map_err(|error| error.to_string())?; println!( "CSV Anonymizer smoke OK: wrote {} rows to {} in {} ms", @@ -197,14 +203,17 @@ pub(crate) fn run_cli(action: CliAction) -> Result<(), String> { force, } => { let result = service - .anonymize_csv(AnonymizeParams { - file_path: input, - output_path: output, - columns, - controls: vec![], - force, - preview_smart_replacements: vec![], - }) + .anonymize_csv( + AnonymizeParams { + file_path: input, + output_path: output, + columns, + controls: vec![], + force, + preview_smart_replacements: vec![], + }, + csv_anonymizer_core::CsvRunOptions::default(), + ) .map_err(|error| error.to_string())?; println!( "Wrote {} rows to {} in {} ms", diff --git a/crates/csv-anonymizer-core/benches/csv_streaming.rs b/crates/csv-anonymizer-core/benches/csv_streaming.rs index f929c45..54fc8ea 100644 --- a/crates/csv-anonymizer-core/benches/csv_streaming.rs +++ b/crates/csv-anonymizer-core/benches/csv_streaming.rs @@ -38,25 +38,28 @@ fn bench_standard_csv_streaming(c: &mut Criterion) { c.bench_function("anonymize_standard_streaming_10k", |b| { b.iter(|| { service - .anonymize_csv(AnonymizeParams { - file_path: black_box(input_path.clone()), - output_path: output_path.clone(), - columns: vec![1, 2], - controls: vec![ - ColumnControl { - column_index: 1, - type_override: Some(DataType::Email), - strategy: AnonymizationStrategy::Auto, - }, - ColumnControl { - column_index: 2, - type_override: Some(DataType::FullName), - strategy: AnonymizationStrategy::Auto, - }, - ], - force: true, - preview_smart_replacements: vec![], - }) + .anonymize_csv( + AnonymizeParams { + file_path: black_box(input_path.clone()), + output_path: output_path.clone(), + columns: vec![1, 2], + controls: vec![ + ColumnControl { + column_index: 1, + type_override: Some(DataType::Email), + strategy: AnonymizationStrategy::Auto, + }, + ColumnControl { + column_index: 2, + type_override: Some(DataType::FullName), + strategy: AnonymizationStrategy::Auto, + }, + ], + force: true, + preview_smart_replacements: vec![], + }, + csv_anonymizer_core::CsvRunOptions::default(), + ) .expect("standard anonymization should succeed") }) }); @@ -108,18 +111,21 @@ fn bench_cardinality(c: &mut Criterion) { group.bench_function(format!("{strategy_label}_{shape_label}_10k"), |b| { b.iter(|| { service - .anonymize_csv(AnonymizeParams { - file_path: black_box(input_path.clone()), - output_path: output_path.clone(), - columns: vec![1], - controls: vec![ColumnControl { - column_index: 1, - type_override: Some(DataType::String), - strategy, - }], - force: true, - preview_smart_replacements: vec![], - }) + .anonymize_csv( + AnonymizeParams { + file_path: black_box(input_path.clone()), + output_path: output_path.clone(), + columns: vec![1], + controls: vec![ColumnControl { + column_index: 1, + type_override: Some(DataType::String), + strategy, + }], + force: true, + preview_smart_replacements: vec![], + }, + csv_anonymizer_core::CsvRunOptions::default(), + ) .expect("cardinality anonymization should succeed") }) }); diff --git a/crates/csv-anonymizer-core/benches/direct_input_paste.rs b/crates/csv-anonymizer-core/benches/direct_input_paste.rs index 81da338..bca331b 100644 --- a/crates/csv-anonymizer-core/benches/direct_input_paste.rs +++ b/crates/csv-anonymizer-core/benches/direct_input_paste.rs @@ -7,11 +7,14 @@ use std::hint::black_box; fn bench_pasted_json(c: &mut Criterion) { let content = build_json_payload(1_000); - let analysis = direct_input::analyze_paste_data(PasteAnalyzeParams { - content: content.clone(), - format: PasteDataFormat::Json, - sample_row_count: 1_000, - }) + let analysis = direct_input::analyze_paste_data( + PasteAnalyzeParams { + content: content.clone(), + format: PasteDataFormat::Json, + sample_row_count: 1_000, + }, + None, + ) .expect("benchmark JSON analysis should succeed"); let selected_columns = columns_for(&analysis.columns, &["[].email", "[].fullName", "[].userId"]); @@ -23,25 +26,31 @@ fn bench_pasted_json(c: &mut Criterion) { c.bench_function("analyze_pasted_json_1k", |b| { b.iter(|| { - direct_input::analyze_paste_data(PasteAnalyzeParams { - content: black_box(content.clone()), - format: PasteDataFormat::Json, - sample_row_count: 1_000, - }) + direct_input::analyze_paste_data( + PasteAnalyzeParams { + content: black_box(content.clone()), + format: PasteDataFormat::Json, + sample_row_count: 1_000, + }, + None, + ) .expect("JSON analysis should succeed") }) }); c.bench_function("transform_pasted_json_1k", |b| { b.iter(|| { - direct_input::transform_paste_data(PasteTransformParams { - content: black_box(content.clone()), - format: PasteDataFormat::Json, - columns: selected_columns.clone(), - controls: controls.clone(), - sample_row_count: 1_000, - preview_smart_replacements: Vec::new(), - }) + direct_input::transform_paste_data( + PasteTransformParams { + content: black_box(content.clone()), + format: PasteDataFormat::Json, + columns: selected_columns.clone(), + controls: controls.clone(), + sample_row_count: 1_000, + preview_smart_replacements: Vec::new(), + }, + csv_anonymizer_core::TransformRuntime::default(), + ) .expect("JSON transform should succeed") }) }); @@ -49,11 +58,14 @@ fn bench_pasted_json(c: &mut Criterion) { fn bench_pasted_xml(c: &mut Criterion) { let content = build_xml_payload(1_000); - let analysis = direct_input::analyze_paste_data(PasteAnalyzeParams { - content: content.clone(), - format: PasteDataFormat::Xml, - sample_row_count: 1_000, - }) + let analysis = direct_input::analyze_paste_data( + PasteAnalyzeParams { + content: content.clone(), + format: PasteDataFormat::Xml, + sample_row_count: 1_000, + }, + None, + ) .expect("benchmark XML analysis should succeed"); let selected_columns = columns_for(&analysis.columns, &["users.user.@email", "users.user.name"]); @@ -64,25 +76,31 @@ fn bench_pasted_xml(c: &mut Criterion) { c.bench_function("analyze_pasted_xml_1k", |b| { b.iter(|| { - direct_input::analyze_paste_data(PasteAnalyzeParams { - content: black_box(content.clone()), - format: PasteDataFormat::Xml, - sample_row_count: 1_000, - }) + direct_input::analyze_paste_data( + PasteAnalyzeParams { + content: black_box(content.clone()), + format: PasteDataFormat::Xml, + sample_row_count: 1_000, + }, + None, + ) .expect("XML analysis should succeed") }) }); c.bench_function("transform_pasted_xml_1k", |b| { b.iter(|| { - direct_input::transform_paste_data(PasteTransformParams { - content: black_box(content.clone()), - format: PasteDataFormat::Xml, - columns: selected_columns.clone(), - controls: controls.clone(), - sample_row_count: 1_000, - preview_smart_replacements: Vec::new(), - }) + direct_input::transform_paste_data( + PasteTransformParams { + content: black_box(content.clone()), + format: PasteDataFormat::Xml, + columns: selected_columns.clone(), + controls: controls.clone(), + sample_row_count: 1_000, + preview_smart_replacements: Vec::new(), + }, + csv_anonymizer_core::TransformRuntime::default(), + ) .expect("XML transform should succeed") }) }); @@ -90,11 +108,14 @@ fn bench_pasted_xml(c: &mut Criterion) { fn bench_pasted_logs(c: &mut Criterion) { let content = build_log_payload(1_000); - let analysis = direct_input::analyze_paste_data(PasteAnalyzeParams { - content: content.clone(), - format: PasteDataFormat::Logs, - sample_row_count: 1_000, - }) + let analysis = direct_input::analyze_paste_data( + PasteAnalyzeParams { + content: content.clone(), + format: PasteDataFormat::Logs, + sample_row_count: 1_000, + }, + None, + ) .expect("benchmark log analysis should succeed"); let selected_columns = columns_for(&analysis.columns, &["email", "ipAddress"]); let controls = vec![ @@ -104,25 +125,31 @@ fn bench_pasted_logs(c: &mut Criterion) { c.bench_function("analyze_pasted_logs_1k", |b| { b.iter(|| { - direct_input::analyze_paste_data(PasteAnalyzeParams { - content: black_box(content.clone()), - format: PasteDataFormat::Logs, - sample_row_count: 1_000, - }) + direct_input::analyze_paste_data( + PasteAnalyzeParams { + content: black_box(content.clone()), + format: PasteDataFormat::Logs, + sample_row_count: 1_000, + }, + None, + ) .expect("log analysis should succeed") }) }); c.bench_function("transform_pasted_logs_1k", |b| { b.iter(|| { - direct_input::transform_paste_data(PasteTransformParams { - content: black_box(content.clone()), - format: PasteDataFormat::Logs, - columns: selected_columns.clone(), - controls: controls.clone(), - sample_row_count: 1_000, - preview_smart_replacements: Vec::new(), - }) + direct_input::transform_paste_data( + PasteTransformParams { + content: black_box(content.clone()), + format: PasteDataFormat::Logs, + columns: selected_columns.clone(), + controls: controls.clone(), + sample_row_count: 1_000, + preview_smart_replacements: Vec::new(), + }, + csv_anonymizer_core::TransformRuntime::default(), + ) .expect("log transform should succeed") }) }); diff --git a/crates/csv-anonymizer-core/src/direct_input/csv_text.rs b/crates/csv-anonymizer-core/src/direct_input/csv_text.rs index b23d637..4919ade 100644 --- a/crates/csv-anonymizer-core/src/direct_input/csv_text.rs +++ b/crates/csv-anonymizer-core/src/direct_input/csv_text.rs @@ -20,42 +20,14 @@ use super::shared::{ paste_transform_data, preview_rows_with_smart_provider, }; -pub(super) fn analyze_csv_text(content: &str, sample_row_count: usize) -> Result { - analyze_csv_text_with_coverage(content, sample_row_count).map(|(analysis, _)| analysis) -} - -pub(super) fn analyze_csv_text_with_candidate_detector( - content: &str, - sample_row_count: usize, - detector: &mut dyn CandidateDetector, -) -> Result { - analyze_csv_text_with_coverage_and_candidate_detector(content, sample_row_count, Some(detector)) - .map(|(analysis, _)| analysis) -} - -/// [`analyze_csv_text`] plus how much of the paste it classified, in the crate's own -/// coverage type. -/// -/// Still split even though the DTO now carries a summary of the same figures: the -/// transform path feeds `build_privacy_report`, which takes a [`DetectionCoverage`], -/// and rebuilding one from the wire summary would re-open the invariant that type's -/// constructor exists to hold. -fn analyze_csv_text_with_coverage( - content: &str, - sample_row_count: usize, -) -> Result<(PasteAnalyzeData, DetectionCoverage)> { - analyze_csv_text_with_coverage_and_candidate_detector(content, sample_row_count, None) -} - -fn analyze_csv_text_with_coverage_and_candidate_detector( +/// CSV analysis plus how much of the paste it classified. The transform path +/// retains the coverage invariant for the privacy report. +pub(super) fn analyze_csv_text( content: &str, sample_row_count: usize, detector: Option<&mut dyn CandidateDetector>, ) -> Result<(PasteAnalyzeData, DetectionCoverage)> { let sample_row_count = paste_detection_sample_rows(sample_row_count)?; - // Spread the sample over the whole paste: pasted content can exceed the - // sample cap, and a head window would leave detection blind to values that - // only appear in the tail. let sample = read_csv_detection_sample_from_str(content, sample_row_count)?; let (columns, detector_status) = build_column_metadata_with_candidate_detector(&sample.headers, &sample.rows, detector); @@ -108,8 +80,7 @@ pub(super) fn transform_csv_text_with_smart_provider( provider: Option<&mut dyn SmartReplacementProvider>, tokenization_key: Option<&crate::TokenizationKey>, ) -> Result { - let (analysis, coverage) = - analyze_csv_text_with_coverage(&input.content, input.sample_row_count)?; + let (analysis, coverage) = analyze_csv_text(&input.content, input.sample_row_count, None)?; let metadata = select_columns(&analysis.columns, &input.columns, &input.controls)?; let rows = read_csv_sample_from_str(&input.content, usize::MAX)?.rows; let existing_smart_replacements = diff --git a/crates/csv-anonymizer-core/src/direct_input/documents.rs b/crates/csv-anonymizer-core/src/direct_input/documents.rs index 7d0df1e..7d96eea 100644 --- a/crates/csv-anonymizer-core/src/direct_input/documents.rs +++ b/crates/csv-anonymizer-core/src/direct_input/documents.rs @@ -13,10 +13,9 @@ use std::time::Instant; use super::shared::{ FieldSampleLimits, FieldSamples, PreviewSelection, analysis_from_fields, - analysis_from_fields_with_candidate_detector, bounded_preview_sample_count, escape_path_key, - next_row_index, paste_detection_sample_rows, paste_transform_data, preview_field_sample_limits, - preview_from_fields_with_smart_provider, push_identified_field_sample, - selected_columns_by_source, smart_replacements_for_fields, + bounded_preview_sample_count, escape_path_key, next_row_index, paste_detection_sample_rows, + paste_transform_data, preview_field_sample_limits, preview_from_fields_with_smart_provider, + push_identified_field_sample, selected_columns_by_source, smart_replacements_for_fields, }; pub(super) fn preview_value_document_with_smart_provider( @@ -42,39 +41,22 @@ pub(super) fn preview_value_document_with_smart_provider( ) } -pub(super) fn transform_json_with_smart_provider( - input: PasteTransformParams, - provider: Option<&mut dyn SmartReplacementProvider>, - tokenization_key: Option<&crate::TokenizationKey>, -) -> Result { - let value = parse_json(&input.content)?; - let (output, result) = transform_value_document( - input, - value, - PasteDataFormat::Json, - provider, - tokenization_key, - )?; - let output = serde_json::to_string_pretty(&output) - .map_err(|error| AnonymizerError::input_parse("JSON", error.to_string()))?; - Ok(PasteTransformData { output, ..result }) -} - -pub(super) fn transform_yaml_with_smart_provider( +pub(super) fn transform_value_document_with_smart_provider( input: PasteTransformParams, + format: PasteDataFormat, provider: Option<&mut dyn SmartReplacementProvider>, tokenization_key: Option<&crate::TokenizationKey>, ) -> Result { - let value = parse_yaml(&input.content)?; - let (output, result) = transform_value_document( - input, - value, - PasteDataFormat::Yaml, - provider, - tokenization_key, - )?; - let output = yaml_serde::to_string(&output) - .map_err(|error| AnonymizerError::input_parse("YAML", error.to_string()))?; + let value = parse_value_document(format, &input.content)?; + let (output, result) = + transform_value_document(input, value, format, provider, tokenization_key)?; + let output = match format { + PasteDataFormat::Json => serde_json::to_string_pretty(&output) + .map_err(|error| AnonymizerError::input_parse("JSON", error.to_string()))?, + PasteDataFormat::Yaml => yaml_serde::to_string(&output) + .map_err(|error| AnonymizerError::input_parse("YAML", error.to_string()))?, + _ => unreachable!("value documents require JSON or YAML"), + }; Ok(PasteTransformData { output, ..result }) } @@ -87,7 +69,7 @@ fn transform_value_document( ) -> Result<(Value, PasteTransformData)> { let start_time = Instant::now(); let (analysis, coverage) = - analyze_value_document_with_coverage(format, &value, input.sample_row_count)?; + analyze_value_document(format, &value, input.sample_row_count, None)?; let metadata = select_columns(&analysis.columns, &input.columns, &input.controls)?; let selected_by_path = selected_columns_by_source(&metadata); let smart_replacements = @@ -146,41 +128,7 @@ pub(super) fn analyze_value_document( format: PasteDataFormat, value: &Value, sample_row_count: usize, -) -> Result { - analyze_value_document_with_coverage(format, value, sample_row_count) - .map(|(analysis, _)| analysis) -} - -pub(super) fn analyze_value_document_with_candidate_detector( - format: PasteDataFormat, - value: &Value, - sample_row_count: usize, - detector: &mut dyn CandidateDetector, -) -> Result { - let sample_row_count = paste_detection_sample_rows(sample_row_count)?; - let mut fields = Vec::new(); - collect_json_fields( - value, - format, - &mut Vec::new(), - &mut fields, - FieldSampleLimits::detection_only(sample_row_count), - )?; - Ok(analysis_from_fields_with_candidate_detector( - format, - &fields, - infer_value_row_count(value), - Some(detector), - ) - .0) -} - -/// [`analyze_value_document`] plus how much of the input it classified. Split for -/// the same reason as the XML pair: only the transform path reports coverage. -fn analyze_value_document_with_coverage( - format: PasteDataFormat, - value: &Value, - sample_row_count: usize, + detector: Option<&mut dyn CandidateDetector>, ) -> Result<(PasteAnalyzeData, DetectionCoverage)> { let sample_row_count = paste_detection_sample_rows(sample_row_count)?; let mut fields = Vec::new(); @@ -196,6 +144,7 @@ fn analyze_value_document_with_coverage( format, &fields, infer_value_row_count(value), + detector, )) } @@ -259,6 +208,14 @@ pub(super) fn parse_yaml(content: &str) -> Result { .map_err(|error| AnonymizerError::input_parse("YAML", error.to_string())) } +pub(super) fn parse_value_document(format: PasteDataFormat, content: &str) -> Result { + match format { + PasteDataFormat::Json => parse_json(content), + PasteDataFormat::Yaml => parse_yaml(content), + _ => unreachable!("value documents require JSON or YAML"), + } +} + fn collect_json_fields( value: &Value, format: PasteDataFormat, diff --git a/crates/csv-anonymizer-core/src/direct_input/mod.rs b/crates/csv-anonymizer-core/src/direct_input/mod.rs index 219af39..e2cf589 100644 --- a/crates/csv-anonymizer-core/src/direct_input/mod.rs +++ b/crates/csv-anonymizer-core/src/direct_input/mod.rs @@ -11,86 +11,36 @@ mod tests; use crate::detection::CandidateDetector; use crate::error::Result; +use crate::execution::TransformRuntime; use crate::metadata::should_auto_select_column; -use crate::smart::SmartReplacementProvider; use crate::types::{ PasteAnalyzeData, PasteAnalyzeParams, PasteDataFormat, PastePreviewParams, PasteTransformData, PasteTransformParams, PreviewData, }; -pub fn analyze_paste_data(input: PasteAnalyzeParams) -> Result { - analyze_paste_data_impl(input, None) -} - -pub fn analyze_paste_data_with_candidate_detector( - input: PasteAnalyzeParams, - detector: &mut dyn CandidateDetector, -) -> Result { - analyze_paste_data_impl(input, Some(detector)) -} - -fn analyze_paste_data_impl( +pub fn analyze_paste_data( input: PasteAnalyzeParams, detector: Option<&mut dyn CandidateDetector>, ) -> Result { shared::validate_paste_content(&input.content)?; let format = format_detection::resolve_format(input.format, &input.content); - let mut analysis = match (format, detector) { - (PasteDataFormat::Csv, Some(detector)) => { - csv_text::analyze_csv_text_with_candidate_detector( - &input.content, - input.sample_row_count, - detector, - ) - } - (PasteDataFormat::Csv, None) => { - csv_text::analyze_csv_text(&input.content, input.sample_row_count) - } - (PasteDataFormat::Json, Some(detector)) => { - let value = documents::parse_json(&input.content)?; - documents::analyze_value_document_with_candidate_detector( - format, - &value, - input.sample_row_count, - detector, - ) - } - (PasteDataFormat::Json, None) => { - let value = documents::parse_json(&input.content)?; - documents::analyze_value_document(format, &value, input.sample_row_count) - } - (PasteDataFormat::Yaml, Some(detector)) => { - let value = documents::parse_yaml(&input.content)?; - documents::analyze_value_document_with_candidate_detector( - format, - &value, - input.sample_row_count, - detector, - ) - } - (PasteDataFormat::Yaml, None) => { - let value = documents::parse_yaml(&input.content)?; - documents::analyze_value_document(format, &value, input.sample_row_count) + let mut analysis = match format { + PasteDataFormat::Csv => { + csv_text::analyze_csv_text(&input.content, input.sample_row_count, detector) + .map(|(analysis, _)| analysis) } - (PasteDataFormat::Xml, Some(detector)) => xml::analyze_xml_with_candidate_detector( - &input.content, - input.sample_row_count, - detector, - ), - (PasteDataFormat::Xml, None) => xml::analyze_xml(&input.content, input.sample_row_count), - (PasteDataFormat::PlainText | PasteDataFormat::Logs, Some(detector)) => { - text::analyze_text_content_with_candidate_detector( - &input.content, - format, - input.sample_row_count, - detector, - ) + PasteDataFormat::Json | PasteDataFormat::Yaml => { + let value = documents::parse_value_document(format, &input.content)?; + documents::analyze_value_document(format, &value, input.sample_row_count, detector) + .map(|(analysis, _)| analysis) } - (PasteDataFormat::PlainText | PasteDataFormat::Logs, None) => { - text::analyze_text_content(&input.content, format, input.sample_row_count) + PasteDataFormat::Xml => xml::analyze_xml(&input.content, input.sample_row_count, detector) + .map(|(analysis, _)| analysis), + PasteDataFormat::PlainText | PasteDataFormat::Logs => { + text::analyze_text_content(&input.content, format, input.sample_row_count, detector) } - (PasteDataFormat::Auto, _) => unreachable!("auto format must resolve before analysis"), + PasteDataFormat::Auto => unreachable!("auto format must resolve before analysis"), }?; for column in &mut analysis.columns { @@ -100,22 +50,14 @@ fn analyze_paste_data_impl( Ok(analysis) } -pub fn preview_paste_data(input: PastePreviewParams) -> Result { - preview_paste_data_with_smart_provider(input, None) -} - -pub fn preview_paste_data_with_smart_provider( +pub fn preview_paste_data( input: PastePreviewParams, - provider: Option<&mut dyn SmartReplacementProvider>, -) -> Result { - preview_paste_data_with_run_secrets(input, provider, None) -} - -pub fn preview_paste_data_with_run_secrets( - input: PastePreviewParams, - provider: Option<&mut dyn SmartReplacementProvider>, - tokenization_key: Option<&crate::TokenizationKey>, + runtime: TransformRuntime<'_, '_>, ) -> Result { + let TransformRuntime { + provider, + tokenization_key, + } = runtime; shared::validate_paste_content(&input.content)?; let format = format_detection::resolve_format(input.format, &input.content); @@ -123,22 +65,12 @@ pub fn preview_paste_data_with_run_secrets( PasteDataFormat::Csv => { csv_text::preview_csv_text_with_smart_provider(input, provider, tokenization_key) } - PasteDataFormat::Json => { - let value = documents::parse_json(&input.content)?; - documents::preview_value_document_with_smart_provider( - input, - value, - PasteDataFormat::Json, - provider, - tokenization_key, - ) - } - PasteDataFormat::Yaml => { - let value = documents::parse_yaml(&input.content)?; + PasteDataFormat::Json | PasteDataFormat::Yaml => { + let value = documents::parse_value_document(format, &input.content)?; documents::preview_value_document_with_smart_provider( input, value, - PasteDataFormat::Yaml, + format, provider, tokenization_key, ) @@ -158,22 +90,14 @@ pub fn preview_paste_data_with_run_secrets( } } -pub fn transform_paste_data(input: PasteTransformParams) -> Result { - transform_paste_data_with_smart_provider(input, None) -} - -pub fn transform_paste_data_with_smart_provider( - input: PasteTransformParams, - provider: Option<&mut dyn SmartReplacementProvider>, -) -> Result { - transform_paste_data_with_run_secrets(input, provider, None) -} - -pub fn transform_paste_data_with_run_secrets( +pub fn transform_paste_data( input: PasteTransformParams, - provider: Option<&mut dyn SmartReplacementProvider>, - tokenization_key: Option<&crate::TokenizationKey>, + runtime: TransformRuntime<'_, '_>, ) -> Result { + let TransformRuntime { + provider, + tokenization_key, + } = runtime; shared::validate_paste_content(&input.content)?; let format = format_detection::resolve_format(input.format, &input.content); @@ -181,11 +105,13 @@ pub fn transform_paste_data_with_run_secrets( PasteDataFormat::Csv => { csv_text::transform_csv_text_with_smart_provider(input, provider, tokenization_key) } - PasteDataFormat::Json => { - documents::transform_json_with_smart_provider(input, provider, tokenization_key) - } - PasteDataFormat::Yaml => { - documents::transform_yaml_with_smart_provider(input, provider, tokenization_key) + PasteDataFormat::Json | PasteDataFormat::Yaml => { + documents::transform_value_document_with_smart_provider( + input, + format, + provider, + tokenization_key, + ) } PasteDataFormat::Xml => { xml::transform_xml_with_smart_provider(input, provider, tokenization_key) @@ -197,46 +123,18 @@ pub fn transform_paste_data_with_run_secrets( } } -pub use quick::{ - generate_quick_values, generate_quick_values_with_run_secrets, - generate_quick_values_with_smart_provider, -}; +pub use quick::generate_quick_values; pub fn replay_paste_text_candidate_evidence( input: &PasteTransformParams, snapshot: &crate::PreparedAnalysisSnapshot, confirmed_candidate_ids: &[String], + runtime: TransformRuntime<'_, '_>, ) -> Result { - replay_paste_text_candidate_evidence_with_smart_provider( - input, - snapshot, - confirmed_candidate_ids, - None, - ) -} - -pub fn replay_paste_text_candidate_evidence_with_smart_provider( - input: &PasteTransformParams, - snapshot: &crate::PreparedAnalysisSnapshot, - confirmed_candidate_ids: &[String], - provider: Option<&mut dyn SmartReplacementProvider>, -) -> Result { - replay_paste_text_candidate_evidence_with_run_secrets( - input, - snapshot, - confirmed_candidate_ids, + let TransformRuntime { provider, - None, - ) -} - -pub fn replay_paste_text_candidate_evidence_with_run_secrets( - input: &PasteTransformParams, - snapshot: &crate::PreparedAnalysisSnapshot, - confirmed_candidate_ids: &[String], - provider: Option<&mut dyn SmartReplacementProvider>, - tokenization_key: Option<&crate::TokenizationKey>, -) -> Result { + tokenization_key, + } = runtime; shared::validate_paste_content(&input.content)?; let format = format_detection::resolve_format(input.format, &input.content); if !matches!(format, PasteDataFormat::PlainText | PasteDataFormat::Logs) { @@ -259,37 +157,12 @@ pub fn preview_paste_text_candidate_evidence( input: &PastePreviewParams, snapshot: &crate::PreparedAnalysisSnapshot, confirmed_candidate_ids: &[String], + runtime: TransformRuntime<'_, '_>, ) -> Result { - preview_paste_text_candidate_evidence_with_smart_provider( - input, - snapshot, - confirmed_candidate_ids, - None, - ) -} - -pub fn preview_paste_text_candidate_evidence_with_smart_provider( - input: &PastePreviewParams, - snapshot: &crate::PreparedAnalysisSnapshot, - confirmed_candidate_ids: &[String], - provider: Option<&mut dyn SmartReplacementProvider>, -) -> Result { - preview_paste_text_candidate_evidence_with_run_secrets( - input, - snapshot, - confirmed_candidate_ids, + let TransformRuntime { provider, - None, - ) -} - -pub fn preview_paste_text_candidate_evidence_with_run_secrets( - input: &PastePreviewParams, - snapshot: &crate::PreparedAnalysisSnapshot, - confirmed_candidate_ids: &[String], - provider: Option<&mut dyn SmartReplacementProvider>, - tokenization_key: Option<&crate::TokenizationKey>, -) -> Result { + tokenization_key, + } = runtime; shared::validate_paste_content(&input.content)?; let format = format_detection::resolve_format(input.format, &input.content); if !matches!(format, PasteDataFormat::PlainText | PasteDataFormat::Logs) { diff --git a/crates/csv-anonymizer-core/src/direct_input/quick.rs b/crates/csv-anonymizer-core/src/direct_input/quick.rs index 53074d8..b9494e7 100644 --- a/crates/csv-anonymizer-core/src/direct_input/quick.rs +++ b/crates/csv-anonymizer-core/src/direct_input/quick.rs @@ -1,8 +1,9 @@ use crate::detection::{classify_pii_risk, detect_column_type_with_name, detect_empty_format}; use crate::error::{AnonymizerError, Result}; +use crate::execution::TransformRuntime; use crate::random::{random_string, random_uuid_v4}; use crate::service::build_privacy_report; -use crate::smart::{SmartReplacementProvider, prepare_smart_replacements_from_rows}; +use crate::smart::prepare_smart_replacements_from_rows; use crate::strategies::{TransformState, transform_value_with_state}; use crate::types::{ AnonymizationStrategy, ColumnMetadata, ColumnValueDistribution, DataType, DetectionCoverage, @@ -13,22 +14,14 @@ use rand::Rng; const QUICK_GENERATE_MAX_COUNT: usize = 1_000; const HEX_CHARSET: &str = "0123456789abcdef"; -pub fn generate_quick_values(input: QuickGenerateParams) -> Result { - generate_quick_values_with_smart_provider(input, None) -} - -pub fn generate_quick_values_with_smart_provider( - input: QuickGenerateParams, - provider: Option<&mut dyn SmartReplacementProvider>, -) -> Result { - generate_quick_values_with_run_secrets(input, provider, None) -} - -pub fn generate_quick_values_with_run_secrets( +pub fn generate_quick_values( input: QuickGenerateParams, - provider: Option<&mut dyn SmartReplacementProvider>, - tokenization_key: Option<&crate::TokenizationKey>, + runtime: TransformRuntime<'_, '_>, ) -> Result { + let TransformRuntime { + provider, + tokenization_key, + } = runtime; if input.count == 0 { return Err(AnonymizerError::input_parse( "quick generation", diff --git a/crates/csv-anonymizer-core/src/direct_input/shared.rs b/crates/csv-anonymizer-core/src/direct_input/shared.rs index af539dd..1d27484 100644 --- a/crates/csv-anonymizer-core/src/direct_input/shared.rs +++ b/crates/csv-anonymizer-core/src/direct_input/shared.rs @@ -82,7 +82,7 @@ pub(super) fn preview_from_fields_with_smart_provider( selection: PreviewSelection<'_, '_>, ) -> Result { let (headers, detection_rows) = fields_to_rows(fields, FieldWindow::Detection); - let metadata = metadata_from_fields(fields, &headers, &detection_rows); + let metadata = metadata_from_fields(fields, &headers, &detection_rows, None).0; let (_, display_rows) = fields_to_rows(fields, FieldWindow::Display); // A field-shaped paste knows how many values it sampled, not how many the source // document holds, so the cardinality warning here rests on the absolute test @@ -183,19 +183,10 @@ pub(super) fn analysis_from_fields( format: PasteDataFormat, fields: &[FieldSamples], row_count: usize, -) -> (PasteAnalyzeData, DetectionCoverage) { - analysis_from_fields_with_candidate_detector(format, fields, row_count, None) -} - -pub(super) fn analysis_from_fields_with_candidate_detector( - format: PasteDataFormat, - fields: &[FieldSamples], - row_count: usize, detector: Option<&mut dyn CandidateDetector>, ) -> (PasteAnalyzeData, DetectionCoverage) { let (headers, rows) = fields_to_rows(fields, FieldWindow::Detection); - let (columns, detector_status) = - metadata_from_fields_with_candidate_detector(fields, &headers, &rows, detector); + let (columns, detector_status) = metadata_from_fields(fields, &headers, &rows, detector); let coverage = detection_coverage(fields, rows.len()); ( @@ -473,14 +464,6 @@ pub(super) fn metadata_from_fields( fields: &[FieldSamples], headers: &[String], rows: &[Vec], -) -> Vec { - metadata_from_fields_with_candidate_detector(fields, headers, rows, None).0 -} - -fn metadata_from_fields_with_candidate_detector( - fields: &[FieldSamples], - headers: &[String], - rows: &[Vec], detector: Option<&mut dyn CandidateDetector>, ) -> (Vec, CandidateDetectorRunStatus) { let (mut metadata, detector_status) = diff --git a/crates/csv-anonymizer-core/src/direct_input/tests.rs b/crates/csv-anonymizer-core/src/direct_input/tests.rs index 973a3d0..c33bfd2 100644 --- a/crates/csv-anonymizer-core/src/direct_input/tests.rs +++ b/crates/csv-anonymizer-core/src/direct_input/tests.rs @@ -42,11 +42,14 @@ impl CandidateDetector for FirstPasteCellNameDetector { #[test] fn transforms_csv_text_with_existing_csv_rules() { let input = "email,name\nada@example.com,Ada Lovelace\n"; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Csv, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Csv, + sample_row_count: 10, + }, + None, + ) .unwrap(); let selected = analysis .columns @@ -55,14 +58,17 @@ fn transforms_csv_text_with_existing_csv_rules() { .map(|column| column.index) .collect::>(); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::Csv, - columns: selected, - controls: Vec::new(), - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Csv, + columns: selected, + controls: Vec::new(), + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!(result.output.starts_with("email,name\n")); @@ -81,13 +87,13 @@ fn paste_analysis_reports_completed_candidate_detection_for_all_supported_format (PasteDataFormat::PlainText, "Contact ada@example.com"), ] { let mut detector = FirstPasteCellNameDetector; - let analysis = analyze_paste_data_with_candidate_detector( + let analysis = analyze_paste_data( PasteAnalyzeParams { content: content.to_string(), format, sample_row_count: 100, }, - &mut detector, + Some(&mut detector), ) .unwrap(); @@ -144,13 +150,13 @@ fn plain_text_candidate_replay_handles_unicode_without_rerunning_detector() { let mut detector = NamedSpanDetector { needle: "Élodie Dupont", }; - let analysis = analyze_paste_data_with_candidate_detector( + let analysis = analyze_paste_data( PasteAnalyzeParams { content: content.to_string(), format: PasteDataFormat::PlainText, sample_row_count: 1, }, - &mut detector, + Some(&mut detector), ) .unwrap(); let snapshot = analysis.prepared_analysis.unwrap(); @@ -166,6 +172,7 @@ fn plain_text_candidate_replay_handles_unicode_without_rerunning_detector() { }, &snapshot, &[candidate_id], + crate::TransformRuntime::default(), ) .unwrap(); @@ -179,13 +186,13 @@ fn plain_text_replay_transforms_deterministic_and_confirmed_candidate_spans_toge let mut detector = NamedSpanDetector { needle: "Ada Lovelace", }; - let analysis = analyze_paste_data_with_candidate_detector( + let analysis = analyze_paste_data( PasteAnalyzeParams { content: content.to_string(), format: PasteDataFormat::PlainText, sample_row_count: 100, }, - &mut detector, + Some(&mut detector), ) .unwrap(); let snapshot = analysis.prepared_analysis.unwrap(); @@ -201,6 +208,7 @@ fn plain_text_replay_transforms_deterministic_and_confirmed_candidate_spans_toge }, &snapshot, std::slice::from_ref(&evidence.id), + crate::TransformRuntime::default(), ) .unwrap(); @@ -220,6 +228,7 @@ fn plain_text_replay_transforms_deterministic_and_confirmed_candidate_spans_toge }, &snapshot, std::slice::from_ref(&evidence.id), + crate::TransformRuntime::default(), ) .unwrap(); let preview_text = preview.previews[0] @@ -238,13 +247,13 @@ fn plain_text_candidate_replay_rejects_stale_source_and_requires_selected_column let mut detector = NamedSpanDetector { needle: "Ada Lovelace", }; - let analysis = analyze_paste_data_with_candidate_detector( + let analysis = analyze_paste_data( PasteAnalyzeParams { content: content.to_string(), format: PasteDataFormat::PlainText, sample_row_count: 100, }, - &mut detector, + Some(&mut detector), ) .unwrap(); let snapshot = analysis.prepared_analysis.unwrap(); @@ -258,14 +267,26 @@ fn plain_text_candidate_replay_rejects_stale_source_and_requires_selected_column preview_smart_replacements: Vec::new(), }; - let unchanged = - replay_paste_text_candidate_evidence(&base, &snapshot, std::slice::from_ref(&candidate_id)) - .unwrap(); + let unchanged = replay_paste_text_candidate_evidence( + &base, + &snapshot, + std::slice::from_ref(&candidate_id), + crate::TransformRuntime::default(), + ) + .unwrap(); assert_eq!(unchanged.output, content); let mut stale = base; stale.content.push('!'); - assert!(replay_paste_text_candidate_evidence(&stale, &snapshot, &[candidate_id]).is_err()); + assert!( + replay_paste_text_candidate_evidence( + &stale, + &snapshot, + &[candidate_id], + crate::TransformRuntime::default() + ) + .is_err() + ); } #[test] @@ -274,13 +295,13 @@ fn plain_text_detector_rejects_candidate_overlapping_deterministic_evidence() { let mut detector = NamedSpanDetector { needle: "ada@example.com", }; - let analysis = analyze_paste_data_with_candidate_detector( + let analysis = analyze_paste_data( PasteAnalyzeParams { content: content.to_string(), format: PasteDataFormat::PlainText, sample_row_count: 100, }, - &mut detector, + Some(&mut detector), ) .unwrap(); @@ -298,18 +319,21 @@ fn plain_text_detector_rejects_candidate_overlapping_deterministic_evidence() { fn csv_text_type_override_preserves_direct_identifier_reporting() { let input = "value\nalpha\nbravo\ncharlie\ndelta\necho\nlate@example.com\n"; - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::Csv, - columns: vec![0], - controls: vec![ColumnControl { - column_index: 0, - type_override: Some(DataType::String), - strategy: AnonymizationStrategy::Redact, - }], - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Csv, + columns: vec![0], + controls: vec![ColumnControl { + column_index: 0, + type_override: Some(DataType::String), + strategy: AnonymizationStrategy::Redact, + }], + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert_eq!(result.privacy_report.direct_identifiers, 1); @@ -320,11 +344,14 @@ fn csv_text_type_override_preserves_direct_identifier_reporting() { #[test] fn analyze_paste_data_auto_selects_columns_with_core_policy() { let input = "email,notes\nada@example.com,\n"; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Csv, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Csv, + sample_row_count: 10, + }, + None, + ) .unwrap(); let email = analysis @@ -372,14 +399,17 @@ fn direct_input_preview_classifies_on_the_whole_paste_not_the_display_window() { content.push_str(&format!("user{row}@example.com\n")); } - let preview = preview_paste_data(PastePreviewParams { - content, - format: PasteDataFormat::Csv, - columns: vec![0], - controls: vec![], - sample_count: 3, - sample_row_count: 100, - }) + let preview = preview_paste_data( + PastePreviewParams { + content, + format: PasteDataFormat::Csv, + columns: vec![0], + controls: vec![], + sample_count: 3, + sample_row_count: 100, + }, + crate::TransformRuntime::default(), + ) .unwrap(); let samples = &preview.previews[0].samples; @@ -418,11 +448,14 @@ fn paste_analysis_classifies_on_the_detection_floor_not_the_requested_sample() { } for sample_row_count in [1, 3, 6, 100] { - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: content.clone(), - format: PasteDataFormat::Csv, - sample_row_count, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: content.clone(), + format: PasteDataFormat::Csv, + sample_row_count, + }, + None, + ) .unwrap(); assert_eq!( @@ -455,11 +488,14 @@ fn paste_preview_and_transform_follow_a_raised_sample_row_count() { } } - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: content.clone(), - format: PasteDataFormat::Csv, - sample_row_count: SAMPLE_ROWS, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: content.clone(), + format: PasteDataFormat::Csv, + sample_row_count: SAMPLE_ROWS, + }, + None, + ) .unwrap(); assert_eq!( analysis.columns[0].pii_risk, @@ -467,14 +503,17 @@ fn paste_preview_and_transform_follow_a_raised_sample_row_count() { "the fixture is meant to read as PII only on the larger basis" ); - let preview = preview_paste_data(PastePreviewParams { - content: content.clone(), - format: PasteDataFormat::Csv, - columns: vec![0], - controls: vec![], - sample_count: 3, - sample_row_count: SAMPLE_ROWS, - }) + let preview = preview_paste_data( + PastePreviewParams { + content: content.clone(), + format: PasteDataFormat::Csv, + columns: vec![0], + controls: vec![], + sample_count: 3, + sample_row_count: SAMPLE_ROWS, + }, + crate::TransformRuntime::default(), + ) .unwrap(); for sample in &preview.previews[0].samples { assert_eq!( @@ -483,14 +522,17 @@ fn paste_preview_and_transform_follow_a_raised_sample_row_count() { ); } - let result = transform_paste_data(PasteTransformParams { - content, - format: PasteDataFormat::Csv, - columns: vec![0], - controls: Vec::new(), - sample_row_count: SAMPLE_ROWS, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content, + format: PasteDataFormat::Csv, + columns: vec![0], + controls: Vec::new(), + sample_row_count: SAMPLE_ROWS, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!( !result.output.contains("@example.com"), @@ -520,11 +562,14 @@ fn field_shaped_paste_detects_pii_that_starts_past_the_detection_basis() { ] { let content = build(2_000, 500); - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: content.clone(), - format, - sample_row_count: 100, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: content.clone(), + format, + sample_row_count: 100, + }, + None, + ) .unwrap(); let column = &analysis.columns[0]; @@ -540,14 +585,17 @@ fn field_shaped_paste_detects_pii_that_starts_past_the_detection_basis() { "{format:?}: the field was not offered for anonymization" ); - let preview = preview_paste_data(PastePreviewParams { - content, - format, - columns: vec![0], - controls: vec![], - sample_count: 3, - sample_row_count: 100, - }) + let preview = preview_paste_data( + PastePreviewParams { + content, + format, + columns: vec![0], + controls: vec![], + sample_count: 3, + sample_row_count: 100, + }, + crate::TransformRuntime::default(), + ) .unwrap(); for sample in &preview.previews[0].samples { assert!( @@ -571,11 +619,14 @@ fn field_shaped_paste_detects_pii_that_starts_past_the_detection_basis() { /// exact. #[test] fn xml_paste_reports_the_documents_record_count_not_the_sample_size() { - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: xml_records(2_000, 2_000), - format: PasteDataFormat::Xml, - sample_row_count: 100, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: xml_records(2_000, 2_000), + format: PasteDataFormat::Xml, + sample_row_count: 100, + }, + None, + ) .unwrap(); assert_eq!(analysis.row_count, 2_000); @@ -635,14 +686,17 @@ fn field_shaped_preview_classifies_on_the_detection_floor_not_the_display_window } content.push_str("\n"); - let preview = preview_paste_data(PastePreviewParams { - content, - format: PasteDataFormat::Xml, - columns: vec![0], - controls: vec![], - sample_count: 3, - sample_row_count: 100, - }) + let preview = preview_paste_data( + PastePreviewParams { + content, + format: PasteDataFormat::Xml, + columns: vec![0], + controls: vec![], + sample_count: 3, + sample_row_count: 100, + }, + crate::TransformRuntime::default(), + ) .unwrap(); let samples = &preview.previews[0].samples; @@ -664,25 +718,28 @@ fn field_shaped_preview_classifies_on_the_detection_floor_not_the_display_window #[test] fn direct_input_preview_includes_selected_column_warnings() { - let preview = preview_paste_data(PastePreviewParams { - content: "email,country\nada@example.com,US\n".to_string(), - format: PasteDataFormat::Csv, - columns: vec![0, 1], - controls: vec![ - ColumnControl { - column_index: 0, - type_override: None, - strategy: AnonymizationStrategy::PassThrough, - }, - ColumnControl { - column_index: 1, - type_override: Some(DataType::CountryCode), - strategy: AnonymizationStrategy::Auto, - }, - ], - sample_count: 3, - sample_row_count: 100, - }) + let preview = preview_paste_data( + PastePreviewParams { + content: "email,country\nada@example.com,US\n".to_string(), + format: PasteDataFormat::Csv, + columns: vec![0, 1], + controls: vec![ + ColumnControl { + column_index: 0, + type_override: None, + strategy: AnonymizationStrategy::PassThrough, + }, + ColumnControl { + column_index: 1, + type_override: Some(DataType::CountryCode), + strategy: AnonymizationStrategy::Auto, + }, + ], + sample_count: 3, + sample_row_count: 100, + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert_eq!(preview.warnings.len(), 2); @@ -703,11 +760,14 @@ fn direct_input_preview_includes_selected_column_warnings() { #[test] fn generates_quick_values_without_user_input() { - let result = generate_quick_values(QuickGenerateParams { - data_type: DataType::Email, - strategy: AnonymizationStrategy::Auto, - count: 2, - }) + let result = generate_quick_values( + QuickGenerateParams { + data_type: DataType::Email, + strategy: AnonymizationStrategy::Auto, + count: 2, + }, + crate::TransformRuntime::default(), + ) .unwrap(); let lines = result.output.lines().collect::>(); @@ -719,11 +779,14 @@ fn generates_quick_values_without_user_input() { #[test] fn generates_type_shaped_quick_values() { - let uuid_result = generate_quick_values(QuickGenerateParams { - data_type: DataType::Uuid, - strategy: AnonymizationStrategy::Auto, - count: 1, - }) + let uuid_result = generate_quick_values( + QuickGenerateParams { + data_type: DataType::Uuid, + strategy: AnonymizationStrategy::Auto, + count: 1, + }, + crate::TransformRuntime::default(), + ) .unwrap(); let uuid = uuid_result.output.as_str(); @@ -731,20 +794,26 @@ fn generates_type_shaped_quick_values() { assert_eq!(uuid.chars().nth(14), Some('4')); assert!(matches!(uuid.chars().nth(19), Some('8' | '9' | 'a' | 'b'))); - let ip_result = generate_quick_values(QuickGenerateParams { - data_type: DataType::IpAddress, - strategy: AnonymizationStrategy::Auto, - count: 1, - }) + let ip_result = generate_quick_values( + QuickGenerateParams { + data_type: DataType::IpAddress, + strategy: AnonymizationStrategy::Auto, + count: 1, + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!(ip_result.output.starts_with("198.51.100.")); - let phone_result = generate_quick_values(QuickGenerateParams { - data_type: DataType::Phone, - strategy: AnonymizationStrategy::Auto, - count: 1, - }) + let phone_result = generate_quick_values( + QuickGenerateParams { + data_type: DataType::Phone, + strategy: AnonymizationStrategy::Auto, + count: 1, + }, + crate::TransformRuntime::default(), + ) .unwrap(); let phone = phone_result.output.as_str(); @@ -758,22 +827,28 @@ fn generates_type_shaped_quick_values() { .all(|(index, character)| character.is_ascii_digit() || matches!(index, 3 | 7)) ); - let name_result = generate_quick_values(QuickGenerateParams { - data_type: DataType::FullName, - strategy: AnonymizationStrategy::Auto, - count: 1, - }) + let name_result = generate_quick_values( + QuickGenerateParams { + data_type: DataType::FullName, + strategy: AnonymizationStrategy::Auto, + count: 1, + }, + crate::TransformRuntime::default(), + ) .unwrap(); let name = name_result.output.as_str(); assert_ne!(name, "First1 Last1"); assert_eq!(name.split_whitespace().count(), 2); - let timestamp_result = generate_quick_values(QuickGenerateParams { - data_type: DataType::Timestamp, - strategy: AnonymizationStrategy::Auto, - count: 1, - }) + let timestamp_result = generate_quick_values( + QuickGenerateParams { + data_type: DataType::Timestamp, + strategy: AnonymizationStrategy::Auto, + count: 1, + }, + crate::TransformRuntime::default(), + ) .unwrap(); let timestamp = timestamp_result.output.as_str(); @@ -786,11 +861,14 @@ fn generates_type_shaped_quick_values() { #[test] fn generates_tokenized_quick_values() { - let result = generate_quick_values(QuickGenerateParams { - data_type: DataType::Email, - strategy: AnonymizationStrategy::Tokenize, - count: 2, - }) + let result = generate_quick_values( + QuickGenerateParams { + data_type: DataType::Email, + strategy: AnonymizationStrategy::Tokenize, + count: 2, + }, + crate::TransformRuntime::default(), + ) .unwrap(); let lines = result.output.lines().collect::>(); @@ -818,9 +896,22 @@ fn paste_tokenization_repeats_across_explicit_run_contexts_without_serializing_t }; let first_key = crate::TokenizationKey::parse_hex(&key_text).unwrap(); let second_key = crate::TokenizationKey::parse_hex(&key_text).unwrap(); - let first = - transform_paste_data_with_run_secrets(input.clone(), None, Some(&first_key)).unwrap(); - let second = transform_paste_data_with_run_secrets(input, None, Some(&second_key)).unwrap(); + let first = transform_paste_data( + input.clone(), + crate::TransformRuntime { + provider: None, + tokenization_key: Some(&first_key), + }, + ) + .unwrap(); + let second = transform_paste_data( + input, + crate::TransformRuntime { + provider: None, + tokenization_key: Some(&second_key), + }, + ) + .unwrap(); assert_eq!(first.output, second.output); assert_eq!(first.privacy_report.keyed_token_values, 1); @@ -831,11 +922,14 @@ fn paste_tokenization_repeats_across_explicit_run_contexts_without_serializing_t #[test] fn quick_generation_rejects_input_only_strategies() { - let error = generate_quick_values(QuickGenerateParams { - data_type: DataType::Email, - strategy: AnonymizationStrategy::Mask, - count: 1, - }) + let error = generate_quick_values( + QuickGenerateParams { + data_type: DataType::Email, + strategy: AnonymizationStrategy::Mask, + count: 1, + }, + crate::TransformRuntime::default(), + ) .unwrap_err(); assert!( @@ -848,11 +942,14 @@ fn quick_generation_rejects_input_only_strategies() { #[test] fn transforms_xml_attributes_and_text() { let input = r#"Ada Lovelace"#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Xml, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + sample_row_count: 10, + }, + None, + ) .unwrap(); let selected = analysis .columns @@ -861,14 +958,17 @@ fn transforms_xml_attributes_and_text() { .map(|column| column.index) .collect::>(); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::Xml, - columns: selected, - controls: Vec::new(), - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + columns: selected, + controls: Vec::new(), + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!(!result.output.contains("ada@example.com")); @@ -878,11 +978,14 @@ fn transforms_xml_attributes_and_text() { #[test] fn transforms_selected_xml_cdata() { let input = r#""#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Xml, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + sample_row_count: 10, + }, + None, + ) .unwrap(); let name = analysis .columns @@ -890,18 +993,21 @@ fn transforms_selected_xml_cdata() { .find(|column| column.name == "users.user.name") .unwrap(); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::Xml, - columns: vec![name.index], - controls: vec![ColumnControl { - column_index: name.index, - type_override: Some(DataType::FullName), - strategy: AnonymizationStrategy::Redact, - }], - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + columns: vec![name.index], + controls: vec![ColumnControl { + column_index: name.index, + type_override: Some(DataType::FullName), + strategy: AnonymizationStrategy::Redact, + }], + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!(!result.output.contains("Ada Lovelace")); @@ -911,29 +1017,35 @@ fn transforms_selected_xml_cdata() { #[test] fn transforms_xml_text_cdata_and_references_as_one_scalar() { let input = r#"Ada &keep"#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Xml, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + sample_row_count: 10, + }, + None, + ) .unwrap(); let name = analysis .columns .iter() .find(|column| column.name == "users.user.name") .unwrap(); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::Xml, - columns: vec![name.index], - controls: vec![ColumnControl { - column_index: name.index, - type_override: Some(DataType::FullName), - strategy: AnonymizationStrategy::Redact, - }], - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + columns: vec![name.index], + controls: vec![ColumnControl { + column_index: name.index, + type_override: Some(DataType::FullName), + strategy: AnonymizationStrategy::Redact, + }], + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); let mut reader = quick_xml::Reader::from_str(&result.output); loop { @@ -947,11 +1059,14 @@ fn transforms_xml_text_cdata_and_references_as_one_scalar() { #[test] fn smart_xml_replacements_use_combined_text_cdata_and_references() { let input = r#"Ada &"#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Xml, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + sample_row_count: 10, + }, + None, + ) .unwrap(); let name = analysis .columns @@ -960,7 +1075,7 @@ fn smart_xml_replacements_use_combined_text_cdata_and_references() { .unwrap(); let mut provider = RecordingSmartProvider::default(); - let result = transform_paste_data_with_smart_provider( + let result = transform_paste_data( PasteTransformParams { content: input.to_string(), format: PasteDataFormat::Xml, @@ -973,7 +1088,10 @@ fn smart_xml_replacements_use_combined_text_cdata_and_references() { sample_row_count: 100, preview_smart_replacements: Vec::new(), }, - Some(&mut provider), + crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, ) .unwrap(); @@ -984,25 +1102,31 @@ fn smart_xml_replacements_use_combined_text_cdata_and_references() { #[test] fn scalar_document_marks_unmeasured_residual_audit_as_review() { let input = r#"{"first":"alice@example.com","second":"alice@example.com"}"#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Json, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Json, + sample_row_count: 10, + }, + None, + ) .unwrap(); let selected = analysis .columns .iter() .map(|column| column.index) .collect::>(); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::Json, - columns: selected, - controls: Vec::new(), - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Json, + columns: selected, + controls: Vec::new(), + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); let audit = result .privacy_report @@ -1022,11 +1146,14 @@ fn json_paths_distinguish_literal_dotted_keys_from_nested_keys() { "items[]": "literal-brackets@example.com", "items": ["array@example.com"] }"#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Json, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Json, + sample_row_count: 10, + }, + None, + ) .unwrap(); let literal = analysis @@ -1053,14 +1180,17 @@ fn json_paths_distinguish_literal_dotted_keys_from_nested_keys() { assert_ne!(literal.source_path, nested.source_path); assert_ne!(literal_brackets.source_path, array_value.source_path); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::Json, - columns: vec![nested.index, array_value.index], - controls: Vec::new(), - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Json, + columns: vec![nested.index, array_value.index], + controls: Vec::new(), + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!(result.output.contains("literal@example.com")); @@ -1072,11 +1202,14 @@ fn json_paths_distinguish_literal_dotted_keys_from_nested_keys() { #[test] fn json_transform_preserves_scalar_value_types() { let input = r#"{"age":42,"ratio":12.5,"flag":true}"#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Json, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Json, + sample_row_count: 10, + }, + None, + ) .unwrap(); let age = analysis @@ -1095,30 +1228,33 @@ fn json_transform_preserves_scalar_value_types() { .find(|column| column.name == "flag") .unwrap(); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::Json, - columns: vec![age.index, ratio.index, flag.index], - controls: vec![ - ColumnControl { - column_index: age.index, - type_override: Some(DataType::NumericId), - strategy: AnonymizationStrategy::Auto, - }, - ColumnControl { - column_index: ratio.index, - type_override: Some(DataType::NumericValue), - strategy: AnonymizationStrategy::Auto, - }, - ColumnControl { - column_index: flag.index, - type_override: Some(DataType::Boolean), - strategy: AnonymizationStrategy::Auto, - }, - ], - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Json, + columns: vec![age.index, ratio.index, flag.index], + controls: vec![ + ColumnControl { + column_index: age.index, + type_override: Some(DataType::NumericId), + strategy: AnonymizationStrategy::Auto, + }, + ColumnControl { + column_index: ratio.index, + type_override: Some(DataType::NumericValue), + strategy: AnonymizationStrategy::Auto, + }, + ColumnControl { + column_index: flag.index, + type_override: Some(DataType::Boolean), + strategy: AnonymizationStrategy::Auto, + }, + ], + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); let output: serde_json::Value = serde_json::from_str(&result.output).unwrap(); @@ -1130,11 +1266,14 @@ fn json_transform_preserves_scalar_value_types() { #[test] fn rejects_oversized_pasted_payloads() { let input = "x".repeat(super::shared::PASTE_MAX_CONTENT_BYTES + 1); - let error = analyze_paste_data(PasteAnalyzeParams { - content: input, - format: PasteDataFormat::PlainText, - sample_row_count: 10, - }) + let error = analyze_paste_data( + PasteAnalyzeParams { + content: input, + format: PasteDataFormat::PlainText, + sample_row_count: 10, + }, + None, + ) .unwrap_err(); assert!(error.to_string().contains("at most 5 MiB")); @@ -1149,19 +1288,25 @@ fn paste_sample_counts_accept_the_settings_ceiling_and_reject_more() { let content = "email\nada@example.com\n".to_string(); assert!( - analyze_paste_data(PasteAnalyzeParams { - content: content.clone(), - format: PasteDataFormat::Csv, - sample_row_count: MAX_SAMPLE_ROW_COUNT, - }) + analyze_paste_data( + PasteAnalyzeParams { + content: content.clone(), + format: PasteDataFormat::Csv, + sample_row_count: MAX_SAMPLE_ROW_COUNT, + }, + None + ) .is_ok() ); - let error = analyze_paste_data(PasteAnalyzeParams { - content, - format: PasteDataFormat::Csv, - sample_row_count: MAX_SAMPLE_ROW_COUNT + 1, - }) + let error = analyze_paste_data( + PasteAnalyzeParams { + content, + format: PasteDataFormat::Csv, + sample_row_count: MAX_SAMPLE_ROW_COUNT + 1, + }, + None, + ) .unwrap_err(); assert!(error.to_string().contains("sample row count")); @@ -1173,11 +1318,14 @@ fn rejects_too_many_structured_fields() { .map(|index| format!(r#""field{index}":"value{index}@example.com""#)) .collect::>() .join(","); - let error = analyze_paste_data(PasteAnalyzeParams { - content: format!("{{{fields}}}"), - format: PasteDataFormat::Json, - sample_row_count: 10, - }) + let error = analyze_paste_data( + PasteAnalyzeParams { + content: format!("{{{fields}}}"), + format: PasteDataFormat::Json, + sample_row_count: 10, + }, + None, + ) .unwrap_err(); assert!(error.to_string().contains("Detected more than")); @@ -1186,11 +1334,14 @@ fn rejects_too_many_structured_fields() { #[test] fn xml_paths_distinguish_dotted_element_names_from_nested_elements() { let input = r#"LiteralNested"#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Xml, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + sample_row_count: 10, + }, + None, + ) .unwrap(); let literal = analysis @@ -1206,14 +1357,17 @@ fn xml_paths_distinguish_dotted_element_names_from_nested_elements() { assert_ne!(literal.source_path, nested.source_path); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::Xml, - columns: vec![nested.index], - controls: Vec::new(), - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + columns: vec![nested.index], + controls: Vec::new(), + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!(result.output.contains("literal@example.com")); @@ -1224,11 +1378,14 @@ fn xml_paths_distinguish_dotted_element_names_from_nested_elements() { fn xml_paths_distinguish_dotted_attribute_names_from_nested_paths() { let input = r#""#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Xml, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + sample_row_count: 10, + }, + None, + ) .unwrap(); let literal_attribute = analysis @@ -1244,14 +1401,17 @@ fn xml_paths_distinguish_dotted_attribute_names_from_nested_paths() { assert_ne!(literal_attribute.source_path, nested_attribute.source_path); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::Xml, - columns: vec![nested_attribute.index], - controls: Vec::new(), - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + columns: vec![nested_attribute.index], + controls: Vec::new(), + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!(result.output.contains("literal@example.com")); @@ -1261,11 +1421,14 @@ fn xml_paths_distinguish_dotted_attribute_names_from_nested_paths() { #[test] fn previews_pasted_json_fields() { let input = r#"[{"email":"ada@example.com"}]"#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Auto, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Auto, + sample_row_count: 10, + }, + None, + ) .unwrap(); assert_eq!(analysis.format, PasteDataFormat::Json); @@ -1274,14 +1437,17 @@ fn previews_pasted_json_fields() { .iter() .find(|column| column.name == "[].email") .unwrap(); - let preview = preview_paste_data(PastePreviewParams { - content: input.to_string(), - format: PasteDataFormat::Json, - columns: vec![email.index], - controls: Vec::new(), - sample_count: 5, - sample_row_count: 100, - }) + let preview = preview_paste_data( + PastePreviewParams { + content: input.to_string(), + format: PasteDataFormat::Json, + columns: vec![email.index], + controls: Vec::new(), + sample_count: 5, + sample_row_count: 100, + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert_eq!(preview.previews[0].column_name, "[].email"); @@ -1303,11 +1469,14 @@ fn previews_plain_text_and_logs_through_shared_orchestration() { } fn assert_shared_smart_preview(input: &str, format: PasteDataFormat) { - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format, + sample_row_count: 10, + }, + None, + ) .unwrap(); let column = analysis.columns.first().expect("detected preview column"); let original = column @@ -1317,7 +1486,7 @@ fn assert_shared_smart_preview(input: &str, format: PasteDataFormat) { .clone(); let mut provider = PrefixSmartProvider; - let preview = preview_paste_data_with_smart_provider( + let preview = preview_paste_data( PastePreviewParams { content: input.to_string(), format, @@ -1330,7 +1499,10 @@ fn assert_shared_smart_preview(input: &str, format: PasteDataFormat) { sample_count: 5, sample_row_count: 100, }, - Some(&mut provider), + crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, ) .unwrap(); @@ -1345,11 +1517,14 @@ fn assert_shared_smart_preview(input: &str, format: PasteDataFormat) { #[test] fn previews_and_transforms_paste_data_with_smart_replacements() { let input = r#"[{"name":"Ada Lovelace"},{"name":"Grace Hopper"}]"#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Json, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Json, + sample_row_count: 10, + }, + None, + ) .unwrap(); let name = analysis .columns @@ -1363,7 +1538,7 @@ fn previews_and_transforms_paste_data_with_smart_replacements() { }]; let mut preview_provider = PrefixSmartProvider; - let preview = preview_paste_data_with_smart_provider( + let preview = preview_paste_data( PastePreviewParams { content: input.to_string(), format: PasteDataFormat::Json, @@ -1372,7 +1547,10 @@ fn previews_and_transforms_paste_data_with_smart_replacements() { sample_count: 5, sample_row_count: 100, }, - Some(&mut preview_provider), + crate::TransformRuntime { + provider: Some(&mut preview_provider), + tokenization_key: None, + }, ) .unwrap(); @@ -1386,14 +1564,17 @@ fn previews_and_transforms_paste_data_with_smart_replacements() { && warning.message.contains("Local AI")) ); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::Json, - columns: vec![name.index], - controls, - sample_row_count: 100, - preview_smart_replacements: preview.smart_replacements, - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Json, + columns: vec![name.index], + controls, + sample_row_count: 100, + preview_smart_replacements: preview.smart_replacements, + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!(result.output.contains("Smart Person 1")); @@ -1410,11 +1591,14 @@ fn paste_transform_reuses_preview_smart_replacements_and_generates_missing_value {"name":"Grace Hopper"}, {"name":"Katherine Johnson"} ]"#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Json, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Json, + sample_row_count: 10, + }, + None, + ) .unwrap(); let name = analysis .columns @@ -1427,7 +1611,7 @@ fn paste_transform_reuses_preview_smart_replacements_and_generates_missing_value strategy: AnonymizationStrategy::LocalAi, }]; let mut preview_provider = PrefixSmartProvider; - let preview = preview_paste_data_with_smart_provider( + let preview = preview_paste_data( PastePreviewParams { content: input.to_string(), format: PasteDataFormat::Json, @@ -1436,12 +1620,15 @@ fn paste_transform_reuses_preview_smart_replacements_and_generates_missing_value sample_count: 1, sample_row_count: 100, }, - Some(&mut preview_provider), + crate::TransformRuntime { + provider: Some(&mut preview_provider), + tokenization_key: None, + }, ) .unwrap(); let mut transform_provider = RecordingSmartProvider::default(); - let result = transform_paste_data_with_smart_provider( + let result = transform_paste_data( PasteTransformParams { content: input.to_string(), format: PasteDataFormat::Json, @@ -1450,7 +1637,10 @@ fn paste_transform_reuses_preview_smart_replacements_and_generates_missing_value sample_row_count: 100, preview_smart_replacements: preview.smart_replacements, }, - Some(&mut transform_provider), + crate::TransformRuntime { + provider: Some(&mut transform_provider), + tokenization_key: None, + }, ) .unwrap(); @@ -1470,11 +1660,14 @@ fn paste_transform_reuses_preview_smart_replacements_and_generates_missing_value #[test] fn paste_transform_rejects_invalid_preview_smart_replacements() { let input = r#"[{"name":"Ada Lovelace"}]"#; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Json, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Json, + sample_row_count: 10, + }, + None, + ) .unwrap(); let name = analysis .columns @@ -1488,7 +1681,7 @@ fn paste_transform_rejects_invalid_preview_smart_replacements() { }]; let mut provider = RecordingSmartProvider::default(); - let result = transform_paste_data_with_smart_provider( + let result = transform_paste_data( PasteTransformParams { content: input.to_string(), format: PasteDataFormat::Json, @@ -1501,7 +1694,10 @@ fn paste_transform_rejects_invalid_preview_smart_replacements() { replacement: "Ada Lovelace".to_string(), }], }, - Some(&mut provider), + crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, ) .unwrap(); @@ -1521,13 +1717,16 @@ fn paste_transform_rejects_invalid_preview_smart_replacements() { #[test] fn quick_generation_uses_smart_replacements_when_requested() { let mut provider = PrefixSmartProvider; - let result = generate_quick_values_with_smart_provider( + let result = generate_quick_values( QuickGenerateParams { data_type: DataType::FullName, strategy: AnonymizationStrategy::LocalAi, count: 2, }, - Some(&mut provider), + crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, ) .unwrap(); let lines = result.output.lines().collect::>(); @@ -1542,11 +1741,14 @@ fn quick_generation_uses_smart_replacements_when_requested() { #[test] fn quick_generation_requires_provider_for_smart_replacement() { - let error = generate_quick_values(QuickGenerateParams { - data_type: DataType::FullName, - strategy: AnonymizationStrategy::LocalAi, - count: 1, - }) + let error = generate_quick_values( + QuickGenerateParams { + data_type: DataType::FullName, + strategy: AnonymizationStrategy::LocalAi, + count: 1, + }, + crate::TransformRuntime::default(), + ) .unwrap_err(); assert!( @@ -1560,11 +1762,14 @@ fn quick_generation_requires_provider_for_smart_replacement() { fn transforms_plain_text_and_preserves_surrounding_text() { let input = "contact ada@example.com from 192.168.0.10 request 550e8400-e29b-41d4-a716-446655440000"; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::PlainText, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::PlainText, + sample_row_count: 10, + }, + None, + ) .unwrap(); let selected = analysis .columns @@ -1573,14 +1778,17 @@ fn transforms_plain_text_and_preserves_surrounding_text() { .map(|column| column.index) .collect::>(); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::PlainText, - columns: selected, - controls: Vec::new(), - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::PlainText, + columns: selected, + controls: Vec::new(), + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!(result.output.starts_with("contact ")); @@ -1596,11 +1804,14 @@ fn transforms_plain_text_and_preserves_surrounding_text() { #[test] fn plain_text_detection_keeps_overlapping_tokens_single_pass() { let input = "profile=https://ada@example.com/users/42"; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::PlainText, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::PlainText, + sample_row_count: 10, + }, + None, + ) .unwrap(); let url = analysis @@ -1610,14 +1821,17 @@ fn plain_text_detection_keeps_overlapping_tokens_single_pass() { .unwrap(); assert!(!analysis.columns.iter().any(|column| column.name == "email")); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: PasteDataFormat::PlainText, - columns: vec![url.index], - controls: Vec::new(), - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::PlainText, + columns: vec![url.index], + controls: Vec::new(), + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!(result.output.starts_with("profile=")); @@ -1627,11 +1841,14 @@ fn plain_text_detection_keeps_overlapping_tokens_single_pass() { #[test] fn auto_detects_logs_and_replaces_inline_values() { let input = "2026-06-25T12:00:00 ERROR user=jane@example.com ip=10.1.2.3"; - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: input.to_string(), - format: PasteDataFormat::Auto, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Auto, + sample_row_count: 10, + }, + None, + ) .unwrap(); assert_eq!(analysis.format, PasteDataFormat::Logs); @@ -1641,14 +1858,17 @@ fn auto_detects_logs_and_replaces_inline_values() { .filter(|column| column.name == "email" || column.name == "ipAddress") .map(|column| column.index) .collect::>(); - let result = transform_paste_data(PasteTransformParams { - content: input.to_string(), - format: analysis.format, - columns: selected, - controls: Vec::new(), - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + let result = transform_paste_data( + PasteTransformParams { + content: input.to_string(), + format: analysis.format, + columns: selected, + controls: Vec::new(), + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!(result.output.contains("ERROR user=")); @@ -1714,11 +1934,14 @@ fn csv_paste_analysis_discloses_that_detection_only_sampled() { content.push_str(&format!("{row},order shipped ok\n")); } - let analysis = analyze_paste_data(PasteAnalyzeParams { - content, - format: PasteDataFormat::Csv, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content, + format: PasteDataFormat::Csv, + sample_row_count: 10, + }, + None, + ) .unwrap(); assert!(analysis.detection_coverage.is_partial); @@ -1734,11 +1957,14 @@ fn csv_paste_analysis_discloses_that_detection_only_sampled() { /// given a warning that does not apply to it. #[test] fn small_csv_paste_analysis_reports_complete_detection_coverage() { - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: "email\nada@example.com\nbob@example.com\n".to_string(), - format: PasteDataFormat::Csv, - sample_row_count: 10, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: "email\nada@example.com\nbob@example.com\n".to_string(), + format: PasteDataFormat::Csv, + sample_row_count: 10, + }, + None, + ) .unwrap(); assert!(!analysis.detection_coverage.is_partial); @@ -1757,11 +1983,14 @@ fn json_paste_analysis_counts_detection_coverage_in_values_not_rows() { .collect::>() .join(","); - let analysis = analyze_paste_data(PasteAnalyzeParams { - content: format!(r#"{{"users":[{records}]}}"#), - format: PasteDataFormat::Json, - sample_row_count: 100, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content: format!(r#"{{"users":[{records}]}}"#), + format: PasteDataFormat::Json, + sample_row_count: 100, + }, + None, + ) .unwrap(); assert_eq!(analysis.row_count, 1, "a top-level object is one row"); @@ -1789,11 +2018,14 @@ fn free_text_paste_analysis_counts_detection_coverage_in_values_not_rows() { )); } - let analysis = analyze_paste_data(PasteAnalyzeParams { - content, - format: PasteDataFormat::PlainText, - sample_row_count: 100, - }) + let analysis = analyze_paste_data( + PasteAnalyzeParams { + content, + format: PasteDataFormat::PlainText, + sample_row_count: 100, + }, + None, + ) .unwrap(); assert!(analysis.detection_coverage.is_partial); @@ -1813,11 +2045,14 @@ fn free_text_paste_analysis_counts_detection_coverage_in_values_not_rows() { /// value. #[test] fn quick_generation_reports_no_sampling_caveat() { - let result = generate_quick_values(QuickGenerateParams { - data_type: DataType::Email, - strategy: AnonymizationStrategy::Pseudonymize, - count: 3, - }) + let result = generate_quick_values( + QuickGenerateParams { + data_type: DataType::Email, + strategy: AnonymizationStrategy::Pseudonymize, + count: 3, + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!( diff --git a/crates/csv-anonymizer-core/src/direct_input/tests/redaction.rs b/crates/csv-anonymizer-core/src/direct_input/tests/redaction.rs index ce7f6ca..b8f64b4 100644 --- a/crates/csv-anonymizer-core/src/direct_input/tests/redaction.rs +++ b/crates/csv-anonymizer-core/src/direct_input/tests/redaction.rs @@ -12,11 +12,14 @@ use crate::types::{ const SCALAR_WARNING: &str = "may change scalar value types"; fn analyze(content: &str, format: PasteDataFormat) -> PasteAnalyzeData { - analyze_paste_data(PasteAnalyzeParams { - content: content.to_string(), - format, - sample_row_count: 10, - }) + analyze_paste_data( + PasteAnalyzeParams { + content: content.to_string(), + format, + sample_row_count: 10, + }, + None, + ) .unwrap() } @@ -34,14 +37,17 @@ fn preview( columns: Vec, controls: Vec, ) -> PreviewData { - preview_paste_data(PastePreviewParams { - content: content.to_string(), - format, - columns, - controls, - sample_count: 3, - sample_row_count: 100, - }) + preview_paste_data( + PastePreviewParams { + content: content.to_string(), + format, + columns, + controls, + sample_count: 3, + sample_row_count: 100, + }, + crate::TransformRuntime::default(), + ) .unwrap() } @@ -51,14 +57,17 @@ fn transform( columns: Vec, controls: Vec, ) -> PasteTransformData { - transform_paste_data(PasteTransformParams { - content: content.to_string(), - format, - columns, - controls, - sample_row_count: 100, - preview_smart_replacements: Vec::new(), - }) + transform_paste_data( + PasteTransformParams { + content: content.to_string(), + format, + columns, + controls, + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + crate::TransformRuntime::default(), + ) .unwrap() } @@ -364,7 +373,7 @@ fn typed_field_override_preserves_existing_privacy_risk_before_redact_default() &["johndoe"], )]; let (headers, rows) = fields_to_rows(&fields, FieldWindow::Detection); - let metadata = metadata_from_fields(&fields, &headers, &rows); + let metadata = metadata_from_fields(&fields, &headers, &rows, None).0; let username = &metadata[0]; assert_eq!(username.detected_type, DataType::String); diff --git a/crates/csv-anonymizer-core/src/direct_input/text.rs b/crates/csv-anonymizer-core/src/direct_input/text.rs index 236082c..21ff626 100644 --- a/crates/csv-anonymizer-core/src/direct_input/text.rs +++ b/crates/csv-anonymizer-core/src/direct_input/text.rs @@ -17,11 +17,10 @@ use std::time::Instant; use super::shared::{ FieldSampleLimits, FieldSamples, PASTE_MAX_TEXT_MATCHES, PreviewSelection, - analysis_from_fields, analysis_from_fields_with_candidate_detector, - bounded_preview_sample_count, next_row_index, paste_detection_sample_rows, - paste_transform_data, preview_field_sample_limits, preview_from_fields_with_smart_provider, - push_identified_field_sample, push_typed_field_sample, selected_columns_by_source, - smart_replacements_for_fields, + analysis_from_fields, bounded_preview_sample_count, next_row_index, + paste_detection_sample_rows, paste_transform_data, preview_field_sample_limits, + preview_from_fields_with_smart_provider, push_identified_field_sample, push_typed_field_sample, + selected_columns_by_source, smart_replacements_for_fields, }; const UNSTRUCTURED_TEXT_COLUMN: &str = "Unstructured text"; @@ -31,28 +30,19 @@ pub(super) fn analyze_text_content( content: &str, format: PasteDataFormat, sample_row_count: usize, + detector: Option<&mut dyn CandidateDetector>, ) -> Result { - let sample_row_count = paste_detection_sample_rows(sample_row_count)?; - let matches = collect_text_matches(content)?; - let fields = text_fields_from_matches( - &matches, - FieldSampleLimits::detection_only(sample_row_count), - )?; - // Free text is the input the coverage disclosure matters most on: a long log whose - // detection window is thinned would otherwise report its types with no caveat, and - // the user would find out only after the output existed. It comes with the analysis - // here rather than being computed separately. - Ok(analysis_from_fields(format, &fields, matches.len()).0) -} + let Some(detector) = detector else { + let sample_row_count = paste_detection_sample_rows(sample_row_count)?; + let matches = collect_text_matches(content)?; + let fields = text_fields_from_matches( + &matches, + FieldSampleLimits::detection_only(sample_row_count), + )?; + return Ok(analysis_from_fields(format, &fields, matches.len(), None).0); + }; -pub(super) fn analyze_text_content_with_candidate_detector( - content: &str, - format: PasteDataFormat, - sample_row_count: usize, - detector: &mut dyn CandidateDetector, -) -> Result { - // A line gives the model useful context and stable replay coordinates. Paste - // bytes are bounded by the public entry point, so every line is retained. + // A line gives the model useful context and stable replay coordinates. let line_count = content.split('\n').count(); let limits = FieldSampleLimits::detection_only(line_count.max(1)); let mut fields = Vec::with_capacity(1); @@ -65,8 +55,7 @@ pub(super) fn analyze_text_content_with_candidate_detector( limits, )?; } - let (mut analysis, _) = - analysis_from_fields_with_candidate_detector(format, &fields, line_count, Some(detector)); + let (mut analysis, _) = analysis_from_fields(format, &fields, line_count, Some(detector)); if matches!( analysis.detection_run_summary.local_ner, crate::types::LocalNerRunStatus::Completed | crate::types::LocalNerRunStatus::Incomplete @@ -381,7 +370,7 @@ pub(super) fn transform_text_with_smart_provider( &matches, FieldSampleLimits::detection_only(detection_sample_rows), )?; - let (analysis, coverage) = analysis_from_fields(format, &fields, matches.len()); + let (analysis, coverage) = analysis_from_fields(format, &fields, matches.len(), None); let metadata = select_columns(&analysis.columns, &input.columns, &input.controls)?; let selected_by_name = selected_columns_by_source(&metadata); // Every match, not the detection window: a span the sample dropped would reach the diff --git a/crates/csv-anonymizer-core/src/direct_input/xml.rs b/crates/csv-anonymizer-core/src/direct_input/xml.rs index 1b295ff..11030d4 100644 --- a/crates/csv-anonymizer-core/src/direct_input/xml.rs +++ b/crates/csv-anonymizer-core/src/direct_input/xml.rs @@ -14,49 +14,26 @@ use std::time::Instant; use super::shared::{ FieldSampleLimits, FieldSamples, PreviewSelection, analysis_from_fields, - analysis_from_fields_with_candidate_detector, bounded_preview_sample_count, escape_path_key, - format_path, next_row_index, paste_detection_sample_rows, paste_transform_data, - preview_field_sample_limits, preview_from_fields_with_smart_provider, - push_identified_field_sample, selected_columns_by_source, smart_replacements_for_fields, + bounded_preview_sample_count, escape_path_key, format_path, next_row_index, + paste_detection_sample_rows, paste_transform_data, preview_field_sample_limits, + preview_from_fields_with_smart_provider, push_identified_field_sample, + selected_columns_by_source, smart_replacements_for_fields, }; -pub(super) fn analyze_xml(content: &str, sample_row_count: usize) -> Result { - analyze_xml_with_coverage(content, sample_row_count).map(|(analysis, _)| analysis) -} - -pub(super) fn analyze_xml_with_candidate_detector( - content: &str, - sample_row_count: usize, - detector: &mut dyn CandidateDetector, -) -> Result { - let sample_row_count = paste_detection_sample_rows(sample_row_count)?; - let fields = collect_xml_fields(content, FieldSampleLimits::detection_only(sample_row_count))?; - Ok(analysis_from_fields_with_candidate_detector( - PasteDataFormat::Xml, - &fields, - infer_xml_row_count(&fields), - Some(detector), - ) - .0) -} - -/// [`analyze_xml`] plus how much of the input it classified. -/// -/// Split out rather than widening `analyze_xml` because only the transform path -/// builds a privacy report and so only it needs the coverage; the analyze command -/// returns the DTO alone. -fn analyze_xml_with_coverage( +/// XML analysis plus the coverage used by transforms for the privacy report. +pub(super) fn analyze_xml( content: &str, sample_row_count: usize, + detector: Option<&mut dyn CandidateDetector>, ) -> Result<(PasteAnalyzeData, DetectionCoverage)> { let sample_row_count = paste_detection_sample_rows(sample_row_count)?; let fields = collect_xml_fields(content, FieldSampleLimits::detection_only(sample_row_count))?; let row_count = infer_xml_row_count(&fields); - Ok(analysis_from_fields( PasteDataFormat::Xml, &fields, row_count, + detector, )) } @@ -82,7 +59,7 @@ pub(super) fn transform_xml_with_smart_provider( provider: Option<&mut dyn SmartReplacementProvider>, tokenization_key: Option<&crate::TokenizationKey>, ) -> Result { - let (analysis, coverage) = analyze_xml_with_coverage(&input.content, input.sample_row_count)?; + let (analysis, coverage) = analyze_xml(&input.content, input.sample_row_count, None)?; let metadata = select_columns(&analysis.columns, &input.columns, &input.controls)?; let selected_by_path = selected_columns_by_source(&metadata); // Collected over every value rather than over the detection window, so each value diff --git a/crates/csv-anonymizer-core/src/execution.rs b/crates/csv-anonymizer-core/src/execution.rs new file mode 100644 index 0000000..522d5c4 --- /dev/null +++ b/crates/csv-anonymizer-core/src/execution.rs @@ -0,0 +1,40 @@ +use crate::TokenizationKey; +use crate::detection::CandidateDetector; +use crate::smart::SmartReplacementProvider; +use crate::types::{DETECTION_SAMPLE_ROW_FLOOR, ProcessControl}; + +pub struct CsvAnalysisOptions<'detector> { + pub sample_rows: usize, + pub candidate_detector: Option<&'detector mut dyn CandidateDetector>, +} + +impl Default for CsvAnalysisOptions<'_> { + fn default() -> Self { + Self { + sample_rows: DETECTION_SAMPLE_ROW_FLOOR, + candidate_detector: None, + } + } +} + +#[derive(Default)] +pub struct TransformRuntime<'provider, 'key> { + pub provider: Option<&'provider mut dyn SmartReplacementProvider>, + pub tokenization_key: Option<&'key TokenizationKey>, +} + +pub struct CsvRunOptions<'control, 'callbacks, 'provider, 'key> { + pub sample_rows: usize, + pub control: Option<&'control mut ProcessControl<'callbacks>>, + pub transform: TransformRuntime<'provider, 'key>, +} + +impl Default for CsvRunOptions<'_, '_, '_, '_> { + fn default() -> Self { + Self { + sample_rows: DETECTION_SAMPLE_ROW_FLOOR, + control: None, + transform: TransformRuntime::default(), + } + } +} diff --git a/crates/csv-anonymizer-core/src/lib.rs b/crates/csv-anonymizer-core/src/lib.rs index 5c9fe4b..00d8da7 100644 --- a/crates/csv-anonymizer-core/src/lib.rs +++ b/crates/csv-anonymizer-core/src/lib.rs @@ -16,6 +16,7 @@ pub(crate) mod csv_io; pub mod detection; pub mod direct_input; pub mod error; +mod execution; mod file_ops; pub(crate) mod metadata; mod prepared_snapshot; @@ -34,6 +35,7 @@ pub(crate) mod types; mod uniqueness; pub use error::{AnonymizerError, Result}; +pub use execution::{CsvAnalysisOptions, CsvRunOptions, TransformRuntime}; pub use metadata::should_auto_select_column; pub use prepared_snapshot::{ PREPARED_ANALYSIS_VERSION, PreparedAnalysisSnapshot, PreparedCandidateEvidence, diff --git a/crates/csv-anonymizer-core/src/service.rs b/crates/csv-anonymizer-core/src/service.rs index 5bfb80c..55da0d6 100644 --- a/crates/csv-anonymizer-core/src/service.rs +++ b/crates/csv-anonymizer-core/src/service.rs @@ -1,16 +1,13 @@ use crate::csv_io::{count_csv_data_rows, read_detection_sample, read_sample}; -use crate::detection::{CandidateDetector, CandidateDetectorRunStatus}; +use crate::detection::CandidateDetectorRunStatus; use crate::error::Result; +use crate::execution::{CsvAnalysisOptions, CsvRunOptions, TransformRuntime}; use crate::metadata::{build_column_metadata, build_column_metadata_with_candidate_detector}; -use crate::smart::{ - SmartReplacementProvider, prepare_smart_replacements_from_csv, - reusable_preview_smart_replacements, -}; +use crate::smart::{prepare_smart_replacements_from_csv, reusable_preview_smart_replacements}; use crate::types::{ AnonymizeData, AnonymizeParams, DETECTION_SAMPLE_ROW_FLOOR, DetectionCoverage, DetectionReviewReason, DetectionRunSummary, DeterministicDetectionStatus, HeadersData, - LocalNerRunStatus, PreflightData, PreflightParams, PreviewData, PreviewParams, ProcessControl, - ProcessOptions, + LocalNerRunStatus, PreflightData, PreflightParams, PreviewData, PreviewParams, ProcessOptions, }; use std::path::Path; @@ -156,45 +153,25 @@ impl AnonymizerService { &self.version } - pub fn analyze_csv(&self, file_path: impl AsRef) -> Result { - self.analyze_csv_with_sample_rows(file_path, DETECTION_SAMPLE_ROW_FLOOR) - } - /// Detection reads the whole file in one streaming pass and keeps - /// `sample_rows` values spread across it, so the exact row count falls out - /// of the same pass that classifies the columns. - /// - /// `sample_rows` is a request, floored by `detection_sample_rows`. - pub fn analyze_csv_with_sample_rows( - &self, - file_path: impl AsRef, - sample_rows: usize, - ) -> Result { - self.analyze_csv_with_sample_rows_and_candidate_detector(file_path, sample_rows, None) - } - - pub fn analyze_csv_with_candidate_detector( + /// `options.sample_rows` values spread across it, so the exact row count falls + /// out of the same pass that classifies the columns. + pub fn analyze_csv( &self, file_path: impl AsRef, - detector: &mut dyn CandidateDetector, - ) -> Result { - self.analyze_csv_with_sample_rows_and_candidate_detector( - file_path, - DETECTION_SAMPLE_ROW_FLOOR, - Some(detector), - ) - } - - pub fn analyze_csv_with_sample_rows_and_candidate_detector( - &self, - file_path: impl AsRef, - sample_rows: usize, - detector: Option<&mut dyn CandidateDetector>, + options: CsvAnalysisOptions<'_>, ) -> Result { + let CsvAnalysisOptions { + sample_rows, + candidate_detector, + } = options; let file_path = normalize_path(file_path.as_ref())?; let sample = read_detection_sample(&file_path, detection_sample_rows(sample_rows))?; - let (metadata, detector_status) = - build_column_metadata_with_candidate_detector(&sample.headers, &sample.rows, detector); + let (metadata, detector_status) = build_column_metadata_with_candidate_detector( + &sample.headers, + &sample.rows, + candidate_detector, + ); Ok(HeadersData { file_path: file_path.clone(), @@ -219,7 +196,13 @@ impl AnonymizerService { /// is the price of that split rather than something to optimize away here. pub fn preflight_anonymization(&self, input: PreflightParams) -> Result { let file_path = normalize_path(&input.file_path)?; - let headers = self.analyze_csv_with_sample_rows(&file_path, input.sample_row_count)?; + let headers = self.analyze_csv( + &file_path, + CsvAnalysisOptions { + sample_rows: input.sample_row_count, + ..Default::default() + }, + )?; // Coverage of the pass that just classified the file, not a fresh count: the // detection sample reads every row and keeps a bounded spread, so both figures // fell out of work already done. @@ -230,24 +213,15 @@ impl AnonymizerService { run_preflight(&file_path, headers.columns, input, coverage) } - pub fn preview_anonymization(&self, input: PreviewParams) -> Result { - self.preview_anonymization_with_smart_provider(input, None) - } - - pub fn preview_anonymization_with_smart_provider( + pub fn preview_anonymization( &self, input: PreviewParams, - provider: Option<&mut dyn SmartReplacementProvider>, - ) -> Result { - self.preview_anonymization_with_run_secrets(input, provider, None) - } - - pub fn preview_anonymization_with_run_secrets( - &self, - input: PreviewParams, - provider: Option<&mut dyn SmartReplacementProvider>, - tokenization_key: Option<&crate::TokenizationKey>, + runtime: TransformRuntime<'_, '_>, ) -> Result { + let TransformRuntime { + provider, + tokenization_key, + } = runtime; let file_path = normalize_path(&input.file_path)?; // Detect on `sample_row_count`, the figure analyze and the run are given, // so the preview cannot show a different detected type — and therefore a @@ -257,8 +231,6 @@ impl AnonymizerService { let detection_sample = read_detection_sample(&file_path, detection_sample_rows(input.sample_row_count))?; let metadata = build_column_metadata(&detection_sample.headers, &detection_sample.rows); - // Displayed rows are a separate, head-anchored window: the user expects - // the preview to show the file's opening rows, not the detection spread. let display = read_sample(&file_path, display_row_count(input.sample_count))?; preview::preview_rows_with_smart_provider( &metadata, @@ -267,9 +239,6 @@ impl AnonymizerService { columns: &input.columns, controls: &input.controls, sample_count: input.sample_count, - // The file's row count, from the pass that just classified it — not the - // sample size, which is what made the cardinality warning miss columns - // whose values repeat across a file far larger than the sample. population_values: detection_sample.data_rows_scanned, }, provider, @@ -282,62 +251,20 @@ impl AnonymizerService { count_csv_data_rows(&file_path) } - pub fn anonymize_csv(&self, input: AnonymizeParams) -> Result { - self.anonymize_csv_with_sample_rows(input, DETECTION_SAMPLE_ROW_FLOOR) - } - - pub fn anonymize_csv_with_sample_rows( - &self, - input: AnonymizeParams, - sample_rows: usize, - ) -> Result { - self.anonymize_csv_with_sample_rows_and_control(input, sample_rows, None) - } - - pub fn anonymize_csv_with_control( - &self, - input: AnonymizeParams, - control: &mut ProcessControl<'_>, - ) -> Result { - self.anonymize_csv_with_sample_rows_and_control( - input, - DETECTION_SAMPLE_ROW_FLOOR, - Some(control), - ) - } - - pub fn anonymize_csv_with_sample_rows_and_control( + pub fn anonymize_csv( &self, input: AnonymizeParams, - sample_rows: usize, - control: Option<&mut ProcessControl<'_>>, + options: CsvRunOptions<'_, '_, '_, '_>, ) -> Result { - self.anonymize_csv_with_sample_rows_and_control_and_smart_provider( - input, + let CsvRunOptions { sample_rows, - control, - None, - ) - } - - pub fn anonymize_csv_with_sample_rows_and_control_and_smart_provider( - &self, - input: AnonymizeParams, - sample_rows: usize, - control: Option<&mut ProcessControl<'_>>, - provider: Option<&mut dyn SmartReplacementProvider>, - ) -> Result { - self.anonymize_csv_with_run_secrets(input, sample_rows, control, provider, None) - } - - pub fn anonymize_csv_with_run_secrets( - &self, - input: AnonymizeParams, - sample_rows: usize, - mut control: Option<&mut ProcessControl<'_>>, - provider: Option<&mut dyn SmartReplacementProvider>, - tokenization_key: Option<&crate::TokenizationKey>, - ) -> Result { + mut control, + transform: + TransformRuntime { + provider, + tokenization_key, + }, + } = options; let input_path = normalize_path(&input.file_path)?; ensure_output_differs_from_input(&input_path, &input.output_path)?; let output_path = validate_output_path(&input.output_path, input.force)?; diff --git a/crates/csv-anonymizer-core/src/service/tests/analysis_preview.rs b/crates/csv-anonymizer-core/src/service/tests/analysis_preview.rs index 5d98f7a..2baea6d 100644 --- a/crates/csv-anonymizer-core/src/service/tests/analysis_preview.rs +++ b/crates/csv-anonymizer-core/src/service/tests/analysis_preview.rs @@ -35,7 +35,9 @@ impl CandidateDetector for FirstCellNameDetector { #[test] fn analyzes_csv_headers_and_default_output_path() { let service = AnonymizerService::new("test-version"); - let result = service.analyze_csv(fixture("sample.csv")).unwrap(); + let result = service + .analyze_csv(fixture("sample.csv"), crate::CsvAnalysisOptions::default()) + .unwrap(); assert_eq!(result.row_count, 5); assert!(result.row_count_is_complete); @@ -53,7 +55,13 @@ fn file_analysis_can_report_additive_candidate_detection() { let mut detector = FirstCellNameDetector; let result = service - .analyze_csv_with_candidate_detector(fixture("sample.csv"), &mut detector) + .analyze_csv( + fixture("sample.csv"), + crate::CsvAnalysisOptions { + candidate_detector: Some(&mut detector), + ..Default::default() + }, + ) .unwrap(); assert_eq!( @@ -74,7 +82,13 @@ fn file_analysis_can_report_additive_candidate_detection() { fn sampled_analysis_still_reports_the_exact_row_count() { let service = AnonymizerService::new("test-version"); let result = service - .analyze_csv_with_sample_rows(fixture("large.csv"), 25) + .analyze_csv( + fixture("large.csv"), + crate::CsvAnalysisOptions { + sample_rows: 25, + ..Default::default() + }, + ) .unwrap(); // The sample is capped at 25 rows, but detection streams every row, so the @@ -109,7 +123,13 @@ fn detects_pii_that_only_starts_after_the_sample_window() { let result = workspace .service - .analyze_csv_with_sample_rows(&input_path, 100) + .analyze_csv( + &input_path, + crate::CsvAnalysisOptions { + sample_rows: 100, + ..Default::default() + }, + ) .unwrap(); let column = &result.columns[0]; @@ -148,7 +168,10 @@ fn detects_pii_that_repeats_on_a_power_of_two_period() { } fs::write(&input_path, &content).unwrap(); - let result = workspace.service.analyze_csv(&input_path).unwrap(); + let result = workspace + .service + .analyze_csv(&input_path, crate::CsvAnalysisOptions::default()) + .unwrap(); let column = &result.columns[0]; assert_eq!( @@ -195,13 +218,25 @@ fn a_small_sample_row_request_cannot_lower_the_detection_basis() { let floored = workspace .service - .analyze_csv_with_sample_rows(&input_path, 100) + .analyze_csv( + &input_path, + crate::CsvAnalysisOptions { + sample_rows: 100, + ..Default::default() + }, + ) .unwrap(); for requested in [1, 2, 3, 5, 10, 50] { let result = workspace .service - .analyze_csv_with_sample_rows(&input_path, requested) + .analyze_csv( + &input_path, + crate::CsvAnalysisOptions { + sample_rows: requested, + ..Default::default() + }, + ) .unwrap(); assert_eq!( @@ -245,7 +280,13 @@ fn preview_classifies_on_the_same_basis_the_setting_gave_analyze() { let analyzed = workspace .service - .analyze_csv_with_sample_rows(&input_path, SAMPLE_ROWS) + .analyze_csv( + &input_path, + crate::CsvAnalysisOptions { + sample_rows: SAMPLE_ROWS, + ..Default::default() + }, + ) .unwrap(); assert_eq!( analyzed.columns[0].pii_risk, @@ -255,12 +296,15 @@ fn preview_classifies_on_the_same_basis_the_setting_gave_analyze() { let preview = workspace .service - .preview_anonymization(PreviewParams { - controls: Vec::new(), - sample_count: 3, - sample_row_count: SAMPLE_ROWS, - ..preview_params(input_path, vec![0]) - }) + .preview_anonymization( + PreviewParams { + controls: Vec::new(), + sample_count: 3, + sample_row_count: SAMPLE_ROWS, + ..preview_params(input_path, vec![0]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); for sample in &preview.previews[0].samples { @@ -285,15 +329,18 @@ fn preview_reuses_repeated_values_within_one_run() { let preview = workspace .service - .preview_anonymization(PreviewParams { - controls: vec![typed_control( - 0, - DataType::Email, - AnonymizationStrategy::Auto, - )], - sample_count: 3, - ..preview_params(input_path, vec![0]) - }) + .preview_anonymization( + PreviewParams { + controls: vec![typed_control( + 0, + DataType::Email, + AnonymizationStrategy::Auto, + )], + sample_count: 3, + ..preview_params(input_path, vec![0]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert_eq!(preview.previews[0].samples.len(), 3); @@ -314,11 +361,14 @@ fn preview_preserves_short_numeric_code_shape() { let preview = workspace .service - .preview_anonymization(PreviewParams { - controls: vec![control(0, AnonymizationStrategy::Auto)], - sample_count: 3, - ..preview_params(input_path, vec![0]) - }) + .preview_anonymization( + PreviewParams { + controls: vec![control(0, AnonymizationStrategy::Auto)], + sample_count: 3, + ..preview_params(input_path, vec![0]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert_eq!(preview.previews[0].samples.len(), 3); @@ -337,10 +387,13 @@ fn preview_preserves_decimal_numeric_shape() { let preview = workspace .service - .preview_anonymization(PreviewParams { - sample_count: 3, - ..preview_params(input_path, vec![0]) - }) + .preview_anonymization( + PreviewParams { + sample_count: 3, + ..preview_params(input_path, vec![0]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert_eq!(preview.previews[0].samples.len(), 3); @@ -364,10 +417,13 @@ fn preview_skips_empty_and_null_samples() { let preview = workspace .service - .preview_anonymization(PreviewParams { - sample_count: 3, - ..preview_params(input_path, vec![0]) - }) + .preview_anonymization( + PreviewParams { + sample_count: 3, + ..preview_params(input_path, vec![0]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert_eq!(preview.previews[0].samples.len(), 1); @@ -386,16 +442,19 @@ fn preview_uses_type_specific_phone_and_name_strategies() { let preview = workspace .service - .preview_anonymization(PreviewParams { - controls: vec![ - control(0, AnonymizationStrategy::Auto), - control(1, AnonymizationStrategy::Auto), - control(2, AnonymizationStrategy::Auto), - control(3, AnonymizationStrategy::Auto), - ], - sample_count: 1, - ..preview_params(input_path, vec![0, 1, 2, 3]) - }) + .preview_anonymization( + PreviewParams { + controls: vec![ + control(0, AnonymizationStrategy::Auto), + control(1, AnonymizationStrategy::Auto), + control(2, AnonymizationStrategy::Auto), + control(3, AnonymizationStrategy::Auto), + ], + sample_count: 1, + ..preview_params(input_path, vec![0, 1, 2, 3]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); let phone = &preview.previews[0].samples[0].anonymized; @@ -418,10 +477,13 @@ fn people_names_fixture_previews_name_like_full_names() { let service = AnonymizerService::new("test-version"); let preview = service - .preview_anonymization(PreviewParams { - controls: vec![control(2, AnonymizationStrategy::Auto)], - ..preview_params(fixture("people-names.csv"), vec![2]) - }) + .preview_anonymization( + PreviewParams { + controls: vec![control(2, AnonymizationStrategy::Auto)], + ..preview_params(fixture("people-names.csv"), vec![2]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert_eq!(preview.previews[0].column_name, "full_name"); @@ -452,15 +514,18 @@ fn people_names_fixture_treats_single_token_name_column_as_name() { let service = AnonymizerService::new("test-version"); let preview = service - .preview_anonymization(PreviewParams { - controls: vec![ - control(0, AnonymizationStrategy::Auto), - control(1, AnonymizationStrategy::Auto), - control(2, AnonymizationStrategy::Auto), - control(3, AnonymizationStrategy::Auto), - ], - ..preview_params(fixture("people-names.csv"), vec![0, 1, 2, 3]) - }) + .preview_anonymization( + PreviewParams { + controls: vec![ + control(0, AnonymizationStrategy::Auto), + control(1, AnonymizationStrategy::Auto), + control(2, AnonymizationStrategy::Auto), + control(3, AnonymizationStrategy::Auto), + ], + ..preview_params(fixture("people-names.csv"), vec![0, 1, 2, 3]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert_eq!(preview.previews[3].column_name, "name"); @@ -512,11 +577,14 @@ fn preview_name_mappings_are_consistent_within_previewed_rows() { let preview = workspace .service - .preview_anonymization(PreviewParams { - controls: controls.clone(), - sample_count: 2, - ..preview_params(input_path.clone(), vec![0, 1, 2]) - }) + .preview_anonymization( + PreviewParams { + controls: controls.clone(), + sample_count: 2, + ..preview_params(input_path.clone(), vec![0, 1, 2]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); for row_index in 0..2 { @@ -538,15 +606,18 @@ fn preview_applies_per_column_type_and_strategy_controls() { let preview = workspace .service - .preview_anonymization(PreviewParams { - controls: vec![typed_control( - 0, - DataType::Email, - AnonymizationStrategy::Mask, - )], - sample_count: 1, - ..preview_params(input_path, vec![0]) - }) + .preview_anonymization( + PreviewParams { + controls: vec![typed_control( + 0, + DataType::Email, + AnonymizationStrategy::Mask, + )], + sample_count: 1, + ..preview_params(input_path, vec![0]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert_eq!(preview.previews[0].samples[0].anonymized, "***"); @@ -573,14 +644,17 @@ fn column_report_for( let result = workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![ColumnControl { - column_index: 0, - type_override, - strategy, - }], - ..anonymize_params(input_path, output_path, vec![0]) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![ColumnControl { + column_index: 0, + type_override, + strategy, + }], + ..anonymize_params(input_path, output_path, vec![0]) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); result.privacy_report.column_reports[0].clone() @@ -697,15 +771,18 @@ fn the_preview_discloses_structure_preservation_before_the_run() { let preview = workspace .service - .preview_anonymization(PreviewParams { - controls: vec![typed_control( - 0, - DataType::Timestamp, - AnonymizationStrategy::Pseudonymize, - )], - sample_count: 1, - ..preview_params(input_path, vec![0]) - }) + .preview_anonymization( + PreviewParams { + controls: vec![typed_control( + 0, + DataType::Timestamp, + AnonymizationStrategy::Pseudonymize, + )], + sample_count: 1, + ..preview_params(input_path, vec![0]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert!( @@ -727,18 +804,21 @@ fn type_override_updates_report_risk_for_effective_type() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - file_path: input_path, - output_path, - columns: vec![0], - controls: vec![typed_control( - 0, - DataType::Email, - AnonymizationStrategy::Redact, - )], - force: false, - preview_smart_replacements: vec![], - }) + .anonymize_csv( + AnonymizeParams { + file_path: input_path, + output_path, + columns: vec![0], + controls: vec![typed_control( + 0, + DataType::Email, + AnonymizationStrategy::Redact, + )], + force: false, + preview_smart_replacements: vec![], + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let report = &result.privacy_report.column_reports[0]; @@ -831,11 +911,14 @@ fn preview_warns_for_pass_through_and_no_op_columns() { let preview = workspace .service - .preview_anonymization(PreviewParams { - controls: vec![control(1, AnonymizationStrategy::PassThrough)], - sample_count: 1, - ..preview_params(input_path, vec![0, 1]) - }) + .preview_anonymization( + PreviewParams { + controls: vec![control(1, AnonymizationStrategy::PassThrough)], + sample_count: 1, + ..preview_params(input_path, vec![0, 1]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); assert_eq!(preview.warnings.len(), 2); diff --git a/crates/csv-anonymizer-core/src/service/tests/anonymize.rs b/crates/csv-anonymizer-core/src/service/tests/anonymize.rs index ee625e1..86bd7fc 100644 --- a/crates/csv-anonymizer-core/src/service/tests/anonymize.rs +++ b/crates/csv-anonymizer-core/src/service/tests/anonymize.rs @@ -1,4 +1,5 @@ use super::*; +use crate::ProcessControl; use crate::types::{MatchedPart, ReleaseEvidenceStatus, ReleaseReadinessStatus}; use std::io::Write; @@ -9,11 +10,10 @@ fn anonymizes_selected_columns_without_web_runtime() { let result = workspace .service - .anonymize_csv(anonymize_params( - fixture("sample.csv"), - output_path.clone(), - vec![1], - )) + .anonymize_csv( + anonymize_params(fixture("sample.csv"), output_path.clone(), vec![1]), + crate::CsvRunOptions::default(), + ) .unwrap(); assert_eq!(result.output_path, output_path); @@ -47,7 +47,10 @@ fn late_starting_pii_is_offered_for_selection_and_not_written_to_the_output() { fs::write(&input_path, &content).unwrap(); // Select exactly the columns the app would offer to anonymize by default. - let headers = workspace.service.analyze_csv(&input_path).unwrap(); + let headers = workspace + .service + .analyze_csv(&input_path, crate::CsvAnalysisOptions::default()) + .unwrap(); let auto_selected: Vec = headers .columns .iter() @@ -62,11 +65,10 @@ fn late_starting_pii_is_offered_for_selection_and_not_written_to_the_output() { workspace .service - .anonymize_csv(anonymize_params( - input_path, - output_path.clone(), - auto_selected, - )) + .anonymize_csv( + anonymize_params(input_path, output_path.clone(), auto_selected), + crate::CsvRunOptions::default(), + ) .unwrap(); let output = fs::read_to_string(&output_path).unwrap(); @@ -90,7 +92,10 @@ fn one_rare_validated_identifier_is_anchored_in_the_detection_basis() { } fs::write(&input_path, content).unwrap(); - let headers = workspace.service.analyze_csv(&input_path).unwrap(); + let headers = workspace + .service + .analyze_csv(&input_path, crate::CsvAnalysisOptions::default()) + .unwrap(); let column = &headers.columns[0]; assert_eq!(column.pii_risk, crate::types::PiiRisk::High); assert_eq!( @@ -116,9 +121,12 @@ fn anonymize_csv_with_control_reports_progress() { workspace .service - .anonymize_csv_with_control( + .anonymize_csv( anonymize_params(fixture("sample.csv"), output_path.clone(), vec![1]), - &mut control, + crate::CsvRunOptions { + control: Some(&mut control), + ..Default::default() + }, ) .unwrap() }; @@ -136,9 +144,12 @@ fn selected_sample_empty_columns_transform_later_values() { let result = workspace .service - .anonymize_csv_with_sample_rows( + .anonymize_csv( anonymize_params(input_path, output_path.clone(), vec![1]), - 2, + crate::CsvRunOptions { + sample_rows: 2, + ..Default::default() + }, ) .unwrap(); @@ -163,16 +174,19 @@ fn anonymize_preserves_numeric_shapes_in_output_file() { workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![ - control(0, AnonymizationStrategy::Auto), - control(1, AnonymizationStrategy::Auto), - control(2, AnonymizationStrategy::Auto), - control(3, AnonymizationStrategy::Auto), - control(4, AnonymizationStrategy::Auto), - ], - ..anonymize_params(input_path, output_path.clone(), vec![0, 1, 2, 3, 4]) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![ + control(0, AnonymizationStrategy::Auto), + control(1, AnonymizationStrategy::Auto), + control(2, AnonymizationStrategy::Auto), + control(3, AnonymizationStrategy::Auto), + control(4, AnonymizationStrategy::Auto), + ], + ..anonymize_params(input_path, output_path.clone(), vec![0, 1, 2, 3, 4]) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let output = read_sample(&output_path, 10).unwrap(); @@ -214,14 +228,17 @@ fn anonymize_reuses_repeated_name_sources_in_random_mode() { workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![typed_control( - 0, - DataType::FirstName, - AnonymizationStrategy::Auto, - )], - ..anonymize_params(input_path, output_path.clone(), vec![0]) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![typed_control( + 0, + DataType::FirstName, + AnonymizationStrategy::Auto, + )], + ..anonymize_params(input_path, output_path.clone(), vec![0]) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let output = read_sample(&output_path, 10).unwrap(); @@ -242,14 +259,17 @@ fn anonymize_random_mode_avoids_duplicate_names_for_distinct_sources() { workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![typed_control( - 0, - DataType::FirstName, - AnonymizationStrategy::Auto, - )], - ..anonymize_params(input_path, output_path.clone(), vec![0]) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![typed_control( + 0, + DataType::FirstName, + AnonymizationStrategy::Auto, + )], + ..anonymize_params(input_path, output_path.clone(), vec![0]) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let output = read_sample(&output_path, 20).unwrap(); @@ -276,14 +296,17 @@ fn anonymize_reuses_repeated_values_in_single_output() { workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![ - typed_control(0, DataType::FirstName, AnonymizationStrategy::Auto), - typed_control(1, DataType::LastName, AnonymizationStrategy::Auto), - typed_control(2, DataType::Email, AnonymizationStrategy::Auto), - ], - ..anonymize_params(input_path.clone(), output_path.clone(), vec![0, 1, 2]) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![ + typed_control(0, DataType::FirstName, AnonymizationStrategy::Auto), + typed_control(1, DataType::LastName, AnonymizationStrategy::Auto), + typed_control(2, DataType::Email, AnonymizationStrategy::Auto), + ], + ..anonymize_params(input_path.clone(), output_path.clone(), vec![0, 1, 2]) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let output = read_sample(&output_path, 10).unwrap(); @@ -304,10 +327,13 @@ fn anonymize_applies_pass_through_control() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![control(0, AnonymizationStrategy::PassThrough)], - ..anonymize_params(input_path, output_path.clone(), vec![0]) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![control(0, AnonymizationStrategy::PassThrough)], + ..anonymize_params(input_path, output_path.clone(), vec![0]) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); assert_eq!(result.columns_anonymized, 0); @@ -328,14 +354,17 @@ fn anonymize_does_not_count_auto_noop_selected_columns() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![ - typed_control(0, DataType::Email, AnonymizationStrategy::PassThrough), - typed_control(1, DataType::CountryCode, AnonymizationStrategy::Auto), - typed_control(2, DataType::String, AnonymizationStrategy::Mask), - ], - ..anonymize_params(input_path, output_path.clone(), vec![0, 1, 2]) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![ + typed_control(0, DataType::Email, AnonymizationStrategy::PassThrough), + typed_control(1, DataType::CountryCode, AnonymizationStrategy::Auto), + typed_control(2, DataType::String, AnonymizationStrategy::Mask), + ], + ..anonymize_params(input_path, output_path.clone(), vec![0, 1, 2]) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let output = read_sample(&output_path, 10).unwrap(); @@ -353,10 +382,13 @@ fn anonymize_rejects_output_path_equal_to_input_even_with_force() { let error = workspace .service - .anonymize_csv(AnonymizeParams { - force: true, - ..anonymize_params(input_path.clone(), input_path.clone(), vec![0]) - }) + .anonymize_csv( + AnonymizeParams { + force: true, + ..anonymize_params(input_path.clone(), input_path.clone(), vec![0]) + }, + crate::CsvRunOptions::default(), + ) .unwrap_err(); assert!( @@ -376,14 +408,17 @@ fn anonymize_rejects_absolute_input_matching_bare_relative_output() { let output_path = input_path.file_name().unwrap().into(); let error = service - .anonymize_csv(AnonymizeParams { - file_path: input_path.clone(), - output_path, - columns: vec![0], - controls: vec![], - force: true, - preview_smart_replacements: vec![], - }) + .anonymize_csv( + AnonymizeParams { + file_path: input_path.clone(), + output_path, + columns: vec![0], + controls: vec![], + force: true, + preview_smart_replacements: vec![], + }, + crate::CsvRunOptions::default(), + ) .unwrap_err(); assert!( @@ -407,14 +442,17 @@ fn anonymize_counts_iban_evidence_as_quasi_identifier() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - file_path: input_path, - output_path, - columns: vec![0], - controls: vec![], - force: false, - preview_smart_replacements: vec![], - }) + .anonymize_csv( + AnonymizeParams { + file_path: input_path, + output_path, + columns: vec![0], + controls: vec![], + force: false, + preview_smart_replacements: vec![], + }, + crate::CsvRunOptions::default(), + ) .unwrap(); assert_eq!(result.privacy_report.direct_identifiers, 0); @@ -443,9 +481,12 @@ fn anonymize_reports_partial_detection_coverage_in_privacy_notes() { let result = workspace .service - .anonymize_csv_with_sample_rows( + .anonymize_csv( anonymize_params(input_path, output_path.clone(), vec![1]), - 10, + crate::CsvRunOptions { + sample_rows: 10, + ..Default::default() + }, ) .unwrap(); @@ -468,9 +509,12 @@ fn anonymize_omits_coverage_note_when_every_row_was_examined() { let result = workspace .service - .anonymize_csv_with_sample_rows( + .anonymize_csv( anonymize_params(input_path, output_path.clone(), vec![1]), - 10, + crate::CsvRunOptions { + sample_rows: 10, + ..Default::default() + }, ) .unwrap(); @@ -500,9 +544,12 @@ fn anonymize_states_the_mistyping_risk_when_no_column_was_left_unselected() { let result = workspace .service - .anonymize_csv_with_sample_rows( + .anonymize_csv( anonymize_params(input_path, output_path.clone(), vec![0, 1]), - 10, + crate::CsvRunOptions { + sample_rows: 10, + ..Default::default() + }, ) .unwrap(); @@ -552,7 +599,10 @@ fn the_release_report_states_how_many_rows_the_released_columns_single_out() { let output_path = workspace.path("quasi-identifiers-anonymized.csv"); write_quasi_identifier_fixture(&input_path); - let headers = workspace.service.analyze_csv(&input_path).unwrap(); + let headers = workspace + .service + .analyze_csv(&input_path, crate::CsvAnalysisOptions::default()) + .unwrap(); let name_index = headers .columns .iter() @@ -562,14 +612,17 @@ fn the_release_report_states_how_many_rows_the_released_columns_single_out() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - file_path: input_path, - output_path, - columns: vec![name_index], - controls: vec![control(name_index, AnonymizationStrategy::Redact)], - force: false, - preview_smart_replacements: vec![], - }) + .anonymize_csv( + AnonymizeParams { + file_path: input_path, + output_path, + columns: vec![name_index], + controls: vec![control(name_index, AnonymizationStrategy::Redact)], + force: false, + preview_smart_replacements: vec![], + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let summary = result @@ -658,7 +711,10 @@ fn a_fully_redacted_release_reports_no_applicable_measure_rather_than_a_pass() { let output_path = workspace.path("all-redacted-anonymized.csv"); write_quasi_identifier_fixture(&input_path); - let headers = workspace.service.analyze_csv(&input_path).unwrap(); + let headers = workspace + .service + .analyze_csv(&input_path, crate::CsvAnalysisOptions::default()) + .unwrap(); let all: Vec = headers.columns.iter().map(|column| column.index).collect(); let controls = all .iter() @@ -667,14 +723,17 @@ fn a_fully_redacted_release_reports_no_applicable_measure_rather_than_a_pass() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - file_path: input_path, - output_path, - columns: all, - controls, - force: false, - preview_smart_replacements: vec![], - }) + .anonymize_csv( + AnonymizeParams { + file_path: input_path, + output_path, + columns: all, + controls, + force: false, + preview_smart_replacements: vec![], + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let item = result @@ -719,7 +778,10 @@ fn the_finding_separates_released_values_from_surviving_formats() { content.push_str("123456789,1011AB\n"); fs::write(&input_path, content).unwrap(); - let headers = workspace.service.analyze_csv(&input_path).unwrap(); + let headers = workspace + .service + .analyze_csv(&input_path, crate::CsvAnalysisOptions::default()) + .unwrap(); let customer_id = headers .columns .iter() @@ -729,18 +791,21 @@ fn the_finding_separates_released_values_from_surviving_formats() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - file_path: input_path, - output_path, - columns: vec![customer_id], - controls: vec![typed_control( - customer_id, - DataType::NumericId, - AnonymizationStrategy::Pseudonymize, - )], - force: false, - preview_smart_replacements: vec![], - }) + .anonymize_csv( + AnonymizeParams { + file_path: input_path, + output_path, + columns: vec![customer_id], + controls: vec![typed_control( + customer_id, + DataType::NumericId, + AnonymizationStrategy::Pseudonymize, + )], + force: false, + preview_smart_replacements: vec![], + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let summary = result @@ -801,7 +866,9 @@ fn city_fixture_uniqueness( output_path: &std::path::Path, ) -> crate::types::AnonymizeData { let service = AnonymizerService::new("test-version"); - let headers = service.analyze_csv(input_path).unwrap(); + let headers = service + .analyze_csv(input_path, crate::CsvAnalysisOptions::default()) + .unwrap(); let name_index = headers .columns .iter() @@ -810,14 +877,17 @@ fn city_fixture_uniqueness( .index; service - .anonymize_csv(AnonymizeParams { - controls: vec![control(name_index, AnonymizationStrategy::Redact)], - ..anonymize_params( - input_path.to_path_buf(), - output_path.to_path_buf(), - vec![name_index], - ) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![control(name_index, AnonymizationStrategy::Redact)], + ..anonymize_params( + input_path.to_path_buf(), + output_path.to_path_buf(), + vec![name_index], + ) + }, + crate::CsvRunOptions::default(), + ) .unwrap() } @@ -1005,7 +1075,10 @@ fn a_partial_match_is_named_as_the_part_it_matched() { } fs::write(&input_path, content).unwrap(); - let headers = workspace.service.analyze_csv(&input_path).unwrap(); + let headers = workspace + .service + .analyze_csv(&input_path, crate::CsvAnalysisOptions::default()) + .unwrap(); let indices: Vec = headers.columns.iter().map(|column| column.index).collect(); let controls = headers .columns @@ -1025,14 +1098,17 @@ fn a_partial_match_is_named_as_the_part_it_matched() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - file_path: input_path, - output_path, - columns: indices, - controls, - force: false, - preview_smart_replacements: vec![], - }) + .anonymize_csv( + AnonymizeParams { + file_path: input_path, + output_path, + columns: indices, + controls, + force: false, + preview_smart_replacements: vec![], + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let item = result diff --git a/crates/csv-anonymizer-core/src/service/tests/cardinality.rs b/crates/csv-anonymizer-core/src/service/tests/cardinality.rs index 521b50c..5ba2203 100644 --- a/crates/csv-anonymizer-core/src/service/tests/cardinality.rs +++ b/crates/csv-anonymizer-core/src/service/tests/cardinality.rs @@ -41,17 +41,23 @@ const INVERSION_MECHANISM: &str = FREQUENCY_INVERSION_MECHANISM; fn repeated_values_columns() -> Vec { let service = AnonymizerService::new("test-version"); service - .analyze_csv(fixture("repeated-values.csv")) + .analyze_csv( + fixture("repeated-values.csv"), + crate::CsvAnalysisOptions::default(), + ) .unwrap() .columns } fn cardinality_warnings(columns: Vec, controls: Vec) -> Vec { let preview = AnonymizerService::new("test-version") - .preview_anonymization(PreviewParams { - controls, - ..preview_params(fixture("repeated-values.csv"), columns) - }) + .preview_anonymization( + PreviewParams { + controls, + ..preview_params(fixture("repeated-values.csv"), columns) + }, + crate::TransformRuntime::default(), + ) .unwrap(); preview @@ -206,14 +212,17 @@ fn warning_about_cardinality_does_not_count_the_column_as_pass_through() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![control(1, AnonymizationStrategy::Pseudonymize)], - ..anonymize_params( - fixture("repeated-values.csv"), - workspace.path("repeated-values-anonymized.csv"), - vec![1], - ) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![control(1, AnonymizationStrategy::Pseudonymize)], + ..anonymize_params( + fixture("repeated-values.csv"), + workspace.path("repeated-values-anonymized.csv"), + vec![1], + ) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let report = result.privacy_report; @@ -230,14 +239,17 @@ fn the_run_reports_the_exact_distribution_of_each_pseudonymized_column() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![control(1, AnonymizationStrategy::Pseudonymize)], - ..anonymize_params( - fixture("repeated-values.csv"), - workspace.path("repeated-values-anonymized.csv"), - vec![1], - ) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![control(1, AnonymizationStrategy::Pseudonymize)], + ..anonymize_params( + fixture("repeated-values.csv"), + workspace.path("repeated-values-anonymized.csv"), + vec![1], + ) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let distributions = result.privacy_report.column_value_distributions; @@ -256,14 +268,17 @@ fn a_redacted_column_reports_no_distribution() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![control(1, AnonymizationStrategy::Redact)], - ..anonymize_params( - fixture("repeated-values.csv"), - workspace.path("repeated-values-redacted.csv"), - vec![1], - ) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![control(1, AnonymizationStrategy::Redact)], + ..anonymize_params( + fixture("repeated-values.csv"), + workspace.path("repeated-values-redacted.csv"), + vec![1], + ) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); assert!( @@ -283,14 +298,17 @@ fn the_report_names_the_invertible_columns_and_the_pseudonymization_caveat() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![control(1, AnonymizationStrategy::Pseudonymize)], - ..anonymize_params( - fixture("repeated-values.csv"), - workspace.path("repeated-values-reported.csv"), - vec![1], - ) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![control(1, AnonymizationStrategy::Pseudonymize)], + ..anonymize_params( + fixture("repeated-values.csv"), + workspace.path("repeated-values-reported.csv"), + vec![1], + ) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let report = result.privacy_report; @@ -332,14 +350,17 @@ fn a_redacted_run_reports_no_frequency_or_pseudonymization_caveat() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![control(1, AnonymizationStrategy::Redact)], - ..anonymize_params( - fixture("repeated-values.csv"), - workspace.path("repeated-values-redacted-notes.csv"), - vec![1], - ) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![control(1, AnonymizationStrategy::Redact)], + ..anonymize_params( + fixture("repeated-values.csv"), + workspace.path("repeated-values-redacted-notes.csv"), + vec![1], + ) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); for note in &result.privacy_report.notes { @@ -373,10 +394,13 @@ fn cardinality_warnings_for( strategy: AnonymizationStrategy, ) -> Vec { AnonymizerService::new("test-version") - .preview_anonymization(PreviewParams { - controls: vec![control(1, strategy)], - ..preview_params(file_path, vec![1]) - }) + .preview_anonymization( + PreviewParams { + controls: vec![control(1, strategy)], + ..preview_params(file_path, vec![1]) + }, + crate::TransformRuntime::default(), + ) .unwrap() .warnings .into_iter() @@ -557,7 +581,12 @@ fn dominant_value_column_file( #[test] fn the_dominant_value_fixture_has_the_shape_these_tests_depend_on() { let service = AnonymizerService::new("test-version"); - let analysis = service.analyze_csv(fixture("dominant-value.csv")).unwrap(); + let analysis = service + .analyze_csv( + fixture("dominant-value.csv"), + crate::CsvAnalysisOptions::default(), + ) + .unwrap(); assert_eq!(analysis.row_count, 200); let queue = &analysis.columns[1]; @@ -690,14 +719,17 @@ fn the_post_run_report_names_the_dominant_value_rather_than_the_distinct_count() let result = workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![control(1, AnonymizationStrategy::Pseudonymize)], - ..anonymize_params( - fixture("dominant-value.csv"), - workspace.path("dominant-value-reported.csv"), - vec![1], - ) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![control(1, AnonymizationStrategy::Pseudonymize)], + ..anonymize_params( + fixture("dominant-value.csv"), + workspace.path("dominant-value-reported.csv"), + vec![1], + ) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let report = result.privacy_report; @@ -798,14 +830,17 @@ fn the_run_reports_the_dominant_value_the_preview_warned_about() { let result = workspace .service - .anonymize_csv(AnonymizeParams { - controls: vec![control(1, AnonymizationStrategy::Pseudonymize)], - ..anonymize_params( - fixture("dominant-value.csv"), - workspace.path("dominant-value-anonymized.csv"), - vec![1], - ) - }) + .anonymize_csv( + AnonymizeParams { + controls: vec![control(1, AnonymizationStrategy::Pseudonymize)], + ..anonymize_params( + fixture("dominant-value.csv"), + workspace.path("dominant-value-anonymized.csv"), + vec![1], + ) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); let distributions = result.privacy_report.column_value_distributions; @@ -910,7 +945,9 @@ fn a_dominant_value_its_runner_up_nearly_matches_is_warned_about_anyway() { // distinct values is well past the absolute term's ten, and 33 singletons put coverage // at 0.67, below the ratio term's gate. let service = AnonymizerService::new("test-version"); - let analysis = service.analyze_csv(path.clone()).unwrap(); + let analysis = service + .analyze_csv(path.clone(), crate::CsvAnalysisOptions::default()) + .unwrap(); assert_eq!(analysis.row_count, 100); let column = &analysis.columns[1]; diff --git a/crates/csv-anonymizer-core/src/service/tests/label_output.rs b/crates/csv-anonymizer-core/src/service/tests/label_output.rs index 11c3e24..20af517 100644 --- a/crates/csv-anonymizer-core/src/service/tests/label_output.rs +++ b/crates/csv-anonymizer-core/src/service/tests/label_output.rs @@ -23,10 +23,13 @@ fn placeholder_ordinal(cell: &str, label: &str) -> Option { fn anonymize_to(input_path: &Path, output_path: &Path, columns: Vec) { let controls = columns.iter().copied().map(label_control).collect(); AnonymizerService::new("test-version") - .anonymize_csv(AnonymizeParams { - controls, - ..anonymize_params(input_path.to_path_buf(), output_path.to_path_buf(), columns) - }) + .anonymize_csv( + AnonymizeParams { + controls, + ..anonymize_params(input_path.to_path_buf(), output_path.to_path_buf(), columns) + }, + crate::CsvRunOptions::default(), + ) .unwrap(); } @@ -56,7 +59,11 @@ fn a_labelled_column_reaches_the_written_file_with_one_placeholder_per_distinct_ // Pins the reason this column is a fair subject: no validator claims free-form // prose, so `String` is what a real free-text column detects as, and the header // is genuinely the only surviving evidence about what the cells held. - let columns = workspace.service.analyze_csv(&input_path).unwrap().columns; + let columns = workspace + .service + .analyze_csv(&input_path, crate::CsvAnalysisOptions::default()) + .unwrap() + .columns; assert_eq!(columns[1].name, "customer notes"); assert_eq!(columns[1].detected_type, DataType::String); @@ -134,10 +141,13 @@ fn the_preview_and_the_run_agree_on_what_each_placeholder_stands_for() { let preview = workspace .service - .preview_anonymization(PreviewParams { - controls: vec![label_control(1)], - ..preview_params(input_path.clone(), vec![1]) - }) + .preview_anonymization( + PreviewParams { + controls: vec![label_control(1)], + ..preview_params(input_path.clone(), vec![1]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); anonymize_to(&input_path, &output_path, vec![1]); let rows = written_rows(&output_path); @@ -179,10 +189,13 @@ fn the_preview_numbers_the_files_opening_rows() { let preview = workspace .service - .preview_anonymization(PreviewParams { - controls: vec![label_control(1)], - ..preview_params(input_path, vec![1]) - }) + .preview_anonymization( + PreviewParams { + controls: vec![label_control(1)], + ..preview_params(input_path, vec![1]) + }, + crate::TransformRuntime::default(), + ) .unwrap(); let shown: Vec<(String, String)> = preview.previews[0] diff --git a/crates/csv-anonymizer-core/src/service/tests/possible_names.rs b/crates/csv-anonymizer-core/src/service/tests/possible_names.rs index a6d3c65..d8bd71c 100644 --- a/crates/csv-anonymizer-core/src/service/tests/possible_names.rs +++ b/crates/csv-anonymizer-core/src/service/tests/possible_names.rs @@ -25,12 +25,15 @@ fn name_column_file( fn reviewed_columns(header: &str, values: &[&str]) -> Vec { let workspace = Workspace::new(); AnonymizerService::new("test-version") - .preview_anonymization(PreviewParams { - ..preview_params( - name_column_file(workspace.directory.path(), header, values), - vec![], - ) - }) + .preview_anonymization( + PreviewParams { + ..preview_params( + name_column_file(workspace.directory.path(), header, values), + vec![], + ) + }, + crate::TransformRuntime::default(), + ) .unwrap() .warnings .iter() @@ -126,7 +129,7 @@ fn a_surfaced_column_is_still_neither_selected_nor_redacted() { let workspace = Workspace::new(); let path = name_column_file(workspace.directory.path(), "agent_name", PEOPLE); let column = AnonymizerService::new("test-version") - .analyze_csv(path) + .analyze_csv(path, crate::CsvAnalysisOptions::default()) .unwrap() .columns .remove(1); diff --git a/crates/csv-anonymizer-core/src/service/tests/smart_replacement.rs b/crates/csv-anonymizer-core/src/service/tests/smart_replacement.rs index 042d3f3..20fdf20 100644 --- a/crates/csv-anonymizer-core/src/service/tests/smart_replacement.rs +++ b/crates/csv-anonymizer-core/src/service/tests/smart_replacement.rs @@ -187,7 +187,7 @@ fn preview_uses_local_ai_provider_for_smart_replacement_columns() { let preview = workspace .service - .preview_anonymization_with_smart_provider( + .preview_anonymization( PreviewParams { controls: vec![typed_control( 0, @@ -197,7 +197,10 @@ fn preview_uses_local_ai_provider_for_smart_replacement_columns() { sample_count: 2, ..preview_params(input_path, vec![0]) }, - Some(&mut provider), + crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, ) .unwrap(); @@ -221,7 +224,7 @@ fn anonymize_uses_local_ai_provider_and_reports_smart_replacements() { let result = workspace .service - .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + .anonymize_csv( AnonymizeParams { controls: vec![typed_control( 0, @@ -230,9 +233,14 @@ fn anonymize_uses_local_ai_provider_and_reports_smart_replacements() { )], ..anonymize_params(input_path, output_path.clone(), vec![0]) }, - 10, - None, - Some(&mut provider), + crate::CsvRunOptions { + sample_rows: 10, + control: None, + transform: crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, + }, ) .unwrap(); @@ -259,20 +267,23 @@ fn anonymize_reuses_preview_smart_replacements_and_generates_missing_values() { let preview = workspace .service - .preview_anonymization_with_smart_provider( + .preview_anonymization( PreviewParams { controls: controls.clone(), sample_count: 1, ..preview_params(input_path.clone(), vec![0]) }, - Some(&mut preview_provider), + crate::TransformRuntime { + provider: Some(&mut preview_provider), + tokenization_key: None, + }, ) .unwrap(); let mut final_provider = RecordingSmartProvider::new("Final"); workspace .service - .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + .anonymize_csv( AnonymizeParams { file_path: input_path, output_path: output_path.clone(), @@ -281,9 +292,14 @@ fn anonymize_reuses_preview_smart_replacements_and_generates_missing_values() { force: false, preview_smart_replacements: preview.smart_replacements.clone(), }, - 10, - None, - Some(&mut final_provider), + crate::CsvRunOptions { + sample_rows: 10, + control: None, + transform: crate::TransformRuntime { + provider: Some(&mut final_provider), + tokenization_key: None, + }, + }, ) .unwrap(); @@ -308,7 +324,7 @@ fn anonymize_rejects_invalid_preview_smart_replacements_and_generates_missing_va let result = workspace .service - .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + .anonymize_csv( AnonymizeParams { controls: vec![typed_control( 0, @@ -322,9 +338,14 @@ fn anonymize_rejects_invalid_preview_smart_replacements_and_generates_missing_va }], ..anonymize_params(input_path, output_path.clone(), vec![0]) }, - 10, - None, - Some(&mut provider), + crate::CsvRunOptions { + sample_rows: 10, + control: None, + transform: crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, + }, ) .unwrap(); @@ -352,7 +373,7 @@ fn anonymize_revalidates_preview_replacement_against_tail_values() { let result = workspace .service - .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + .anonymize_csv( AnonymizeParams { controls: vec![typed_control( 0, @@ -366,9 +387,14 @@ fn anonymize_revalidates_preview_replacement_against_tail_values() { }], ..anonymize_params(input_path, output_path.clone(), vec![0]) }, - 10, - None, - Some(&mut provider), + crate::CsvRunOptions { + sample_rows: 10, + control: None, + transform: crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, + }, ) .unwrap(); @@ -392,7 +418,7 @@ fn anonymize_rejects_smart_replacements_carrying_another_rows_value() { let result = workspace .service - .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + .anonymize_csv( AnonymizeParams { controls: vec![typed_control( 0, @@ -401,9 +427,14 @@ fn anonymize_rejects_smart_replacements_carrying_another_rows_value() { )], ..anonymize_params(input_path, output_path.clone(), vec![0]) }, - 10, - None, - Some(&mut provider), + crate::CsvRunOptions { + sample_rows: 10, + control: None, + transform: crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, + }, ) .unwrap(); @@ -518,7 +549,7 @@ fn anonymize_rejects_smart_replacements_carrying_an_earlier_chunks_value() { let result = workspace .service - .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + .anonymize_csv( AnonymizeParams { controls: vec![typed_control( 0, @@ -527,9 +558,14 @@ fn anonymize_rejects_smart_replacements_carrying_an_earlier_chunks_value() { )], ..anonymize_params(input_path, output_path.clone(), vec![0]) }, - 30, - None, - Some(&mut provider), + crate::CsvRunOptions { + sample_rows: 30, + control: None, + transform: crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, + }, ) .unwrap(); @@ -638,7 +674,7 @@ fn anonymize_reports_all_rejected_smart_replacement_batches() { let result = workspace .service - .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + .anonymize_csv( AnonymizeParams { controls: vec![typed_control( 0, @@ -647,9 +683,14 @@ fn anonymize_reports_all_rejected_smart_replacement_batches() { )], ..anonymize_params(input_path, output_path.clone(), vec![0]) }, - 10, - None, - Some(&mut provider), + crate::CsvRunOptions { + sample_rows: 10, + control: None, + transform: crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, + }, ) .unwrap(); @@ -682,7 +723,7 @@ fn anonymize_caps_local_ai_unique_values_and_falls_back_for_excess_values() { let result = workspace .service - .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + .anonymize_csv( AnonymizeParams { file_path: input_path, output_path, @@ -695,9 +736,14 @@ fn anonymize_caps_local_ai_unique_values_and_falls_back_for_excess_values() { force: false, preview_smart_replacements: vec![], }, - 10, - None, - Some(&mut provider), + crate::CsvRunOptions { + sample_rows: 10, + control: None, + transform: crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, + }, ) .unwrap(); let requested_values = provider.requests.iter().map(Vec::len).sum::(); @@ -724,7 +770,7 @@ fn anonymize_rejects_duplicate_smart_outputs_across_provider_chunks() { let result = workspace .service - .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + .anonymize_csv( AnonymizeParams { file_path: input_path, output_path, @@ -737,9 +783,14 @@ fn anonymize_rejects_duplicate_smart_outputs_across_provider_chunks() { force: false, preview_smart_replacements: vec![], }, - 30, - None, - Some(&mut provider), + crate::CsvRunOptions { + sample_rows: 30, + control: None, + transform: crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, + }, ) .unwrap(); @@ -762,15 +813,18 @@ fn local_ai_strategy_requires_provider_before_processing() { let error = workspace .service - .preview_anonymization(PreviewParams { - controls: vec![typed_control( - 0, - DataType::FullName, - AnonymizationStrategy::LocalAi, - )], - sample_count: 1, - ..preview_params(input_path, vec![0]) - }) + .preview_anonymization( + PreviewParams { + controls: vec![typed_control( + 0, + DataType::FullName, + AnonymizationStrategy::LocalAi, + )], + sample_count: 1, + ..preview_params(input_path, vec![0]) + }, + crate::TransformRuntime::default(), + ) .unwrap_err(); assert!(error.to_string().contains("Local AI")); @@ -828,14 +882,19 @@ fn a_local_ai_column_rejected_wholesale_releases_no_source_value() { let result = workspace .service - .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + .anonymize_csv( AnonymizeParams { controls: vec![control(0, AnonymizationStrategy::LocalAi)], ..anonymize_params(input_path, output_path.clone(), vec![0]) }, - 100, - None, - Some(&mut provider), + crate::CsvRunOptions { + sample_rows: 100, + control: None, + transform: crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, + }, ) .unwrap(); @@ -908,13 +967,16 @@ fn pass_through_types_are_still_unchanged_under_auto_and_pseudonymize() { workspace .service - .anonymize_csv_with_sample_rows_and_control( + .anonymize_csv( AnonymizeParams { controls: vec![control(0, strategy)], ..anonymize_params(input_path, output_path.clone(), vec![0]) }, - 100, - None, + crate::CsvRunOptions { + sample_rows: 100, + control: None, + ..Default::default() + }, ) .unwrap(); @@ -940,7 +1002,7 @@ fn wholesale_leak_guard_fallback_is_explained_without_naming_unproven_columns() let result = workspace .service - .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + .anonymize_csv( AnonymizeParams { controls: vec![typed_control( 0, @@ -949,9 +1011,14 @@ fn wholesale_leak_guard_fallback_is_explained_without_naming_unproven_columns() )], ..anonymize_params(input_path, output_path.clone(), vec![0]) }, - 10, - None, - Some(&mut provider), + crate::CsvRunOptions { + sample_rows: 10, + control: None, + transform: crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, + }, ) .unwrap(); @@ -986,14 +1053,19 @@ fn accepted_local_ai_replacements_draw_no_wholesale_fallback_note() { let result = workspace .service - .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + .anonymize_csv( AnonymizeParams { controls: vec![control(0, AnonymizationStrategy::LocalAi)], ..anonymize_params(input_path, output_path.clone(), vec![0]) }, - 100, - None, - Some(&mut provider), + crate::CsvRunOptions { + sample_rows: 100, + control: None, + transform: crate::TransformRuntime { + provider: Some(&mut provider), + tokenization_key: None, + }, + }, ) .unwrap(); diff --git a/crates/csv-anonymizer-core/src/strategies/tests/mapping_budget.rs b/crates/csv-anonymizer-core/src/strategies/tests/mapping_budget.rs index 925d3a0..51cec7e 100644 --- a/crates/csv-anonymizer-core/src/strategies/tests/mapping_budget.rs +++ b/crates/csv-anonymizer-core/src/strategies/tests/mapping_budget.rs @@ -399,18 +399,21 @@ fn measure(strategy: AnonymizationStrategy, distinct_values: usize) -> (u64, usi let service = AnonymizerService::new("mapping-budget-measurement"); service - .anonymize_csv(AnonymizeParams { - file_path: input_path, - output_path, - columns: vec![1], - controls: vec![ColumnControl { - column_index: 1, - type_override: Some(DataType::String), - strategy, - }], - force: true, - preview_smart_replacements: vec![], - }) + .anonymize_csv( + AnonymizeParams { + file_path: input_path, + output_path, + columns: vec![1], + controls: vec![ColumnControl { + column_index: 1, + type_override: Some(DataType::String), + strategy, + }], + force: true, + preview_smart_replacements: vec![], + }, + crate::CsvRunOptions::default(), + ) .expect("measurement run should succeed"); // Recomputed here rather than read from the run, because the run's state is diff --git a/crates/csv-anonymizer-core/src/types.rs b/crates/csv-anonymizer-core/src/types.rs index 117bbc6..078ff63 100644 --- a/crates/csv-anonymizer-core/src/types.rs +++ b/crates/csv-anonymizer-core/src/types.rs @@ -1,2555 +1,44 @@ -use serde::{Deserialize, Serialize}; -use std::path::PathBuf; - -/// The largest detection basis any entry point accepts, for any input kind. -/// -/// One limit rather than one per workflow, because the figure comes from one -/// setting. "Sample rows" is not per-workflow: a user who raises it to work on a -/// large CSV file has raised it for the paste workflow too, so a value that reaches -/// the setting has to be a value every entry point will honour. -/// -/// Enforced twice, and both sites read this: `settings::sanitize_settings` clamps -/// what can be stored, and the paste entry points reject an oversized request -/// outright, since they are reachable by callers that never went through settings. -pub const MAX_SAMPLE_ROW_COUNT: usize = 10_000; - -/// The largest display window any entry point accepts, for any input kind. One -/// limit for the same reason as [`MAX_SAMPLE_ROW_COUNT`] — it comes from the -/// "Preview rows" setting, which is likewise not per-workflow. -pub const MAX_PREVIEW_SAMPLE_COUNT: usize = 100; - -/// The smallest detection basis any entry point classifies on, for any input kind. -/// -/// A floor rather than a default: `service::detection_sample_rows` and -/// `direct_input::shared::paste_detection_sample_rows` both raise a caller's "Sample -/// rows" to at least this, so the setting can only ask for more evidence than the -/// default, never less. -/// -/// One constant rather than one per workflow, because the file and paste workflows -/// promise the user that they classify on the same basis. Two literals held that -/// promise by coincidence: changing either alone left a pasted CSV and the same file -/// on disk detecting different types, and so being offered different strategies, with -/// nothing failing to say so. -pub(crate) const DETECTION_SAMPLE_ROW_FLOOR: usize = 100; - -/// What `strategies::structured::transform_generic_string` keeps of a value. -/// -/// Named once because two callers state it about the same transform: the detected -/// types that have no transformer of their own, and the Local AI fallback for a -/// pass-through type, which lands on the same function. See -/// [`DataType::pseudonymization_preserves_structure`]. -pub(crate) const GENERIC_STRING_STRUCTURE_DISCLOSURE: &str = "the replacement is random text of roughly the original's length (within about 20%), so value length survives approximately"; - -/// The default is [`DataType::Unknown`], and it is the only variant that may be one. -/// -/// A default is reached when nothing said what a column holds, so it has to be the -/// answer that claims nothing. Every other variant is a claim, and the ones that would -/// tempt a reader — `String`, `Enum` — are claims in the wrong direction: `Enum` is -/// pass-through, so a defaulted column would be described as deliberately kept -/// unchanged. `Unknown` takes the generic-string transform and states no structure it -/// preserves, which is the honest reading of a column nobody classified. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum DataType { - Email, - Uuid, - Timestamp, - NumericId, - NumericValue, - PostalCode, - Address, - IpAddress, - Url, - MacAddress, - TaxId, - Boolean, - Currency, - Percentage, - CountryCode, - Phone, - FirstName, - LastName, - FullName, - Enum, - String, - #[default] - Unknown, -} - -impl DataType { - pub(crate) fn privacy_finding_kind_and_reason( - self, - ) -> Option<(PrivacyFindingKind, &'static str)> { - match self { - DataType::Email | DataType::Phone => Some(( - PrivacyFindingKind::Contact, - "Column type indicates contact information.", - )), - DataType::FirstName | DataType::LastName | DataType::FullName => Some(( - PrivacyFindingKind::Person, - "Column type indicates person names.", - )), - DataType::Address => Some(( - PrivacyFindingKind::PrivateAddress, - "Column type indicates private address data.", - )), - DataType::PostalCode => Some(( - PrivacyFindingKind::AddressRegion, - "Column type indicates postal address context.", - )), - DataType::TaxId => Some(( - PrivacyFindingKind::GovernmentId, - "Column type indicates government or tax identifier data.", - )), - // Identifier-shaped, but nothing here says *what* it identifies. A - // financial classification needs evidence: the `account_number` header - // kind, or the IBAN or payment-card validator. Absent that, this is a - // surrogate key. - DataType::NumericId => Some(( - PrivacyFindingKind::RecordIdentifier, - "Column type indicates identifier-shaped values; review context.", - )), - DataType::Uuid => Some(( - PrivacyFindingKind::RecordIdentifier, - "Column type indicates persistent identifier-shaped values; what they identify is unknown.", - )), - DataType::IpAddress | DataType::MacAddress => Some(( - PrivacyFindingKind::NetworkOrDeviceId, - "Column type indicates network or device identifiers.", - )), - DataType::Url => Some((PrivacyFindingKind::Url, "Column type indicates URLs.")), - DataType::NumericValue - | DataType::Timestamp - | DataType::Boolean - | DataType::Currency - | DataType::Percentage - | DataType::CountryCode - | DataType::Enum - | DataType::String - | DataType::Unknown => None, - } - } - - pub(crate) fn report_identifier_class(self) -> Option { - match self { - DataType::Email - | DataType::Phone - | DataType::FullName - | DataType::FirstName - | DataType::LastName - | DataType::TaxId - | DataType::Address => Some(ReportIdentifierClass::Direct), - DataType::Uuid - | DataType::NumericId - | DataType::PostalCode - | DataType::IpAddress - | DataType::Url - | DataType::MacAddress - | DataType::Timestamp - | DataType::CountryCode => Some(ReportIdentifierClass::Quasi), - DataType::NumericValue - | DataType::Boolean - | DataType::Currency - | DataType::Percentage - | DataType::Enum - | DataType::String - | DataType::Unknown => None, - } - } - - pub(crate) fn uses_default_pass_through(self) -> bool { - matches!( - self, - DataType::CountryCode - | DataType::Enum - | DataType::Boolean - | DataType::Currency - | DataType::Percentage - ) - } - - /// What a rule-based pseudonym for this type keeps of the original value, or - /// `None` when it keeps nothing worth disclosing. - /// - /// Every transformer in `strategies` is format-preserving on purpose: a - /// pseudonymized timestamp has to parse as a timestamp, a pseudonymized amount has - /// to stay a number, or the output is unusable. This sentence is what the release - /// report states about the part of the source that comes through intact. - /// - /// The sharpest case is `Timestamp`: `strategies::structured::transform_timestamp` - /// splits the value at its ten-byte ISO date prefix and concatenates the remainder - /// verbatim, so `2024-06-15 10:30:45.123450` keeps `10:30:45.123450` exactly. In an - /// event log a microsecond time-of-day is very nearly a primary key, so the - /// "anonymized" file joins straight back to the source on a column the report - /// marked verified. The date moves by at most 365 days, which also means a date of - /// birth keeps its year to within one and the subject keeps their age. - /// - /// Returning a sentence rather than a bool because the sentence is the point: a - /// caller cannot write one wording that is true of "keeps the domain after @" and - /// of "keeps the digit count and sign" at once. - /// - /// No wildcard arm. A data type added to the enum has to be classified here rather - /// than defaulting into the silent half — silence is the defect this closes. - pub(crate) fn pseudonymization_preserves_structure(self) -> Option<&'static str> { - match self { - DataType::Email => Some( - "the local part is replaced but the domain after @ is kept verbatim, so recipients of a rare or personal domain stay identifiable", - ), - DataType::Timestamp => Some( - "the time of day is kept exactly — including sub-second digits, which are close to unique per event — and only the date moves, by at most a year, so an age or a year of birth survives", - ), - DataType::Phone => Some( - "only the digits are redrawn: the digit count, country prefix punctuation and separator layout are kept", - ), - DataType::NumericId => Some( - "the digit count and any leading zeros are kept, so the magnitude of the identifier is preserved", - ), - DataType::NumericValue => Some( - "the sign, digit count and number of decimal places are kept, so the magnitude of the value is preserved", - ), - DataType::FirstName | DataType::LastName | DataType::FullName => Some( - "replacements are drawn from a fixed name pool and the number of name parts is kept", - ), - // Generic-string pseudonymization draws a random value of 80–120% of the - // original's length, so the length survives approximately. Everything that - // is not handled by a transformer of its own lands here. - DataType::Address - | DataType::PostalCode - | DataType::IpAddress - | DataType::Url - | DataType::MacAddress - | DataType::TaxId - | DataType::String - | DataType::Unknown => Some(GENERIC_STRING_STRUCTURE_DISCLOSURE), - // A UUID is machine-generated and carries no structure about its subject; - // the transform keeps only the UUID format and the original's letter case, - // neither of which narrows anyone down. - DataType::Uuid => None, - // Returned unchanged under Auto and Pseudonymize, so there is no transform - // to describe — `uses_default_pass_through` is what the report says about - // these, and it says it plainly. Under a rejected Local AI candidate they - // take the generic-string path instead; the Local AI column report states - // that separately, because it is a property of the strategy rather than of - // the type. - DataType::Enum - | DataType::CountryCode - | DataType::Boolean - | DataType::Currency - | DataType::Percentage => None, - } - } - - pub(crate) fn transforms_generated_quick_value(self) -> bool { - matches!( - self, - DataType::Email - | DataType::Uuid - | DataType::Timestamp - | DataType::NumericId - | DataType::NumericValue - | DataType::Phone - | DataType::FirstName - | DataType::LastName - | DataType::FullName - | DataType::String - | DataType::Unknown - ) - } - - pub(crate) fn redaction_changes_structured_scalar_type(self) -> bool { - matches!( - self, - DataType::NumericId - | DataType::NumericValue - | DataType::Boolean - | DataType::Currency - | DataType::Percentage - ) - } - - pub(crate) fn redaction_placeholder(self) -> Option { - match self { - DataType::Email => Some(RedactionPlaceholder::Email), - DataType::Phone => Some(RedactionPlaceholder::Phone), - DataType::FirstName | DataType::LastName | DataType::FullName => { - Some(RedactionPlaceholder::Person) - } - DataType::Address => Some(RedactionPlaceholder::Address), - DataType::TaxId => Some(RedactionPlaceholder::GovernmentId), - DataType::Url => Some(RedactionPlaceholder::Url), - DataType::IpAddress | DataType::MacAddress => Some(RedactionPlaceholder::NetworkId), - DataType::String - | DataType::Unknown - | DataType::Enum - | DataType::Uuid - | DataType::Timestamp - | DataType::NumericId - | DataType::PostalCode - | DataType::NumericValue - | DataType::Boolean - | DataType::Currency - | DataType::Percentage - | DataType::CountryCode => None, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ReportIdentifierClass { - Direct, - Quasi, -} - -/// Placeholders a column's *detected type* can justify on its own. -/// -/// Deliberately has no account variant. "This is a bank account" is a claim about -/// what a value means, not about its shape, so it can only come from evidence — the -/// IBAN or card validator, or an `account_number` header — which reaches -/// `[ACCOUNT_ID]` through `placeholder_from_evidence` instead. A column of plain -/// integers gets a non-linkable placeholder derived from its column header. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum RedactionPlaceholder { - Email, - Phone, - Person, - Address, - GovernmentId, - Url, - NetworkId, -} - -/// The default is [`Confidence::Low`], the only reading that asserts nothing. -/// -/// `is_actionable` is false at Low, so a detection nobody measured cannot act as -/// evidence. Defaulting to High or Medium would let an unmeasured column carry the -/// weight of a measured one. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum Confidence { - High, - Medium, - #[default] - Low, -} - -/// How much a finding exposes, most severe first. -/// -/// Declaration order is load-bearing: `Ord` is derived, so `High < Medium < Low` and sorting -/// ascending puts the most severe finding first. The privacy report relies on that when it -/// breaks ties between findings — a report that shows a Medium above a High has under-sold -/// what it found, which is the one direction these figures may not be wrong in. -/// -/// The default is [`PiiRisk::High`], deliberately the most severe rather than the most -/// common. A default stands in for "nobody assessed this column", and the only reading -/// of that which cannot make a file look safer than it is, is the worst case. `Low` -/// would silently un-flag a column: `is_elevated` is what auto-selects a column, -/// defaults it to Redact, and names it when it is released unchanged, and a defaulted -/// `Low` turns all three off without anything having decided so. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum PiiRisk { - #[default] - High, - Medium, - Low, -} - -impl PiiRisk { - /// Whether this risk is one the app acts on: auto-selects the column, defaults it - /// to Redact, and names it when it is released unchanged. - /// - /// Named once because it is the app's privacy threshold rather than a comparison. - /// Spelled out at each site, the five copies could be changed apart, and a site - /// left behind would go on treating a Medium-risk column as ordinary — selecting - /// it out of the run, or leaving it out of the report that says what the released - /// file still exposes. - pub(crate) fn is_elevated(self) -> bool { - match self { - Self::High | Self::Medium => true, - Self::Low => false, - } - } -} - -/// The categories of privacy evidence the report can attribute to a column. -/// -/// Declaration order is load-bearing, but it is the *last* word rather than the first: the -/// privacy report breaks a tie on score and match count by `PiiRisk` first, so a High finding -/// always leads a Medium one, and only then by kind, which on a derived enum is declaration -/// order. Reordering these variants therefore changes which of two equally-risky findings a -/// reader sees first, so treat the order as user-facing output rather than as a free choice. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum PrivacyFindingKind { - Person, - Contact, - PrivateAddress, - /// An address component that narrows someone down to an area rather than to a - /// doorstep: a postal code on its own. - /// - /// Separate from [`PrivacyFindingKind::PrivateAddress`] because a street address - /// locates a person and a postal code locates a neighbourhood. Both matter, but - /// only one is a direct identifier, and reporting a zip column as a private - /// address overstated what the file actually contains. - AddressRegion, - PrivateDate, - AccountOrFinancialId, - /// A key that identifies a row without being sensitive in itself: an order - /// number, a record id, a customer sequence number. - /// - /// Distinct from [`PrivacyFindingKind::AccountOrFinancialId`], which is for - /// bank accounts, cards and IBANs. Both are identifiers, but only one exposes - /// a payment instrument, and collapsing them made every column of order - /// numbers report as financial data. A surrogate key still re-identifies a - /// row, so it is Medium rather than Low. - RecordIdentifier, - GovernmentId, - CredentialOrSecret, - NetworkOrDeviceId, - Url, - MixedSensitiveText, -} - -/// The default is [`EmptyFormat::EmptyString`], which is also what -/// [`crate::detection::detect_empty_format`] answers for a column in which nothing -/// null-shaped was seen. It carries no privacy claim either way — it decides how a blank -/// cell is written, not what survives in it. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum EmptyFormat { - #[default] - EmptyString, - Null, - Mixed, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DetectionResult { - pub data_type: DataType, - pub confidence: Confidence, - pub sample_matches: usize, - pub total_samples: usize, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub trace: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DetectionTrace { - pub summary: String, - pub selected_reason: String, - pub total_non_empty: usize, - pub candidates: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DetectionTraceItem { - pub data_type: DataType, - pub reason: String, - pub match_count: usize, - pub total_considered: usize, - pub confidence: Confidence, - pub accepted: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PrivacyFinding { - pub kind: PrivacyFindingKind, - pub data_type: DataType, - pub row_index: usize, - pub start: usize, - pub end: usize, - pub match_value: String, - pub sample_value: String, - pub confidence: Confidence, - pub score: u8, - pub detector: String, - pub reason: String, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PrivacyEvidenceSummary { - pub kind: PrivacyFindingKind, - pub data_type: DataType, - pub confidence: Confidence, - pub match_count: usize, - pub sample_count: usize, - pub score: u8, - #[serde(default)] - pub detector: String, - #[serde(default)] - pub reason: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub detectors: Vec, -} - -impl PrivacyEvidenceSummary { - /// Whether the app may act on this finding. - /// - /// A Low-confidence finding is a shape that resembles the thing more often than - /// it is the thing — `pattern:phone-digits` fires on any bare digit run, and most - /// bare digit runs are order numbers. Such a finding is worth *recording*, so a - /// reviewer can see it and so free text still gets redacted span by span, but it - /// is not worth *asserting*: it may not raise a column's risk and it may not put - /// a specific type's name in the output. - /// - /// This lives on the summary because two modules decide it and they have to agree. - /// `analyze_column_privacy` folds risk over the evidence; `placeholder_from_evidence` - /// names the redaction placeholder from the same list. When only the risk fold - /// filtered, a column the risk model had explicitly declined to trust still - /// redacted to `[PHONE]` — the output file asserting a phone number was there on - /// the evidence the app had just rejected. One predicate, so the next change to - /// the threshold cannot reach one consumer and miss the other. - pub(crate) fn is_actionable(&self) -> bool { - self.confidence != Confidence::Low - } -} - -/// `Default` is for constructing one, not for accepting one. -/// -/// Fifteen fields of which a given caller varies two or three is what -/// `crate::test_support::column` exists to absorb, and `Default` is what lets it. It -/// widens nothing on the deserialization side: the serde contract is unchanged, so a -/// field this struct requires on the wire is still required — only the fields already -/// carrying `#[serde(default)]`, each with its own reason stated below, may be absent. -/// -/// Every defaulted field is the least-privileged reading of "nobody decided this": -/// `PiiRisk::High`, `DataType::Unknown`, `Confidence::Low`, -/// `AnonymizationStrategy::PassThrough`, `is_selected: false`, and a zeroed -/// distribution. A default may leave a column looking more exposed than it is; it may -/// never leave one looking safer. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ColumnReviewReason { - DetectorsDisagree, - LocalNerLowConfidence, - AmbiguousContext, - InsufficientSample, -} - -/// How conclusive the analysis of a column was, independently of its privacy risk. -/// -/// Risk answers "what would this expose if the finding is correct?" while this -/// disposition answers "did the available evidence justify a conclusion?" Keeping -/// them separate prevents an unrecognised string column from being presented as -/// tested-benign merely because no detector raised its risk. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum EvidenceDisposition { - DetectedSensitive, - TestedBenign, - #[default] - Uncertain, - AnalysisIncomplete, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct FormatEvidence { - pub data_type: DataType, - pub confidence: Confidence, - pub match_count: usize, - pub sample_count: usize, - pub basis: FormatEvidenceBasis, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub detectors: Vec, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum FormatEvidenceBasis { - DetectionSample, - UserOverride, - #[default] - RetainedPreviewValues, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum SemanticSpecificity { - Specific, - #[default] - Generic, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum SemanticStatus { - Resolved, - #[default] - Uncertain, - Conflicting, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SemanticDecision { - pub kind: String, - pub confidence: Confidence, - pub specificity: SemanticSpecificity, - pub status: SemanticStatus, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub supporting_evidence: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub conflicting_evidence: Vec, - pub reason: String, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PrivacyDecision { - pub risk: PiiRisk, - pub recommended_strategy: AnonymizationStrategy, - pub auto_selected: bool, - pub reason: String, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum RedactionPlaceholderSource { - Typed, - ColumnHeader, - #[default] - Generic, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct RedactionDecision { - pub placeholder: String, - pub source: RedactionPlaceholderSource, - pub is_typed: bool, - pub preserves_equality: bool, - pub reason: String, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ColumnEvidenceProfile { - pub format_evidence: FormatEvidence, - pub semantic_decision: SemanticDecision, - pub privacy_decision: PrivacyDecision, - pub redaction_decision: RedactionDecision, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ColumnMetadata { - pub name: String, - /// Whether some other column's header reduces to the same placeholder label as - /// this one's. - /// - /// A duplicate header is legal in CSV, and on its own it is only untidy — the - /// column table shows two rows with one name and the index tells them apart. - /// It stops being cosmetic once a label reaches a *cell*: labels number each - /// distinct value per column, so two columns named `notes` would both start at - /// `[NOTES_1]`, and a reader comparing those cells would conclude the source - /// values were equal when nothing of the kind was measured. Columns marked here - /// fold their position into the label instead. - /// - /// Measured on the label rather than the raw header, because that is where the - /// collision happens: `Notes`, `notes` and `notes!` all reduce to `NOTES`. - /// - /// Defaulted rather than optional so a caller that never compared headers gets - /// the unqualified label, which is the behaviour for a column whose header is - /// unique — the common case, and the readable one. - #[serde(default)] - pub header_label_is_ambiguous: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub source_path: Option, - pub index: usize, - pub detected_type: DataType, - pub confidence: Confidence, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detection_trace: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub privacy_findings: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub privacy_evidence: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub review_reasons: Vec, - #[serde(default)] - pub evidence_disposition: EvidenceDisposition, - #[serde(default)] - pub evidence_profile: ColumnEvidenceProfile, - pub pii_risk: PiiRisk, - pub sample_values: Vec, - /// The value distribution of the *detection sample*, not of the whole input. - /// - /// This is what lets the column table warn before a run rather than after one, - /// which is the only point at which the user can still change strategy. A - /// sampled distinct count is a lower bound on the true one, which is the right - /// direction to be wrong in: a high-cardinality column cannot be made to look - /// low-cardinality by sampling, while a genuinely low-cardinality one saturates - /// within a few dozen rows. - /// - /// Defaulted rather than optional so a caller that never measured produces a - /// zeroed distribution, which fails the sample-size floor and therefore cannot - /// raise a warning it has no evidence for. - #[serde(default)] - pub sample_value_distribution: ColumnValueDistribution, - pub empty_format: EmptyFormat, - pub is_selected: bool, - pub strategy: AnonymizationStrategy, -} - -/// The default is [`AnonymizationStrategy::PassThrough`], which is the variant that -/// claims the least rather than the one a user most often ends on. -/// -/// A default stands in for "no strategy was chosen", and the report has to read that -/// as "nothing was done to this column". `PassThrough` does exactly that: the release -/// report calls it Review and says the values are kept unchanged, and -/// `uniqueness::LinkableProjection::for_column` treats it as `WholeValue`, the most -/// linkable projection there is. Every other variant would be a claim in the opposite -/// direction — `Redact` is reported Verified, so a defaulted column would carry a green -/// tick nobody earned for it. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum AnonymizationStrategy { - Auto, - Pseudonymize, - Tokenize, - LocalAi, - Mask, - Label, - Redact, - #[default] - PassThrough, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ColumnControl { - pub column_index: usize, - pub type_override: Option, - pub strategy: AnonymizationStrategy, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ParsedSample { - pub headers: Vec, - pub rows: Vec>, - /// Data rows read while building the sample. This is the input's full - /// data-row count only when `scanned_entire_input` is true; it is always - /// >= `rows.len()`, because a spread sample thins what it keeps. - pub data_rows_scanned: usize, - /// Whether every data row was read. False only for a head window that hit - /// its cap; detection samples always scan the whole input. - pub scanned_entire_input: bool, -} - -/// What the two figures in a detection-coverage disclosure count. -/// -/// The unit is carried rather than assumed because "rows" is only true for the two -/// tabular entry points. A field-based paste — JSON, YAML, XML, or free text scanned -/// for privacy spans — has no rows to sample: `direct_input::shared::detection_coverage` -/// counts the values of the busiest field, and `PasteAnalyzeData::row_count` is derived -/// separately and can disagree outright. A pasted `{"users": [500 objects]}` shows one -/// row in the UI, and free text shows a match count that is neither its line count nor -/// its row count, so a disclosure hard-coded to "rows" states a figure the user cannot -/// find anywhere on screen and cannot check. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum DetectionCoverageUnit { - Rows, - Values, -} - -impl DetectionCoverageUnit { - /// The noun a disclosure sentence uses for these figures. - pub(crate) fn plural_noun(self) -> &'static str { - match self { - Self::Rows => "rows", - Self::Values => "values", - } - } -} - -/// How much of an input detection actually classified. -/// -/// Detection votes on a bounded sample, so a value occurring in few rows can be -/// missed — and a column whose sensitive values were all missed is never -/// auto-selected, so it is written unchanged. That is a deliberate trade for -/// bounded memory on inputs of any size, but it is only an honest one if the user -/// is told the verdict rests on a sample. This carries the figures needed to say so -/// and the unit they are counted in. -/// -/// Not a DTO. It is `pub(crate)` on purpose: [`Self::new`] is the only place the -/// `examined <= total` invariant is established, and a `Deserialize` impl would let -/// a wire value walk straight past it and report a sample larger than the input it -/// was drawn from. What crosses the IPC boundary is [`DetectionCoverageSummary`], -/// a flat snapshot taken after clamping, plus the report notes and preflight review -/// item built from this. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct DetectionCoverage { - /// Rows or values kept for classification. - examined: usize, - /// Rows or values the input holds. - total: usize, - unit: DetectionCoverageUnit, -} - -impl DetectionCoverage { - /// Coverage of the sample a detection pass actually kept. - /// - /// Read off the sample rather than recomputed from the requested row count, - /// because the request is a ceiling and the sample is what happened: a file with - /// fewer rows than the request is fully covered, and asking the request would - /// claim otherwise. - pub(crate) fn from_detection_sample(sample: &ParsedSample) -> Self { - Self::rows(sample.rows.len(), sample.data_rows_scanned) - } - - /// Coverage for an input that has nothing to sample. - /// - /// Both figures are zero, which is not a missing figure but the only truthful - /// pair here: quick-generate has no source input, so no row went unexamined and - /// none was examined either. Only [`Self::is_partial`] is consulted for such an - /// input, and it answers false — the disclosure stays silent, which is right, - /// because there is no sampling to disclose. Nothing may print - /// [`Self::examined`] and [`Self::total`] without gating on - /// [`Self::is_partial`] first, or this constructor renders as "0 of 0". - pub(crate) fn complete() -> Self { - Self::rows(0, 0) - } - - /// Coverage counted in data rows: the two tabular entry points, CSV file and - /// pasted CSV text. - pub(crate) fn rows(examined: usize, total: usize) -> Self { - Self::new(examined, total, DetectionCoverageUnit::Rows) - } - - /// Coverage counted in field values: JSON, YAML, XML and free-text pastes, which - /// have fields rather than rows. See [`DetectionCoverageUnit`]. - pub(crate) fn values(examined: usize, total: usize) -> Self { - Self::new(examined, total, DetectionCoverageUnit::Values) - } - - fn new(examined: usize, total: usize, unit: DetectionCoverageUnit) -> Self { - Self { - examined: examined.min(total), - total, - unit, - } - } - - /// Whether some of the input went unclassified. - pub(crate) fn is_partial(self) -> bool { - self.examined < self.total - } - - pub(crate) fn examined(self) -> usize { - self.examined - } - - pub(crate) fn total(self) -> usize { - self.total - } - - pub(crate) fn unit(self) -> DetectionCoverageUnit { - self.unit - } - - /// The IPC-facing snapshot of this coverage. - pub(crate) fn summary(self) -> DetectionCoverageSummary { - DetectionCoverageSummary { - examined: self.examined, - total: self.total, - unit: self.unit, - is_partial: self.is_partial(), - } - } -} - -/// What detection classified, as the paste analyze result reports it. -/// -/// This exists so a paste user learns that detection sampled before choosing -/// columns, not after the output already exists. The file workflow gets the same -/// disclosure from preflight; the paste workflow has no preflight, so until this -/// crossed the boundary the only place it appeared was the post-transform privacy -/// report — advice ("raise Sample rows") that by then costs a whole second run. -/// -/// Counts only. No value, column name or excerpt goes on the wire here, so the -/// disclosure cannot itself leak what detection missed. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DetectionCoverageSummary { - pub examined: usize, - pub total: usize, - pub unit: DetectionCoverageUnit, - /// `examined < total`, decided here rather than in the client. - /// - /// The comparison is trivial, which is exactly why it would drift: a client that - /// re-derives it is free to write `<=` or to compare against a row count from a - /// different field, and either mistake silences the warning rather than - /// producing a visible error. - pub is_partial: bool, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct ProcessOptions<'a> { - pub smart_replacements: Option<&'a crate::smart::SmartReplacementMap>, - /// Validated run-only secret. The key type cannot be serialized. - pub tokenization_key: Option<&'a crate::strategies::TokenizationKey>, - /// Mapping entries this run may hold before it refuses to continue, or `None` for - /// `TransformState::MAPPING_ENTRY_CEILING`. - /// - /// A run option rather than a constant read at the point of use because the - /// ceiling is a resource limit, and a resource limit that cannot be set to a - /// reachable value cannot be tested: the real one stands for about 5 GB of - /// mapping, so the only way to show that the run loop *consults* it — and that - /// refusing leaves no partial output — is to hand the loop a smaller one. - pub mapping_entry_ceiling: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ProcessProgress { - pub rows_processed: usize, -} - -pub struct ProcessControl<'a> { - pub on_progress: Option<&'a mut dyn FnMut(ProcessProgress)>, - pub should_cancel: Option<&'a dyn Fn() -> bool>, -} - -impl ProcessControl<'_> { - pub fn none() -> Self { - Self { - on_progress: None, - should_cancel: None, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProcessResult { - pub row_count: usize, - pub output_path: PathBuf, - pub duration_ms: u128, - pub transform_report: TransformReport, -} - -/// What one column's consistent pseudonyms reveal about the values behind them. -/// -/// A consistent pseudonym preserves equality, which is why the strategies that -/// produce one keep a dataset joinable. The cost is that equality also preserves -/// the *shape* of the value distribution, and the shape is enough to work against -/// the mapping: -/// -/// - Few `distinct_values` over many `total_values` means the mapping can be -/// relabelled by frequency, by anyone who knows how the real field is -/// distributed. The tokens stay opaque; the histogram does not. -/// - A `singleton_values` entry is a pseudonym covering exactly one row, which -/// singles that record out however unguessable the token looks. -/// - `max_value_occurrences` is the most common value's row count, and it is the -/// sharpest of the four for frequency inversion — the dominant value is the -/// attacker's easiest anchor, and inverting that one pseudonym recovers -/// `max_value_occurrences / total_values` of the column in a single step. Acted on -/// as a *share* rather than a count, because a count means something different in a -/// 200-row file than in a 5-million-row one. The threshold it is compared against is -/// `MIN_INVERTIBLE_DOMINANT_SHARE`, which is private, so this names it rather than -/// linking to it. -/// -/// Reported per column because the risk is per column: one low-cardinality column -/// is not made safer by a high-cardinality neighbour. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ColumnValueDistribution { - pub column_index: usize, - pub distinct_values: usize, - pub total_values: usize, - pub singleton_values: usize, - /// Values seen exactly twice. Carried for the same reason as the singleton - /// count: the two together are what let a *sampled* distribution say anything - /// about the input it was drawn from — see `estimated_distinct_values`, which is - /// private, so this names it rather than linking to it. - #[serde(default)] - pub doubleton_values: usize, - pub max_value_occurrences: usize, -} - -/// Minimum values before the cardinality test means anything. -/// -/// A floor is not optional. `distinct_values <= total_values`, so any -/// `distinct_values < K` test is *vacuously true* whenever `total_values < K` — on a -/// 5-row fixture every column looks low-cardinality, including the unique ID -/// columns. Measured on `tests/fixtures/large.csv` truncated to varying lengths: at -/// 5 rows all 7 columns trip the test, including `id`, `email` and `user_uuid`; the -/// mis-flags disappear at 10 rows, i.e. at exactly K. -/// -/// 50 rather than the minimum 10 because the margin matters: at 20 rows the -/// highest-cardinality realistic column (`name`) shows 19 distinct values, one step -/// from the boundary, while at 50 rows it shows 42 — a margin of 32. It also -/// silences every short test fixture, which is where all the noise was. -const CARDINALITY_FLOOR: usize = 50; - -/// Distinct values below which a mapping is treated as frequency-invertible. -/// -/// The two genuinely low-cardinality columns in the corpus saturate at 8 (`country`) -/// and 4 (`status`); the next column up saturates at 100 (`name`). The data pins this -/// constant only to the interval (8, 100] — the decade between is unpopulated — so 10 -/// is the conservative end of the measured interval, and it agrees with the common -/// convention that fewer than about ten buckets makes a frequency table trivially -/// labellable. -const MAX_INVERTIBLE_DISTINCT_VALUES: usize = 10; - -/// Distinct-to-total ratio below which a mapping is treated as frequency-invertible, -/// for columns too large for the absolute test to catch. -/// -/// Reads as "each pseudonym covers more than twenty rows on average". It exists for -/// the case the absolute test misses: `large.csv name` has 100 distinct values over -/// 10500 rows, so `distinct < 10` is false while the ratio is 0.0095. -/// -/// 0.05 rather than a looser 0.20 by a subsumption argument rather than measurement. -/// `distinct/total < 0.05` is already implied by `distinct < 10` for any total below -/// 200, so the ratio term is inert on small and medium inputs and the verdict there -/// rests entirely on the stable statistic — the absolute count, which saturates. The -/// ratio only takes over once there are enough rows for it to have converged. A 0.20 -/// threshold would instead activate immediately above the floor and flag, say, 2000 -/// distinct names in a 10500-row file, which is a five-row group. -const MAX_INVERTIBLE_DISTINCT_RATIO: f64 = 0.05; - -/// Sample coverage below which a *sampled* distribution is treated as saying nothing -/// about the input's distinct count, so the ratio test is skipped. Good–Turing coverage, -/// `1 - singletons/values`; the measurements pin it only to the interval (0.40, 0.87] and -/// 0.75 sits inside with margin at both ends. Not measured on real production data, -/// non-Zipf skew, or columns sitting near the gate itself. -/// See docs/calibration.md#min_sample_coverage for the measurements behind this. -const MIN_SAMPLE_COVERAGE: f64 = 0.75; - -/// Share of a column's values carried by its single most common value, at or above which -/// the mapping is treated as frequency-invertible. Catches the shape neither other term -/// sees: thousands of distinct values, one of them covering most of the rows. A share and -/// not a count, so it means the same in a 200-row file and a 5-million-row one. The -/// measurements pin it to [1/3, 0.35]; not measured on real production columns. -/// See docs/calibration.md#min_invertible_dominant_share for the measurements behind this. -const MIN_INVERTIBLE_DOMINANT_SHARE: f64 = 1.0 / 3.0; - -/// Which of the three tests judged a distribution frequency-invertible. -/// -/// Exists so a warning can name the evidence it actually has. The three terms catch -/// genuinely different shapes — a handful of values, one value dominating a diverse -/// column, and many small groups across a large one — and a single wording cannot -/// describe all three without describing at least two of them wrongly. Reporting a -/// column of 101 values where one covers half the rows as holding "only 101 distinct -/// value(s)" would be true, would read as reassuring, and would name a risk the column -/// does not have while staying silent about the one it does. -/// -/// Each variant carries the figure its wording needs, computed at the point the test -/// fires, because [`ColumnValueDistribution::estimated_distinct_values`] and -/// [`ColumnValueDistribution::dominant_value_share`] are private and a message -/// builder outside this module cannot recover them from the public fields. -#[derive(Debug, Clone, Copy, PartialEq)] -pub(crate) enum FrequencyInversionRisk { - /// Few enough distinct values that the whole frequency table is labellable. - FewDistinctValues, - /// One value carries [`MIN_INVERTIBLE_DOMINANT_SHARE`] or more of the column, so - /// inverting that single pseudonym recovers `share` of it. - DominantValue { share: f64 }, - /// Enough rows per distinct value that the groups can be matched by size, even - /// though no single value dominates and there are too many to enumerate. - LargeGroups { estimated_distinct_values: usize }, -} - -impl ColumnValueDistribution { - /// Builds the distribution of `values`, skipping the ones a transform would skip. - /// - /// Uses [`crate::detection::is_empty_value`] and [`crate::smart::value_identity_key`] - /// so that a distribution measured over a detection sample is comparable with one - /// accumulated during a run: the transform returns early on empty values and folds - /// case and padding, so counting either differently here would make the pre-run - /// warning disagree with the post-run report on the same data. - pub(crate) fn from_values(column_index: usize, values: &[String]) -> Self { - let mut occurrences: std::collections::HashMap = - std::collections::HashMap::new(); - let mut total_values = 0usize; - for value in values { - let trimmed = value.trim(); - if crate::detection::is_empty_value(trimmed) { - continue; - } - total_values += 1; - *occurrences - .entry(crate::smart::value_identity_key(trimmed)) - .or_insert(0) += 1; - } - - Self { - column_index, - distinct_values: occurrences.len(), - total_values, - singleton_values: occurrences.values().filter(|count| **count == 1).count(), - doubleton_values: occurrences.values().filter(|count| **count == 2).count(), - max_value_occurrences: occurrences.values().copied().max().unwrap_or(0), - } - } - - /// Whether consistent pseudonyms over this distribution could be relabelled by - /// frequency analysis. - /// - /// Singleton counts deliberately play no part. A singleton rule reads as the - /// record-isolation risk and sounds right, but measured against the corpus it - /// flags every unique-key column — `id`, `email`, `user_uuid`, `created_at` — - /// which are exactly the columns users pseudonymize as a matter of course. That - /// isolation is a property of the input being unique already, not something - /// consistent pseudonymization introduces, and `distinct == total` says it more - /// directly. A warning that fires on every email column is noise. - /// - /// See [`CARDINALITY_FLOOR`], [`MAX_INVERTIBLE_DISTINCT_VALUES`], - /// [`MIN_INVERTIBLE_DOMINANT_SHARE`], [`MAX_INVERTIBLE_DISTINCT_RATIO`] and - /// [`MIN_SAMPLE_COVERAGE`] for what each constant is measured against. - /// - /// For a distribution measured over the whole column. A sampled one has to say - /// how large the column really is — see [`Self::frequency_inversion_risk_in`]. - pub(crate) fn risks_frequency_inversion(&self) -> bool { - self.frequency_inversion_risk().is_some() - } - - /// [`Self::risks_frequency_inversion`], with the reason it answered yes. - pub(crate) fn frequency_inversion_risk(&self) -> Option { - self.frequency_inversion_risk_in(self.total_values) - } - - /// As [`Self::risks_frequency_inversion`], for a distribution measured over a - /// sample of a column that holds `population_values` values in total. - /// - /// The distinction is not pedantic — getting it wrong is what made the pre-run - /// warning miss the case it exists for. The ratio test divides distinct values by - /// the column's size, and a sample's size is the *sample's*, capped at a hundred - /// or so. Measured against that cap, `distinct / total < 0.05` needs fewer than - /// five distinct values in a hundred, which the absolute test has already caught: - /// the ratio term was inert on every sampled distribution while being the only - /// term that catches a column of thirty values in five thousand rows. Such a - /// column drew no warning until after its output had been written. - /// - /// Dividing by the true row count instead is wrong in the other direction, and - /// worse: a hundred sampled values cannot look like a million distinct ones, so a - /// fully unique column would score 0.005 and be flagged as trivially invertible. - /// [`MIN_SAMPLE_COVERAGE`] is the gate that separates the two, and - /// [`Self::estimated_distinct_values`] is what gets compared once it opens. - /// - /// `population_values` is the column's row count, an upper bound on how many - /// values it holds — empty cells are not values. Erring high makes the ratio - /// smaller and the warning likelier, which is the safe direction for a warning - /// the user can dismiss by choosing another strategy. - /// - /// The dominant-value term needs no equivalent adjustment, and that is the reason it - /// sits where it does. It compares two figures the distribution measured *itself* — - /// a share of its own values — so `population_values` never enters. A share is also - /// the one statistic a small sample can estimate here, unlike its distinct count, - /// which is why only the distinct-count term needs a population figure and a gate. - /// See docs/calibration.md#sample-share-vs-distinct-count for the measurements. - /// - /// The order of the three tests is the order of decreasing certainty, and it is - /// also what each variant means: a later variant implies the earlier tests - /// declined. So `LargeGroups` is only ever reported for a column with at least - /// [`MAX_INVERTIBLE_DISTINCT_VALUES`] distinct values and no dominant one, which is - /// what lets its wording talk about average group size without qualification. - pub(crate) fn frequency_inversion_risk_in( - &self, - population_values: usize, - ) -> Option { - if self.total_values < CARDINALITY_FLOOR { - return None; - } - if self.distinct_values < MAX_INVERTIBLE_DISTINCT_VALUES { - return Some(FrequencyInversionRisk::FewDistinctValues); - } - // Deliberately ahead of the coverage gate. The shape this catches — one dominant - // value over a long tail of near-unique ones — is singleton-heavy and therefore - // *low*-coverage, so behind the gate this term would be silent on exactly the - // See docs/calibration.md#min_sample_coverage for the figures. - let share = self.dominant_value_share(); - if share >= MIN_INVERTIBLE_DOMINANT_SHARE { - return Some(FrequencyInversionRisk::DominantValue { share }); - } - if self.sample_coverage() < MIN_SAMPLE_COVERAGE { - return None; - } - let estimated_distinct_values = self.estimated_distinct_values(); - if (estimated_distinct_values as f64) / (population_values.max(1) as f64) - < MAX_INVERTIBLE_DISTINCT_RATIO - { - return Some(FrequencyInversionRisk::LargeGroups { - estimated_distinct_values, - }); - } - None - } - - /// Share of this distribution's values carried by its single most common value. - /// - /// The fraction of the column that one inverted pseudonym recovers, so it is also - /// the size of the leak the warning is reporting. `0.0` for a distribution that - /// measured nothing, and `1 / total_values` for a fully unique column — which is - /// below [`MIN_INVERTIBLE_DOMINANT_SHARE`] for any column past - /// [`CARDINALITY_FLOOR`], so a unique column cannot trip the term that reads this. - fn dominant_value_share(&self) -> f64 { - if self.total_values == 0 { - return 0.0; - } - (self.max_value_occurrences as f64) / (self.total_values as f64) - } - - /// Good–Turing sample coverage: the estimated share of this column's values that - /// belong to a group the distribution has already seen. - /// - /// Exactly `1.0` for a distribution measured over a whole column with no - /// singletons, and `0.0` for one where every value was seen once — which is both - /// a unique column measured exactly and a sample that has learned nothing. - /// [`Self::frequency_inversion_risk_in`] is what distinguishes those, by only - /// consulting coverage once the absolute test has declined. - fn sample_coverage(&self) -> f64 { - if self.total_values == 0 { - return 0.0; - } - 1.0 - (self.singleton_values as f64) / (self.total_values as f64) - } - - /// Chao1: the distinct count this distribution implies for the whole column, - /// including groups it did not see. - /// - /// `distinct + f1² / 2·f2` over singletons and doubletons — the standard - /// lower-bound estimator, on the reasoning that how many groups you saw *once* - /// tells you how many you missed entirely. Returns the observed count unchanged - /// for a distribution with no singletons, which is what a fully measured column - /// with every value repeated looks like. - /// - /// A lower bound, so it under-estimates rather than over-estimates a diverse - /// column, which is the wrong direction for a warning — hence - /// [`MIN_SAMPLE_COVERAGE`] refusing the comparison before this is consulted. - /// `pub(crate)` because the preflight memory projection needs the same estimate - /// and a second copy of a statistical estimator is a copy that drifts. - pub(crate) fn estimated_distinct_values(&self) -> usize { - if self.doubleton_values > 0 { - return self.distinct_values - + self.singleton_values.saturating_pow(2) / (2 * self.doubleton_values); - } - // The bias-corrected form, for when nothing was seen exactly twice and the - // ratio above would divide by zero. - self.distinct_values - + self - .singleton_values - .saturating_mul(self.singleton_values.saturating_sub(1)) - / 2 - } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TransformReport { - pub unique_pseudonym_values: usize, - pub reused_pseudonym_values: usize, - pub collisions_avoided: usize, - pub exhausted_pseudonym_pools: usize, - pub opaque_token_values: usize, - pub keyed_token_values: usize, - pub keyed_token_columns: Vec, - pub smart_replacement_requests: usize, - pub smart_replacement_values: usize, - pub smart_replacement_rejections: usize, - pub smart_replacement_rejection_reasons: Vec, - pub smart_replacement_fallbacks: usize, - pub shape_fallback_values: usize, - /// Selected values that a strategy claiming to transform returned byte-for-byte - /// unchanged after trimming. This is the exact residual-leak guard; it is not a - /// heuristic scan for merely realistic-looking replacement data. - pub unchanged_sensitive_values: usize, - pub unchanged_sensitive_columns: Vec, - /// Unique selected high/medium-risk source values fingerprinted for the broad - /// residual audit, and released cell fingerprints compared against them. - pub residual_audit_source_values: usize, - pub residual_audit_output_values: usize, - pub residual_audit_matches: usize, - /// True when either fingerprint set reached its independent memory ceiling. - pub residual_audit_incomplete: bool, - pub column_value_distributions: Vec, - pub row_uniqueness: Option, -} - -/// How exposed the released rows are once every column is read together. -/// -/// The rest of this crate's privacy figures are per column, so a file can be reported as -/// having no unselected high or medium risk column while postcode, birth date and job -/// title jointly single out a third of its rows. This is the figure that says so. -/// -/// Measured over the columns an outsider could match against data they already hold — -/// see `crate::uniqueness::LinkableProjection` for how that subset is decided and what it -/// deliberately leaves out. Every count here is over that subset except -/// `distinct_rows_all_columns`. -/// -/// Absent rather than zeroed where there are no rows: unstructured text and single pasted -/// values never populate it, and a summary claiming zero unique rows would read as a clean -/// result from a check that never ran. -/// -/// A DTO, like [`DetectionCoverageSummary`] and unlike the private `DetectionCoverage`: a flat -/// snapshot with no invariant of its own to protect, so `Deserialize` costs nothing here. -/// The relationships between these figures — that the counts are over the columns named, -/// that a stopped measurement zeroes the rest — are established in -/// `crate::uniqueness::RowUniquenessTracker::summary`, which is the only thing that builds -/// one outside tests. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct RowUniquenessSummary { - /// Data rows hashed. Blank rows are written through untransformed and are not counted. - pub rows_measured: usize, - /// The columns the measure read, each with what an outsider could match it on, in - /// column order. Empty means nothing released is matchable — a statement about how the - /// columns were transformed, not a finding that the data is anonymous. - /// - /// Only columns that actually yielded something are listed. A column whose projection - /// came back empty on every row contributed nothing to any class and is named nowhere, - /// so no claim here can rest on a column that in fact carried no signal. - pub matched_columns: Vec, - /// Distinct equivalence classes over the subset. - pub distinct_classes: usize, - /// Rows alone in their class: someone holding those columns for a person finds - /// exactly their row. - pub unique_rows: usize, - /// The k-anonymity floor — the smallest class present. One freak record sets it, which - /// is why `fifth_percentile_class_size` is reported beside it. - pub smallest_class: usize, - /// The class size at or below which the most exposed 5% of rows sit. - pub fifth_percentile_class_size: usize, - /// Distinct rows over *every* released column, subset rule not applied. Answers the - /// separate and simpler question of whether the file could be shuffled or aggregated, - /// and acts as a check on the subset rule: a wide gap between this and - /// `distinct_classes` means the rule is doing a lot of work and deserves a look. - /// - /// `None` when this histogram alone outgrew what the check keeps. It fills faster than - /// the other one by construction — whole rows against projections of a subset of them — - /// so it is suppressed on its own rather than taking the joint measure down with it. - pub distinct_rows_all_columns: Option, - /// The *joint* measurement stopped early because the file held more classes than the - /// check keeps. Every count above is then a lower bound, and no verified claim may rest - /// on them. Set only by the linkable histogram: `distinct_rows_all_columns` going - /// absent does not make the joint figures incomplete. - pub measurement_incomplete: bool, - /// What `unique_rows` would have been with each matched column dropped, ascending by - /// that count, then by column index. - /// - /// The only figure here a reader can act on. `unique_rows` says how exposed the file is; - /// this says which column to change to fix it, which is the difference between a report - /// and an alarm. - /// - /// Empty when `drop_attribution_incomplete` is set, and also when the file has no matched - /// column to drop. The two are told apart by that flag rather than by the emptiness of - /// this list, because "we did not measure" and "there is nothing to drop" are opposite - /// findings that would otherwise look identical. - pub drop_column_effects: Vec, - /// The attribution was not run, or was stopped, so `drop_column_effects` is empty for a - /// reason other than there being nothing to say. - /// - /// Set when the joint measurement itself is incomplete (there is no baseline to compare - /// against), when the file has more columns than the attribution tracks, or when the - /// leave-one-out histograms outgrew their shared budget. Reported rather than hidden: a - /// reader who is told nothing about which column to drop should know whether that is - /// because no column would help or because nobody looked. - pub drop_attribution_incomplete: bool, -} - -/// What dropping one column would do to the count of rows that stand alone. -/// -/// Exact, not an estimate: the count is read off a second equivalence-class histogram built -/// over the same rows with this column's contribution removed, in the same single pass. That -/// matters because the intuitive estimate is wrong in both directions — dropping a column -/// whose projection is nearly constant changes almost nothing however revealing the column -/// looks, and dropping one of two correlated columns can change almost nothing either. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DropColumnEffect { - /// `ColumnMetadata::index`, as everywhere else. - pub column_index: usize, - /// Rows that would still be alone in their class with this column dropped, every other - /// column unchanged. Compare against `RowUniquenessSummary::unique_rows`; it can never - /// be larger, since removing a column only ever merges classes. - pub unique_rows_without: usize, -} - -/// One column the joint measure read, and what it was matched on. -/// -/// The pairing is the point, and it must not be split back into two lists of column -/// indices — value-carrying and format-only. Two categories cannot express three kinds of -/// contribution, and the missing one is the partial match, the most common kind on a -/// pseudonymized file: a report built from two lists tells the reader that rows "share their -/// combination of birth_date, email" when what they share is a decade and a domain. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct MatchedColumn { - /// `ColumnMetadata::index`, which is how every other report structure names a column. - pub column_index: usize, - pub matched_on: MatchedPart, - /// Whether every measured row actually carried `matched_on`, or only some of them did. - /// - /// `matched_on` is decided once per column, from the column's strategy and detected type - /// alone — no cell value can change it. The values can still disagree with it: a cell - /// that does not fit its column's detected shape is pseudonymized generically, and the - /// projection returns nothing for that row. So a `Timestamp` column where one value in a - /// hundred parses is `DateDecadeAndTime`, and the finding said the rows "share the decade - /// and time of birth_date" of ninety-nine rows that carry no decade. - /// - /// A `bool` and not a count, because the report needs to know only whether to qualify the - /// phrase. Quoting "matched on 1 of 100 rows" would invite the reader to weigh a number - /// that is not the one that matters: those rows were counted as sharing nothing there, - /// so the arithmetic is already right and only the wording was over-claiming. - pub matched_every_row: bool, -} - -/// What survived a column that an outsider holding the original could match against. -/// -/// Written so each variant completes the sentence "rows share **…** with each other", which -/// is what forces the distinction: `WholeValue` licenses naming the column bare, and -/// nothing else does. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum MatchedPart { - /// The released cell is the original, or a skeleton of it anyone can derive. The column - /// may be named on its own. - WholeValue, - /// Everything from the last `@`: the employer, not the person. - EmailDomain, - /// The decade of the released date, and the time of day exactly. Approximate on the date - /// half by construction — see `crate::uniqueness::LinkableProjection` — and named as an - /// approximation so a reader is not told their rows share a birth date when they share a - /// decade. - DateDecadeAndTime, - /// No part of the value, only a format property: a digit count, a separator layout, a - /// number of name parts, a mask's word and letter counts. Counted like any other, - /// because a joint measure is where individually weak signals combine, but never named - /// as though it were the value. - SurvivingFormat, - /// Not the cell at all — only whether it was blank, and with which blank token. - /// - /// A cell the engine reads as empty is written through verbatim before any strategy - /// runs, so even a redacted column publishes its missingness pattern, and someone - /// holding the original record knows which of its fields were blank. Named apart because - /// "the blank-cell pattern of address" and "address" are wildly different claims, and - /// because the remedy differs too: no strategy fixes this one. - BlankPattern, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TransformContext<'a> { - pub column_name: &'a str, - pub column_index: usize, - pub row_index: usize, - pub empty_format: EmptyFormat, -} - -impl<'a> TransformContext<'a> { - /// The context for one cell of `column`. - /// - /// Every caller wants exactly this: the column's own name, index and empty - /// format, plus where the value sits. Assembled by hand at each site, the fields - /// are three same-shaped values a mistake can silently swap — a context built - /// with another column's index makes the transform key its mapping under the - /// wrong column, so two columns share replacements and a value redacted in one - /// reappears in the other. - pub fn for_column(column: &'a ColumnMetadata, row_index: usize) -> Self { - Self { - column_name: &column.name, - column_index: column.index, - row_index, - empty_format: column.empty_format, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct HeadersData { - pub file_path: PathBuf, - pub row_count: usize, - pub row_count_is_complete: bool, - pub default_output_path: PathBuf, - #[serde(default)] - pub detection_run_summary: DetectionRunSummary, - pub columns: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum PasteDataFormat { - Auto, - Csv, - Json, - Xml, - Yaml, - PlainText, - Logs, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PasteAnalyzeParams { - pub content: String, - pub format: PasteDataFormat, - pub sample_row_count: usize, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PasteAnalyzeData { - pub format: PasteDataFormat, - pub row_count: usize, - pub row_count_is_complete: bool, - /// How much of the paste the detected types actually rest on. - /// - /// Carried by the analyze result rather than only by the privacy report so the - /// column table can caveat itself *before* the user selects columns and - /// transforms. See [`DetectionCoverageSummary`]. - pub detection_coverage: DetectionCoverageSummary, - #[serde(default)] - pub detection_run_summary: DetectionRunSummary, - pub columns: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub prepared_analysis: Option, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum LocalNerRunStatus { - #[default] - Disabled, - Completed, - Unavailable, - Failed, - Incomplete, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum DeterministicDetectionStatus { - #[default] - Completed, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum DetectionReviewReason { - DetectorFailed, - CandidateRejected, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct DetectionRunSummary { - pub deterministic: DeterministicDetectionStatus, - pub local_ner: LocalNerRunStatus, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detector_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub model_version: Option, - #[serde(default)] - pub examined_cells: usize, - #[serde(default)] - pub total_eligible_cells: usize, - #[serde(default)] - pub skipped_oversized_cells: usize, - #[serde(default)] - pub accepted_candidates: usize, - #[serde(default)] - pub rejected_candidates: usize, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub review_reasons: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub message: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PasteTransformParams { - pub content: String, - pub format: PasteDataFormat, - pub columns: Vec, - #[serde(default)] - pub controls: Vec, - /// Rows to classify on, matching the figure paste analyze was given. - pub sample_row_count: usize, - #[serde(default)] - pub preview_smart_replacements: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PastePreviewParams { - pub content: String, - pub format: PasteDataFormat, - pub columns: Vec, - #[serde(default)] - pub controls: Vec, - /// Rows to display. A window on the paste, not evidence about it. - pub sample_count: usize, - /// Rows to classify on, matching the figure paste analyze was given. - pub sample_row_count: usize, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PasteTransformData { - pub output: String, - pub row_count: usize, - pub columns_anonymized: usize, - pub duration_ms: u128, - pub privacy_report: PrivacyReport, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct QuickTransformParams { - pub input: String, - pub data_type: DataType, - pub strategy: AnonymizationStrategy, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct QuickGenerateParams { - pub data_type: DataType, - pub strategy: AnonymizationStrategy, - pub count: usize, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct QuickTransformData { - pub output: String, - pub row_count: usize, - pub values: Vec, - pub privacy_report: PrivacyReport, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SampleTransform { - pub original: String, - pub anonymized: String, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ColumnPreview { - pub column_index: usize, - pub column_name: String, - pub samples: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PreviewWarning { - pub column_index: usize, - pub column_name: String, - pub message: String, - pub severity: WarningSeverity, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SmartReplacementEntry { - pub column_index: usize, - pub original: String, - pub replacement: String, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum SmartReplacementRejectionReason { - UnexpectedOriginal, - MissingOutput, - EmptyOutput, - SameAsOriginal, - ContainsOriginal, - /// The replacement was, or contained, a source value belonging to a *different* - /// row of the same column. - /// - /// Kept apart from [`Self::ContainsOriginal`] because the two describe different - /// events and only one of them moves a person's data between rows. - /// `ContainsOriginal` means the model echoed back the value it was asked to - /// replace, which wastes the request; this means it emitted somebody else's real - /// value, which would have published that value against the wrong record. A - /// report that merged them could not say which had happened. - MatchesOtherOriginal, - ControlCharacter, - DuplicateOriginal, - DuplicateOutput, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SmartReplacementRejectionCount { - pub reason: SmartReplacementRejectionReason, - pub count: usize, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum WarningSeverity { - Info, - Warning, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PreviewData { - pub previews: Vec, - pub warnings: Vec, - pub smart_replacements: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PreviewParams { - pub file_path: PathBuf, - pub columns: Vec, - #[serde(default)] - pub controls: Vec, - /// Rows to display. A window on the input, not evidence about it. - pub sample_count: usize, - /// Rows to classify on — the same figure analyze and the run are given, so the - /// preview shows the strategies the run will apply rather than the ones a - /// display-sized sample happens to imply. - pub sample_row_count: usize, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AnonymizeParams { - pub file_path: PathBuf, - pub output_path: PathBuf, - pub columns: Vec, - #[serde(default)] - pub controls: Vec, - pub force: bool, - #[serde(default)] - pub preview_smart_replacements: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AnonymizeData { - pub output_path: PathBuf, - pub row_count: usize, - pub columns_anonymized: usize, - pub duration_ms: u128, - pub privacy_report: PrivacyReport, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum PreflightMode { - Preview, - Anonymize, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PreflightParams { - pub mode: PreflightMode, - pub file_path: PathBuf, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub output_path: Option, - pub columns: Vec, - #[serde(default)] - pub controls: Vec, - pub force: bool, - pub sample_row_count: usize, - #[serde(default)] - pub preview_smart_replacements: Vec, - pub local_ai_ready: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub local_ai_message: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PreflightData { - pub mode: PreflightMode, - pub readiness: ReleaseReadiness, - pub evidence: Vec, - pub column_reports: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct PrivacyReport { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detection_run_summary: Option, - pub direct_identifiers: usize, - pub quasi_identifiers: usize, - pub pseudonymized_columns: usize, - pub smart_replacement_columns: usize, - pub opaque_token_columns: usize, - pub masked_columns: usize, - #[serde(default)] - pub labelled_columns: usize, - #[serde(default)] - pub redacted_columns: usize, - pub pass_through_columns: usize, - pub unique_pseudonym_values: usize, - pub reused_pseudonym_values: usize, - pub collisions_avoided: usize, - pub exhausted_pseudonym_pools: usize, - pub opaque_token_values: usize, - #[serde(default)] - pub keyed_token_values: usize, - #[serde(default)] - pub keyed_token_columns: Vec, - pub smart_replacement_values: usize, - #[serde(default)] - pub smart_replacement_rejections: usize, - #[serde(default)] - pub smart_replacement_rejection_reasons: Vec, - pub smart_replacement_fallbacks: usize, - #[serde(default)] - pub shape_fallback_values: usize, - #[serde(default)] - pub readiness: ReleaseReadiness, - #[serde(default)] - pub evidence: Vec, - #[serde(default)] - pub column_reports: Vec, - #[serde(default)] - pub column_value_distributions: Vec, - #[serde(default)] - pub row_uniqueness: Option, - #[serde(default)] - pub utility_metrics: Vec, - pub notes: Vec, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ReleaseReadiness { - pub status: ReleaseReadinessStatus, - pub blockers: Vec, - pub review_items: Vec, - pub verified_items: Vec, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ReleaseReadinessStatus { - Verified, - #[default] - Review, - Blocked, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ReleaseEvidenceItem { - pub id: String, - pub label: String, - pub status: ReleaseEvidenceStatus, - pub detail: String, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ReleaseEvidenceStatus { - Verified, - Review, - Blocked, - Info, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ColumnReleaseReport { - pub column_index: usize, - pub column_name: String, - pub selected: bool, - pub detected_type: DataType, - pub pii_risk: PiiRisk, - pub strategy: AnonymizationStrategy, - pub action: String, - pub status: ReleaseEvidenceStatus, - pub detail: String, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct UtilityMetric { - pub label: String, - pub value: String, - pub status: ReleaseEvidenceStatus, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detail: Option, -} +mod column; +mod processing; +mod report; +mod sampling; +mod workflow; + +pub use column::{ + AnonymizationStrategy, ColumnControl, ColumnEvidenceProfile, ColumnMetadata, + ColumnReviewReason, Confidence, DataType, DetectionResult, DetectionRunSummary, DetectionTrace, + DetectionTraceItem, DeterministicDetectionStatus, EmptyFormat, EvidenceDisposition, + FormatEvidence, FormatEvidenceBasis, LocalNerRunStatus, PiiRisk, PrivacyDecision, + PrivacyEvidenceSummary, PrivacyFinding, PrivacyFindingKind, RedactionDecision, + RedactionPlaceholderSource, SemanticDecision, SemanticSpecificity, SemanticStatus, +}; +pub(crate) use column::{ + DetectionReviewReason, GENERIC_STRING_STRUCTURE_DISCLOSURE, RedactionPlaceholder, + ReportIdentifierClass, +}; +pub use processing::{ + ProcessControl, ProcessOptions, ProcessProgress, ProcessResult, SmartReplacementEntry, + SmartReplacementRejectionCount, SmartReplacementRejectionReason, TransformContext, + TransformReport, +}; +pub(crate) use report::FrequencyInversionRisk; +pub use report::{ + ColumnReleaseReport, ColumnValueDistribution, DropColumnEffect, MatchedColumn, MatchedPart, + PrivacyReport, ReleaseEvidenceItem, ReleaseEvidenceStatus, ReleaseReadiness, + ReleaseReadinessStatus, RowUniquenessSummary, UtilityMetric, +}; +pub(crate) use sampling::{DETECTION_SAMPLE_ROW_FLOOR, DetectionCoverage}; +pub use sampling::{ + DetectionCoverageSummary, DetectionCoverageUnit, MAX_PREVIEW_SAMPLE_COUNT, + MAX_SAMPLE_ROW_COUNT, ParsedSample, +}; +pub use workflow::{ + AnonymizeData, AnonymizeParams, ColumnPreview, HeadersData, PasteAnalyzeData, + PasteAnalyzeParams, PasteDataFormat, PastePreviewParams, PasteTransformData, + PasteTransformParams, PreflightData, PreflightMode, PreflightParams, PreviewData, + PreviewParams, PreviewWarning, QuickGenerateParams, QuickTransformData, QuickTransformParams, + SampleTransform, WarningSeverity, +}; #[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - use std::path::PathBuf; - - #[test] - fn column_metadata_serializes_frontend_contract_shape() { - let column = ColumnMetadata { - header_label_is_ambiguous: false, - name: "email".to_string(), - source_path: Some("$.user.email".to_string()), - index: 2, - detected_type: DataType::Email, - confidence: Confidence::High, - detection_trace: Some(DetectionTrace { - summary: "email evidence".to_string(), - selected_reason: "value matched email".to_string(), - total_non_empty: 1, - candidates: vec![DetectionTraceItem { - data_type: DataType::Email, - reason: "valid email".to_string(), - match_count: 1, - total_considered: 1, - confidence: Confidence::High, - accepted: true, - }], - }), - privacy_findings: Vec::new(), - privacy_evidence: vec![PrivacyEvidenceSummary { - kind: PrivacyFindingKind::Contact, - data_type: DataType::Email, - confidence: Confidence::High, - match_count: 1, - sample_count: 1, - score: 100, - detector: "email".to_string(), - reason: "Column contains contact details.".to_string(), - detectors: vec!["email".to_string()], - }], - review_reasons: vec![ColumnReviewReason::AmbiguousContext], - evidence_disposition: EvidenceDisposition::DetectedSensitive, - evidence_profile: Default::default(), - pii_risk: PiiRisk::High, - sample_values: vec!["ada@example.com".to_string()], - sample_value_distribution: Default::default(), - empty_format: EmptyFormat::EmptyString, - is_selected: true, - strategy: AnonymizationStrategy::Redact, - }; - - let value = serde_json::to_value(&column).unwrap(); - - assert_eq!(value["detectedType"], json!("email")); - assert_eq!(value["detectionTrace"]["totalNonEmpty"], json!(1)); - assert_eq!(value["privacyEvidence"][0]["matchCount"], json!(1)); - assert_eq!(value["reviewReasons"], json!(["ambiguousContext"])); - assert_eq!(value["piiRisk"], json!("high")); - assert_eq!(value["sampleValues"], json!(["ada@example.com"])); - assert_eq!(value["emptyFormat"], json!("emptyString")); - assert_eq!(value["isSelected"], json!(true)); - assert_eq!(value["strategy"], json!("redact")); - assert!(value.get("detected_type").is_none()); - assert!(value.get("pii_risk").is_none()); - - let round_trip: ColumnMetadata = serde_json::from_value(value).unwrap(); - assert_eq!(round_trip, column); - } - - #[test] - fn preflight_params_serialize_optional_and_default_contract_fields() { - let params = PreflightParams { - mode: PreflightMode::Anonymize, - file_path: PathBuf::from("/data/input.csv"), - output_path: None, - columns: vec![0, 2], - controls: Vec::new(), - force: false, - sample_row_count: 10, - preview_smart_replacements: Vec::new(), - local_ai_ready: false, - local_ai_message: None, - }; - - let value = serde_json::to_value(¶ms).unwrap(); - - assert_eq!(value["mode"], json!("anonymize")); - assert_eq!(value["sampleRowCount"], json!(10)); - assert_eq!(value["previewSmartReplacements"], json!([])); - assert_eq!(value["localAiReady"], json!(false)); - assert!(value.get("outputPath").is_none()); - assert!(value.get("localAiMessage").is_none()); - assert!(value.get("sample_row_count").is_none()); - - let minimal = json!({ - "mode": "preview", - "filePath": "/data/input.csv", - "columns": [1], - "force": false, - "sampleRowCount": 5, - "localAiReady": true - }); - let decoded: PreflightParams = serde_json::from_value(minimal).unwrap(); - assert_eq!(decoded.mode, PreflightMode::Preview); - assert_eq!(decoded.controls, Vec::::new()); - assert_eq!( - decoded.preview_smart_replacements, - Vec::::new() - ); - assert_eq!(decoded.output_path, None); - assert_eq!(decoded.local_ai_message, None); - } - - #[test] - fn privacy_report_serializes_nested_release_and_smart_replacement_fields() { - let report = PrivacyReport { - detection_run_summary: None, - direct_identifiers: 1, - quasi_identifiers: 2, - pseudonymized_columns: 1, - smart_replacement_columns: 1, - opaque_token_columns: 0, - masked_columns: 0, - labelled_columns: 0, - redacted_columns: 1, - pass_through_columns: 0, - unique_pseudonym_values: 3, - reused_pseudonym_values: 0, - collisions_avoided: 0, - exhausted_pseudonym_pools: 0, - opaque_token_values: 0, - keyed_token_values: 0, - keyed_token_columns: Vec::new(), - smart_replacement_values: 2, - smart_replacement_rejections: 1, - smart_replacement_rejection_reasons: vec![SmartReplacementRejectionCount { - reason: SmartReplacementRejectionReason::ContainsOriginal, - count: 1, - }], - smart_replacement_fallbacks: 1, - shape_fallback_values: 2, - readiness: ReleaseReadiness { - status: ReleaseReadinessStatus::Review, - blockers: Vec::new(), - review_items: vec!["Review Smart replacement output.".to_string()], - verified_items: Vec::new(), - }, - evidence: vec![ReleaseEvidenceItem { - id: "local-ai".to_string(), - label: "Local AI".to_string(), - status: ReleaseEvidenceStatus::Review, - detail: "Review generated values.".to_string(), - }], - column_reports: vec![ColumnReleaseReport { - column_index: 2, - column_name: "email".to_string(), - selected: true, - detected_type: DataType::Email, - pii_risk: PiiRisk::High, - strategy: AnonymizationStrategy::LocalAi, - action: "Smart replacement".to_string(), - status: ReleaseEvidenceStatus::Review, - detail: "Generated replacements.".to_string(), - }], - column_value_distributions: vec![ColumnValueDistribution { - column_index: 2, - distinct_values: 3, - total_values: 10, - singleton_values: 1, - doubleton_values: 0, - max_value_occurrences: 6, - }], - row_uniqueness: Some(RowUniquenessSummary { - rows_measured: 10, - matched_columns: vec![ - MatchedColumn { - column_index: 0, - matched_on: MatchedPart::WholeValue, - matched_every_row: true, - }, - MatchedColumn { - column_index: 2, - matched_on: MatchedPart::DateDecadeAndTime, - matched_every_row: true, - }, - MatchedColumn { - column_index: 3, - matched_on: MatchedPart::SurvivingFormat, - matched_every_row: true, - }, - ], - distinct_classes: 7, - unique_rows: 4, - smallest_class: 1, - fifth_percentile_class_size: 1, - distinct_rows_all_columns: Some(10), - measurement_incomplete: false, - drop_column_effects: vec![ - DropColumnEffect { - column_index: 2, - unique_rows_without: 1, - }, - DropColumnEffect { - column_index: 0, - unique_rows_without: 3, - }, - ], - drop_attribution_incomplete: false, - }), - utility_metrics: vec![UtilityMetric { - label: "Rows".to_string(), - value: "10".to_string(), - status: ReleaseEvidenceStatus::Info, - detail: Some("sample".to_string()), - }], - notes: vec!["Review generated replacements.".to_string()], - }; - - let value = serde_json::to_value(&report).unwrap(); - - assert_eq!(value["directIdentifiers"], json!(1)); - assert_eq!(value["smartReplacementColumns"], json!(1)); - assert_eq!( - value["smartReplacementRejectionReasons"][0]["reason"], - json!("containsOriginal") - ); - assert_eq!(value["readiness"]["status"], json!("review")); - assert_eq!(value["evidence"][0]["status"], json!("review")); - assert_eq!(value["columnReports"][0]["detectedType"], json!("email")); - assert_eq!(value["utilityMetrics"][0]["status"], json!("info")); - assert_eq!( - value["columnValueDistributions"][0]["distinctValues"], - json!(3) - ); - assert_eq!( - value["columnValueDistributions"][0]["singletonValues"], - json!(1) - ); - assert!(value.get("direct_identifiers").is_none()); - assert!(value.get("smart_replacement_columns").is_none()); - - let round_trip: PrivacyReport = serde_json::from_value(value).unwrap(); - assert_eq!(round_trip, report); - } - - #[test] - fn privacy_report_accepts_defaulted_newer_fields_when_deserializing() { - let value = json!({ - "directIdentifiers": 1, - "quasiIdentifiers": 0, - "pseudonymizedColumns": 1, - "smartReplacementColumns": 0, - "opaqueTokenColumns": 0, - "maskedColumns": 0, - "passThroughColumns": 0, - "uniquePseudonymValues": 1, - "reusedPseudonymValues": 0, - "collisionsAvoided": 0, - "exhaustedPseudonymPools": 0, - "opaqueTokenValues": 0, - "smartReplacementValues": 0, - "smartReplacementFallbacks": 0, - "notes": [] - }); - - let report: PrivacyReport = serde_json::from_value(value).unwrap(); - - assert_eq!(report.redacted_columns, 0); - assert_eq!(report.smart_replacement_rejections, 0); - assert_eq!(report.smart_replacement_rejection_reasons, Vec::new()); - assert_eq!(report.readiness, ReleaseReadiness::default()); - assert_eq!(report.evidence, Vec::new()); - assert_eq!(report.column_reports, Vec::new()); - assert_eq!(report.utility_metrics, Vec::new()); - } - - #[test] - fn selected_enums_use_camel_case_wire_values() { - assert_eq!( - serde_json::to_value(DataType::NumericId).unwrap(), - json!("numericId") - ); - assert_eq!( - serde_json::to_value(PasteDataFormat::PlainText).unwrap(), - json!("plainText") - ); - assert_eq!( - serde_json::to_value(AnonymizationStrategy::PassThrough).unwrap(), - json!("passThrough") - ); - assert_eq!( - serde_json::to_value(ReleaseEvidenceStatus::Info).unwrap(), - json!("info") - ); - } - - fn distribution(distinct: usize, total: usize) -> ColumnValueDistribution { - ColumnValueDistribution { - column_index: 0, - distinct_values: distinct, - total_values: total, - // No singletons, so coverage is 1.0 and the ratio test is reachable. - // The shapes that make coverage matter are pinned separately below. - singleton_values: 0, - doubleton_values: 0, - max_value_occurrences: 0, - } - } - - /// The floor is the whole reason this predicate is not just `distinct < 10`. - /// `distinct_values <= total_values`, so below the floor the absolute test is - /// vacuously true and every short column — including unique-key columns — looks - /// low-cardinality. Measured: at 5 rows all 7 columns of `large.csv` trip it. - #[test] - fn cardinality_risk_ignores_columns_with_too_few_values_to_judge() { - // The vacuous case: every value distinct, yet fewer than the constant. - assert!(!distribution(5, 5).risks_frequency_inversion()); - assert!(!distribution(49, 49).risks_frequency_inversion()); - // Genuinely low cardinality, but still not enough rows to claim it. - assert!(!distribution(4, 49).risks_frequency_inversion()); - // One value more and the same shape is judgeable. - assert!(distribution(4, 50).risks_frequency_inversion()); - } - - #[test] - fn cardinality_risk_flags_few_distinct_values_at_the_boundary() { - assert!(distribution(9, 60).risks_frequency_inversion()); - // Ten distinct over sixty is 0.167, above the ratio threshold too, so this - // pins both terms at once: neither may fire here. - assert!(!distribution(10, 60).risks_frequency_inversion()); - } - - /// The ratio term exists for columns the absolute term cannot reach. Measured on - /// `large.csv name`: 100 distinct over 10500 values. - #[test] - fn cardinality_risk_flags_a_large_column_by_ratio() { - assert!(distribution(100, 10_500).risks_frequency_inversion()); - // Same distinct count, far fewer rows: 0.1 is above the threshold, and the - // absolute term does not reach 100 either. - assert!(!distribution(100, 1_000).risks_frequency_inversion()); - } - - /// Below 200 values the ratio term is subsumed by the absolute one, which is why - /// 0.05 was chosen: small and medium inputs are judged on the statistic that - /// saturates rather than on one that keeps drifting as more rows are read. - #[test] - fn the_ratio_term_is_inert_below_two_hundred_values() { - for total in [50usize, 100, 199] { - for distinct in 10..=total { - let subject = distribution(distinct, total); - assert!( - !subject.risks_frequency_inversion(), - "{distinct} distinct over {total} fired without the absolute term" - ); - } - } - } - - /// Counts the values a transform would count: empties skipped, case and padding - /// folded. A pre-run warning that measured differently from the run would - /// contradict the report built from the same data. - #[test] - fn distribution_from_values_matches_the_transform_value_identity() { - let values = [ - "Sales".to_string(), - " sales ".to_string(), - "SALES".to_string(), - "Legal".to_string(), - String::new(), - "null".to_string(), - "NULL".to_string(), - " ".to_string(), - ]; - - let subject = ColumnValueDistribution::from_values(3, &values); - - assert_eq!(subject.column_index, 3); - assert_eq!(subject.distinct_values, 2); - assert_eq!(subject.total_values, 4); - assert_eq!(subject.singleton_values, 1); - assert_eq!(subject.doubleton_values, 0); - assert_eq!(subject.max_value_occurrences, 3); - } - - /// Singletons and doubletons are counted after identity folding too, since they - /// are what [`ColumnValueDistribution::estimated_distinct_values`] reads. - #[test] - fn distribution_counts_singletons_and_doubletons() { - let values = ["a", "a", "b", "b", "c", "d", "d", "d"] - .map(str::to_string) - .to_vec(); - - let subject = ColumnValueDistribution::from_values(0, &values); - - assert_eq!(subject.distinct_values, 4); - assert_eq!(subject.singleton_values, 1, "c"); - assert_eq!(subject.doubleton_values, 2, "a and b"); - } - - /// A sampled column of a large file is the case the ratio test could not see. The - /// numbers are the measured ones for 30 departments over 5000 rows. - #[test] - fn a_sampled_distribution_is_judged_against_the_columns_real_size() { - let sampled = ColumnValueDistribution { - column_index: 1, - distinct_values: 29, - total_values: 100, - singleton_values: 5, - doubleton_values: 4, - max_value_occurrences: 8, - }; - - // Against the sample's own size the ratio is 0.29 and the absolute term does - // not reach 29 either, so judged this way the column looks safe. - assert!(!sampled.risks_frequency_inversion()); - // Against the file it was drawn from, it is 30-odd values over 5000 rows. - assert!(sampled.frequency_inversion_risk_in(5_000).is_some()); - } - - /// The counterweight, and why the ratio test cannot simply divide by the row - /// count: a sample of a hundred cannot look like a million distinct values, so a - /// fully unique column scores 0.005 on the ratio alone. Coverage is what - /// distinguishes "few distinct values" from "a sample that learned nothing". - #[test] - fn a_sample_that_learned_nothing_cannot_raise_a_warning() { - let every_value_new = ColumnValueDistribution { - column_index: 0, - distinct_values: 100, - total_values: 100, - singleton_values: 100, - doubleton_values: 0, - max_value_occurrences: 1, - }; - - assert!( - every_value_new - .frequency_inversion_risk_in(1_000_000) - .is_none() - ); - } - - /// Coverage gates the ratio term only. A column with few enough distinct values - /// is invertible whatever the coverage figure says, so the absolute term has to - /// answer first. - #[test] - fn the_absolute_term_is_not_gated_by_coverage() { - let sparse_but_unsaturated = ColumnValueDistribution { - column_index: 0, - distinct_values: 9, - total_values: 60, - // Coverage 0.0, far below the gate. - singleton_values: 60, - doubleton_values: 0, - max_value_occurrences: 1, - }; - - assert!( - sparse_but_unsaturated - .frequency_inversion_risk_in(1_000_000) - .is_some() - ); - } - - /// Chao1 estimates the groups a sample missed from how many it saw exactly once. - /// Pinned on the measured shape rather than the formula, so a rewrite that keeps - /// the behaviour passes and one that changes it fails. - #[test] - fn the_estimated_distinct_count_allows_for_unseen_values() { - let sampled = ColumnValueDistribution { - column_index: 0, - distinct_values: 29, - total_values: 100, - singleton_values: 5, - doubleton_values: 4, - max_value_occurrences: 8, - }; - // 29 + 25/8 = 32, against a true count of 30. - assert_eq!(sampled.estimated_distinct_values(), 32); - - // Nothing seen twice: the bias-corrected form, and no division by zero. - let no_doubletons = ColumnValueDistribution { - distinct_values: 10, - total_values: 100, - singleton_values: 4, - doubleton_values: 0, - ..ColumnValueDistribution::default() - }; - assert_eq!(no_doubletons.estimated_distinct_values(), 10 + 6); - - // Every value repeated: nothing was missed. - let fully_measured = ColumnValueDistribution { - distinct_values: 30, - total_values: 5_000, - singleton_values: 0, - doubleton_values: 0, - ..ColumnValueDistribution::default() - }; - assert_eq!(fully_measured.estimated_distinct_values(), 30); - } - - #[test] - fn distribution_from_no_usable_values_is_empty_rather_than_risky() { - let values = [String::new(), "null".to_string(), " ".to_string()]; - - let subject = ColumnValueDistribution::from_values(0, &values); - - assert_eq!(subject.total_values, 0); - assert_eq!(subject.max_value_occurrences, 0); - assert!(!subject.risks_frequency_inversion()); - } - - /// A default distribution is what a caller that never measured produces, so it - /// must never be able to raise a warning it has no evidence for. - #[test] - fn a_defaulted_distribution_cannot_raise_a_warning() { - assert!(!ColumnValueDistribution::default().risks_frequency_inversion()); - } - - /// One dominant value over forty-nine near-unique ones. - /// - /// Not invented: these are the exact figures the detection sample computes for - /// `tests/fixtures/dominant-value.csv`, pinned in - /// `service::tests::cardinality::the_dominant_value_fixture_has_the_shape_these_tests_depend_on`, - /// so the unit tests here and the integration tests there are reasoning about one - /// column rather than two that happen to resemble each other. - fn dominant_value_sample() -> ColumnValueDistribution { - ColumnValueDistribution { - column_index: 1, - distinct_values: 50, - total_values: 100, - singleton_values: 49, - doubleton_values: 0, - max_value_occurrences: 51, - } - } - - /// The gap this term closes. A column that is *diverse* — here 50 distinct values in - /// the sample and thousands in the file — but whose most common value covers half the - /// rows leaks half the column the moment one pseudonym is matched, and until this term - /// existed it drew no warning at all: 50 distinct values is far above the absolute - /// limit, and the Chao1 ratio is high precisely because the column is diverse. - /// - /// Both older terms are asserted against as well as the verdict. Without them a change - /// that made the *absolute* term start firing at 50 distinct values would leave this - /// test green while flagging most columns in the corpus. - #[test] - fn a_dominant_value_is_flagged_however_diverse_the_rest_of_the_column_is() { - let subject = dominant_value_sample(); - - assert!(subject.distinct_values >= MAX_INVERTIBLE_DISTINCT_VALUES); - assert!(subject.sample_coverage() < MIN_SAMPLE_COVERAGE); - - assert!(subject.frequency_inversion_risk_in(5_000_000).is_some()); - } - - /// The same column with the dominance taken out of it, and nothing else changed. - /// - /// Half the rows moved off the dominant value onto values of their own, so the column - /// is *more* diverse than the one above while holding the same 100 values. It has to - /// go silent, or the term is reading something other than dominance. - #[test] - fn the_same_column_without_a_dominant_value_stays_silent() { - let spread_out = ColumnValueDistribution { - distinct_values: 76, - singleton_values: 75, - max_value_occurrences: 25, - ..dominant_value_sample() - }; - - assert!(spread_out.frequency_inversion_risk_in(5_000_000).is_none()); - } - - /// The dominant-value term is deliberately ahead of the coverage gate, and this is - /// the reason: the shape it catches is singleton-heavy, so its coverage is low. The - /// fixture's coverage is 0.51 against a 0.75 gate. Behind the gate this term would have - /// been silent on almost every column it was added for: of 2400 measured columns whose - /// top value covered half the rows, none reached coverage 0.75, and of 2400 covering - /// three fifths, six did. - #[test] - fn the_dominant_value_term_is_not_gated_by_coverage() { - let subject = dominant_value_sample(); - - assert!(subject.sample_coverage() < MIN_SAMPLE_COVERAGE); - assert!(subject.frequency_inversion_risk_in(5_000_000).is_some()); - } - - /// A share and not a count, which is the whole point of the constant: the identical - /// shape has to answer the same way at every file size. A count-based rule would - /// have to be either silent on the small file or noisy on the large one. - #[test] - fn the_dominant_value_verdict_does_not_move_with_the_files_size() { - for population in [100usize, 5_000, 1_000_000, 5_000_000] { - assert!( - dominant_value_sample() - .frequency_inversion_risk_in(population) - .is_some(), - "silent at {population} values" - ); - } - } - - /// The boundary, measured on a fully counted column so no estimate is involved: - /// 20 of 60 is exactly a third and fires, 19 of 60 does not. - /// - /// The surrounding counts are chosen so neither other term can answer — 15 distinct - /// clears the absolute limit, and Chao1 over the column's own 60 values is 0.67, - /// nowhere near the 0.05 ratio limit — so the verdict here is the dominant-value - /// term's alone. - #[test] - fn the_dominant_share_boundary_sits_at_one_third() { - let at_the_boundary = ColumnValueDistribution { - column_index: 0, - distinct_values: 15, - total_values: 60, - singleton_values: 10, - doubleton_values: 2, - max_value_occurrences: 20, - }; - assert!(at_the_boundary.risks_frequency_inversion()); - - let one_row_short = ColumnValueDistribution { - max_value_occurrences: 19, - ..at_the_boundary - }; - assert!(!one_row_short.risks_frequency_inversion()); - } - - /// A genuinely unique column has to stay silent, and the dominant-value term must - /// not be what breaks that. Its most common value covers one row, so its share is - /// `1 / total_values` — below a third for any column past the floor, which is why - /// the term needs no special case for it. Pinned at four sizes because a share is - /// exactly the kind of quantity that misbehaves at the small end. - #[test] - fn a_unique_column_cannot_trip_the_dominant_value_term() { - for total in [50usize, 60, 5_000, 1_000_000] { - let unique = ColumnValueDistribution { - column_index: 0, - distinct_values: total, - total_values: total, - singleton_values: total, - doubleton_values: 0, - max_value_occurrences: 1, - }; - - assert!( - unique.frequency_inversion_risk_in(total).is_none(), - "{total} unique values were flagged" - ); - } - } - - /// The false-positive case the constant was calibrated against. This is a measured - /// 100-value sample of a Zipf column with exponent 1.0 over 1000 labels — the - /// ordinary shape of real categorical data, whose top value takes a seventh of the - /// rows. Over 4000 such samples no draw reached a third, and a threshold low enough - /// to catch this shape would fire on most text columns in most files. - #[test] - fn a_mildly_skewed_high_cardinality_column_stays_silent() { - let ordinary_skew = ColumnValueDistribution { - column_index: 0, - distinct_values: 68, - total_values: 100, - singleton_values: 55, - doubleton_values: 8, - max_value_occurrences: 14, - }; - - assert!(ordinary_skew.frequency_inversion_risk_in(100_000).is_none()); - } -} +mod tests; diff --git a/crates/csv-anonymizer-core/src/types/column.rs b/crates/csv-anonymizer-core/src/types/column.rs new file mode 100644 index 0000000..f1d6138 --- /dev/null +++ b/crates/csv-anonymizer-core/src/types/column.rs @@ -0,0 +1,747 @@ +use super::*; +use serde::{Deserialize, Serialize}; +/// What `strategies::structured::transform_generic_string` keeps of a value. +/// +/// Named once because two callers state it about the same transform: the detected +/// types that have no transformer of their own, and the Local AI fallback for a +/// pass-through type, which lands on the same function. See +/// [`DataType::pseudonymization_preserves_structure`]. +pub(crate) const GENERIC_STRING_STRUCTURE_DISCLOSURE: &str = "the replacement is random text of roughly the original's length (within about 20%), so value length survives approximately"; + +/// The default is [`DataType::Unknown`], and it is the only variant that may be one. +/// +/// A default is reached when nothing said what a column holds, so it has to be the +/// answer that claims nothing. Every other variant is a claim, and the ones that would +/// tempt a reader — `String`, `Enum` — are claims in the wrong direction: `Enum` is +/// pass-through, so a defaulted column would be described as deliberately kept +/// unchanged. `Unknown` takes the generic-string transform and states no structure it +/// preserves, which is the honest reading of a column nobody classified. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum DataType { + Email, + Uuid, + Timestamp, + NumericId, + NumericValue, + PostalCode, + Address, + IpAddress, + Url, + MacAddress, + TaxId, + Boolean, + Currency, + Percentage, + CountryCode, + Phone, + FirstName, + LastName, + FullName, + Enum, + String, + #[default] + Unknown, +} + +impl DataType { + pub(crate) fn privacy_finding_kind_and_reason( + self, + ) -> Option<(PrivacyFindingKind, &'static str)> { + match self { + DataType::Email | DataType::Phone => Some(( + PrivacyFindingKind::Contact, + "Column type indicates contact information.", + )), + DataType::FirstName | DataType::LastName | DataType::FullName => Some(( + PrivacyFindingKind::Person, + "Column type indicates person names.", + )), + DataType::Address => Some(( + PrivacyFindingKind::PrivateAddress, + "Column type indicates private address data.", + )), + DataType::PostalCode => Some(( + PrivacyFindingKind::AddressRegion, + "Column type indicates postal address context.", + )), + DataType::TaxId => Some(( + PrivacyFindingKind::GovernmentId, + "Column type indicates government or tax identifier data.", + )), + // Identifier-shaped, but nothing here says *what* it identifies. A + // financial classification needs evidence: the `account_number` header + // kind, or the IBAN or payment-card validator. Absent that, this is a + // surrogate key. + DataType::NumericId => Some(( + PrivacyFindingKind::RecordIdentifier, + "Column type indicates identifier-shaped values; review context.", + )), + DataType::Uuid => Some(( + PrivacyFindingKind::RecordIdentifier, + "Column type indicates persistent identifier-shaped values; what they identify is unknown.", + )), + DataType::IpAddress | DataType::MacAddress => Some(( + PrivacyFindingKind::NetworkOrDeviceId, + "Column type indicates network or device identifiers.", + )), + DataType::Url => Some((PrivacyFindingKind::Url, "Column type indicates URLs.")), + DataType::NumericValue + | DataType::Timestamp + | DataType::Boolean + | DataType::Currency + | DataType::Percentage + | DataType::CountryCode + | DataType::Enum + | DataType::String + | DataType::Unknown => None, + } + } + + pub(crate) fn report_identifier_class(self) -> Option { + match self { + DataType::Email + | DataType::Phone + | DataType::FullName + | DataType::FirstName + | DataType::LastName + | DataType::TaxId + | DataType::Address => Some(ReportIdentifierClass::Direct), + DataType::Uuid + | DataType::NumericId + | DataType::PostalCode + | DataType::IpAddress + | DataType::Url + | DataType::MacAddress + | DataType::Timestamp + | DataType::CountryCode => Some(ReportIdentifierClass::Quasi), + DataType::NumericValue + | DataType::Boolean + | DataType::Currency + | DataType::Percentage + | DataType::Enum + | DataType::String + | DataType::Unknown => None, + } + } + + pub(crate) fn uses_default_pass_through(self) -> bool { + matches!( + self, + DataType::CountryCode + | DataType::Enum + | DataType::Boolean + | DataType::Currency + | DataType::Percentage + ) + } + + /// What a rule-based pseudonym for this type keeps of the original value, or + /// `None` when it keeps nothing worth disclosing. + /// + /// Every transformer in `strategies` is format-preserving on purpose: a + /// pseudonymized timestamp has to parse as a timestamp, a pseudonymized amount has + /// to stay a number, or the output is unusable. This sentence is what the release + /// report states about the part of the source that comes through intact. + /// + /// The sharpest case is `Timestamp`: `strategies::structured::transform_timestamp` + /// splits the value at its ten-byte ISO date prefix and concatenates the remainder + /// verbatim, so `2024-06-15 10:30:45.123450` keeps `10:30:45.123450` exactly. In an + /// event log a microsecond time-of-day is very nearly a primary key, so the + /// "anonymized" file joins straight back to the source on a column the report + /// marked verified. The date moves by at most 365 days, which also means a date of + /// birth keeps its year to within one and the subject keeps their age. + /// + /// Returning a sentence rather than a bool because the sentence is the point: a + /// caller cannot write one wording that is true of "keeps the domain after @" and + /// of "keeps the digit count and sign" at once. + /// + /// No wildcard arm. A data type added to the enum has to be classified here rather + /// than defaulting into the silent half — silence is the defect this closes. + pub(crate) fn pseudonymization_preserves_structure(self) -> Option<&'static str> { + match self { + DataType::Email => Some( + "the local part is replaced but the domain after @ is kept verbatim, so recipients of a rare or personal domain stay identifiable", + ), + DataType::Timestamp => Some( + "the time of day is kept exactly — including sub-second digits, which are close to unique per event — and only the date moves, by at most a year, so an age or a year of birth survives", + ), + DataType::Phone => Some( + "only the digits are redrawn: the digit count, country prefix punctuation and separator layout are kept", + ), + DataType::NumericId => Some( + "the digit count and any leading zeros are kept, so the magnitude of the identifier is preserved", + ), + DataType::NumericValue => Some( + "the sign, digit count and number of decimal places are kept, so the magnitude of the value is preserved", + ), + DataType::FirstName | DataType::LastName | DataType::FullName => Some( + "replacements are drawn from a fixed name pool and the number of name parts is kept", + ), + // Generic-string pseudonymization draws a random value of 80–120% of the + // original's length, so the length survives approximately. Everything that + // is not handled by a transformer of its own lands here. + DataType::Address + | DataType::PostalCode + | DataType::IpAddress + | DataType::Url + | DataType::MacAddress + | DataType::TaxId + | DataType::String + | DataType::Unknown => Some(GENERIC_STRING_STRUCTURE_DISCLOSURE), + // A UUID is machine-generated and carries no structure about its subject; + // the transform keeps only the UUID format and the original's letter case, + // neither of which narrows anyone down. + DataType::Uuid => None, + // Returned unchanged under Auto and Pseudonymize, so there is no transform + // to describe — `uses_default_pass_through` is what the report says about + // these, and it says it plainly. Under a rejected Local AI candidate they + // take the generic-string path instead; the Local AI column report states + // that separately, because it is a property of the strategy rather than of + // the type. + DataType::Enum + | DataType::CountryCode + | DataType::Boolean + | DataType::Currency + | DataType::Percentage => None, + } + } + + pub(crate) fn transforms_generated_quick_value(self) -> bool { + matches!( + self, + DataType::Email + | DataType::Uuid + | DataType::Timestamp + | DataType::NumericId + | DataType::NumericValue + | DataType::Phone + | DataType::FirstName + | DataType::LastName + | DataType::FullName + | DataType::String + | DataType::Unknown + ) + } + + pub(crate) fn redaction_changes_structured_scalar_type(self) -> bool { + matches!( + self, + DataType::NumericId + | DataType::NumericValue + | DataType::Boolean + | DataType::Currency + | DataType::Percentage + ) + } + + pub(crate) fn redaction_placeholder(self) -> Option { + match self { + DataType::Email => Some(RedactionPlaceholder::Email), + DataType::Phone => Some(RedactionPlaceholder::Phone), + DataType::FirstName | DataType::LastName | DataType::FullName => { + Some(RedactionPlaceholder::Person) + } + DataType::Address => Some(RedactionPlaceholder::Address), + DataType::TaxId => Some(RedactionPlaceholder::GovernmentId), + DataType::Url => Some(RedactionPlaceholder::Url), + DataType::IpAddress | DataType::MacAddress => Some(RedactionPlaceholder::NetworkId), + DataType::String + | DataType::Unknown + | DataType::Enum + | DataType::Uuid + | DataType::Timestamp + | DataType::NumericId + | DataType::PostalCode + | DataType::NumericValue + | DataType::Boolean + | DataType::Currency + | DataType::Percentage + | DataType::CountryCode => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ReportIdentifierClass { + Direct, + Quasi, +} + +/// Placeholders a column's *detected type* can justify on its own. +/// +/// Deliberately has no account variant. "This is a bank account" is a claim about +/// what a value means, not about its shape, so it can only come from evidence — the +/// IBAN or card validator, or an `account_number` header — which reaches +/// `[ACCOUNT_ID]` through `placeholder_from_evidence` instead. A column of plain +/// integers gets a non-linkable placeholder derived from its column header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RedactionPlaceholder { + Email, + Phone, + Person, + Address, + GovernmentId, + Url, + NetworkId, +} + +/// The default is [`Confidence::Low`], the only reading that asserts nothing. +/// +/// `is_actionable` is false at Low, so a detection nobody measured cannot act as +/// evidence. Defaulting to High or Medium would let an unmeasured column carry the +/// weight of a measured one. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Confidence { + High, + Medium, + #[default] + Low, +} + +/// How much a finding exposes, most severe first. +/// +/// Declaration order is load-bearing: `Ord` is derived, so `High < Medium < Low` and sorting +/// ascending puts the most severe finding first. The privacy report relies on that when it +/// breaks ties between findings — a report that shows a Medium above a High has under-sold +/// what it found, which is the one direction these figures may not be wrong in. +/// +/// The default is [`PiiRisk::High`], deliberately the most severe rather than the most +/// common. A default stands in for "nobody assessed this column", and the only reading +/// of that which cannot make a file look safer than it is, is the worst case. `Low` +/// would silently un-flag a column: `is_elevated` is what auto-selects a column, +/// defaults it to Redact, and names it when it is released unchanged, and a defaulted +/// `Low` turns all three off without anything having decided so. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PiiRisk { + #[default] + High, + Medium, + Low, +} + +impl PiiRisk { + /// Whether this risk is one the app acts on: auto-selects the column, defaults it + /// to Redact, and names it when it is released unchanged. + /// + /// Named once because it is the app's privacy threshold rather than a comparison. + /// Spelled out at each site, the five copies could be changed apart, and a site + /// left behind would go on treating a Medium-risk column as ordinary — selecting + /// it out of the run, or leaving it out of the report that says what the released + /// file still exposes. + pub(crate) fn is_elevated(self) -> bool { + match self { + Self::High | Self::Medium => true, + Self::Low => false, + } + } +} + +/// The categories of privacy evidence the report can attribute to a column. +/// +/// Declaration order is load-bearing, but it is the *last* word rather than the first: the +/// privacy report breaks a tie on score and match count by `PiiRisk` first, so a High finding +/// always leads a Medium one, and only then by kind, which on a derived enum is declaration +/// order. Reordering these variants therefore changes which of two equally-risky findings a +/// reader sees first, so treat the order as user-facing output rather than as a free choice. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PrivacyFindingKind { + Person, + Contact, + PrivateAddress, + /// An address component that narrows someone down to an area rather than to a + /// doorstep: a postal code on its own. + /// + /// Separate from [`PrivacyFindingKind::PrivateAddress`] because a street address + /// locates a person and a postal code locates a neighbourhood. Both matter, but + /// only one is a direct identifier, and reporting a zip column as a private + /// address overstated what the file actually contains. + AddressRegion, + PrivateDate, + AccountOrFinancialId, + /// A key that identifies a row without being sensitive in itself: an order + /// number, a record id, a customer sequence number. + /// + /// Distinct from [`PrivacyFindingKind::AccountOrFinancialId`], which is for + /// bank accounts, cards and IBANs. Both are identifiers, but only one exposes + /// a payment instrument, and collapsing them made every column of order + /// numbers report as financial data. A surrogate key still re-identifies a + /// row, so it is Medium rather than Low. + RecordIdentifier, + GovernmentId, + CredentialOrSecret, + NetworkOrDeviceId, + Url, + MixedSensitiveText, +} + +/// The default is [`EmptyFormat::EmptyString`], which is also what +/// [`crate::detection::detect_empty_format`] answers for a column in which nothing +/// null-shaped was seen. It carries no privacy claim either way — it decides how a blank +/// cell is written, not what survives in it. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum EmptyFormat { + #[default] + EmptyString, + Null, + Mixed, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetectionResult { + pub data_type: DataType, + pub confidence: Confidence, + pub sample_matches: usize, + pub total_samples: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trace: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetectionTrace { + pub summary: String, + pub selected_reason: String, + pub total_non_empty: usize, + pub candidates: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetectionTraceItem { + pub data_type: DataType, + pub reason: String, + pub match_count: usize, + pub total_considered: usize, + pub confidence: Confidence, + pub accepted: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PrivacyFinding { + pub kind: PrivacyFindingKind, + pub data_type: DataType, + pub row_index: usize, + pub start: usize, + pub end: usize, + pub match_value: String, + pub sample_value: String, + pub confidence: Confidence, + pub score: u8, + pub detector: String, + pub reason: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PrivacyEvidenceSummary { + pub kind: PrivacyFindingKind, + pub data_type: DataType, + pub confidence: Confidence, + pub match_count: usize, + pub sample_count: usize, + pub score: u8, + #[serde(default)] + pub detector: String, + #[serde(default)] + pub reason: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub detectors: Vec, +} + +impl PrivacyEvidenceSummary { + /// Whether the app may act on this finding. + /// + /// A Low-confidence finding is a shape that resembles the thing more often than + /// it is the thing — `pattern:phone-digits` fires on any bare digit run, and most + /// bare digit runs are order numbers. Such a finding is worth *recording*, so a + /// reviewer can see it and so free text still gets redacted span by span, but it + /// is not worth *asserting*: it may not raise a column's risk and it may not put + /// a specific type's name in the output. + /// + /// This lives on the summary because two modules decide it and they have to agree. + /// `analyze_column_privacy` folds risk over the evidence; `placeholder_from_evidence` + /// names the redaction placeholder from the same list. When only the risk fold + /// filtered, a column the risk model had explicitly declined to trust still + /// redacted to `[PHONE]` — the output file asserting a phone number was there on + /// the evidence the app had just rejected. One predicate, so the next change to + /// the threshold cannot reach one consumer and miss the other. + pub(crate) fn is_actionable(&self) -> bool { + self.confidence != Confidence::Low + } +} + +/// `Default` is for constructing one, not for accepting one. +/// +/// Fifteen fields of which a given caller varies two or three is what +/// `crate::test_support::column` exists to absorb, and `Default` is what lets it. It +/// widens nothing on the deserialization side: the serde contract is unchanged, so a +/// field this struct requires on the wire is still required — only the fields already +/// carrying `#[serde(default)]`, each with its own reason stated below, may be absent. +/// +/// Every defaulted field is the least-privileged reading of "nobody decided this": +/// `PiiRisk::High`, `DataType::Unknown`, `Confidence::Low`, +/// `AnonymizationStrategy::PassThrough`, `is_selected: false`, and a zeroed +/// distribution. A default may leave a column looking more exposed than it is; it may +/// never leave one looking safer. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ColumnReviewReason { + DetectorsDisagree, + LocalNerLowConfidence, + AmbiguousContext, + InsufficientSample, +} + +/// How conclusive the analysis of a column was, independently of its privacy risk. +/// +/// Risk answers "what would this expose if the finding is correct?" while this +/// disposition answers "did the available evidence justify a conclusion?" Keeping +/// them separate prevents an unrecognised string column from being presented as +/// tested-benign merely because no detector raised its risk. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum EvidenceDisposition { + DetectedSensitive, + TestedBenign, + #[default] + Uncertain, + AnalysisIncomplete, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FormatEvidence { + pub data_type: DataType, + pub confidence: Confidence, + pub match_count: usize, + pub sample_count: usize, + pub basis: FormatEvidenceBasis, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub detectors: Vec, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FormatEvidenceBasis { + DetectionSample, + UserOverride, + #[default] + RetainedPreviewValues, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SemanticSpecificity { + Specific, + #[default] + Generic, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SemanticStatus { + Resolved, + #[default] + Uncertain, + Conflicting, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SemanticDecision { + pub kind: String, + pub confidence: Confidence, + pub specificity: SemanticSpecificity, + pub status: SemanticStatus, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub supporting_evidence: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub conflicting_evidence: Vec, + pub reason: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PrivacyDecision { + pub risk: PiiRisk, + pub recommended_strategy: AnonymizationStrategy, + pub auto_selected: bool, + pub reason: String, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum RedactionPlaceholderSource { + Typed, + ColumnHeader, + #[default] + Generic, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RedactionDecision { + pub placeholder: String, + pub source: RedactionPlaceholderSource, + pub is_typed: bool, + pub preserves_equality: bool, + pub reason: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ColumnEvidenceProfile { + pub format_evidence: FormatEvidence, + pub semantic_decision: SemanticDecision, + pub privacy_decision: PrivacyDecision, + pub redaction_decision: RedactionDecision, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ColumnMetadata { + pub name: String, + /// Whether some other column's header reduces to the same placeholder label as + /// this one's. + /// + /// A duplicate header is legal in CSV, and on its own it is only untidy — the + /// column table shows two rows with one name and the index tells them apart. + /// It stops being cosmetic once a label reaches a *cell*: labels number each + /// distinct value per column, so two columns named `notes` would both start at + /// `[NOTES_1]`, and a reader comparing those cells would conclude the source + /// values were equal when nothing of the kind was measured. Columns marked here + /// fold their position into the label instead. + /// + /// Measured on the label rather than the raw header, because that is where the + /// collision happens: `Notes`, `notes` and `notes!` all reduce to `NOTES`. + /// + /// Defaulted rather than optional so a caller that never compared headers gets + /// the unqualified label, which is the behaviour for a column whose header is + /// unique — the common case, and the readable one. + #[serde(default)] + pub header_label_is_ambiguous: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_path: Option, + pub index: usize, + pub detected_type: DataType, + pub confidence: Confidence, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detection_trace: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub privacy_findings: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub privacy_evidence: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub review_reasons: Vec, + #[serde(default)] + pub evidence_disposition: EvidenceDisposition, + #[serde(default)] + pub evidence_profile: ColumnEvidenceProfile, + pub pii_risk: PiiRisk, + pub sample_values: Vec, + /// The value distribution of the *detection sample*, not of the whole input. + /// + /// This is what lets the column table warn before a run rather than after one, + /// which is the only point at which the user can still change strategy. A + /// sampled distinct count is a lower bound on the true one, which is the right + /// direction to be wrong in: a high-cardinality column cannot be made to look + /// low-cardinality by sampling, while a genuinely low-cardinality one saturates + /// within a few dozen rows. + /// + /// Defaulted rather than optional so a caller that never measured produces a + /// zeroed distribution, which fails the sample-size floor and therefore cannot + /// raise a warning it has no evidence for. + #[serde(default)] + pub sample_value_distribution: ColumnValueDistribution, + pub empty_format: EmptyFormat, + pub is_selected: bool, + pub strategy: AnonymizationStrategy, +} + +/// The default is [`AnonymizationStrategy::PassThrough`], which is the variant that +/// claims the least rather than the one a user most often ends on. +/// +/// A default stands in for "no strategy was chosen", and the report has to read that +/// as "nothing was done to this column". `PassThrough` does exactly that: the release +/// report calls it Review and says the values are kept unchanged, and +/// `uniqueness::LinkableProjection::for_column` treats it as `WholeValue`, the most +/// linkable projection there is. Every other variant would be a claim in the opposite +/// direction — `Redact` is reported Verified, so a defaulted column would carry a green +/// tick nobody earned for it. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum AnonymizationStrategy { + Auto, + Pseudonymize, + Tokenize, + LocalAi, + Mask, + Label, + Redact, + #[default] + PassThrough, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ColumnControl { + pub column_index: usize, + pub type_override: Option, + pub strategy: AnonymizationStrategy, +} +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum LocalNerRunStatus { + #[default] + Disabled, + Completed, + Unavailable, + Failed, + Incomplete, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum DeterministicDetectionStatus { + #[default] + Completed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum DetectionReviewReason { + DetectorFailed, + CandidateRejected, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetectionRunSummary { + pub deterministic: DeterministicDetectionStatus, + pub local_ner: LocalNerRunStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detector_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_version: Option, + #[serde(default)] + pub examined_cells: usize, + #[serde(default)] + pub total_eligible_cells: usize, + #[serde(default)] + pub skipped_oversized_cells: usize, + #[serde(default)] + pub accepted_candidates: usize, + #[serde(default)] + pub rejected_candidates: usize, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub review_reasons: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, +} diff --git a/crates/csv-anonymizer-core/src/types/processing.rs b/crates/csv-anonymizer-core/src/types/processing.rs new file mode 100644 index 0000000..a6c4e7e --- /dev/null +++ b/crates/csv-anonymizer-core/src/types/processing.rs @@ -0,0 +1,140 @@ +use super::*; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct ProcessOptions<'a> { + pub smart_replacements: Option<&'a crate::smart::SmartReplacementMap>, + /// Validated run-only secret. The key type cannot be serialized. + pub tokenization_key: Option<&'a crate::strategies::TokenizationKey>, + /// Mapping entries this run may hold before it refuses to continue, or `None` for + /// `TransformState::MAPPING_ENTRY_CEILING`. + /// + /// A run option rather than a constant read at the point of use because the + /// ceiling is a resource limit, and a resource limit that cannot be set to a + /// reachable value cannot be tested: the real one stands for about 5 GB of + /// mapping, so the only way to show that the run loop *consults* it — and that + /// refusing leaves no partial output — is to hand the loop a smaller one. + pub mapping_entry_ceiling: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProcessProgress { + pub rows_processed: usize, +} + +pub struct ProcessControl<'a> { + pub on_progress: Option<&'a mut dyn FnMut(ProcessProgress)>, + pub should_cancel: Option<&'a dyn Fn() -> bool>, +} + +impl ProcessControl<'_> { + pub fn none() -> Self { + Self { + on_progress: None, + should_cancel: None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProcessResult { + pub row_count: usize, + pub output_path: PathBuf, + pub duration_ms: u128, + pub transform_report: TransformReport, +} +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransformReport { + pub unique_pseudonym_values: usize, + pub reused_pseudonym_values: usize, + pub collisions_avoided: usize, + pub exhausted_pseudonym_pools: usize, + pub opaque_token_values: usize, + pub keyed_token_values: usize, + pub keyed_token_columns: Vec, + pub smart_replacement_requests: usize, + pub smart_replacement_values: usize, + pub smart_replacement_rejections: usize, + pub smart_replacement_rejection_reasons: Vec, + pub smart_replacement_fallbacks: usize, + pub shape_fallback_values: usize, + /// Selected values that a strategy claiming to transform returned byte-for-byte + /// unchanged after trimming. This is the exact residual-leak guard; it is not a + /// heuristic scan for merely realistic-looking replacement data. + pub unchanged_sensitive_values: usize, + pub unchanged_sensitive_columns: Vec, + /// Unique selected high/medium-risk source values fingerprinted for the broad + /// residual audit, and released cell fingerprints compared against them. + pub residual_audit_source_values: usize, + pub residual_audit_output_values: usize, + pub residual_audit_matches: usize, + /// True when either fingerprint set reached its independent memory ceiling. + pub residual_audit_incomplete: bool, + pub column_value_distributions: Vec, + pub row_uniqueness: Option, +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TransformContext<'a> { + pub column_name: &'a str, + pub column_index: usize, + pub row_index: usize, + pub empty_format: EmptyFormat, +} + +impl<'a> TransformContext<'a> { + /// The context for one cell of `column`. + /// + /// Every caller wants exactly this: the column's own name, index and empty + /// format, plus where the value sits. Assembled by hand at each site, the fields + /// are three same-shaped values a mistake can silently swap — a context built + /// with another column's index makes the transform key its mapping under the + /// wrong column, so two columns share replacements and a value redacted in one + /// reappears in the other. + pub fn for_column(column: &'a ColumnMetadata, row_index: usize) -> Self { + Self { + column_name: &column.name, + column_index: column.index, + row_index, + empty_format: column.empty_format, + } + } +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SmartReplacementEntry { + pub column_index: usize, + pub original: String, + pub replacement: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SmartReplacementRejectionReason { + UnexpectedOriginal, + MissingOutput, + EmptyOutput, + SameAsOriginal, + ContainsOriginal, + /// The replacement was, or contained, a source value belonging to a *different* + /// row of the same column. + /// + /// Kept apart from [`Self::ContainsOriginal`] because the two describe different + /// events and only one of them moves a person's data between rows. + /// `ContainsOriginal` means the model echoed back the value it was asked to + /// replace, which wastes the request; this means it emitted somebody else's real + /// value, which would have published that value against the wrong record. A + /// report that merged them could not say which had happened. + MatchesOtherOriginal, + ControlCharacter, + DuplicateOriginal, + DuplicateOutput, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SmartReplacementRejectionCount { + pub reason: SmartReplacementRejectionReason, + pub count: usize, +} diff --git a/crates/csv-anonymizer-core/src/types/report.rs b/crates/csv-anonymizer-core/src/types/report.rs new file mode 100644 index 0000000..fe1991f --- /dev/null +++ b/crates/csv-anonymizer-core/src/types/report.rs @@ -0,0 +1,577 @@ +use super::*; +use serde::{Deserialize, Serialize}; +/// What one column's consistent pseudonyms reveal about the values behind them. +/// +/// A consistent pseudonym preserves equality, which is why the strategies that +/// produce one keep a dataset joinable. The cost is that equality also preserves +/// the *shape* of the value distribution, and the shape is enough to work against +/// the mapping: +/// +/// - Few `distinct_values` over many `total_values` means the mapping can be +/// relabelled by frequency, by anyone who knows how the real field is +/// distributed. The tokens stay opaque; the histogram does not. +/// - A `singleton_values` entry is a pseudonym covering exactly one row, which +/// singles that record out however unguessable the token looks. +/// - `max_value_occurrences` is the most common value's row count, and it is the +/// sharpest of the four for frequency inversion — the dominant value is the +/// attacker's easiest anchor, and inverting that one pseudonym recovers +/// `max_value_occurrences / total_values` of the column in a single step. Acted on +/// as a *share* rather than a count, because a count means something different in a +/// 200-row file than in a 5-million-row one. The threshold it is compared against is +/// `MIN_INVERTIBLE_DOMINANT_SHARE`, which is private, so this names it rather than +/// linking to it. +/// +/// Reported per column because the risk is per column: one low-cardinality column +/// is not made safer by a high-cardinality neighbour. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ColumnValueDistribution { + pub column_index: usize, + pub distinct_values: usize, + pub total_values: usize, + pub singleton_values: usize, + /// Values seen exactly twice. Carried for the same reason as the singleton + /// count: the two together are what let a *sampled* distribution say anything + /// about the input it was drawn from — see `estimated_distinct_values`, which is + /// private, so this names it rather than linking to it. + #[serde(default)] + pub doubleton_values: usize, + pub max_value_occurrences: usize, +} + +/// Minimum values before the cardinality test means anything. +/// +/// A floor is not optional. `distinct_values <= total_values`, so any +/// `distinct_values < K` test is *vacuously true* whenever `total_values < K` — on a +/// 5-row fixture every column looks low-cardinality, including the unique ID +/// columns. Measured on `tests/fixtures/large.csv` truncated to varying lengths: at +/// 5 rows all 7 columns trip the test, including `id`, `email` and `user_uuid`; the +/// mis-flags disappear at 10 rows, i.e. at exactly K. +/// +/// 50 rather than the minimum 10 because the margin matters: at 20 rows the +/// highest-cardinality realistic column (`name`) shows 19 distinct values, one step +/// from the boundary, while at 50 rows it shows 42 — a margin of 32. It also +/// silences every short test fixture, which is where all the noise was. +const CARDINALITY_FLOOR: usize = 50; + +/// Distinct values below which a mapping is treated as frequency-invertible. +/// +/// The two genuinely low-cardinality columns in the corpus saturate at 8 (`country`) +/// and 4 (`status`); the next column up saturates at 100 (`name`). The data pins this +/// constant only to the interval (8, 100] — the decade between is unpopulated — so 10 +/// is the conservative end of the measured interval, and it agrees with the common +/// convention that fewer than about ten buckets makes a frequency table trivially +/// labellable. +const MAX_INVERTIBLE_DISTINCT_VALUES: usize = 10; + +/// Distinct-to-total ratio below which a mapping is treated as frequency-invertible, +/// for columns too large for the absolute test to catch. +/// +/// Reads as "each pseudonym covers more than twenty rows on average". It exists for +/// the case the absolute test misses: `large.csv name` has 100 distinct values over +/// 10500 rows, so `distinct < 10` is false while the ratio is 0.0095. +/// +/// 0.05 rather than a looser 0.20 by a subsumption argument rather than measurement. +/// `distinct/total < 0.05` is already implied by `distinct < 10` for any total below +/// 200, so the ratio term is inert on small and medium inputs and the verdict there +/// rests entirely on the stable statistic — the absolute count, which saturates. The +/// ratio only takes over once there are enough rows for it to have converged. A 0.20 +/// threshold would instead activate immediately above the floor and flag, say, 2000 +/// distinct names in a 10500-row file, which is a five-row group. +const MAX_INVERTIBLE_DISTINCT_RATIO: f64 = 0.05; + +/// Sample coverage below which a *sampled* distribution is treated as saying nothing +/// about the input's distinct count, so the ratio test is skipped. Good–Turing coverage, +/// `1 - singletons/values`; the measurements pin it only to the interval (0.40, 0.87] and +/// 0.75 sits inside with margin at both ends. Not measured on real production data, +/// non-Zipf skew, or columns sitting near the gate itself. +/// See docs/calibration.md#min_sample_coverage for the measurements behind this. +const MIN_SAMPLE_COVERAGE: f64 = 0.75; + +/// Share of a column's values carried by its single most common value, at or above which +/// the mapping is treated as frequency-invertible. Catches the shape neither other term +/// sees: thousands of distinct values, one of them covering most of the rows. A share and +/// not a count, so it means the same in a 200-row file and a 5-million-row one. The +/// measurements pin it to [1/3, 0.35]; not measured on real production columns. +/// See docs/calibration.md#min_invertible_dominant_share for the measurements behind this. +const MIN_INVERTIBLE_DOMINANT_SHARE: f64 = 1.0 / 3.0; + +/// Which of the three tests judged a distribution frequency-invertible. +/// +/// Exists so a warning can name the evidence it actually has. The three terms catch +/// genuinely different shapes — a handful of values, one value dominating a diverse +/// column, and many small groups across a large one — and a single wording cannot +/// describe all three without describing at least two of them wrongly. Reporting a +/// column of 101 values where one covers half the rows as holding "only 101 distinct +/// value(s)" would be true, would read as reassuring, and would name a risk the column +/// does not have while staying silent about the one it does. +/// +/// Each variant carries the figure its wording needs, computed at the point the test +/// fires, because [`ColumnValueDistribution::estimated_distinct_values`] and +/// [`ColumnValueDistribution::dominant_value_share`] are private and a message +/// builder outside this module cannot recover them from the public fields. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum FrequencyInversionRisk { + /// Few enough distinct values that the whole frequency table is labellable. + FewDistinctValues, + /// One value carries [`MIN_INVERTIBLE_DOMINANT_SHARE`] or more of the column, so + /// inverting that single pseudonym recovers `share` of it. + DominantValue { share: f64 }, + /// Enough rows per distinct value that the groups can be matched by size, even + /// though no single value dominates and there are too many to enumerate. + LargeGroups { estimated_distinct_values: usize }, +} + +impl ColumnValueDistribution { + /// Builds the distribution of `values`, skipping the ones a transform would skip. + /// + /// Uses [`crate::detection::is_empty_value`] and [`crate::smart::value_identity_key`] + /// so that a distribution measured over a detection sample is comparable with one + /// accumulated during a run: the transform returns early on empty values and folds + /// case and padding, so counting either differently here would make the pre-run + /// warning disagree with the post-run report on the same data. + pub(crate) fn from_values(column_index: usize, values: &[String]) -> Self { + let mut occurrences: std::collections::HashMap = + std::collections::HashMap::new(); + let mut total_values = 0usize; + for value in values { + let trimmed = value.trim(); + if crate::detection::is_empty_value(trimmed) { + continue; + } + total_values += 1; + *occurrences + .entry(crate::smart::value_identity_key(trimmed)) + .or_insert(0) += 1; + } + + Self { + column_index, + distinct_values: occurrences.len(), + total_values, + singleton_values: occurrences.values().filter(|count| **count == 1).count(), + doubleton_values: occurrences.values().filter(|count| **count == 2).count(), + max_value_occurrences: occurrences.values().copied().max().unwrap_or(0), + } + } + + /// Whether consistent pseudonyms over this distribution could be relabelled by + /// frequency analysis. + /// + /// Singleton counts deliberately play no part. A singleton rule reads as the + /// record-isolation risk and sounds right, but measured against the corpus it + /// flags every unique-key column — `id`, `email`, `user_uuid`, `created_at` — + /// which are exactly the columns users pseudonymize as a matter of course. That + /// isolation is a property of the input being unique already, not something + /// consistent pseudonymization introduces, and `distinct == total` says it more + /// directly. A warning that fires on every email column is noise. + /// + /// See [`CARDINALITY_FLOOR`], [`MAX_INVERTIBLE_DISTINCT_VALUES`], + /// [`MIN_INVERTIBLE_DOMINANT_SHARE`], [`MAX_INVERTIBLE_DISTINCT_RATIO`] and + /// [`MIN_SAMPLE_COVERAGE`] for what each constant is measured against. + /// + /// For a distribution measured over the whole column. A sampled one has to say + /// how large the column really is — see [`Self::frequency_inversion_risk_in`]. + pub(crate) fn risks_frequency_inversion(&self) -> bool { + self.frequency_inversion_risk().is_some() + } + + /// [`Self::risks_frequency_inversion`], with the reason it answered yes. + pub(crate) fn frequency_inversion_risk(&self) -> Option { + self.frequency_inversion_risk_in(self.total_values) + } + + /// As [`Self::risks_frequency_inversion`], for a distribution measured over a + /// sample of a column that holds `population_values` values in total. + /// + /// The distinction is not pedantic — getting it wrong is what made the pre-run + /// warning miss the case it exists for. The ratio test divides distinct values by + /// the column's size, and a sample's size is the *sample's*, capped at a hundred + /// or so. Measured against that cap, `distinct / total < 0.05` needs fewer than + /// five distinct values in a hundred, which the absolute test has already caught: + /// the ratio term was inert on every sampled distribution while being the only + /// term that catches a column of thirty values in five thousand rows. Such a + /// column drew no warning until after its output had been written. + /// + /// Dividing by the true row count instead is wrong in the other direction, and + /// worse: a hundred sampled values cannot look like a million distinct ones, so a + /// fully unique column would score 0.005 and be flagged as trivially invertible. + /// [`MIN_SAMPLE_COVERAGE`] is the gate that separates the two, and + /// [`Self::estimated_distinct_values`] is what gets compared once it opens. + /// + /// `population_values` is the column's row count, an upper bound on how many + /// values it holds — empty cells are not values. Erring high makes the ratio + /// smaller and the warning likelier, which is the safe direction for a warning + /// the user can dismiss by choosing another strategy. + /// + /// The dominant-value term needs no equivalent adjustment, and that is the reason it + /// sits where it does. It compares two figures the distribution measured *itself* — + /// a share of its own values — so `population_values` never enters. A share is also + /// the one statistic a small sample can estimate here, unlike its distinct count, + /// which is why only the distinct-count term needs a population figure and a gate. + /// See docs/calibration.md#sample-share-vs-distinct-count for the measurements. + /// + /// The order of the three tests is the order of decreasing certainty, and it is + /// also what each variant means: a later variant implies the earlier tests + /// declined. So `LargeGroups` is only ever reported for a column with at least + /// [`MAX_INVERTIBLE_DISTINCT_VALUES`] distinct values and no dominant one, which is + /// what lets its wording talk about average group size without qualification. + pub(crate) fn frequency_inversion_risk_in( + &self, + population_values: usize, + ) -> Option { + if self.total_values < CARDINALITY_FLOOR { + return None; + } + if self.distinct_values < MAX_INVERTIBLE_DISTINCT_VALUES { + return Some(FrequencyInversionRisk::FewDistinctValues); + } + // Deliberately ahead of the coverage gate. The shape this catches — one dominant + // value over a long tail of near-unique ones — is singleton-heavy and therefore + // *low*-coverage, so behind the gate this term would be silent on exactly the + // See docs/calibration.md#min_sample_coverage for the figures. + let share = self.dominant_value_share(); + if share >= MIN_INVERTIBLE_DOMINANT_SHARE { + return Some(FrequencyInversionRisk::DominantValue { share }); + } + if self.sample_coverage() < MIN_SAMPLE_COVERAGE { + return None; + } + let estimated_distinct_values = self.estimated_distinct_values(); + if (estimated_distinct_values as f64) / (population_values.max(1) as f64) + < MAX_INVERTIBLE_DISTINCT_RATIO + { + return Some(FrequencyInversionRisk::LargeGroups { + estimated_distinct_values, + }); + } + None + } + + /// Share of this distribution's values carried by its single most common value. + /// + /// The fraction of the column that one inverted pseudonym recovers, so it is also + /// the size of the leak the warning is reporting. `0.0` for a distribution that + /// measured nothing, and `1 / total_values` for a fully unique column — which is + /// below [`MIN_INVERTIBLE_DOMINANT_SHARE`] for any column past + /// [`CARDINALITY_FLOOR`], so a unique column cannot trip the term that reads this. + fn dominant_value_share(&self) -> f64 { + if self.total_values == 0 { + return 0.0; + } + (self.max_value_occurrences as f64) / (self.total_values as f64) + } + + /// Good–Turing sample coverage: the estimated share of this column's values that + /// belong to a group the distribution has already seen. + /// + /// Exactly `1.0` for a distribution measured over a whole column with no + /// singletons, and `0.0` for one where every value was seen once — which is both + /// a unique column measured exactly and a sample that has learned nothing. + /// [`Self::frequency_inversion_risk_in`] is what distinguishes those, by only + /// consulting coverage once the absolute test has declined. + fn sample_coverage(&self) -> f64 { + if self.total_values == 0 { + return 0.0; + } + 1.0 - (self.singleton_values as f64) / (self.total_values as f64) + } + + /// Chao1: the distinct count this distribution implies for the whole column, + /// including groups it did not see. + /// + /// `distinct + f1² / 2·f2` over singletons and doubletons — the standard + /// lower-bound estimator, on the reasoning that how many groups you saw *once* + /// tells you how many you missed entirely. Returns the observed count unchanged + /// for a distribution with no singletons, which is what a fully measured column + /// with every value repeated looks like. + /// + /// A lower bound, so it under-estimates rather than over-estimates a diverse + /// column, which is the wrong direction for a warning — hence + /// [`MIN_SAMPLE_COVERAGE`] refusing the comparison before this is consulted. + /// `pub(crate)` because the preflight memory projection needs the same estimate + /// and a second copy of a statistical estimator is a copy that drifts. + pub(crate) fn estimated_distinct_values(&self) -> usize { + if self.doubleton_values > 0 { + return self.distinct_values + + self.singleton_values.saturating_pow(2) / (2 * self.doubleton_values); + } + // The bias-corrected form, for when nothing was seen exactly twice and the + // ratio above would divide by zero. + self.distinct_values + + self + .singleton_values + .saturating_mul(self.singleton_values.saturating_sub(1)) + / 2 + } +} +/// How exposed the released rows are once every column is read together. +/// +/// The rest of this crate's privacy figures are per column, so a file can be reported as +/// having no unselected high or medium risk column while postcode, birth date and job +/// title jointly single out a third of its rows. This is the figure that says so. +/// +/// Measured over the columns an outsider could match against data they already hold — +/// see `crate::uniqueness::LinkableProjection` for how that subset is decided and what it +/// deliberately leaves out. Every count here is over that subset except +/// `distinct_rows_all_columns`. +/// +/// Absent rather than zeroed where there are no rows: unstructured text and single pasted +/// values never populate it, and a summary claiming zero unique rows would read as a clean +/// result from a check that never ran. +/// +/// A DTO, like [`DetectionCoverageSummary`] and unlike the private `DetectionCoverage`: a flat +/// snapshot with no invariant of its own to protect, so `Deserialize` costs nothing here. +/// The relationships between these figures — that the counts are over the columns named, +/// that a stopped measurement zeroes the rest — are established in +/// `crate::uniqueness::RowUniquenessTracker::summary`, which is the only thing that builds +/// one outside tests. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RowUniquenessSummary { + /// Data rows hashed. Blank rows are written through untransformed and are not counted. + pub rows_measured: usize, + /// The columns the measure read, each with what an outsider could match it on, in + /// column order. Empty means nothing released is matchable — a statement about how the + /// columns were transformed, not a finding that the data is anonymous. + /// + /// Only columns that actually yielded something are listed. A column whose projection + /// came back empty on every row contributed nothing to any class and is named nowhere, + /// so no claim here can rest on a column that in fact carried no signal. + pub matched_columns: Vec, + /// Distinct equivalence classes over the subset. + pub distinct_classes: usize, + /// Rows alone in their class: someone holding those columns for a person finds + /// exactly their row. + pub unique_rows: usize, + /// The k-anonymity floor — the smallest class present. One freak record sets it, which + /// is why `fifth_percentile_class_size` is reported beside it. + pub smallest_class: usize, + /// The class size at or below which the most exposed 5% of rows sit. + pub fifth_percentile_class_size: usize, + /// Distinct rows over *every* released column, subset rule not applied. Answers the + /// separate and simpler question of whether the file could be shuffled or aggregated, + /// and acts as a check on the subset rule: a wide gap between this and + /// `distinct_classes` means the rule is doing a lot of work and deserves a look. + /// + /// `None` when this histogram alone outgrew what the check keeps. It fills faster than + /// the other one by construction — whole rows against projections of a subset of them — + /// so it is suppressed on its own rather than taking the joint measure down with it. + pub distinct_rows_all_columns: Option, + /// The *joint* measurement stopped early because the file held more classes than the + /// check keeps. Every count above is then a lower bound, and no verified claim may rest + /// on them. Set only by the linkable histogram: `distinct_rows_all_columns` going + /// absent does not make the joint figures incomplete. + pub measurement_incomplete: bool, + /// What `unique_rows` would have been with each matched column dropped, ascending by + /// that count, then by column index. + /// + /// The only figure here a reader can act on. `unique_rows` says how exposed the file is; + /// this says which column to change to fix it, which is the difference between a report + /// and an alarm. + /// + /// Empty when `drop_attribution_incomplete` is set, and also when the file has no matched + /// column to drop. The two are told apart by that flag rather than by the emptiness of + /// this list, because "we did not measure" and "there is nothing to drop" are opposite + /// findings that would otherwise look identical. + pub drop_column_effects: Vec, + /// The attribution was not run, or was stopped, so `drop_column_effects` is empty for a + /// reason other than there being nothing to say. + /// + /// Set when the joint measurement itself is incomplete (there is no baseline to compare + /// against), when the file has more columns than the attribution tracks, or when the + /// leave-one-out histograms outgrew their shared budget. Reported rather than hidden: a + /// reader who is told nothing about which column to drop should know whether that is + /// because no column would help or because nobody looked. + pub drop_attribution_incomplete: bool, +} + +/// What dropping one column would do to the count of rows that stand alone. +/// +/// Exact, not an estimate: the count is read off a second equivalence-class histogram built +/// over the same rows with this column's contribution removed, in the same single pass. That +/// matters because the intuitive estimate is wrong in both directions — dropping a column +/// whose projection is nearly constant changes almost nothing however revealing the column +/// looks, and dropping one of two correlated columns can change almost nothing either. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DropColumnEffect { + /// `ColumnMetadata::index`, as everywhere else. + pub column_index: usize, + /// Rows that would still be alone in their class with this column dropped, every other + /// column unchanged. Compare against `RowUniquenessSummary::unique_rows`; it can never + /// be larger, since removing a column only ever merges classes. + pub unique_rows_without: usize, +} + +/// One column the joint measure read, and what it was matched on. +/// +/// The pairing is the point, and it must not be split back into two lists of column +/// indices — value-carrying and format-only. Two categories cannot express three kinds of +/// contribution, and the missing one is the partial match, the most common kind on a +/// pseudonymized file: a report built from two lists tells the reader that rows "share their +/// combination of birth_date, email" when what they share is a decade and a domain. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MatchedColumn { + /// `ColumnMetadata::index`, which is how every other report structure names a column. + pub column_index: usize, + pub matched_on: MatchedPart, + /// Whether every measured row actually carried `matched_on`, or only some of them did. + /// + /// `matched_on` is decided once per column, from the column's strategy and detected type + /// alone — no cell value can change it. The values can still disagree with it: a cell + /// that does not fit its column's detected shape is pseudonymized generically, and the + /// projection returns nothing for that row. So a `Timestamp` column where one value in a + /// hundred parses is `DateDecadeAndTime`, and the finding said the rows "share the decade + /// and time of birth_date" of ninety-nine rows that carry no decade. + /// + /// A `bool` and not a count, because the report needs to know only whether to qualify the + /// phrase. Quoting "matched on 1 of 100 rows" would invite the reader to weigh a number + /// that is not the one that matters: those rows were counted as sharing nothing there, + /// so the arithmetic is already right and only the wording was over-claiming. + pub matched_every_row: bool, +} + +/// What survived a column that an outsider holding the original could match against. +/// +/// Written so each variant completes the sentence "rows share **…** with each other", which +/// is what forces the distinction: `WholeValue` licenses naming the column bare, and +/// nothing else does. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum MatchedPart { + /// The released cell is the original, or a skeleton of it anyone can derive. The column + /// may be named on its own. + WholeValue, + /// Everything from the last `@`: the employer, not the person. + EmailDomain, + /// The decade of the released date, and the time of day exactly. Approximate on the date + /// half by construction — see `crate::uniqueness::LinkableProjection` — and named as an + /// approximation so a reader is not told their rows share a birth date when they share a + /// decade. + DateDecadeAndTime, + /// No part of the value, only a format property: a digit count, a separator layout, a + /// number of name parts, a mask's word and letter counts. Counted like any other, + /// because a joint measure is where individually weak signals combine, but never named + /// as though it were the value. + SurvivingFormat, + /// Not the cell at all — only whether it was blank, and with which blank token. + /// + /// A cell the engine reads as empty is written through verbatim before any strategy + /// runs, so even a redacted column publishes its missingness pattern, and someone + /// holding the original record knows which of its fields were blank. Named apart because + /// "the blank-cell pattern of address" and "address" are wildly different claims, and + /// because the remedy differs too: no strategy fixes this one. + BlankPattern, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PrivacyReport { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detection_run_summary: Option, + pub direct_identifiers: usize, + pub quasi_identifiers: usize, + pub pseudonymized_columns: usize, + pub smart_replacement_columns: usize, + pub opaque_token_columns: usize, + pub masked_columns: usize, + #[serde(default)] + pub labelled_columns: usize, + #[serde(default)] + pub redacted_columns: usize, + pub pass_through_columns: usize, + pub unique_pseudonym_values: usize, + pub reused_pseudonym_values: usize, + pub collisions_avoided: usize, + pub exhausted_pseudonym_pools: usize, + pub opaque_token_values: usize, + #[serde(default)] + pub keyed_token_values: usize, + #[serde(default)] + pub keyed_token_columns: Vec, + pub smart_replacement_values: usize, + #[serde(default)] + pub smart_replacement_rejections: usize, + #[serde(default)] + pub smart_replacement_rejection_reasons: Vec, + pub smart_replacement_fallbacks: usize, + #[serde(default)] + pub shape_fallback_values: usize, + #[serde(default)] + pub readiness: ReleaseReadiness, + #[serde(default)] + pub evidence: Vec, + #[serde(default)] + pub column_reports: Vec, + #[serde(default)] + pub column_value_distributions: Vec, + #[serde(default)] + pub row_uniqueness: Option, + #[serde(default)] + pub utility_metrics: Vec, + pub notes: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReleaseReadiness { + pub status: ReleaseReadinessStatus, + pub blockers: Vec, + pub review_items: Vec, + pub verified_items: Vec, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ReleaseReadinessStatus { + Verified, + #[default] + Review, + Blocked, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReleaseEvidenceItem { + pub id: String, + pub label: String, + pub status: ReleaseEvidenceStatus, + pub detail: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ReleaseEvidenceStatus { + Verified, + Review, + Blocked, + Info, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ColumnReleaseReport { + pub column_index: usize, + pub column_name: String, + pub selected: bool, + pub detected_type: DataType, + pub pii_risk: PiiRisk, + pub strategy: AnonymizationStrategy, + pub action: String, + pub status: ReleaseEvidenceStatus, + pub detail: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UtilityMetric { + pub label: String, + pub value: String, + pub status: ReleaseEvidenceStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[cfg(test)] +mod tests; diff --git a/crates/csv-anonymizer-core/src/types/report/tests.rs b/crates/csv-anonymizer-core/src/types/report/tests.rs new file mode 100644 index 0000000..745f2af --- /dev/null +++ b/crates/csv-anonymizer-core/src/types/report/tests.rs @@ -0,0 +1,374 @@ +use super::*; +fn distribution(distinct: usize, total: usize) -> ColumnValueDistribution { + ColumnValueDistribution { + column_index: 0, + distinct_values: distinct, + total_values: total, + // No singletons, so coverage is 1.0 and the ratio test is reachable. + // The shapes that make coverage matter are pinned separately below. + singleton_values: 0, + doubleton_values: 0, + max_value_occurrences: 0, + } +} + +/// The floor is the whole reason this predicate is not just `distinct < 10`. +/// `distinct_values <= total_values`, so below the floor the absolute test is +/// vacuously true and every short column — including unique-key columns — looks +/// low-cardinality. Measured: at 5 rows all 7 columns of `large.csv` trip it. +#[test] +fn cardinality_risk_ignores_columns_with_too_few_values_to_judge() { + // The vacuous case: every value distinct, yet fewer than the constant. + assert!(!distribution(5, 5).risks_frequency_inversion()); + assert!(!distribution(49, 49).risks_frequency_inversion()); + // Genuinely low cardinality, but still not enough rows to claim it. + assert!(!distribution(4, 49).risks_frequency_inversion()); + // One value more and the same shape is judgeable. + assert!(distribution(4, 50).risks_frequency_inversion()); +} + +#[test] +fn cardinality_risk_flags_few_distinct_values_at_the_boundary() { + assert!(distribution(9, 60).risks_frequency_inversion()); + // Ten distinct over sixty is 0.167, above the ratio threshold too, so this + // pins both terms at once: neither may fire here. + assert!(!distribution(10, 60).risks_frequency_inversion()); +} + +/// The ratio term exists for columns the absolute term cannot reach. Measured on +/// `large.csv name`: 100 distinct over 10500 values. +#[test] +fn cardinality_risk_flags_a_large_column_by_ratio() { + assert!(distribution(100, 10_500).risks_frequency_inversion()); + // Same distinct count, far fewer rows: 0.1 is above the threshold, and the + // absolute term does not reach 100 either. + assert!(!distribution(100, 1_000).risks_frequency_inversion()); +} + +/// Below 200 values the ratio term is subsumed by the absolute one, which is why +/// 0.05 was chosen: small and medium inputs are judged on the statistic that +/// saturates rather than on one that keeps drifting as more rows are read. +#[test] +fn the_ratio_term_is_inert_below_two_hundred_values() { + for total in [50usize, 100, 199] { + for distinct in 10..=total { + let subject = distribution(distinct, total); + assert!( + !subject.risks_frequency_inversion(), + "{distinct} distinct over {total} fired without the absolute term" + ); + } + } +} + +/// Counts the values a transform would count: empties skipped, case and padding +/// folded. A pre-run warning that measured differently from the run would +/// contradict the report built from the same data. +#[test] +fn distribution_from_values_matches_the_transform_value_identity() { + let values = [ + "Sales".to_string(), + " sales ".to_string(), + "SALES".to_string(), + "Legal".to_string(), + String::new(), + "null".to_string(), + "NULL".to_string(), + " ".to_string(), + ]; + + let subject = ColumnValueDistribution::from_values(3, &values); + + assert_eq!(subject.column_index, 3); + assert_eq!(subject.distinct_values, 2); + assert_eq!(subject.total_values, 4); + assert_eq!(subject.singleton_values, 1); + assert_eq!(subject.doubleton_values, 0); + assert_eq!(subject.max_value_occurrences, 3); +} + +/// Singletons and doubletons are counted after identity folding too, since they +/// are what [`ColumnValueDistribution::estimated_distinct_values`] reads. +#[test] +fn distribution_counts_singletons_and_doubletons() { + let values = ["a", "a", "b", "b", "c", "d", "d", "d"] + .map(str::to_string) + .to_vec(); + + let subject = ColumnValueDistribution::from_values(0, &values); + + assert_eq!(subject.distinct_values, 4); + assert_eq!(subject.singleton_values, 1, "c"); + assert_eq!(subject.doubleton_values, 2, "a and b"); +} + +/// A sampled column of a large file is the case the ratio test could not see. The +/// numbers are the measured ones for 30 departments over 5000 rows. +#[test] +fn a_sampled_distribution_is_judged_against_the_columns_real_size() { + let sampled = ColumnValueDistribution { + column_index: 1, + distinct_values: 29, + total_values: 100, + singleton_values: 5, + doubleton_values: 4, + max_value_occurrences: 8, + }; + + // Against the sample's own size the ratio is 0.29 and the absolute term does + // not reach 29 either, so judged this way the column looks safe. + assert!(!sampled.risks_frequency_inversion()); + // Against the file it was drawn from, it is 30-odd values over 5000 rows. + assert!(sampled.frequency_inversion_risk_in(5_000).is_some()); +} + +/// The counterweight, and why the ratio test cannot simply divide by the row +/// count: a sample of a hundred cannot look like a million distinct values, so a +/// fully unique column scores 0.005 on the ratio alone. Coverage is what +/// distinguishes "few distinct values" from "a sample that learned nothing". +#[test] +fn a_sample_that_learned_nothing_cannot_raise_a_warning() { + let every_value_new = ColumnValueDistribution { + column_index: 0, + distinct_values: 100, + total_values: 100, + singleton_values: 100, + doubleton_values: 0, + max_value_occurrences: 1, + }; + + assert!( + every_value_new + .frequency_inversion_risk_in(1_000_000) + .is_none() + ); +} + +/// Coverage gates the ratio term only. A column with few enough distinct values +/// is invertible whatever the coverage figure says, so the absolute term has to +/// answer first. +#[test] +fn the_absolute_term_is_not_gated_by_coverage() { + let sparse_but_unsaturated = ColumnValueDistribution { + column_index: 0, + distinct_values: 9, + total_values: 60, + // Coverage 0.0, far below the gate. + singleton_values: 60, + doubleton_values: 0, + max_value_occurrences: 1, + }; + + assert!( + sparse_but_unsaturated + .frequency_inversion_risk_in(1_000_000) + .is_some() + ); +} + +/// Chao1 estimates the groups a sample missed from how many it saw exactly once. +/// Pinned on the measured shape rather than the formula, so a rewrite that keeps +/// the behaviour passes and one that changes it fails. +#[test] +fn the_estimated_distinct_count_allows_for_unseen_values() { + let sampled = ColumnValueDistribution { + column_index: 0, + distinct_values: 29, + total_values: 100, + singleton_values: 5, + doubleton_values: 4, + max_value_occurrences: 8, + }; + // 29 + 25/8 = 32, against a true count of 30. + assert_eq!(sampled.estimated_distinct_values(), 32); + + // Nothing seen twice: the bias-corrected form, and no division by zero. + let no_doubletons = ColumnValueDistribution { + distinct_values: 10, + total_values: 100, + singleton_values: 4, + doubleton_values: 0, + ..ColumnValueDistribution::default() + }; + assert_eq!(no_doubletons.estimated_distinct_values(), 10 + 6); + + // Every value repeated: nothing was missed. + let fully_measured = ColumnValueDistribution { + distinct_values: 30, + total_values: 5_000, + singleton_values: 0, + doubleton_values: 0, + ..ColumnValueDistribution::default() + }; + assert_eq!(fully_measured.estimated_distinct_values(), 30); +} + +#[test] +fn distribution_from_no_usable_values_is_empty_rather_than_risky() { + let values = [String::new(), "null".to_string(), " ".to_string()]; + + let subject = ColumnValueDistribution::from_values(0, &values); + + assert_eq!(subject.total_values, 0); + assert_eq!(subject.max_value_occurrences, 0); + assert!(!subject.risks_frequency_inversion()); +} + +/// A default distribution is what a caller that never measured produces, so it +/// must never be able to raise a warning it has no evidence for. +#[test] +fn a_defaulted_distribution_cannot_raise_a_warning() { + assert!(!ColumnValueDistribution::default().risks_frequency_inversion()); +} + +/// One dominant value over forty-nine near-unique ones. +/// +/// Not invented: these are the exact figures the detection sample computes for +/// `tests/fixtures/dominant-value.csv`, pinned in +/// `service::tests::cardinality::the_dominant_value_fixture_has_the_shape_these_tests_depend_on`, +/// so the unit tests here and the integration tests there are reasoning about one +/// column rather than two that happen to resemble each other. +fn dominant_value_sample() -> ColumnValueDistribution { + ColumnValueDistribution { + column_index: 1, + distinct_values: 50, + total_values: 100, + singleton_values: 49, + doubleton_values: 0, + max_value_occurrences: 51, + } +} + +/// The gap this term closes. A column that is *diverse* — here 50 distinct values in +/// the sample and thousands in the file — but whose most common value covers half the +/// rows leaks half the column the moment one pseudonym is matched, and until this term +/// existed it drew no warning at all: 50 distinct values is far above the absolute +/// limit, and the Chao1 ratio is high precisely because the column is diverse. +/// +/// Both older terms are asserted against as well as the verdict. Without them a change +/// that made the *absolute* term start firing at 50 distinct values would leave this +/// test green while flagging most columns in the corpus. +#[test] +fn a_dominant_value_is_flagged_however_diverse_the_rest_of_the_column_is() { + let subject = dominant_value_sample(); + + assert!(subject.distinct_values >= MAX_INVERTIBLE_DISTINCT_VALUES); + assert!(subject.sample_coverage() < MIN_SAMPLE_COVERAGE); + + assert!(subject.frequency_inversion_risk_in(5_000_000).is_some()); +} + +/// The same column with the dominance taken out of it, and nothing else changed. +/// +/// Half the rows moved off the dominant value onto values of their own, so the column +/// is *more* diverse than the one above while holding the same 100 values. It has to +/// go silent, or the term is reading something other than dominance. +#[test] +fn the_same_column_without_a_dominant_value_stays_silent() { + let spread_out = ColumnValueDistribution { + distinct_values: 76, + singleton_values: 75, + max_value_occurrences: 25, + ..dominant_value_sample() + }; + + assert!(spread_out.frequency_inversion_risk_in(5_000_000).is_none()); +} + +/// The dominant-value term is deliberately ahead of the coverage gate, and this is +/// the reason: the shape it catches is singleton-heavy, so its coverage is low. The +/// fixture's coverage is 0.51 against a 0.75 gate. Behind the gate this term would have +/// been silent on almost every column it was added for: of 2400 measured columns whose +/// top value covered half the rows, none reached coverage 0.75, and of 2400 covering +/// three fifths, six did. +#[test] +fn the_dominant_value_term_is_not_gated_by_coverage() { + let subject = dominant_value_sample(); + + assert!(subject.sample_coverage() < MIN_SAMPLE_COVERAGE); + assert!(subject.frequency_inversion_risk_in(5_000_000).is_some()); +} + +/// A share and not a count, which is the whole point of the constant: the identical +/// shape has to answer the same way at every file size. A count-based rule would +/// have to be either silent on the small file or noisy on the large one. +#[test] +fn the_dominant_value_verdict_does_not_move_with_the_files_size() { + for population in [100usize, 5_000, 1_000_000, 5_000_000] { + assert!( + dominant_value_sample() + .frequency_inversion_risk_in(population) + .is_some(), + "silent at {population} values" + ); + } +} + +/// The boundary, measured on a fully counted column so no estimate is involved: +/// 20 of 60 is exactly a third and fires, 19 of 60 does not. +/// +/// The surrounding counts are chosen so neither other term can answer — 15 distinct +/// clears the absolute limit, and Chao1 over the column's own 60 values is 0.67, +/// nowhere near the 0.05 ratio limit — so the verdict here is the dominant-value +/// term's alone. +#[test] +fn the_dominant_share_boundary_sits_at_one_third() { + let at_the_boundary = ColumnValueDistribution { + column_index: 0, + distinct_values: 15, + total_values: 60, + singleton_values: 10, + doubleton_values: 2, + max_value_occurrences: 20, + }; + assert!(at_the_boundary.risks_frequency_inversion()); + + let one_row_short = ColumnValueDistribution { + max_value_occurrences: 19, + ..at_the_boundary + }; + assert!(!one_row_short.risks_frequency_inversion()); +} + +/// A genuinely unique column has to stay silent, and the dominant-value term must +/// not be what breaks that. Its most common value covers one row, so its share is +/// `1 / total_values` — below a third for any column past the floor, which is why +/// the term needs no special case for it. Pinned at four sizes because a share is +/// exactly the kind of quantity that misbehaves at the small end. +#[test] +fn a_unique_column_cannot_trip_the_dominant_value_term() { + for total in [50usize, 60, 5_000, 1_000_000] { + let unique = ColumnValueDistribution { + column_index: 0, + distinct_values: total, + total_values: total, + singleton_values: total, + doubleton_values: 0, + max_value_occurrences: 1, + }; + + assert!( + unique.frequency_inversion_risk_in(total).is_none(), + "{total} unique values were flagged" + ); + } +} + +/// The false-positive case the constant was calibrated against. This is a measured +/// 100-value sample of a Zipf column with exponent 1.0 over 1000 labels — the +/// ordinary shape of real categorical data, whose top value takes a seventh of the +/// rows. Over 4000 such samples no draw reached a third, and a threshold low enough +/// to catch this shape would fire on most text columns in most files. +#[test] +fn a_mildly_skewed_high_cardinality_column_stays_silent() { + let ordinary_skew = ColumnValueDistribution { + column_index: 0, + distinct_values: 68, + total_values: 100, + singleton_values: 55, + doubleton_values: 8, + max_value_occurrences: 14, + }; + + assert!(ordinary_skew.frequency_inversion_risk_in(100_000).is_none()); +} diff --git a/crates/csv-anonymizer-core/src/types/sampling.rs b/crates/csv-anonymizer-core/src/types/sampling.rs new file mode 100644 index 0000000..f479622 --- /dev/null +++ b/crates/csv-anonymizer-core/src/types/sampling.rs @@ -0,0 +1,192 @@ +use serde::{Deserialize, Serialize}; +/// The largest detection basis any entry point accepts, for any input kind. +/// +/// One limit rather than one per workflow, because the figure comes from one +/// setting. "Sample rows" is not per-workflow: a user who raises it to work on a +/// large CSV file has raised it for the paste workflow too, so a value that reaches +/// the setting has to be a value every entry point will honour. +/// +/// Enforced twice, and both sites read this: `settings::sanitize_settings` clamps +/// what can be stored, and the paste entry points reject an oversized request +/// outright, since they are reachable by callers that never went through settings. +pub const MAX_SAMPLE_ROW_COUNT: usize = 10_000; + +/// The largest display window any entry point accepts, for any input kind. One +/// limit for the same reason as [`MAX_SAMPLE_ROW_COUNT`] — it comes from the +/// "Preview rows" setting, which is likewise not per-workflow. +pub const MAX_PREVIEW_SAMPLE_COUNT: usize = 100; + +/// The smallest detection basis any entry point classifies on, for any input kind. +/// +/// A floor rather than a default: `service::detection_sample_rows` and +/// `direct_input::shared::paste_detection_sample_rows` both raise a caller's "Sample +/// rows" to at least this, so the setting can only ask for more evidence than the +/// default, never less. +/// +/// One constant rather than one per workflow, because the file and paste workflows +/// promise the user that they classify on the same basis. Two literals held that +/// promise by coincidence: changing either alone left a pasted CSV and the same file +/// on disk detecting different types, and so being offered different strategies, with +/// nothing failing to say so. +pub(crate) const DETECTION_SAMPLE_ROW_FLOOR: usize = 100; +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ParsedSample { + pub headers: Vec, + pub rows: Vec>, + /// Data rows read while building the sample. This is the input's full + /// data-row count only when `scanned_entire_input` is true; it is always + /// >= `rows.len()`, because a spread sample thins what it keeps. + pub data_rows_scanned: usize, + /// Whether every data row was read. False only for a head window that hit + /// its cap; detection samples always scan the whole input. + pub scanned_entire_input: bool, +} + +/// What the two figures in a detection-coverage disclosure count. +/// +/// The unit is carried rather than assumed because "rows" is only true for the two +/// tabular entry points. A field-based paste — JSON, YAML, XML, or free text scanned +/// for privacy spans — has no rows to sample: `direct_input::shared::detection_coverage` +/// counts the values of the busiest field, and `PasteAnalyzeData::row_count` is derived +/// separately and can disagree outright. A pasted `{"users": [500 objects]}` shows one +/// row in the UI, and free text shows a match count that is neither its line count nor +/// its row count, so a disclosure hard-coded to "rows" states a figure the user cannot +/// find anywhere on screen and cannot check. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum DetectionCoverageUnit { + Rows, + Values, +} + +impl DetectionCoverageUnit { + /// The noun a disclosure sentence uses for these figures. + pub(crate) fn plural_noun(self) -> &'static str { + match self { + Self::Rows => "rows", + Self::Values => "values", + } + } +} + +/// How much of an input detection actually classified. +/// +/// Detection votes on a bounded sample, so a value occurring in few rows can be +/// missed — and a column whose sensitive values were all missed is never +/// auto-selected, so it is written unchanged. That is a deliberate trade for +/// bounded memory on inputs of any size, but it is only an honest one if the user +/// is told the verdict rests on a sample. This carries the figures needed to say so +/// and the unit they are counted in. +/// +/// Not a DTO. It is `pub(crate)` on purpose: [`Self::new`] is the only place the +/// `examined <= total` invariant is established, and a `Deserialize` impl would let +/// a wire value walk straight past it and report a sample larger than the input it +/// was drawn from. What crosses the IPC boundary is [`DetectionCoverageSummary`], +/// a flat snapshot taken after clamping, plus the report notes and preflight review +/// item built from this. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DetectionCoverage { + /// Rows or values kept for classification. + examined: usize, + /// Rows or values the input holds. + total: usize, + unit: DetectionCoverageUnit, +} + +impl DetectionCoverage { + /// Coverage of the sample a detection pass actually kept. + /// + /// Read off the sample rather than recomputed from the requested row count, + /// because the request is a ceiling and the sample is what happened: a file with + /// fewer rows than the request is fully covered, and asking the request would + /// claim otherwise. + pub(crate) fn from_detection_sample(sample: &ParsedSample) -> Self { + Self::rows(sample.rows.len(), sample.data_rows_scanned) + } + + /// Coverage for an input that has nothing to sample. + /// + /// Both figures are zero, which is not a missing figure but the only truthful + /// pair here: quick-generate has no source input, so no row went unexamined and + /// none was examined either. Only [`Self::is_partial`] is consulted for such an + /// input, and it answers false — the disclosure stays silent, which is right, + /// because there is no sampling to disclose. Nothing may print + /// [`Self::examined`] and [`Self::total`] without gating on + /// [`Self::is_partial`] first, or this constructor renders as "0 of 0". + pub(crate) fn complete() -> Self { + Self::rows(0, 0) + } + + /// Coverage counted in data rows: the two tabular entry points, CSV file and + /// pasted CSV text. + pub(crate) fn rows(examined: usize, total: usize) -> Self { + Self::new(examined, total, DetectionCoverageUnit::Rows) + } + + /// Coverage counted in field values: JSON, YAML, XML and free-text pastes, which + /// have fields rather than rows. See [`DetectionCoverageUnit`]. + pub(crate) fn values(examined: usize, total: usize) -> Self { + Self::new(examined, total, DetectionCoverageUnit::Values) + } + + fn new(examined: usize, total: usize, unit: DetectionCoverageUnit) -> Self { + Self { + examined: examined.min(total), + total, + unit, + } + } + + /// Whether some of the input went unclassified. + pub(crate) fn is_partial(self) -> bool { + self.examined < self.total + } + + pub(crate) fn examined(self) -> usize { + self.examined + } + + pub(crate) fn total(self) -> usize { + self.total + } + + pub(crate) fn unit(self) -> DetectionCoverageUnit { + self.unit + } + + /// The IPC-facing snapshot of this coverage. + pub(crate) fn summary(self) -> DetectionCoverageSummary { + DetectionCoverageSummary { + examined: self.examined, + total: self.total, + unit: self.unit, + is_partial: self.is_partial(), + } + } +} + +/// What detection classified, as the paste analyze result reports it. +/// +/// This exists so a paste user learns that detection sampled before choosing +/// columns, not after the output already exists. The file workflow gets the same +/// disclosure from preflight; the paste workflow has no preflight, so until this +/// crossed the boundary the only place it appeared was the post-transform privacy +/// report — advice ("raise Sample rows") that by then costs a whole second run. +/// +/// Counts only. No value, column name or excerpt goes on the wire here, so the +/// disclosure cannot itself leak what detection missed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetectionCoverageSummary { + pub examined: usize, + pub total: usize, + pub unit: DetectionCoverageUnit, + /// `examined < total`, decided here rather than in the client. + /// + /// The comparison is trivial, which is exactly why it would drift: a client that + /// re-derives it is free to write `<=` or to compare against a row count from a + /// different field, and either mistake silences the warning rather than + /// producing a visible error. + pub is_partial: bool, +} diff --git a/crates/csv-anonymizer-core/src/types/tests.rs b/crates/csv-anonymizer-core/src/types/tests.rs new file mode 100644 index 0000000..ec67aef --- /dev/null +++ b/crates/csv-anonymizer-core/src/types/tests.rs @@ -0,0 +1,292 @@ +use super::*; +use serde_json::json; +use std::path::PathBuf; +#[test] +fn column_metadata_serializes_frontend_contract_shape() { + let column = ColumnMetadata { + header_label_is_ambiguous: false, + name: "email".to_string(), + source_path: Some("$.user.email".to_string()), + index: 2, + detected_type: DataType::Email, + confidence: Confidence::High, + detection_trace: Some(DetectionTrace { + summary: "email evidence".to_string(), + selected_reason: "value matched email".to_string(), + total_non_empty: 1, + candidates: vec![DetectionTraceItem { + data_type: DataType::Email, + reason: "valid email".to_string(), + match_count: 1, + total_considered: 1, + confidence: Confidence::High, + accepted: true, + }], + }), + privacy_findings: Vec::new(), + privacy_evidence: vec![PrivacyEvidenceSummary { + kind: PrivacyFindingKind::Contact, + data_type: DataType::Email, + confidence: Confidence::High, + match_count: 1, + sample_count: 1, + score: 100, + detector: "email".to_string(), + reason: "Column contains contact details.".to_string(), + detectors: vec!["email".to_string()], + }], + review_reasons: vec![ColumnReviewReason::AmbiguousContext], + evidence_disposition: EvidenceDisposition::DetectedSensitive, + evidence_profile: Default::default(), + pii_risk: PiiRisk::High, + sample_values: vec!["ada@example.com".to_string()], + sample_value_distribution: Default::default(), + empty_format: EmptyFormat::EmptyString, + is_selected: true, + strategy: AnonymizationStrategy::Redact, + }; + + let value = serde_json::to_value(&column).unwrap(); + + assert_eq!(value["detectedType"], json!("email")); + assert_eq!(value["detectionTrace"]["totalNonEmpty"], json!(1)); + assert_eq!(value["privacyEvidence"][0]["matchCount"], json!(1)); + assert_eq!(value["reviewReasons"], json!(["ambiguousContext"])); + assert_eq!(value["piiRisk"], json!("high")); + assert_eq!(value["sampleValues"], json!(["ada@example.com"])); + assert_eq!(value["emptyFormat"], json!("emptyString")); + assert_eq!(value["isSelected"], json!(true)); + assert_eq!(value["strategy"], json!("redact")); + assert!(value.get("detected_type").is_none()); + assert!(value.get("pii_risk").is_none()); + + let round_trip: ColumnMetadata = serde_json::from_value(value).unwrap(); + assert_eq!(round_trip, column); +} + +#[test] +fn preflight_params_serialize_optional_and_default_contract_fields() { + let params = PreflightParams { + mode: PreflightMode::Anonymize, + file_path: PathBuf::from("/data/input.csv"), + output_path: None, + columns: vec![0, 2], + controls: Vec::new(), + force: false, + sample_row_count: 10, + preview_smart_replacements: Vec::new(), + local_ai_ready: false, + local_ai_message: None, + }; + + let value = serde_json::to_value(¶ms).unwrap(); + + assert_eq!(value["mode"], json!("anonymize")); + assert_eq!(value["sampleRowCount"], json!(10)); + assert_eq!(value["previewSmartReplacements"], json!([])); + assert_eq!(value["localAiReady"], json!(false)); + assert!(value.get("outputPath").is_none()); + assert!(value.get("localAiMessage").is_none()); + assert!(value.get("sample_row_count").is_none()); + + let minimal = json!({ + "mode": "preview", + "filePath": "/data/input.csv", + "columns": [1], + "force": false, + "sampleRowCount": 5, + "localAiReady": true + }); + let decoded: PreflightParams = serde_json::from_value(minimal).unwrap(); + assert_eq!(decoded.mode, PreflightMode::Preview); + assert_eq!(decoded.controls, Vec::::new()); + assert_eq!( + decoded.preview_smart_replacements, + Vec::::new() + ); + assert_eq!(decoded.output_path, None); + assert_eq!(decoded.local_ai_message, None); +} + +#[test] +fn privacy_report_serializes_nested_release_and_smart_replacement_fields() { + let report = PrivacyReport { + detection_run_summary: None, + direct_identifiers: 1, + quasi_identifiers: 2, + pseudonymized_columns: 1, + smart_replacement_columns: 1, + opaque_token_columns: 0, + masked_columns: 0, + labelled_columns: 0, + redacted_columns: 1, + pass_through_columns: 0, + unique_pseudonym_values: 3, + reused_pseudonym_values: 0, + collisions_avoided: 0, + exhausted_pseudonym_pools: 0, + opaque_token_values: 0, + keyed_token_values: 0, + keyed_token_columns: Vec::new(), + smart_replacement_values: 2, + smart_replacement_rejections: 1, + smart_replacement_rejection_reasons: vec![SmartReplacementRejectionCount { + reason: SmartReplacementRejectionReason::ContainsOriginal, + count: 1, + }], + smart_replacement_fallbacks: 1, + shape_fallback_values: 2, + readiness: ReleaseReadiness { + status: ReleaseReadinessStatus::Review, + blockers: Vec::new(), + review_items: vec!["Review Smart replacement output.".to_string()], + verified_items: Vec::new(), + }, + evidence: vec![ReleaseEvidenceItem { + id: "local-ai".to_string(), + label: "Local AI".to_string(), + status: ReleaseEvidenceStatus::Review, + detail: "Review generated values.".to_string(), + }], + column_reports: vec![ColumnReleaseReport { + column_index: 2, + column_name: "email".to_string(), + selected: true, + detected_type: DataType::Email, + pii_risk: PiiRisk::High, + strategy: AnonymizationStrategy::LocalAi, + action: "Smart replacement".to_string(), + status: ReleaseEvidenceStatus::Review, + detail: "Generated replacements.".to_string(), + }], + column_value_distributions: vec![ColumnValueDistribution { + column_index: 2, + distinct_values: 3, + total_values: 10, + singleton_values: 1, + doubleton_values: 0, + max_value_occurrences: 6, + }], + row_uniqueness: Some(RowUniquenessSummary { + rows_measured: 10, + matched_columns: vec![ + MatchedColumn { + column_index: 0, + matched_on: MatchedPart::WholeValue, + matched_every_row: true, + }, + MatchedColumn { + column_index: 2, + matched_on: MatchedPart::DateDecadeAndTime, + matched_every_row: true, + }, + MatchedColumn { + column_index: 3, + matched_on: MatchedPart::SurvivingFormat, + matched_every_row: true, + }, + ], + distinct_classes: 7, + unique_rows: 4, + smallest_class: 1, + fifth_percentile_class_size: 1, + distinct_rows_all_columns: Some(10), + measurement_incomplete: false, + drop_column_effects: vec![ + DropColumnEffect { + column_index: 2, + unique_rows_without: 1, + }, + DropColumnEffect { + column_index: 0, + unique_rows_without: 3, + }, + ], + drop_attribution_incomplete: false, + }), + utility_metrics: vec![UtilityMetric { + label: "Rows".to_string(), + value: "10".to_string(), + status: ReleaseEvidenceStatus::Info, + detail: Some("sample".to_string()), + }], + notes: vec!["Review generated replacements.".to_string()], + }; + + let value = serde_json::to_value(&report).unwrap(); + + assert_eq!(value["directIdentifiers"], json!(1)); + assert_eq!(value["smartReplacementColumns"], json!(1)); + assert_eq!( + value["smartReplacementRejectionReasons"][0]["reason"], + json!("containsOriginal") + ); + assert_eq!(value["readiness"]["status"], json!("review")); + assert_eq!(value["evidence"][0]["status"], json!("review")); + assert_eq!(value["columnReports"][0]["detectedType"], json!("email")); + assert_eq!(value["utilityMetrics"][0]["status"], json!("info")); + assert_eq!( + value["columnValueDistributions"][0]["distinctValues"], + json!(3) + ); + assert_eq!( + value["columnValueDistributions"][0]["singletonValues"], + json!(1) + ); + assert!(value.get("direct_identifiers").is_none()); + assert!(value.get("smart_replacement_columns").is_none()); + + let round_trip: PrivacyReport = serde_json::from_value(value).unwrap(); + assert_eq!(round_trip, report); +} + +#[test] +fn privacy_report_accepts_defaulted_newer_fields_when_deserializing() { + let value = json!({ + "directIdentifiers": 1, + "quasiIdentifiers": 0, + "pseudonymizedColumns": 1, + "smartReplacementColumns": 0, + "opaqueTokenColumns": 0, + "maskedColumns": 0, + "passThroughColumns": 0, + "uniquePseudonymValues": 1, + "reusedPseudonymValues": 0, + "collisionsAvoided": 0, + "exhaustedPseudonymPools": 0, + "opaqueTokenValues": 0, + "smartReplacementValues": 0, + "smartReplacementFallbacks": 0, + "notes": [] + }); + + let report: PrivacyReport = serde_json::from_value(value).unwrap(); + + assert_eq!(report.redacted_columns, 0); + assert_eq!(report.smart_replacement_rejections, 0); + assert_eq!(report.smart_replacement_rejection_reasons, Vec::new()); + assert_eq!(report.readiness, ReleaseReadiness::default()); + assert_eq!(report.evidence, Vec::new()); + assert_eq!(report.column_reports, Vec::new()); + assert_eq!(report.utility_metrics, Vec::new()); +} + +#[test] +fn selected_enums_use_camel_case_wire_values() { + assert_eq!( + serde_json::to_value(DataType::NumericId).unwrap(), + json!("numericId") + ); + assert_eq!( + serde_json::to_value(PasteDataFormat::PlainText).unwrap(), + json!("plainText") + ); + assert_eq!( + serde_json::to_value(AnonymizationStrategy::PassThrough).unwrap(), + json!("passThrough") + ); + assert_eq!( + serde_json::to_value(ReleaseEvidenceStatus::Info).unwrap(), + json!("info") + ); +} diff --git a/crates/csv-anonymizer-core/src/types/workflow.rs b/crates/csv-anonymizer-core/src/types/workflow.rs new file mode 100644 index 0000000..4779095 --- /dev/null +++ b/crates/csv-anonymizer-core/src/types/workflow.rs @@ -0,0 +1,226 @@ +use super::*; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HeadersData { + pub file_path: PathBuf, + pub row_count: usize, + pub row_count_is_complete: bool, + pub default_output_path: PathBuf, + #[serde(default)] + pub detection_run_summary: DetectionRunSummary, + pub columns: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PasteDataFormat { + Auto, + Csv, + Json, + Xml, + Yaml, + PlainText, + Logs, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PasteAnalyzeParams { + pub content: String, + pub format: PasteDataFormat, + pub sample_row_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PasteAnalyzeData { + pub format: PasteDataFormat, + pub row_count: usize, + pub row_count_is_complete: bool, + /// How much of the paste the detected types actually rest on. + /// + /// Carried by the analyze result rather than only by the privacy report so the + /// column table can caveat itself *before* the user selects columns and + /// transforms. See [`DetectionCoverageSummary`]. + pub detection_coverage: DetectionCoverageSummary, + #[serde(default)] + pub detection_run_summary: DetectionRunSummary, + pub columns: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prepared_analysis: Option, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PasteTransformParams { + pub content: String, + pub format: PasteDataFormat, + pub columns: Vec, + #[serde(default)] + pub controls: Vec, + /// Rows to classify on, matching the figure paste analyze was given. + pub sample_row_count: usize, + #[serde(default)] + pub preview_smart_replacements: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PastePreviewParams { + pub content: String, + pub format: PasteDataFormat, + pub columns: Vec, + #[serde(default)] + pub controls: Vec, + /// Rows to display. A window on the paste, not evidence about it. + pub sample_count: usize, + /// Rows to classify on, matching the figure paste analyze was given. + pub sample_row_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PasteTransformData { + pub output: String, + pub row_count: usize, + pub columns_anonymized: usize, + pub duration_ms: u128, + pub privacy_report: PrivacyReport, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct QuickTransformParams { + pub input: String, + pub data_type: DataType, + pub strategy: AnonymizationStrategy, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct QuickGenerateParams { + pub data_type: DataType, + pub strategy: AnonymizationStrategy, + pub count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct QuickTransformData { + pub output: String, + pub row_count: usize, + pub values: Vec, + pub privacy_report: PrivacyReport, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SampleTransform { + pub original: String, + pub anonymized: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ColumnPreview { + pub column_index: usize, + pub column_name: String, + pub samples: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PreviewWarning { + pub column_index: usize, + pub column_name: String, + pub message: String, + pub severity: WarningSeverity, +} +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum WarningSeverity { + Info, + Warning, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PreviewData { + pub previews: Vec, + pub warnings: Vec, + pub smart_replacements: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PreviewParams { + pub file_path: PathBuf, + pub columns: Vec, + #[serde(default)] + pub controls: Vec, + /// Rows to display. A window on the input, not evidence about it. + pub sample_count: usize, + /// Rows to classify on — the same figure analyze and the run are given, so the + /// preview shows the strategies the run will apply rather than the ones a + /// display-sized sample happens to imply. + pub sample_row_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AnonymizeParams { + pub file_path: PathBuf, + pub output_path: PathBuf, + pub columns: Vec, + #[serde(default)] + pub controls: Vec, + pub force: bool, + #[serde(default)] + pub preview_smart_replacements: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AnonymizeData { + pub output_path: PathBuf, + pub row_count: usize, + pub columns_anonymized: usize, + pub duration_ms: u128, + pub privacy_report: PrivacyReport, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PreflightMode { + Preview, + Anonymize, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PreflightParams { + pub mode: PreflightMode, + pub file_path: PathBuf, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_path: Option, + pub columns: Vec, + #[serde(default)] + pub controls: Vec, + pub force: bool, + pub sample_row_count: usize, + #[serde(default)] + pub preview_smart_replacements: Vec, + pub local_ai_ready: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_ai_message: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PreflightData { + pub mode: PreflightMode, + pub readiness: ReleaseReadiness, + pub evidence: Vec, + pub column_reports: Vec, +} diff --git a/docs/architecture.md b/docs/architecture.md index 2a27431..766564a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -26,6 +26,13 @@ CSV Anonymizer is a local-first desktop application with three runtime boundarie 6. UI workflow phases must not encode contradictory combinations of busy, job, and result state. 7. Showing a preview must preserve the user's reading position. +## Source and workflow boundaries + +- Core CSV entry points take explicit execution inputs: `CsvAnalysisOptions` carries sampling and optional candidate detection, `TransformRuntime` carries run-only Smart replacement and tokenization dependencies, and `CsvRunOptions` adds processing control. CLI and Tauri construct those borrowed inputs at their boundary; lower-level processing inputs remain separate. +- `types.rs` remains the internal wire façade. Its declarations are physically owned by `types/column.rs`, `types/sampling.rs`, `types/processing.rs`, `types/workflow.rs`, and `types/report.rs`, while crate-root type paths and serialized contracts remain unchanged. +- Frontend protection readiness is derived by the pure `workflowReadiness` helper. Each CSV, paste, and quick workflow combines it with its own source, selection, and busy prerequisites so disabled controls and direct actions make the same decision. +- `useAnonymizeJob` submits and handles terminal workflow outcomes. `useAnonymizeJobTracker` owns live-channel updates, stale-channel polling, cancellation requests, and lost-contact recovery without changing the parent hook's public shape. + ## Change policy Boundary refactors land separately from behavioral changes. Every phase must pass Rust tests, Clippy, TypeScript, ESLint, frontend tests, IPC contract checks, comment narration checks, and browser workflow tests before the next phase begins. diff --git a/docs/calibration.md b/docs/calibration.md index d5ab717..d1798b8 100644 --- a/docs/calibration.md +++ b/docs/calibration.md @@ -12,10 +12,10 @@ the constant. If you change a constant, update the table and re-run its harness. --- -# `types.rs` — frequency-inversion constants +# `types/report.rs` — frequency-inversion constants -Source: the frequency-inversion constants in `types.rs`. Every figure below is copied -verbatim from the doc comments that previously sat above each constant. +Source: the frequency-inversion constants in `types/report.rs`. Every figure below is copied +verbatim from the doc comments that define each constant. ## MIN_SAMPLE_COVERAGE diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4fa2e89..1d39ee7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "csv-anonymizer-frontend", - "version": "1.0.93", + "version": "2.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "csv-anonymizer-frontend", - "version": "1.0.93", + "version": "2.0.0", "hasInstallScript": true, "dependencies": { "@tauri-apps/api": "^2.11.1", diff --git a/frontend/package.json b/frontend/package.json index 1642418..0c4358e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "csv-anonymizer-frontend", "private": true, - "version": "1.0.93", + "version": "2.0.0", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/hooks/useAnonymizeJob.ts b/frontend/src/hooks/useAnonymizeJob.ts index 9d62b5a..fe5813b 100644 --- a/frontend/src/hooks/useAnonymizeJob.ts +++ b/frontend/src/hooks/useAnonymizeJob.ts @@ -1,8 +1,5 @@ -import { useEffect, useReducer, useRef } from 'react' import { - cancelAnonymizeJob, firstPreflightBlocker, - getAnonymizeJobStatus, preflightAnonymization, startAnonymizeJob, } from '../tauri' @@ -17,97 +14,11 @@ import type { } from '../types' import { messageFrom } from '../utils/errors' import { confirmEphemeralTokenizationKey } from '../utils/tokenizationKey' -import { isValidTokenizationKey } from '../utils/tokenizationKey' import { directoryOf } from '../utils/paths' +import { getProtectionReadiness } from './workflowReadiness' +import { useAnonymizeJobTracker } from './useAnonymizeJobTracker' import type { WorkflowShell } from './workflowTypes' -/** Gap between successful status polls. */ -const POLL_INTERVAL_MS = 300 -/** Status-query fallback begins only after the live progress channel falls silent. */ -const CHANNEL_SILENCE_BEFORE_POLL_MS = 5_000 -/** - * Failed polls tolerated silently before the user is told contact was lost. - * - * Above one so a single dropped response — common enough while the backend is - * busy writing — never surfaces as an error the run then recovers from anyway. - */ -const POLL_FAILURES_BEFORE_REPORTING = 3 -/** Ceiling on the backoff, so a long outage still notices recovery promptly. */ -const MAX_POLL_RETRY_MS = 5_000 -/** - * How long contact may stay lost before the client stops polling and hands the UI back. - * - * Without a deadline, polling has no terminal condition at all: `snapshot_job` returns a - * permanent error for an id that is no longer in the registry, and a poisoned registry - * mutex fails forever too — neither can ever succeed on a retry. Retrying those every - * five seconds keeps `busy === 'running'` for good, and that state has no exit: the view - * gates every other control on it, and Cancel cannot rescue it either — pressing it only - * sets a flag the worker reads, and the terminal `canceled` state that would release the - * view can only be learned by the polling that is broken. Only killing the app recovers. - * - * Two minutes is measured against continuous unreachability, not against run length: a - * healthy hour-long run answers every poll in microseconds (a status snapshot is an - * in-memory lock and clone), so a long run never approaches this. Two minutes of silence - * is roughly thirty consecutive failed attempts at the capped 5s backoff — far past any - * plausible transient hiccup — while still being short enough that a wedged user is not - * held hostage. - * - * Deliberately a wall clock and not a failure count: the backoff makes "how many failures" - * a poor proxy for "how long the user has been stuck", which is the thing that matters. - * - * An unknown-job error is not treated as immediately terminal. The only signal Tauri hands - * back is the Rust error prose ("Unknown anonymization job: …"); matching on it would stop - * working silently the day someone rewords that string, leaving exactly the freeze this - * constant exists to prevent. The deadline costs an unknown-job outage two minutes of - * waiting and works no matter how the backend words its errors. - */ -const LOST_CONTACT_DEADLINE_MS = 120_000 - -/** - * Says tracking stopped without claiming the run failed or finished. - * - * The backend job is unaffected by the client giving up on it — it may still be streaming - * rows into the output file — so this must not read as a failure that invites a retry over - * the same output path. - */ -function formatMinutes(milliseconds: number) { - const minutes = milliseconds / 60_000 - return `${minutes} ${minutes === 1 ? 'minute' : 'minutes'}` -} - -const LOST_CONTACT_GIVE_UP_MESSAGE = - // The duration is read off the deadline rather than written out, because the two had - // already been written out twice and would part company the first time the deadline moved: - // the user would be told "two minutes" by a timer that waited five. Pluralised for the same - // reason — a one-minute deadline would otherwise render "over 1 minutes". - `Lost contact with the running job for over ${formatMinutes(LOST_CONTACT_DEADLINE_MS)}, so ` + - 'this app stopped tracking it. The job may still be running and writing to the output ' + - 'file: check that file before starting another run over the same path.' - -/** - * How long to wait before retrying after `failures` consecutive failed polls. - * - * Backs off rather than retrying flat at the poll interval, so a backend that is - * unresponsive because it is overloaded is not asked 200 more times a minute while - * it recovers. Capped, because the job may finish at any point and the client has - * no other way to find out. - */ -function pollRetryDelay(failures: number) { - return Math.min(POLL_INTERVAL_MS * 2 ** (failures - 1), MAX_POLL_RETRY_MS) -} - -/** - * Says contact was lost without claiming the run failed, and keeps the underlying - * reason where it is useful. - * - * The distinction matters: the job may well still be streaming rows, and telling - * someone their run failed when it has not is what would make them start a second - * one over the same output. - */ -function lostContactMessage(caught: unknown) { - return `Lost contact with the running job, still retrying. ${messageFrom(caught)}` -} - type AnonymizeJobArgs = { inputPath: string outputPath: string @@ -116,42 +27,15 @@ type AnonymizeJobArgs = { hasColumns: boolean hasSelectedColumns: boolean headers: HeadersData | null - previewSmartReplacements: SmartReplacementEntry[] preparedAnalysis: PreparedAnalysis | null + previewSmartReplacements: SmartReplacementEntry[] tokenizationKey: string | null + selectedUsesLocalAi: boolean selectedUsesTokenization: boolean - localAiBlocked: boolean persistSettings: (settings: AppSettings) => Promise refreshSettings: () => Promise } -type JobTrackerState = - | { tag: 'idle'; status: null } - | { tag: 'running'; jobId: string; status: AnonymizeJobStatus } - -type JobTrackerEvent = - | { type: 'started'; status: AnonymizeJobStatus } - | { type: 'progress'; status: AnonymizeJobStatus } - | { type: 'cleared' } - -const idleJobTracker: JobTrackerState = { tag: 'idle', status: null } - -function jobTrackerReducer(state: JobTrackerState, event: JobTrackerEvent): JobTrackerState { - switch (event.type) { - case 'started': - return event.status.state === 'running' - ? { tag: 'running', jobId: event.status.jobId, status: event.status } - : idleJobTracker - case 'progress': - if (state.tag !== 'running' || event.status.jobId !== state.jobId) return state - return event.status.state === 'running' - ? { ...state, status: event.status } - : idleJobTracker - case 'cleared': - return idleJobTracker - } -} - export function useAnonymizeJob( shell: WorkflowShell, { @@ -166,7 +50,7 @@ export function useAnonymizeJob( preparedAnalysis, tokenizationKey, selectedUsesTokenization, - localAiBlocked, + selectedUsesLocalAi, persistSettings, refreshSettings, }: AnonymizeJobArgs, @@ -174,180 +58,50 @@ export function useAnonymizeJob( const { busy, setBusy, setError, setResult, settings, localAi } = shell const localAiRequest = localAi.request - const [jobTracker, dispatchJob] = useReducer(jobTrackerReducer, idleJobTracker) - const activeJobId = jobTracker.tag === 'running' ? jobTracker.jobId : null - const jobStatus = jobTracker.status - const handleJobStatusRef = useRef(handleJobStatus) - const terminalJobIdsRef = useRef(new Set()) - const lastChannelUpdateRef = useRef(null) - const consecutivePollFailuresRef = useRef(0) - /** When the current run of consecutive poll failures started, for the lost-contact deadline. */ - const lostContactSinceRef = useRef(null) - /** - * The exact message this hook last wrote, so recovery clears only that message. - * - * Clearing on "polling failed recently" instead would erase whatever error is on screen, - * including ones written after it by other code paths — a rejected Cancel, or a Local AI - * download failure during a run — leaving the user believing an action succeeded. - */ - const lostContactMessageRef = useRef(null) - const canAnonymize = Boolean( - hasColumns && - hasSelectedColumns && - inputPath && - outputPath && - busy === 'idle' && - (!settings.localNerEnabled || Boolean(preparedAnalysis)) && - !localAiBlocked && - isValidTokenizationKey(selectedUsesTokenization ? tokenizationKey : null), - ) - - useEffect(() => { - handleJobStatusRef.current = handleJobStatus + const trackerReadiness = getProtectionReadiness({ + usesLocalAi: selectedUsesLocalAi, + localAi, + requiresPreparedAnalysis: settings.localNerEnabled, + hasPreparedAnalysis: Boolean(preparedAnalysis), + usesTokenization: selectedUsesTokenization, + tokenizationKey, }) + const tracker = useAnonymizeJobTracker({ busy, setBusy, setError, onTerminal: handleTerminal }) + const jobStatus = tracker.jobStatus - useEffect(() => { - if (busy !== 'running' || !activeJobId) return - - const jobId = activeJobId - let isMounted = true - let timeoutId: number | undefined - consecutivePollFailuresRef.current = 0 - lostContactSinceRef.current = null - lostContactMessageRef.current = null - - function reportLostContact(message: string) { - lostContactMessageRef.current = message - setError(message) - } - - function clearLostContactReport() { - const reported = lostContactMessageRef.current - if (!reported) return - lostContactMessageRef.current = null - // Only retract our own message: anything written after it belongs to another - // code path and is still news to the user. - setError((current) => (current === reported ? null : current)) - } - - async function pollJob() { - const lastChannelUpdate = lastChannelUpdateRef.current - if (lastChannelUpdate !== null) { - const channelAge = Date.now() - lastChannelUpdate - if (channelAge < CHANNEL_SILENCE_BEFORE_POLL_MS) { - timeoutId = window.setTimeout( - pollJob, - CHANNEL_SILENCE_BEFORE_POLL_MS - channelAge, - ) - return - } - } - try { - const status = await getAnonymizeJobStatus(jobId) - if (!isMounted) return - // Clear a lost-contact message the moment contact returns, so a recovered - // run does not finish while still showing the reason it looked stuck. - clearLostContactReport() - consecutivePollFailuresRef.current = 0 - lostContactSinceRef.current = null - const finished = handleJobStatusRef.current(status) - if (!finished) { - timeoutId = window.setTimeout(pollJob, POLL_INTERVAL_MS) - } - } catch (caught) { - if (!isMounted) return - // Losing contact with the job is not the job failing. Cancelling here used - // to abort the run after two missed polls — under a second — discarding work - // that can legitimately stream for an hour. The run is cancel-safe and leaves - // no partial output either way, so there is nothing for the client to protect - // by ending it; it only has to keep asking and say that it is out of touch. - consecutivePollFailuresRef.current += 1 - lostContactSinceRef.current ??= Date.now() - - if (Date.now() - lostContactSinceRef.current >= LOST_CONTACT_DEADLINE_MS) { - // Give the UI back rather than poll a job we can provably never hear from - // again. Not rescheduling is the point: this is the only exit from `running` - // when the failure is permanent. - lostContactMessageRef.current = null - dispatchJob({ type: 'cleared' }) - setBusy('idle') - setError(LOST_CONTACT_GIVE_UP_MESSAGE) - return - } - - if (consecutivePollFailuresRef.current >= POLL_FAILURES_BEFORE_REPORTING) { - reportLostContact(lostContactMessage(caught)) - } - timeoutId = window.setTimeout(pollJob, pollRetryDelay(consecutivePollFailuresRef.current)) - } - } - - timeoutId = window.setTimeout(pollJob, POLL_INTERVAL_MS) - - return () => { - isMounted = false - if (timeoutId) window.clearTimeout(timeoutId) - } - }, [activeJobId, busy, setBusy, setError]) - - function handleJobStatus(status: AnonymizeJobStatus) { - if (status.state === 'running') { - // A channel can deliver completion before the start command resolves with - // its earlier running snapshot. Never let that stale response resurrect a - // terminal job. - if (terminalJobIdsRef.current.has(status.jobId)) return true - dispatchJob({ type: activeJobId ? 'progress' : 'started', status }) - return false - } - - terminalJobIdsRef.current.add(status.jobId) - dispatchJob({ type: 'progress', status }) - setBusy('idle') - + function handleTerminal(status: AnonymizeJobStatus) { if (status.state === 'succeeded' && status.result) { setResult(status.result) const nextSettings = settingsAfterSuccessfulRun(settings, status.result) - if (nextSettings !== settings) { - void persistSettings(nextSettings) - } else { - void refreshSettings() - } - return true - } - - if (status.state === 'canceled') { - setError('Output creation canceled.') - } else { - setError(status.error ? messageFrom(status.error) : 'Output creation failed.') + if (nextSettings !== settings) void persistSettings(nextSettings) + else void refreshSettings() + return } - return true + if (status.state === 'canceled') setError('Output creation canceled.') + else setError(status.error ? messageFrom(status.error) : 'Output creation failed.') } - function anonymizeBlockedMessage() { - if (localAiBlocked) return 'Set up Local AI before creating output with Smart replacement columns.' - if (settings.localNerEnabled && !preparedAnalysis) return 'Analyze the source again before creating output.' - if (!inputPath || !hasColumns) return 'Load a CSV file first.' - if (!hasSelectedColumns) return 'Select at least one column to anonymize.' - if (!outputPath) return 'Choose an output path.' - return 'Wait for the current operation to finish.' - } + const canAnonymize = Boolean( + hasColumns && hasSelectedColumns && inputPath && outputPath && busy === 'idle' && !trackerReadiness.blocker, + ) async function runAnonymization() { if (!canAnonymize) { - setError(anonymizeBlockedMessage()) - return - } - - const activeTokenizationKey = selectedUsesTokenization ? tokenizationKey : null - if (!isValidTokenizationKey(activeTokenizationKey)) { - setError('Enter a valid 64-character hexadecimal tokenization key before creating output.') + if (trackerReadiness.blocker === 'localAi') setError('Set up Local AI before creating output with Smart replacement columns.') + else if (trackerReadiness.blocker === 'preparedAnalysis') setError('Analyze the source again before creating output.') + else if (trackerReadiness.blocker === 'tokenizationKey') setError('Enter a valid 64-character hexadecimal tokenization key before creating output.') + else if (!inputPath || !hasColumns) setError('Load a CSV file first.') + else if (!hasSelectedColumns) setError('Select at least one column to anonymize.') + else if (!outputPath) setError('Choose an output path.') + else setError('Wait for the current operation to finish.') return } + const activeTokenizationKey = trackerReadiness.tokenizationKey if (!confirmEphemeralTokenizationKey(activeTokenizationKey)) return setBusy('running') setError(null) setResult(null) - dispatchJob({ type: 'cleared' }) + tracker.clearJobState() try { const preflight = await preflightAnonymization({ @@ -382,33 +136,18 @@ export function useAnonymizeJob( preparedAnalysis, tokenizationKey: activeTokenizationKey, }, - onProgress: (nextStatus) => { - lastChannelUpdateRef.current = Date.now() - handleJobStatusRef.current(nextStatus) - }, + onProgress: tracker.onProgress, }) - handleJobStatus(status) + tracker.acceptStatus(status) } catch (caught) { - dispatchJob({ type: 'cleared' }) + tracker.clearJobState() setBusy('idle') setError(messageFrom(caught)) } } - async function cancelCurrentJob() { - if (!activeJobId || busy !== 'running') return - - try { - const status = await cancelAnonymizeJob(activeJobId) - handleJobStatus(status) - } catch (caught) { - setError(messageFrom(caught)) - } - } - - function clearJobState() { - dispatchJob({ type: 'cleared' }) - } + const cancelCurrentJob = tracker.cancelCurrentJob + const clearJobState = tracker.clearJobState return { jobStatus, diff --git a/frontend/src/hooks/useAnonymizeJobTracker.ts b/frontend/src/hooks/useAnonymizeJobTracker.ts new file mode 100644 index 0000000..f2c3e6e --- /dev/null +++ b/frontend/src/hooks/useAnonymizeJobTracker.ts @@ -0,0 +1,82 @@ +import { useEffect, useReducer, useRef } from 'react' +import { cancelAnonymizeJob, getAnonymizeJobStatus } from '../tauri' +import type { AnonymizeJobStatus } from '../types' +import { messageFrom } from '../utils/errors' +import type { WorkflowShell } from './workflowTypes' + +const POLL_INTERVAL_MS = 300 +const CHANNEL_SILENCE_BEFORE_POLL_MS = 5_000 +const POLL_FAILURES_BEFORE_REPORTING = 3 +const MAX_POLL_RETRY_MS = 5_000 +const LOST_CONTACT_DEADLINE_MS = 120_000 +function formatMinutes(milliseconds: number) { const minutes = milliseconds / 60_000; return `${minutes} ${minutes === 1 ? 'minute' : 'minutes'}` } +const LOST_CONTACT_GIVE_UP_MESSAGE = `Lost contact with the running job for over ${formatMinutes(LOST_CONTACT_DEADLINE_MS)}, so this app stopped tracking it. The job may still be running and writing to the output file: check that file before starting another run over the same path.` +function pollRetryDelay(failures: number) { return Math.min(POLL_INTERVAL_MS * 2 ** (failures - 1), MAX_POLL_RETRY_MS) } +function lostContactMessage(caught: unknown) { return `Lost contact with the running job, still retrying. ${messageFrom(caught)}` } + +type State = { tag: 'idle'; status: null } | { tag: 'running'; jobId: string; status: AnonymizeJobStatus } +type Event = { type: 'started' | 'progress'; status: AnonymizeJobStatus } | { type: 'cleared' } +const idle: State = { tag: 'idle', status: null } +function reducer(state: State, event: Event): State { + if (event.type === 'cleared') return idle + if (event.type === 'started') return event.status.state === 'running' ? { tag: 'running', jobId: event.status.jobId, status: event.status } : idle + if (state.tag !== 'running' || event.status.jobId !== state.jobId) return state + return event.status.state === 'running' ? { ...state, status: event.status } : idle +} + +type AnonymizeJobTrackerArgs = Pick & { onTerminal: (status: AnonymizeJobStatus) => void } +export function useAnonymizeJobTracker({ busy, setBusy, setError, onTerminal }: AnonymizeJobTrackerArgs) { + const [state, dispatch] = useReducer(reducer, idle) + const activeJobId = state.tag === 'running' ? state.jobId : null + const onTerminalRef = useRef(onTerminal) + const handlerRef = useRef<(status: AnonymizeJobStatus) => boolean>(() => false) + const terminalIds = useRef(new Set()) + const lastChannelUpdate = useRef(null) + const failures = useRef(0) + const lostSince = useRef(null) + const lostMessage = useRef(null) + useEffect(() => { onTerminalRef.current = onTerminal }) + useEffect(() => { handlerRef.current = acceptStatus }) + + function acceptStatus(status: AnonymizeJobStatus) { + if (status.state === 'running') { + if (terminalIds.current.has(status.jobId)) return true + dispatch({ type: activeJobId ? 'progress' : 'started', status }); return false + } + terminalIds.current.add(status.jobId) + dispatch({ type: 'progress', status }); setBusy('idle'); onTerminalRef.current(status); return true + } + function onProgress(status: AnonymizeJobStatus) { lastChannelUpdate.current = Date.now(); handlerRef.current(status) } + + useEffect(() => { + if (busy !== 'running' || !activeJobId) return + const jobId = activeJobId; let mounted = true; let timeout: number | undefined + failures.current = 0; lostSince.current = null; lostMessage.current = null + const report = (message: string) => { lostMessage.current = message; setError(message) } + const clearReport = () => { const reported = lostMessage.current; if (!reported) return; lostMessage.current = null; setError(current => current === reported ? null : current) } + async function poll() { + const channel = lastChannelUpdate.current + if (channel !== null && Date.now() - channel < CHANNEL_SILENCE_BEFORE_POLL_MS) { timeout = window.setTimeout(poll, CHANNEL_SILENCE_BEFORE_POLL_MS - (Date.now() - channel)); return } + try { + const status = await getAnonymizeJobStatus(jobId); if (!mounted) return + clearReport(); failures.current = 0; lostSince.current = null + if (!handlerRef.current(status)) timeout = window.setTimeout(poll, POLL_INTERVAL_MS) + } catch (caught) { + if (!mounted) return + failures.current += 1; lostSince.current ??= Date.now() + if (Date.now() - lostSince.current >= LOST_CONTACT_DEADLINE_MS) { lostMessage.current = null; dispatch({ type: 'cleared' }); setBusy('idle'); setError(LOST_CONTACT_GIVE_UP_MESSAGE); return } + if (failures.current >= POLL_FAILURES_BEFORE_REPORTING) report(lostContactMessage(caught)) + timeout = window.setTimeout(poll, pollRetryDelay(failures.current)) + } + } + timeout = window.setTimeout(poll, POLL_INTERVAL_MS) + return () => { mounted = false; if (timeout) window.clearTimeout(timeout) } + }, [activeJobId, busy, setBusy, setError]) + + async function cancelCurrentJob() { + if (!activeJobId || busy !== 'running') return + try { const status = await cancelAnonymizeJob(activeJobId); acceptStatus(status) } catch (caught) { setError(messageFrom(caught)) } + } + function clearJobState() { dispatch({ type: 'cleared' }) } + return { jobStatus: state.status, acceptStatus, onProgress, cancelCurrentJob, clearJobState } +} diff --git a/frontend/src/hooks/useAnonymizerWorkflow.ts b/frontend/src/hooks/useAnonymizerWorkflow.ts index 4461134..2f832da 100644 --- a/frontend/src/hooks/useAnonymizerWorkflow.ts +++ b/frontend/src/hooks/useAnonymizerWorkflow.ts @@ -11,6 +11,7 @@ import { useCsvSelection } from './useCsvSelection' import { useLocalAi } from './useLocalAi' import { usePersistentSettings } from './usePersistentSettings' import { usePreviewWorkflow } from './usePreviewWorkflow' +import { isSmartReplacementBlocked } from './workflowReadiness' import { useWorkflowArtifacts, useSelectionInvalidation } from './useWorkflowArtifacts' import type { BusyState, WorkflowShell } from './workflowTypes' @@ -79,14 +80,11 @@ export function useAnonymizerWorkflow() { const hasFile = Boolean(inputPath.trim()) const isLoading = busy !== 'idle' const settingsDisabled = isLoading || !settingsLoaded - const localAiBlocked = - selectionUsesLocalAi(selectedColumns) && (!localAi.ready || localAi.downloadRunning) + const localAiBlocked = isSmartReplacementBlocked(selectionUsesLocalAi(selectedColumns), localAi) const previewWorkflow = usePreviewWorkflow(shell, { inputPath, selectedColumns, hasColumns, - hasSelectedColumns, - localAiBlocked, controlsForColumns, selectionUsesLocalAi, selectionUsesTokenization, @@ -103,12 +101,12 @@ export function useAnonymizerWorkflow() { hasSelectedColumns, headers, previewSmartReplacements: preview?.smartReplacements ?? [], - localAiBlocked, persistSettings, refreshSettings, preparedAnalysis, tokenizationKey, selectedUsesTokenization: selectionUsesTokenization(selectedColumns), + selectedUsesLocalAi: selectionUsesLocalAi(selectedColumns), }) const invalidatingSelection = useSelectionInvalidation( { setSelectedColumns: setCsvSelectedColumns, toggleColumn: toggleCsvColumn, updateColumnStrategy: updateCsvColumnStrategy }, diff --git a/frontend/src/hooks/usePasteDataWorkflow.ts b/frontend/src/hooks/usePasteDataWorkflow.ts index fb1ca2b..446afde 100644 --- a/frontend/src/hooks/usePasteDataWorkflow.ts +++ b/frontend/src/hooks/usePasteDataWorkflow.ts @@ -9,12 +9,12 @@ import type { PreviewData, } from '../types' import { messageFrom } from '../utils/errors' -import { confirmEphemeralTokenizationKey, isValidTokenizationKey } from '../utils/tokenizationKey' +import { confirmEphemeralTokenizationKey } from '../utils/tokenizationKey' import { useColumnSelection } from './useColumnSelection' import { useCopyOutput } from './useCopyOutput' import type { LocalAiState } from './useLocalAi' +import { getProtectionReadiness } from './workflowReadiness' import { useSelectionInvalidation } from './useWorkflowArtifacts' - export type PasteBusyState = 'idle' | 'analyzing' | 'previewing' | 'transforming' | 'copying' type PasteDataWorkflowOptions = { @@ -49,21 +49,26 @@ export function usePasteDataWorkflow({ const isContentTooLarge = contentByteLength > MAX_PASTE_CONTENT_BYTES const selectedUsesLocalAi = selection.selectionUsesLocalAi(selection.selectedColumns) const selectedUsesTokenization = selection.selectionUsesTokenization(selection.selectedColumns) - const activeTokenizationKey = selectedUsesTokenization ? tokenizationKey : null - const localAiBlocked = selectedUsesLocalAi && (!localAi.ready || localAi.downloadRunning) + const getRunReadiness = () => { + const protection = getProtectionReadiness({ + usesLocalAi: selectedUsesLocalAi, + localAi, + requiresPreparedAnalysis: settings.localNerEnabled, + hasPreparedAnalysis: Boolean(analysis?.preparedAnalysis), + usesTokenization: selectedUsesTokenization, + tokenizationKey, + }) + const blocker = + !settingsLoaded || !analysis || selection.selectedColumns.length === 0 || isBusy + ? 'unavailable' + : protection.blocker + return { blocker, protection } + } + const readiness = getRunReadiness() + const activeTokenizationKey = readiness.protection.tokenizationKey const canAnalyze = settingsLoaded && content.trim().length > 0 && !isBusy && !isContentTooLarge const canClear = !isBusy && (content.length > 0 || analysis !== null || preview !== null || result !== null || copyStatus !== null) - // Preview and transform are gated on exactly the same conditions: both send the - // current selection to the backend, so anything that makes one unsafe makes the - // other unsafe too. They were two identical expressions that could drift apart. - const canRun = - settingsLoaded && - Boolean(analysis) && - selection.selectedColumns.length > 0 && - (!settings.localNerEnabled || Boolean(analysis?.preparedAnalysis)) && - !isBusy && - !localAiBlocked && - isValidTokenizationKey(activeTokenizationKey) + const canRun = readiness.blocker === null useEffect(() => () => { operationSequence.current += 1 @@ -136,18 +141,25 @@ export function usePasteDataWorkflow({ setContentState('') resetDerivedState() } - async function showPreview() { - if (!settingsLoaded || !analysis || selection.selectedColumns.length === 0 || isBusy) return - if (localAiBlocked) { + const current = getRunReadiness() + if (current.blocker === 'unavailable') { + if (settingsLoaded && analysis && selection.selectedColumns.length > 0 && !isBusy) { + if (current.protection.blocker === 'localAi') onError('Set up Local AI before previewing Smart replacement fields.') + else if (current.protection.blocker === 'preparedAnalysis') onError('Analyze the content again before previewing.') + else if (current.protection.blocker === 'tokenizationKey') onError('Enter a valid 64-character hexadecimal tokenization key before previewing.') + } + return + } + if (current.blocker === 'localAi') { onError('Set up Local AI before previewing Smart replacement fields.') return } - if (settings.localNerEnabled && !analysis.preparedAnalysis) { + if (current.blocker === 'preparedAnalysis') { onError('Analyze the content again before previewing.') return } - if (!isValidTokenizationKey(activeTokenizationKey)) { + if (current.blocker === 'tokenizationKey') { onError('Enter a valid 64-character hexadecimal tokenization key before previewing.') return } @@ -159,13 +171,13 @@ export function usePasteDataWorkflow({ try { const nextPreview = await previewPasteData({ content, - format: analysis.format, + format: analysis!.format, columns: selection.selectedColumns, controls: selection.controlsForColumns(selection.selectedColumns), sampleCount: settings.previewSampleCount, sampleRowCount: settings.sampleRowCount, localAi: localAi.request, - preparedAnalysis: analysis.preparedAnalysis, + preparedAnalysis: analysis!.preparedAnalysis, tokenizationKey: activeTokenizationKey, }) if (sequence === operationSequence.current) setPreview(nextPreview) @@ -177,16 +189,24 @@ export function usePasteDataWorkflow({ } async function transform() { - if (!settingsLoaded || !analysis || selection.selectedColumns.length === 0 || isBusy) return - if (localAiBlocked) { + const current = getRunReadiness() + if (current.blocker === 'unavailable') { + if (settingsLoaded && analysis && selection.selectedColumns.length > 0 && !isBusy) { + if (current.protection.blocker === 'localAi') onError('Set up Local AI before anonymizing Smart replacement fields.') + else if (current.protection.blocker === 'preparedAnalysis') onError('Analyze the content again before transforming.') + else if (current.protection.blocker === 'tokenizationKey') onError('Enter a valid 64-character hexadecimal tokenization key before transforming.') + } + return + } + if (current.blocker === 'localAi') { onError('Set up Local AI before anonymizing Smart replacement fields.') return } - if (settings.localNerEnabled && !analysis.preparedAnalysis) { + if (current.blocker === 'preparedAnalysis') { onError('Analyze the content again before transforming.') return } - if (!isValidTokenizationKey(activeTokenizationKey)) { + if (current.blocker === 'tokenizationKey') { onError('Enter a valid 64-character hexadecimal tokenization key before transforming.') return } @@ -198,13 +218,13 @@ export function usePasteDataWorkflow({ try { const nextResult = await transformPasteData({ content, - format: analysis.format, + format: analysis!.format, columns: selection.selectedColumns, controls: selection.controlsForColumns(selection.selectedColumns), sampleRowCount: settings.sampleRowCount, previewSmartReplacements: preview?.smartReplacements ?? [], localAi: localAi.request, - preparedAnalysis: analysis.preparedAnalysis, + preparedAnalysis: analysis!.preparedAnalysis, tokenizationKey: activeTokenizationKey, }) if (sequence === operationSequence.current) setResult(nextResult) @@ -249,7 +269,7 @@ export function usePasteDataWorkflow({ contentByteLength, isContentTooLarge, selectedUsesLocalAi, - localAiBlocked, + localAiBlocked: readiness.protection.localAiBlocked, isBusy, canAnalyze, canClear, diff --git a/frontend/src/hooks/usePreviewWorkflow.ts b/frontend/src/hooks/usePreviewWorkflow.ts index 1acc166..232fc7b 100644 --- a/frontend/src/hooks/usePreviewWorkflow.ts +++ b/frontend/src/hooks/usePreviewWorkflow.ts @@ -7,15 +7,13 @@ import type { PreparedAnalysis, } from '../types' import { messageFrom } from '../utils/errors' -import { isValidTokenizationKey } from '../utils/tokenizationKey' +import { getProtectionReadiness } from './workflowReadiness' import type { WorkflowShell } from './workflowTypes' type PreviewWorkflowArgs = { inputPath: string selectedColumns: number[] hasColumns: boolean - hasSelectedColumns: boolean - localAiBlocked: boolean controlsForColumns: (columns: number[]) => ColumnControl[] selectionUsesLocalAi: (columns: number[]) => boolean selectionUsesTokenization: (columns: number[]) => boolean @@ -30,8 +28,6 @@ export function usePreviewWorkflow( inputPath, selectedColumns, hasColumns, - hasSelectedColumns, - localAiBlocked, controlsForColumns, selectionUsesLocalAi, selectionUsesTokenization, @@ -42,9 +38,6 @@ export function usePreviewWorkflow( ) { const { busy, setBusy, setError, setResult, settings, localAi } = shell const localAiRequest = localAi.request - const localAiReady = localAi.ready - const selectedUsesTokenization = selectionUsesTokenization(selectedColumns) - const selectedTokenizationKey = selectedUsesTokenization ? tokenizationKey : null const operationSequence = useRef(0) const selectedColumnsFingerprint = selectedColumns.join(',') @@ -56,33 +49,56 @@ export function usePreviewWorkflow( operationSequence.current += 1 }, []) - const canPreview = Boolean( - hasColumns && - hasSelectedColumns && - inputPath && - busy === 'idle' && - (!settings.localNerEnabled || Boolean(preparedAnalysis)) && - !localAiBlocked && - isValidTokenizationKey(selectedTokenizationKey), - ) + function getPreviewReadiness(path: string, columns: number[]) { + const protection = getProtectionReadiness({ + usesLocalAi: selectionUsesLocalAi(columns), + localAi, + requiresPreparedAnalysis: settings.localNerEnabled, + hasPreparedAnalysis: Boolean(preparedAnalysis), + usesTokenization: selectionUsesTokenization(columns), + tokenizationKey, + }) + const blocker = !path + ? 'missingSource' + : columns.length === 0 + ? 'noSelection' + : !hasColumns + ? 'missingColumns' + : busy !== 'idle' + ? 'busy' + : protection.blocker + return { blocker, protection } + } + + const canPreview = getPreviewReadiness(inputPath, selectedColumns).blocker === null async function previewCsv(path = inputPath, columnsToPreview = selectedColumns) { - if (!path || columnsToPreview.length === 0) { + const readiness = getPreviewReadiness(path, columnsToPreview) + if (readiness.blocker === 'missingSource' || readiness.blocker === 'noSelection') { setPreview(null) return } - if (selectionUsesLocalAi(columnsToPreview) && !localAiReady) { + if (readiness.blocker === 'missingColumns') { + setError('Load a CSV file first.') + return + } + if (readiness.blocker === 'busy') { + setError('Wait for the current operation to finish.') + return + } + if (readiness.blocker === 'localAi') { setError('Set up Local AI before previewing Smart replacement columns.') return } - const tokenizationKeyForPreview = selectionUsesTokenization(columnsToPreview) - ? tokenizationKey - : null - if (!isValidTokenizationKey(tokenizationKeyForPreview)) { + if (readiness.blocker === 'preparedAnalysis') { + setError('Analyze the source again before previewing.') + return + } + if (readiness.blocker === 'tokenizationKey') { setError('Enter a valid 64-character hexadecimal tokenization key before previewing.') return } - + const tokenizationKeyForPreview = readiness.protection.tokenizationKey const sequence = ++operationSequence.current setBusy('preview') setError(null) @@ -127,6 +143,7 @@ export function usePreviewWorkflow( } } + return { canPreview, previewCsv, diff --git a/frontend/src/hooks/useQuickGenerateWorkflow.ts b/frontend/src/hooks/useQuickGenerateWorkflow.ts index cf19c4d..24a9a3b 100644 --- a/frontend/src/hooks/useQuickGenerateWorkflow.ts +++ b/frontend/src/hooks/useQuickGenerateWorkflow.ts @@ -2,10 +2,10 @@ import { useEffect, useRef, useState } from 'react' import { generateQuickValues } from '../tauri' import type { AnonymizationStrategy, DataType, QuickTransformData } from '../types' import { messageFrom } from '../utils/errors' -import { confirmEphemeralTokenizationKey, isValidTokenizationKey } from '../utils/tokenizationKey' +import { confirmEphemeralTokenizationKey } from '../utils/tokenizationKey' import { useCopyOutput } from './useCopyOutput' import type { LocalAiState } from './useLocalAi' - +import { getProtectionReadiness } from './workflowReadiness' export type QuickBusyState = 'idle' | 'generating' | 'copying' export const QUICK_MIN_COUNT = 1 @@ -39,11 +39,24 @@ export function useQuickGenerateWorkflow({ const { copyOutput, copyStatus, setCopyStatus } = useCopyOutput({ isBusy, onError, setBusy }) const usesLocalAi = strategy === 'localAi' const usesTokenization = strategy === 'tokenize' - const activeTokenizationKey = usesTokenization ? tokenizationKey : null - const localAiBlocked = usesLocalAi && (!localAi.ready || localAi.downloadRunning) - const canGenerate = - settingsLoaded && count >= QUICK_MIN_COUNT && count <= QUICK_MAX_COUNT && !isBusy && !localAiBlocked && - isValidTokenizationKey(activeTokenizationKey) + const getGenerateReadiness = () => { + const protection = getProtectionReadiness({ + usesLocalAi, + localAi, + requiresPreparedAnalysis: false, + hasPreparedAnalysis: true, + usesTokenization, + tokenizationKey, + }) + const blocker = !settingsLoaded || count < QUICK_MIN_COUNT || count > QUICK_MAX_COUNT || isBusy + ? 'unavailable' + : protection.blocker + return { blocker, protection } + } + const readiness = getGenerateReadiness() + const activeTokenizationKey = readiness.protection.tokenizationKey + const localAiBlocked = readiness.protection.localAiBlocked + const canGenerate = readiness.blocker === null /** Any input change invalidates the values on screen: they were generated for the old settings. */ function clearOutput() { @@ -55,8 +68,9 @@ export function useQuickGenerateWorkflow({ useEffect(() => { if (previousTokenizationKey.current === activeTokenizationKey) return previousTokenizationKey.current = activeTokenizationKey - clearOutput() - }, [activeTokenizationKey, setCopyStatus]) + setResult(null) + setCopyStatus(null) + }, [activeTokenizationKey, setCopyStatus, setResult]) function setDataType(nextDataType: DataType) { setDataTypeState(nextDataType) @@ -74,16 +88,17 @@ export function useQuickGenerateWorkflow({ } async function generate() { - if (!settingsLoaded || count < QUICK_MIN_COUNT || count > QUICK_MAX_COUNT || isBusy) return - if (localAiBlocked) { + const current = getGenerateReadiness() + if (current.blocker === 'unavailable') return + if (current.blocker === 'localAi') { onError('Set up Local AI before generating Smart replacement values.') return } - if (!isValidTokenizationKey(activeTokenizationKey)) { + if (current.blocker === 'tokenizationKey') { onError('Enter a valid 64-character hexadecimal tokenization key before generating values.') return } - if (!confirmEphemeralTokenizationKey(activeTokenizationKey)) return + if (!confirmEphemeralTokenizationKey(current.protection.tokenizationKey)) return onError(null) setBusy('generating') setCopyStatus(null) diff --git a/frontend/src/hooks/workflowReadiness.ts b/frontend/src/hooks/workflowReadiness.ts new file mode 100644 index 0000000..5b0615d --- /dev/null +++ b/frontend/src/hooks/workflowReadiness.ts @@ -0,0 +1,30 @@ +import { isValidTokenizationKey } from '../utils/tokenizationKey' + +type LocalAiReadiness = { ready: boolean; downloadRunning: boolean } +export type ProtectionBlocker = 'localAi' | 'preparedAnalysis' | 'tokenizationKey' +export type ProtectionReadinessInput = { + usesLocalAi: boolean + localAi: LocalAiReadiness + requiresPreparedAnalysis: boolean + hasPreparedAnalysis: boolean + usesTokenization: boolean + tokenizationKey: string | null | undefined +} + +export function isSmartReplacementBlocked(usesLocalAi: boolean, localAi: LocalAiReadiness): boolean { + return usesLocalAi && (!localAi.ready || localAi.downloadRunning) +} + +export function getProtectionReadiness(input: ProtectionReadinessInput): { + blocker: ProtectionBlocker | null + tokenizationKey: string | null + localAiBlocked: boolean +} { + const localAiBlocked = isSmartReplacementBlocked(input.usesLocalAi, input.localAi) + const tokenizationKey = input.usesTokenization ? input.tokenizationKey ?? null : null + let blocker: ProtectionBlocker | null = null + if (localAiBlocked) blocker = 'localAi' + else if (input.requiresPreparedAnalysis && !input.hasPreparedAnalysis) blocker = 'preparedAnalysis' + else if (!isValidTokenizationKey(tokenizationKey)) blocker = 'tokenizationKey' + return { blocker, tokenizationKey, localAiBlocked } +} diff --git a/package.json b/package.json index d241c59..4685a7c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "csv-anonymizer", - "version": "1.0.93", + "version": "2.0.0", "description": "Desktop CSV anonymizer for local-first data transformation workflows.", "desktopName": "csv-anonymizer.desktop", "homepage": "https://github.com/ddv1982/csv-data-anonymizer", diff --git a/scripts/__tests__/check-contracts.test.mjs b/scripts/__tests__/check-contracts.test.mjs index eeb2a31..af6deaa 100644 --- a/scripts/__tests__/check-contracts.test.mjs +++ b/scripts/__tests__/check-contracts.test.mjs @@ -3,7 +3,7 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import test from 'node:test' -import { checkContracts, enumContracts, limitContracts, structContracts } from '../check-contracts.mjs' +import { checkContracts, enumContracts, limitContracts, rustSources, structContracts } from '../check-contracts.mjs' // The fixture is generated from the contract lists the gate itself exports, so it // agrees with them by construction. A contract added to the gate is covered here @@ -56,28 +56,27 @@ function generatedTsDefaults() { return `${limitContracts.map(([, tsName]) => `export const ${tsName} = 10`).join('\n')}\n` } -// `jobs` lands in src-tauri/src/jobs.rs, the third entry in the gate's rustSources, -// which makes it the natural place to plant a second declaration: it is read after -// the core types file, so the copy is the later one and a first-match lookup would -// never see it. +// `jobs` lands in src-tauri/src/jobs.rs. Every checked source is materialized +// below, so adding a source to the gate cannot silently omit it from fixtures. function fixture({ coreTypes = generatedRustTypes(), + reportTypes = '', jobs = '', types = generatedTsTypes(), defaults = generatedTsDefaults(), } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'csv-anonymizer-contracts-')) - fs.mkdirSync(path.join(root, 'crates/csv-anonymizer-core/src'), { recursive: true }) - fs.mkdirSync(path.join(root, 'src-tauri/src/commands'), { recursive: true }) - fs.mkdirSync(path.join(root, 'src-tauri/src/local_ai'), { recursive: true }) - fs.mkdirSync(path.join(root, 'src-tauri/src/settings'), { recursive: true }) + const rustContent = new Map([ + ['crates/csv-anonymizer-core/src/types.rs', coreTypes], + ['crates/csv-anonymizer-core/src/types/report.rs', reportTypes], + ['src-tauri/src/jobs.rs', jobs], + ]) + for (const source of rustSources) { + const filename = path.join(root, source) + fs.mkdirSync(path.dirname(filename), { recursive: true }) + fs.writeFileSync(filename, rustContent.get(source) ?? '') + } fs.mkdirSync(path.join(root, 'frontend/src'), { recursive: true }) - - fs.writeFileSync(path.join(root, 'crates/csv-anonymizer-core/src/types.rs'), coreTypes) - fs.writeFileSync(path.join(root, 'src-tauri/src/jobs.rs'), jobs) - fs.writeFileSync(path.join(root, 'src-tauri/src/commands/csv.rs'), '') - fs.writeFileSync(path.join(root, 'src-tauri/src/local_ai/types.rs'), '') - fs.writeFileSync(path.join(root, 'src-tauri/src/settings/model.rs'), '') fs.writeFileSync(path.join(root, 'frontend/src/types.ts'), types) fs.writeFileSync(path.join(root, 'frontend/src/defaults.ts'), defaults) return root diff --git a/scripts/check-contracts.mjs b/scripts/check-contracts.mjs index 4137bd8..4f65a91 100644 --- a/scripts/check-contracts.mjs +++ b/scripts/check-contracts.mjs @@ -8,8 +8,13 @@ import { pathToFileURL } from 'node:url' // owns the analysis/transform vocabulary; the Tauri crate owns the settings, // job and Local AI surfaces, which reach the frontend over IPC just the same // and drift just as silently when they are left unchecked. -const rustSources = [ +export const rustSources = [ 'crates/csv-anonymizer-core/src/types.rs', + 'crates/csv-anonymizer-core/src/types/column.rs', + 'crates/csv-anonymizer-core/src/types/sampling.rs', + 'crates/csv-anonymizer-core/src/types/processing.rs', + 'crates/csv-anonymizer-core/src/types/workflow.rs', + 'crates/csv-anonymizer-core/src/types/report.rs', 'src-tauri/src/commands/csv.rs', 'src-tauri/src/jobs.rs', 'src-tauri/src/local_ai/types.rs', diff --git a/src-tauri/src/commands/csv.rs b/src-tauri/src/commands/csv.rs index 43a3a08..640f561 100644 --- a/src-tauri/src/commands/csv.rs +++ b/src-tauri/src/commands/csv.rs @@ -132,7 +132,13 @@ fn analyze_csv_data( let mut headers = if local_ner_enabled { if let Some(message) = local_ner_unavailable_message(local_ner_model)? { let mut headers = service - .analyze_csv_with_sample_rows(analysis_path, sample_row_count) + .analyze_csv( + analysis_path, + csv_anonymizer_core::CsvAnalysisOptions { + sample_rows: sample_row_count, + ..Default::default() + }, + ) .map_err(|error| error.to_string())?; headers.detection_run_summary.local_ner = LocalNerRunStatus::Unavailable; headers.detection_run_summary.message = Some(message); @@ -140,16 +146,24 @@ fn analyze_csv_data( } else { let mut detector = local_candidate_detector(local_ner_model)?; service - .analyze_csv_with_sample_rows_and_candidate_detector( + .analyze_csv( analysis_path, - sample_row_count, - Some(&mut detector), + csv_anonymizer_core::CsvAnalysisOptions { + sample_rows: sample_row_count, + candidate_detector: Some(&mut detector), + }, ) .map_err(|error| error.to_string())? } } else { service - .analyze_csv_with_sample_rows(analysis_path, sample_row_count) + .analyze_csv( + analysis_path, + csv_anonymizer_core::CsvAnalysisOptions { + sample_rows: sample_row_count, + ..Default::default() + }, + ) .map_err(|error| error.to_string())? }; // The staged path is an implementation detail. Keep the public analysis and @@ -269,7 +283,7 @@ pub async fn preview_anonymization( .as_mut() .map(|provider| provider as &mut dyn SmartReplacementProvider); service() - .preview_anonymization_with_run_secrets( + .preview_anonymization( PreviewParams { file_path: processing_path, columns: request.columns, @@ -277,8 +291,10 @@ pub async fn preview_anonymization( sample_count: request.sample_count, sample_row_count: request.sample_row_count, }, - provider, - tokenization_key.as_ref(), + csv_anonymizer_core::TransformRuntime { + provider, + tokenization_key: tokenization_key.as_ref(), + }, ) .map_err(|error| error.to_string()) }) @@ -598,13 +614,16 @@ mod tests { std::fs::write(&input_path, "name\nGrace\n").expect("replace original"); let preview = service() - .preview_anonymization(PreviewParams { - file_path: validated.processing_path(), - columns: vec![0], - controls: Vec::new(), - sample_count: 1, - sample_row_count: 100, - }) + .preview_anonymization( + PreviewParams { + file_path: validated.processing_path(), + columns: vec![0], + controls: Vec::new(), + sample_count: 1, + sample_row_count: 100, + }, + csv_anonymizer_core::TransformRuntime::default(), + ) .expect("preview staged input"); assert_eq!(preview.previews[0].samples[0].original, "Alice"); diff --git a/src-tauri/src/commands/csv/paste.rs b/src-tauri/src/commands/csv/paste.rs index f5c4eae..3f43a5b 100644 --- a/src-tauri/src/commands/csv/paste.rs +++ b/src-tauri/src/commands/csv/paste.rs @@ -15,7 +15,7 @@ use crate::settings::SettingsStore; use csv_anonymizer_core::{ LocalNerRunStatus, PasteAnalyzeData, PasteAnalyzeParams, PastePreviewParams, PasteTransformData, PasteTransformParams, PreparedAnalysisSnapshot, PreviewData, - SmartReplacementProvider, + SmartReplacementProvider, TransformRuntime, }; use serde::Deserialize; use std::sync::Arc; @@ -54,21 +54,19 @@ pub async fn analyze_pasted_data( let sample_row_count = request.sample_row_count; let mut analysis = if local_ner_enabled { if let Some(message) = local_ner_unavailable_message(&local_ner_model)? { - let mut analysis = csv_anonymizer_core::direct_input::analyze_paste_data(request) - .map_err(|error| error.to_string())?; + let mut analysis = + csv_anonymizer_core::direct_input::analyze_paste_data(request, None) + .map_err(|error| error.to_string())?; analysis.detection_run_summary.local_ner = LocalNerRunStatus::Unavailable; analysis.detection_run_summary.message = Some(message); analysis } else { let mut detector = local_candidate_detector(&local_ner_model)?; - csv_anonymizer_core::direct_input::analyze_paste_data_with_candidate_detector( - request, - &mut detector, - ) - .map_err(|error| error.to_string())? + csv_anonymizer_core::direct_input::analyze_paste_data(request, Some(&mut detector)) + .map_err(|error| error.to_string())? } } else { - csv_anonymizer_core::direct_input::analyze_paste_data(request) + csv_anonymizer_core::direct_input::analyze_paste_data(request, None) .map_err(|error| error.to_string())? }; let prepared_analysis = if local_ner_enabled { @@ -139,19 +137,23 @@ pub async fn preview_pasted_data( .filter(|snapshot| matches!(snapshot.format.as_str(), "plainText" | "logs")) { let confirmed = selected_candidate_ids(snapshot, &request.params.columns); - csv_anonymizer_core::direct_input::preview_paste_text_candidate_evidence_with_run_secrets( + csv_anonymizer_core::direct_input::preview_paste_text_candidate_evidence( &request.params, snapshot, &confirmed, - provider, - tokenization_key.as_ref(), + TransformRuntime { + provider, + tokenization_key: tokenization_key.as_ref(), + }, ) .map_err(|error| error.to_string()) } else { - csv_anonymizer_core::direct_input::preview_paste_data_with_run_secrets( + csv_anonymizer_core::direct_input::preview_paste_data( request.params, - provider, - tokenization_key.as_ref(), + TransformRuntime { + provider, + tokenization_key: tokenization_key.as_ref(), + }, ) .map_err(|error| error.to_string()) } @@ -199,19 +201,23 @@ pub async fn anonymize_pasted_data( .filter(|snapshot| matches!(snapshot.format.as_str(), "plainText" | "logs")) { let confirmed = selected_candidate_ids(snapshot, &request.params.columns); - csv_anonymizer_core::direct_input::replay_paste_text_candidate_evidence_with_run_secrets( + csv_anonymizer_core::direct_input::replay_paste_text_candidate_evidence( &request.params, snapshot, &confirmed, - provider, - tokenization_key.as_ref(), + TransformRuntime { + provider, + tokenization_key: tokenization_key.as_ref(), + }, ) .map_err(|error| error.to_string()) } else { - csv_anonymizer_core::direct_input::transform_paste_data_with_run_secrets( + csv_anonymizer_core::direct_input::transform_paste_data( request.params, - provider, - tokenization_key.as_ref(), + TransformRuntime { + provider, + tokenization_key: tokenization_key.as_ref(), + }, ) .map_err(|error| error.to_string()) }?; diff --git a/src-tauri/src/commands/csv/quick.rs b/src-tauri/src/commands/csv/quick.rs index e86c9db..0ae05eb 100644 --- a/src-tauri/src/commands/csv/quick.rs +++ b/src-tauri/src/commands/csv/quick.rs @@ -3,7 +3,9 @@ use crate::command_error::CommandError; use crate::commands::shared::run_blocking; use crate::local_ai::{LocalAiRequest, smart_provider_for_strategy}; use crate::settings::SettingsStore; -use csv_anonymizer_core::{QuickGenerateParams, QuickTransformData, SmartReplacementProvider}; +use csv_anonymizer_core::{ + QuickGenerateParams, QuickTransformData, SmartReplacementProvider, TransformRuntime, +}; use serde::Deserialize; use std::sync::Arc; use tauri::State; @@ -35,10 +37,12 @@ pub async fn generate_quick_values( let provider = provider .as_mut() .map(|provider| provider as &mut dyn SmartReplacementProvider); - csv_anonymizer_core::direct_input::generate_quick_values_with_run_secrets( + csv_anonymizer_core::direct_input::generate_quick_values( request.params, - provider, - tokenization_key.as_ref(), + TransformRuntime { + provider, + tokenization_key: tokenization_key.as_ref(), + }, ) .map_err(|error| error.to_string()) }) diff --git a/src-tauri/src/jobs.rs b/src-tauri/src/jobs.rs index fd634e4..86ea65d 100644 --- a/src-tauri/src/jobs.rs +++ b/src-tauri/src/jobs.rs @@ -355,12 +355,16 @@ pub fn run_anonymize_job( let provider = provider .as_mut() .map(|provider| provider as &mut dyn SmartReplacementProvider); - service().anonymize_csv_with_run_secrets( + service().anonymize_csv( input, - sample_row_count, - Some(&mut control), - provider, - tokenization_key.as_ref(), + csv_anonymizer_core::CsvRunOptions { + sample_rows: sample_row_count, + control: Some(&mut control), + transform: csv_anonymizer_core::TransformRuntime { + provider, + tokenization_key: tokenization_key.as_ref(), + }, + }, ) } Err(error) => Err(AnonymizerError::SmartReplacement(error)), diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 38b7d77..829bb65 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -71,5 +71,5 @@ "productName": "CSV Anonymizer", "mainBinaryName": "csv-anonymizer", "identifier": "io.github.ddv1982.csv-data-anonymizer", - "version": "1.0.93" + "version": "2.0.0" }