diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cae9f7..6103a6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v1.0.93 - 2026-09-08 + +- Close anonymization disclosure paths in pasted text and XML, including complete PEM redaction, character-reference transformation, and honest residual-audit evidence. +- Revalidate Smart preview replacements against full-run values, retain output handles through publication, and prevent malformed keys from consuming the only anonymization job slot. +- Harden Local AI transport and download completion, invalidate stale frontend artifacts, correct Label report counts, and update the audited Browserslist dependency. + ## v1.0.92 - 2026-08-19 - Stop the per-column Action dropdowns from clipping their text: a short control box inherited the body line-height, so the line box overflowed the select and cut the glyphs. diff --git a/Cargo.lock b/Cargo.lock index e60961f..a8db20d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -681,7 +681,7 @@ dependencies = [ [[package]] name = "csv-anonymizer-app" -version = "1.0.92" +version = "1.0.93" dependencies = [ "csv-anonymizer-core", "tempfile", @@ -689,7 +689,7 @@ dependencies = [ [[package]] name = "csv-anonymizer-core" -version = "1.0.92" +version = "1.0.93" dependencies = [ "blake3", "card-validate", @@ -720,7 +720,7 @@ dependencies = [ [[package]] name = "csv-anonymizer-tauri" -version = "1.0.92" +version = "1.0.93" dependencies = [ "csv-anonymizer-core", "directories", diff --git a/Cargo.toml b/Cargo.toml index c752cd3..dcd0582 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ resolver = "3" [workspace.package] -version = "1.0.92" +version = "1.0.93" edition = "2024" license = "MIT" authors = ["Douwe de Vries "] diff --git a/build/linux/io.github.ddv1982.csv-data-anonymizer.metainfo.xml b/build/linux/io.github.ddv1982.csv-data-anonymizer.metainfo.xml index 81071ef..e56d2f2 100644 --- a/build/linux/io.github.ddv1982.csv-data-anonymizer.metainfo.xml +++ b/build/linux/io.github.ddv1982.csv-data-anonymizer.metainfo.xml @@ -17,6 +17,7 @@ Utility + diff --git a/crates/csv-anonymizer-core/src/csv_io.rs b/crates/csv-anonymizer-core/src/csv_io.rs index c535f07..607739b 100644 --- a/crates/csv-anonymizer-core/src/csv_io.rs +++ b/crates/csv-anonymizer-core/src/csv_io.rs @@ -1,5 +1,5 @@ use crate::error::{AnonymizerError, Result, csv_error}; -use crate::file_ops::replace_file_atomically; +use crate::file_ops::replace_file_atomically_with_handle; use crate::process_control::{check_canceled, report_progress}; use crate::sampling::SpreadSampler; use crate::strategies::{TransformState, transform_row_with_state}; @@ -509,23 +509,28 @@ pub(crate) fn process_file_with_control_and_overwrite( ) -> Result { validate_file(input_path)?; let start_time = Instant::now(); - let mut result = replace_file_atomically(output_path, overwrite, |temporary_output_path| { - process_file_to_temporary_output( - input_path, - temporary_output_path, - columns, - options, - control, - start_time, - ) - })?; + let mut result = replace_file_atomically_with_handle( + output_path, + overwrite, + |temporary_output_path, temporary_output_file| { + process_file_to_temporary_output( + input_path, + temporary_output_path, + temporary_output_file, + columns, + options, + control, + start_time, + ) + }, + )?; result.output_path = output_path.to_path_buf(); Ok(result) } - fn process_file_to_temporary_output( input_path: &Path, temporary_output_path: &Path, + temporary_output_file: &mut std::fs::File, columns: &[ColumnMetadata], options: ProcessOptions<'_>, control: Option<&mut ProcessControl<'_>>, @@ -538,8 +543,7 @@ fn process_file_to_temporary_output( .map_err(csv_error)?; let mut writer = WriterBuilder::new() .has_headers(false) - .from_path(temporary_output_path) - .map_err(csv_error)?; + .from_writer(&mut *temporary_output_file); process_csv_reader_to_writer( &mut reader, diff --git a/crates/csv-anonymizer-core/src/detection/spans.rs b/crates/csv-anonymizer-core/src/detection/spans.rs index d0427e7..26ae693 100644 --- a/crates/csv-anonymizer-core/src/detection/spans.rs +++ b/crates/csv-anonymizer-core/src/detection/spans.rs @@ -250,18 +250,33 @@ fn push_secret_spans<'a>(content: &'a str, candidates: &mut Vec> } } - for regex_match in private_key_marker_pattern().find_iter(content) { + for begin in private_key_marker_pattern().find_iter(content) { + // Redact the complete PEM object, not just its BEGIN line. If a log truncates + // before the END marker, conservatively treat the remainder as key material: + // publishing an unterminated fragment can still disclose the private key. + let label = begin + .as_str() + .strip_prefix("-----BEGIN ") + .and_then(|label| label.strip_suffix("-----")); + let end = label + .and_then(|label| { + let marker = format!("-----END {label}-----"); + content[begin.end()..] + .find(&marker) + .map(|offset| begin.end() + offset + marker.len()) + }) + .unwrap_or(content.len()); candidates.push(PrivacySpan { field_name: "secret", kind: PrivacyFindingKind::CredentialOrSecret, data_type: DataType::String, - start: regex_match.start(), - end: regex_match.end(), - value: regex_match.as_str(), + start: begin.start(), + end, + value: &content[begin.start()..end], confidence: Confidence::High, score: 99, detector: "pattern:private-key", - reason: "Private key marker pattern.", + reason: "Private key PEM block, including delimiter and payload.", priority: 2, }); } @@ -594,6 +609,31 @@ mod tests { )); } + #[test] + fn private_key_spans_cover_complete_and_unterminated_pem_blocks() { + let complete = "prefix\n-----BEGIN RSA PRIVATE KEY-----\nabc123\n-----END RSA PRIVATE KEY-----\nsuffix"; + let spans = collect_privacy_spans(complete); + let span = spans + .iter() + .find(|span| span.detector == "pattern:private-key") + .expect("PEM block should be detected"); + assert_eq!( + span.value, + "-----BEGIN RSA PRIVATE KEY-----\nabc123\n-----END RSA PRIVATE KEY-----" + ); + assert_eq!(&complete[..span.start], "prefix\n"); + assert_eq!(&complete[span.end..], "\nsuffix"); + + let truncated = "-----BEGIN PRIVATE KEY-----\nsecret payload"; + let span = collect_privacy_spans(truncated) + .into_iter() + .find(|span| span.detector == "pattern:private-key") + .expect("unterminated PEM block should be detected"); + assert_eq!(span.start, 0); + assert_eq!(span.end, truncated.len()); + assert_eq!(span.value, truncated); + } + fn phone_span_confidence(content: &str) -> Option { collect_privacy_spans(content) .iter() diff --git a/crates/csv-anonymizer-core/src/direct_input/documents.rs b/crates/csv-anonymizer-core/src/direct_input/documents.rs index d100ae8..7d0df1e 100644 --- a/crates/csv-anonymizer-core/src/direct_input/documents.rs +++ b/crates/csv-anonymizer-core/src/direct_input/documents.rs @@ -85,13 +85,13 @@ fn transform_value_document( provider: Option<&mut dyn SmartReplacementProvider>, tokenization_key: Option<&crate::TokenizationKey>, ) -> Result<(Value, PasteTransformData)> { + let start_time = Instant::now(); let (analysis, coverage) = analyze_value_document_with_coverage(format, &value, input.sample_row_count)?; let metadata = select_columns(&analysis.columns, &input.columns, &input.controls)?; let selected_by_path = selected_columns_by_source(&metadata); let smart_replacements = prepare_value_smart_replacements(&value, format, &metadata, &input, provider)?; - let start_time = Instant::now(); let mut state = TransformState::with_smart_replacements_if_active(smart_replacements) .with_tokenization_key(tokenization_key.cloned()); let mut row_indices = HashMap::new(); @@ -106,13 +106,15 @@ fn transform_value_document( transform_json_value(&mut value, &mut Vec::new(), &mut context); let row_count = infer_value_row_count(&value); - // The output is serialized by the caller, which knows whether this document is - // JSON or YAML; everything else about the run is settled here. + let mut report = state.report(); + // Scalar document traversal does not run the row-level residual audit; mark that + // distinction explicitly so an empty fingerprint intersection is not a verified pass. + report.residual_audit_incomplete = true; let result = paste_transform_data( String::new(), row_count, &metadata, - state.report(), + report, coverage, start_time, ); diff --git a/crates/csv-anonymizer-core/src/direct_input/tests.rs b/crates/csv-anonymizer-core/src/direct_input/tests.rs index 61f97e2..973a3d0 100644 --- a/crates/csv-anonymizer-core/src/direct_input/tests.rs +++ b/crates/csv-anonymizer-core/src/direct_input/tests.rs @@ -908,6 +908,112 @@ fn transforms_selected_xml_cdata() { assert!(result.output.contains("Ada &keep"#; + let analysis = analyze_paste_data(PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + sample_row_count: 10, + }) + .unwrap(); + let name = analysis + .columns + .iter() + .find(|column| column.name == "users.user.name") + .unwrap(); + let result = transform_paste_data(PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + columns: vec![name.index], + controls: vec![ColumnControl { + column_index: name.index, + type_override: Some(DataType::FullName), + strategy: AnonymizationStrategy::Redact, + }], + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }) + .unwrap(); + let mut reader = quick_xml::Reader::from_str(&result.output); + loop { + if matches!(reader.read_event().unwrap(), quick_xml::events::Event::Eof) { + break; + } + } + assert!(result.output.contains("keep")); +} + +#[test] +fn smart_xml_replacements_use_combined_text_cdata_and_references() { + let input = r#"Ada &"#; + let analysis = analyze_paste_data(PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + sample_row_count: 10, + }) + .unwrap(); + let name = analysis + .columns + .iter() + .find(|column| column.name == "users.user.name") + .unwrap(); + let mut provider = RecordingSmartProvider::default(); + + let result = transform_paste_data_with_smart_provider( + PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Xml, + columns: vec![name.index], + controls: vec![ColumnControl { + column_index: name.index, + type_override: Some(DataType::FullName), + strategy: AnonymizationStrategy::LocalAi, + }], + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }, + Some(&mut provider), + ) + .unwrap(); + + assert_eq!(provider.requests, vec![vec!["Ada Lovelace&".to_string()]]); + assert!(result.output.contains("Generated Person 1")); +} + +#[test] +fn scalar_document_marks_unmeasured_residual_audit_as_review() { + let input = r#"{"first":"alice@example.com","second":"alice@example.com"}"#; + let analysis = analyze_paste_data(PasteAnalyzeParams { + content: input.to_string(), + format: PasteDataFormat::Json, + sample_row_count: 10, + }) + .unwrap(); + let selected = analysis + .columns + .iter() + .map(|column| column.index) + .collect::>(); + let result = transform_paste_data(PasteTransformParams { + content: input.to_string(), + format: PasteDataFormat::Json, + columns: selected, + controls: Vec::new(), + sample_row_count: 100, + preview_smart_replacements: Vec::new(), + }) + .unwrap(); + let audit = result + .privacy_report + .evidence + .iter() + .find(|item| item.id == "broad-residual-value-audit") + .expect("scalar report should include residual audit evidence"); + assert_eq!(audit.status, crate::types::ReleaseEvidenceStatus::Review); + assert!(audit.detail.contains("was not run")); +} + #[test] fn json_paths_distinguish_literal_dotted_keys_from_nested_keys() { let input = r#"{ diff --git a/crates/csv-anonymizer-core/src/direct_input/text.rs b/crates/csv-anonymizer-core/src/direct_input/text.rs index ea30441..236082c 100644 --- a/crates/csv-anonymizer-core/src/direct_input/text.rs +++ b/crates/csv-anonymizer-core/src/direct_input/text.rs @@ -104,12 +104,14 @@ pub(super) fn replay_text_candidate_evidence( provider, tokenization_key, )?; - let row_count = input.content.split('\n').count(); + let mut report = replay.report; + report.residual_audit_incomplete = true; + let row_count = input.content.lines().count().max(1); Ok(paste_transform_data( replay.output, row_count, &replay.columns, - replay.report, + report, DetectionCoverage::values(row_count, row_count), start_time, )) @@ -415,11 +417,14 @@ pub(super) fn transform_text_with_smart_provider( } output.push_str(&input.content[last_end..]); + let mut report = state.report(); + // Span replacement is scalar-oriented and does not drive the row-level residual audit. + report.residual_audit_incomplete = true; Ok(paste_transform_data( output, analysis.row_count, &metadata, - state.report(), + report, coverage, start_time, )) diff --git a/crates/csv-anonymizer-core/src/direct_input/xml.rs b/crates/csv-anonymizer-core/src/direct_input/xml.rs index 6866352..1b295ff 100644 --- a/crates/csv-anonymizer-core/src/direct_input/xml.rs +++ b/crates/csv-anonymizer-core/src/direct_input/xml.rs @@ -98,11 +98,14 @@ pub(super) fn transform_xml_with_smart_provider( .with_tokenization_key(tokenization_key.cloned()); let output = transform_xml_content(&input.content, &selected_by_path, &mut state)?; + let mut report = state.report(); + // XML scalar content is transformed node-by-node, not through the row-level audit. + report.residual_audit_incomplete = true; Ok(paste_transform_data( output, analysis.row_count, &metadata, - state.report(), + report, coverage, start_time, )) @@ -116,14 +119,17 @@ pub(super) fn collect_xml_fields( reader.config_mut().trim_text(false); let mut path = Vec::new(); let mut fields = Vec::new(); + let mut pending_scalar = None; loop { match reader.read_event().map_err(xml_error)? { Event::Start(event) => { + flush_xml_field_sample(&mut pending_scalar, &mut fields, limits)?; path.push(xml_name(event.name().as_ref())); collect_xml_attributes(&reader, &event, &path, &mut fields, limits)?; } Event::Empty(event) => { + flush_xml_field_sample(&mut pending_scalar, &mut fields, limits)?; path.push(xml_name(event.name().as_ref())); collect_xml_attributes(&reader, &event, &path, &mut fields, limits)?; path.pop(); @@ -132,23 +138,50 @@ pub(super) fn collect_xml_fields( let value = event .xml_content(XmlVersion::Implicit1_0) .map_err(xml_error)?; - push_xml_text_sample(&mut fields, &path, value.trim(), limits)?; + append_pending_xml_scalar(&mut pending_scalar, &path, value.into_owned(), false); } Event::CData(event) => { let value = event.decode().map_err(xml_error)?; - push_xml_text_sample(&mut fields, &path, value.trim(), limits)?; + append_pending_xml_scalar(&mut pending_scalar, &path, value.into_owned(), true); + } + Event::GeneralRef(event) => { + let value = decode_xml_general_ref(event.as_ref()) + .ok_or_else(|| xml_error("unsupported XML general reference"))?; + append_pending_xml_scalar(&mut pending_scalar, &path, value, false); } Event::End(_) => { + flush_xml_field_sample(&mut pending_scalar, &mut fields, limits)?; path.pop(); } - Event::Eof => break, - _ => {} + Event::Eof => { + flush_xml_field_sample(&mut pending_scalar, &mut fields, limits)?; + break; + } + _ => { + flush_xml_field_sample(&mut pending_scalar, &mut fields, limits)?; + } } } Ok(fields) } +fn flush_xml_field_sample( + pending: &mut Option, + fields: &mut Vec, + limits: FieldSampleLimits, +) -> Result<()> { + let Some(pending_scalar) = pending.take() else { + return Ok(()); + }; + push_xml_text_sample( + fields, + &pending_scalar.path, + pending_scalar.raw.trim(), + limits, + ) +} + fn push_xml_text_sample( fields: &mut Vec, path: &[String], @@ -204,54 +237,83 @@ fn transform_xml_content( state, }; + let mut pending_scalar: Option = None; loop { let event = reader.read_event().map_err(xml_error)?; match event { Event::Start(event) => { + flush_pending_xml_scalar(&mut pending_scalar, &mut writer, &mut transform_context)?; path.push(xml_name(event.name().as_ref())); let event = transform_xml_attributes(&reader, event, &path, &mut transform_context)?; writer.write_event(Event::Start(event)).map_err(xml_error)?; } Event::Empty(event) => { + flush_pending_xml_scalar(&mut pending_scalar, &mut writer, &mut transform_context)?; path.push(xml_name(event.name().as_ref())); let event = transform_xml_attributes(&reader, event, &path, &mut transform_context)?; writer.write_event(Event::Empty(event)).map_err(xml_error)?; path.pop(); } - // Text and CDATA decode before the path is checked, so a node that - // cannot be decoded fails the run even when it was never selected. - // That is safe only because `collect_xml_fields` decodes every node - // unconditionally too, and `transform_xml_with_smart_provider` always - // analyzes before it transforms — so an undecodable document has - // already been rejected by the time it reaches here. + // Text, CDATA, and general references are one logical scalar in XML. Buffer adjacent + // content so a value split as `Ada Lovelace` is transformed once, rather than + // leaving the reference untouched or assigning separate replacements to each piece. Event::Text(event) => { let raw = event .xml_content(XmlVersion::Implicit1_0) .map_err(xml_error)?; - let replacement = xml_text_replacement(&path, &raw, &mut transform_context); - let event = match replacement.as_deref() { - Some(anonymized) => Event::Text(BytesText::new(anonymized)), - None => Event::Text(event), - }; - writer.write_event(event).map_err(xml_error)?; + if selected_xml_text_path(&path, selected_by_path) { + append_pending_xml_scalar(&mut pending_scalar, &path, raw.into_owned(), false); + } else { + flush_pending_xml_scalar( + &mut pending_scalar, + &mut writer, + &mut transform_context, + )?; + writer.write_event(Event::Text(event)).map_err(xml_error)?; + } } Event::CData(event) => { let raw = event.decode().map_err(xml_error)?; - let replacement = xml_text_replacement(&path, &raw, &mut transform_context); - let event = match replacement.as_deref() { - Some(anonymized) => Event::CData(BytesCData::new(anonymized)), - None => Event::CData(event), - }; - writer.write_event(event).map_err(xml_error)?; + if selected_xml_text_path(&path, selected_by_path) { + append_pending_xml_scalar(&mut pending_scalar, &path, raw.into_owned(), true); + } else { + flush_pending_xml_scalar( + &mut pending_scalar, + &mut writer, + &mut transform_context, + )?; + writer.write_event(Event::CData(event)).map_err(xml_error)?; + } + } + Event::GeneralRef(event) => { + if selected_xml_text_path(&path, selected_by_path) { + let raw = decode_xml_general_ref(event.as_ref()) + .ok_or_else(|| xml_error("unsupported XML general reference"))?; + append_pending_xml_scalar(&mut pending_scalar, &path, raw, false); + } else { + flush_pending_xml_scalar( + &mut pending_scalar, + &mut writer, + &mut transform_context, + )?; + writer + .write_event(Event::GeneralRef(event)) + .map_err(xml_error)?; + } } Event::End(event) => { + flush_pending_xml_scalar(&mut pending_scalar, &mut writer, &mut transform_context)?; writer.write_event(Event::End(event)).map_err(xml_error)?; path.pop(); } - Event::Eof => break, + Event::Eof => { + flush_pending_xml_scalar(&mut pending_scalar, &mut writer, &mut transform_context)?; + break; + } other => { + flush_pending_xml_scalar(&mut pending_scalar, &mut writer, &mut transform_context)?; writer.write_event(other).map_err(xml_error)?; } } @@ -260,6 +322,80 @@ fn transform_xml_content( String::from_utf8(writer.into_inner()).map_err(xml_error) } +struct PendingXmlScalar { + path: Vec, + raw: String, + all_cdata: bool, +} + +fn selected_xml_text_path(path: &[String], selected: &HashMap) -> bool { + selected.contains_key(&xml_text_source_path(path)) +} + +fn append_pending_xml_scalar( + pending: &mut Option, + path: &[String], + raw: String, + is_cdata: bool, +) { + match pending { + Some(existing) if existing.path == path => { + existing.raw.push_str(&raw); + existing.all_cdata &= is_cdata; + } + _ => { + *pending = Some(PendingXmlScalar { + path: path.to_vec(), + raw, + all_cdata: is_cdata, + }); + } + } +} + +fn flush_pending_xml_scalar( + pending: &mut Option, + writer: &mut Writer>, + context: &mut XmlTransformContext<'_>, +) -> Result<()> { + let Some(pending_scalar) = pending.take() else { + return Ok(()); + }; + let replacement = xml_text_replacement(&pending_scalar.path, &pending_scalar.raw, context); + match replacement.as_deref() { + Some(anonymized) if pending_scalar.all_cdata => writer + .write_event(Event::CData(BytesCData::new(anonymized))) + .map_err(xml_error), + Some(anonymized) => writer + .write_event(Event::Text(BytesText::new(anonymized))) + .map_err(xml_error), + None if pending_scalar.all_cdata => writer + .write_event(Event::CData(BytesCData::new(&pending_scalar.raw))) + .map_err(xml_error), + None => writer + .write_event(Event::Text(BytesText::new(&pending_scalar.raw))) + .map_err(xml_error), + } +} + +fn decode_xml_general_ref(raw: &[u8]) -> Option { + let name = std::str::from_utf8(raw).ok()?; + match name { + "amp" => Some("&".to_string()), + "lt" => Some("<".to_string()), + "gt" => Some(">".to_string()), + "apos" => Some("'".to_string()), + "quot" => Some("\"".to_string()), + value if let Some(hex) = value.strip_prefix("#x") => { + char::from_u32(u32::from_str_radix(hex, 16).ok()?).map(|c| c.to_string()) + } + value if let Some(decimal) = value.strip_prefix('#') => { + char::from_u32(decimal.parse().ok()?).map(|c| c.to_string()) + } + _ => None, + } +} + /// Wraps any XML reader/writer failure as an input-parse error. /// /// Every fallible XML call reports the same way, so the message stays uniform diff --git a/crates/csv-anonymizer-core/src/file_ops.rs b/crates/csv-anonymizer-core/src/file_ops.rs index a49525a..c4b20bf 100644 --- a/crates/csv-anonymizer-core/src/file_ops.rs +++ b/crates/csv-anonymizer-core/src/file_ops.rs @@ -1,30 +1,50 @@ use crate::error::{AnonymizerError, Result}; -use std::fs; +use std::fs::{self, File}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; static NEXT_TEMPORARY_OUTPUT_FILE: AtomicU64 = AtomicU64::new(0); +#[cfg(test)] pub(crate) fn replace_file_atomically( output_path: &Path, overwrite: bool, write_temporary: impl FnOnce(&Path) -> Result, ) -> Result { - let temporary_output_path = reserve_temporary_output_path(output_path)?; - match write_temporary(&temporary_output_path) { + replace_file_atomically_with_handle(output_path, overwrite, |path, _file| write_temporary(path)) +} + +pub(crate) fn replace_file_atomically_with_handle( + output_path: &Path, + overwrite: bool, + write_temporary: impl FnOnce(&Path, &mut File) -> Result, +) -> Result { + let (temporary_output_path, mut temporary_output_file) = + reserve_temporary_output_path(output_path)?; + match write_temporary(&temporary_output_path, &mut temporary_output_file) { Ok(result) => { - // Only the overwrite branch adopts a mode. It is the one that replaces a file - // the user already placed; on the other branch a destination existing is the - // error, so there is no mode there to inherit and reading one would describe a - // file publication is about to refuse to touch. - let publish_result = if overwrite { - adopt_destination_permissions(&temporary_output_path, output_path).and_then(|()| { - fs::rename(&temporary_output_path, output_path).map_err(AnonymizerError::from) - }) - } else { - fs::hard_link(&temporary_output_path, output_path) - .map_err(|error| no_clobber_publish_error(error, output_path)) - }; + let publish_result = (|| { + temporary_output_file.sync_all()?; + ensure_temporary_path_matches_handle( + &temporary_output_path, + &temporary_output_file, + )?; + // Only the overwrite branch adopts a mode. It is the one that replaces a file + // the user already placed; on the other branch a destination existing is the + // error, so there is no mode there to inherit and reading one would describe a + // file publication is about to refuse to touch. + if overwrite { + adopt_destination_permissions(&temporary_output_path, output_path).and_then( + |()| { + fs::rename(&temporary_output_path, output_path) + .map_err(AnonymizerError::from) + }, + ) + } else { + fs::hard_link(&temporary_output_path, output_path) + .map_err(|error| no_clobber_publish_error(error, output_path)) + } + })(); match publish_result { Ok(()) => { if !overwrite { @@ -45,6 +65,25 @@ pub(crate) fn replace_file_atomically( } } +#[cfg(unix)] +fn ensure_temporary_path_matches_handle(path: &Path, file: &File) -> Result<()> { + use std::os::unix::fs::MetadataExt; + + let path_metadata = fs::metadata(path)?; + let file_metadata = file.metadata()?; + if path_metadata.dev() != file_metadata.dev() || path_metadata.ino() != file_metadata.ino() { + return Err(AnonymizerError::Io(std::io::Error::other( + "Temporary output path was replaced before publication.", + ))); + } + Ok(()) +} + +#[cfg(not(unix))] +fn ensure_temporary_path_matches_handle(_path: &Path, _file: &File) -> Result<()> { + Ok(()) +} + /// Explains a no-clobber publish that failed for a reason other than the /// destination existing. /// @@ -163,7 +202,7 @@ fn adopt_destination_permissions(_temporary_path: &Path, _output_path: &Path) -> Ok(()) } -fn reserve_temporary_output_path(output_path: &Path) -> Result { +fn reserve_temporary_output_path(output_path: &Path) -> Result<(PathBuf, File)> { let parent = output_path.parent().unwrap_or_else(|| Path::new(".")); let file_name = output_path .file_name() @@ -177,18 +216,13 @@ fn reserve_temporary_output_path(output_path: &Path) -> Result { )); let mut options = fs::OpenOptions::new(); options.create_new(true).write(true); - // Owner-only from the moment it exists, not set afterwards: between creation - // and a later chmod the file would be readable at the directory's default - // mode, and it is holding transformed source data for the whole of that - // window. `create_new` above means this mode applies to a file this call just - // made, never to one it found. #[cfg(unix)] { use std::os::unix::fs::OpenOptionsExt; options.mode(0o600); } match options.open(&path) { - Ok(_) => return Ok(path), + Ok(file) => return Ok((path, file)), Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, Err(error) => return Err(error.into()), } @@ -221,7 +255,6 @@ mod tests { .ends_with(".tmp") })); } - #[test] fn failed_publish_removes_the_temporary_file() { let temp_dir = tempfile::tempdir().unwrap(); @@ -243,6 +276,30 @@ mod tests { })); } + #[cfg(unix)] + #[test] + fn retained_output_handle_survives_temporary_path_symlink_substitution() { + use std::io::Write; + + let temp_dir = tempfile::tempdir().unwrap(); + let output_path = temp_dir.path().join("output.csv"); + let victim_path = temp_dir.path().join("victim.csv"); + fs::write(&victim_path, "victim").unwrap(); + + let error = + replace_file_atomically_with_handle(&output_path, true, |temporary_path, file| { + fs::remove_file(temporary_path).unwrap(); + std::os::unix::fs::symlink(&victim_path, temporary_path).unwrap(); + file.write_all(b"anonymized").unwrap(); + Ok(()) + }) + .unwrap_err(); + + assert!(matches!(error, AnonymizerError::Io(_))); + assert_eq!(fs::read_to_string(&victim_path).unwrap(), "victim"); + assert!(!output_path.exists()); + } + #[test] fn publication_matrix_preserves_overwrite_contracts() { let temp_dir = tempfile::tempdir().unwrap(); @@ -461,7 +518,7 @@ mod tests { let threads = (0..16) .map(|_| { let output_path = output_path.clone(); - std::thread::spawn(move || reserve_temporary_output_path(&output_path).unwrap()) + std::thread::spawn(move || reserve_temporary_output_path(&output_path).unwrap().0) }) .collect::>(); let paths = threads diff --git a/crates/csv-anonymizer-core/src/release_report.rs b/crates/csv-anonymizer-core/src/release_report.rs index 0bc96c1..cf921e4 100644 --- a/crates/csv-anonymizer-core/src/release_report.rs +++ b/crates/csv-anonymizer-core/src/release_report.rs @@ -508,10 +508,14 @@ pub(crate) fn build_readiness( if let Some(report) = context.transform_report && report.residual_audit_incomplete { - review_items.push( + review_items.push(if report.residual_audit_source_values == 0 + && report.residual_audit_output_values == 0 + { + "The broad residual-value audit was not run for this scalar workflow; its empty result must not be treated as a pass.".to_string() + } else { "The broad residual-value audit reached its memory bound and is incomplete." - .to_string(), - ); + .to_string() + }); } // A review item rather than a blocker. Whether few distinct values matter depends // on what the column holds — a six-valued column may carry nothing sensitive — so @@ -668,7 +672,12 @@ pub(crate) fn build_evidence( } else { ReleaseEvidenceStatus::Verified }, - detail: if report.residual_audit_incomplete { + detail: if report.residual_audit_incomplete + && report.residual_audit_source_values == 0 + && report.residual_audit_output_values == 0 + { + "The broad residual-value audit was not run for this scalar workflow; its empty result must not be treated as a pass.".to_string() + } else if report.residual_audit_incomplete { format!( "The bounded audit reached its capacity after fingerprinting {} protected source value(s) and {} released value(s); its result is incomplete and must not be treated as a pass.", report.residual_audit_source_values, report.residual_audit_output_values diff --git a/crates/csv-anonymizer-core/src/service/tests/smart_replacement.rs b/crates/csv-anonymizer-core/src/service/tests/smart_replacement.rs index 2c0e656..042d3f3 100644 --- a/crates/csv-anonymizer-core/src/service/tests/smart_replacement.rs +++ b/crates/csv-anonymizer-core/src/service/tests/smart_replacement.rs @@ -342,6 +342,46 @@ fn anonymize_rejects_invalid_preview_smart_replacements_and_generates_missing_va ); } +#[test] +fn anonymize_revalidates_preview_replacement_against_tail_values() { + let workspace = Workspace::new(); + let input_path = workspace.path("smart-preview-tail-collision.csv"); + let output_path = workspace.path("smart-preview-tail-collision-output.csv"); + fs::write(&input_path, "name\nAlice Smith\nCharlie Ray\n").unwrap(); + let mut provider = RecordingSmartProvider::new("Generated"); + + let result = workspace + .service + .anonymize_csv_with_sample_rows_and_control_and_smart_provider( + AnonymizeParams { + controls: vec![typed_control( + 0, + DataType::FullName, + AnonymizationStrategy::LocalAi, + )], + preview_smart_replacements: vec![SmartReplacementEntry { + column_index: 0, + original: "Alice Smith".to_string(), + replacement: "Charlie Ray".to_string(), + }], + ..anonymize_params(input_path, output_path.clone(), vec![0]) + }, + 10, + None, + Some(&mut provider), + ) + .unwrap(); + + let output = read_sample(&output_path, 10).unwrap(); + assert_ne!(output.rows[0][0], "Charlie Ray"); + assert_ne!(output.rows[1][0], "Charlie Ray"); + assert_eq!( + provider.requests, + vec![vec!["Alice Smith".to_string(), "Charlie Ray".to_string()]] + ); + assert!(result.privacy_report.smart_replacement_rejections >= 1); +} + #[test] fn anonymize_rejects_smart_replacements_carrying_another_rows_value() { let workspace = Workspace::new(); diff --git a/crates/csv-anonymizer-core/src/smart.rs b/crates/csv-anonymizer-core/src/smart.rs index 59fb60b..716f5be 100644 --- a/crates/csv-anonymizer-core/src/smart.rs +++ b/crates/csv-anonymizer-core/src/smart.rs @@ -192,6 +192,37 @@ impl SmartReplacementMap { .map(|replacement| value_identity_key(&replacement.original)) .collect() } + /// Drops preview entries whose output carries any source value from the + /// complete final column. A preview only sees its display window, so an + /// output that looked synthetic there can collide with a value in the tail. + /// Those originals must be requested again rather than reused. + fn reject_replacements_containing_source_values( + &mut self, + column_index: usize, + source_keys: &BTreeSet, + ) { + let mut rejected = Vec::new(); + self.replacements.retain(|key, replacement| { + if key.column_index != column_index { + return true; + } + let reason = invalid_replacement_reason( + &replacement.original, + &replacement.replacement, + source_keys, + ); + if let Some(reason) = reason { + rejected.push(reason); + false + } else { + true + } + }); + self.rejected_values += rejected.len(); + for reason in rejected { + *self.rejection_counts.entry(reason).or_default() += 1; + } + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -383,6 +414,8 @@ fn build_replacement_map( let mut map = existing.cloned().unwrap_or_default(); for (column_index, values) in batches { + let final_source_keys = source_keys(&values); + map.reject_replacements_containing_source_values(column_index, &final_source_keys); let missing_values = values .into_iter() .filter(|value| !map.contains(column_index, value)) diff --git a/crates/csv-anonymizer-core/src/strategies/state.rs b/crates/csv-anonymizer-core/src/strategies/state.rs index 1810ceb..3046c54 100644 --- a/crates/csv-anonymizer-core/src/strategies/state.rs +++ b/crates/csv-anonymizer-core/src/strategies/state.rs @@ -30,7 +30,9 @@ impl TokenizationKey { return Err(AnonymizerError::InvalidTokenizationKey); } let mut bytes = [0_u8; 32]; - for (index, pair) in value.as_bytes().chunks_exact(2).enumerate() { + let (pairs, remainder) = value.as_bytes().as_chunks::<2>(); + debug_assert!(remainder.is_empty()); + for (index, pair) in pairs.iter().enumerate() { let pair = std::str::from_utf8(pair).map_err(|_| AnonymizerError::InvalidTokenizationKey)?; bytes[index] = u8::from_str_radix(pair, 16) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index ec9156f..4fa2e89 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "csv-anonymizer-frontend", - "version": "1.0.92", + "version": "1.0.93", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "csv-anonymizer-frontend", - "version": "1.0.92", + "version": "1.0.93", "hasInstallScript": true, "dependencies": { "@tauri-apps/api": "^2.11.1", @@ -3168,9 +3168,9 @@ } }, "node_modules/baseline-browser-mapping": { - "version": "2.10.38", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.38.tgz", - "integrity": "sha512-31/02mVB4yuQU6adKk5SlY6m+mxDwUq5KZkyYgnLrrKl7TEm1+3PyDtDBz2kOv/wxZz41GHsvV1A/u6RmiyBvw==", + "version": "2.11.21", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.21.tgz", + "integrity": "sha512-uh8vpY/1/YyFkunIDFH/12p7/7VdPKA1hejMVEbdkEaWnUz0Hesvx5EbiU6XxjyHZIOju+ZMbQJkRh+es3/spQ==", "dev": true, "license": "Apache-2.0", "bin": { @@ -3204,9 +3204,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.4", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.4.tgz", - "integrity": "sha512-MTc8i/x9jBQd1iMw2CFGS+rwMa07eYjLR0CCTLDACl9xhxy+nIs3KeML/biicXtk9JrZ6dnnTatmc7ErPXIxqw==", + "version": "4.28.9", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.9.tgz", + "integrity": "sha512-EWazOblFYUvlGZcfGhPUPmYh3nikUxBVb+y9MJun5f3hBi812X+8MSQTujLBtgK3cf51fJWbWfOjyeO954d+Eg==", "dev": true, "funding": [ { @@ -3224,11 +3224,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.10.38", - "caniuse-lite": "^1.0.30001799", - "electron-to-chromium": "^1.5.376", - "node-releases": "^2.0.48", - "update-browserslist-db": "^1.2.3" + "baseline-browser-mapping": "^2.11.20", + "caniuse-lite": "^1.0.30001810", + "electron-to-chromium": "^1.5.420", + "node-releases": "^2.0.54", + "update-browserslist-db": "^1.3.2" }, "bin": { "browserslist": "cli.js" @@ -3238,9 +3238,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001799", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001799.tgz", - "integrity": "sha512-hG1bReV+OUU+MOqK4t/ZWI0tZOyz3rqS9XuhOUz1cIcbwBKjOyJEJuw9ER5JuNyqxNk8u/JUVbGibBOL1yrjFw==", + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", "dev": true, "funding": [ { @@ -3393,9 +3393,9 @@ "peer": true }, "node_modules/electron-to-chromium": { - "version": "1.5.378", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.378.tgz", - "integrity": "sha512-VinvOAuuPmdD1guEgGv5f2Qp7/vlfqOrUOMYNnOD4wj3pit8kRsQHzfIf6teyUGWo15Tg5+bOJaRunvyltpVWQ==", + "version": "1.5.423", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.423.tgz", + "integrity": "sha512-rRZfTSY8ptHYMQxa+uIycJMFKmY1T0GIApNMXJYGehguTZa56TEEl19pKPCoBqk5Gpf7QizZn/jt7xur+DYxag==", "dev": true, "license": "ISC" }, @@ -4528,9 +4528,9 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.50", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.50.tgz", - "integrity": "sha512-J6l92tKHX6w8Jy5nO1Vuc01NoIiRGi/d6qBKVxh+IQ8Cr3b6HbVNfKiF8ZpFKufTwpwxMmce2W3iQZ861ZRyTg==", + "version": "2.0.54", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz", + "integrity": "sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ==", "dev": true, "license": "MIT", "engines": { @@ -5461,9 +5461,9 @@ "license": "MIT" }, "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.2.tgz", + "integrity": "sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw==", "dev": true, "funding": [ { diff --git a/frontend/package.json b/frontend/package.json index 40d4385..1642418 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "csv-anonymizer-frontend", "private": true, - "version": "1.0.92", + "version": "1.0.93", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src/components/PrivacyReportSummary.test.tsx b/frontend/src/components/PrivacyReportSummary.test.tsx index 6e53993..7a0c50a 100644 --- a/frontend/src/components/PrivacyReportSummary.test.tsx +++ b/frontend/src/components/PrivacyReportSummary.test.tsx @@ -66,6 +66,19 @@ describe('PrivacyReportSummary', () => { expect(screen.queryByText('Smart Replacement')).not.toBeInTheDocument() }) + it('counts labelled columns as transformed and describes an all-labelled report', () => { + render( + , + ) + + expect(screen.getByText('2')).toBeInTheDocument() + expect(screen.getByText('2 labelled')).toBeInTheDocument() + }) + it('prints a count Rust sent as a string exactly as Rust wrote it', () => { render( 0 ? parts.join(', ') : 'No transformed columns' } - export function sensitiveSummary(privacyReport: PrivacyReport) { const parts = [ metricPart(privacyReport.directIdentifiers, 'direct'), diff --git a/frontend/src/components/workflow/AnonymizerWorkflowView.tsx b/frontend/src/components/workflow/AnonymizerWorkflowView.tsx index d048c3b..5bbf0ef 100644 --- a/frontend/src/components/workflow/AnonymizerWorkflowView.tsx +++ b/frontend/src/components/workflow/AnonymizerWorkflowView.tsx @@ -141,27 +141,27 @@ function ColumnSelectionStep({ actions={[ { label: 'Select All', - disabled: workflow.busy === 'loading' || workflow.allSelected || workflow.columns.length === 0, + disabled: workflow.isLoading || workflow.allSelected || workflow.columns.length === 0, onClick: () => workflow.setColumnSelection(workflow.columns.map((column) => column.index)), }, { label: 'Deselect All', - disabled: workflow.busy === 'loading' || workflow.selectedColumns.length === 0, + disabled: workflow.isLoading || workflow.selectedColumns.length === 0, onClick: () => workflow.setColumnSelection([]), }, { label: 'Select High Risk', - disabled: workflow.busy === 'loading' || workflow.highRiskColumns.length === 0, + disabled: workflow.isLoading || workflow.highRiskColumns.length === 0, onClick: () => workflow.setColumnSelection(workflow.highRiskColumns), }, { label: 'Select Detected Risk', - disabled: workflow.busy === 'loading' || workflow.detectedRiskColumns.length === 0, + disabled: workflow.isLoading || workflow.detectedRiskColumns.length === 0, onClick: () => workflow.setColumnSelection(workflow.detectedRiskColumns), }, { label: 'Select Uncertain', - disabled: workflow.busy === 'loading' || workflow.uncertainColumns.length === 0, + disabled: workflow.isLoading || workflow.uncertainColumns.length === 0, onClick: () => workflow.setColumnSelection(workflow.uncertainColumns), }, ]} @@ -191,6 +191,7 @@ function ColumnSelectionStep({ allColumnCount={workflow.columns.length} selectedSet={workflow.selectedSet} loading={workflow.busy === 'loading'} + disabled={workflow.isLoading} showAllColumns={workflow.showAllColumns} hiddenColumnCount={workflow.hiddenColumnCount} onToggleColumn={workflow.toggleColumn} diff --git a/frontend/src/hooks/useAnonymizerWorkflow.ts b/frontend/src/hooks/useAnonymizerWorkflow.ts index dd74dc0..4461134 100644 --- a/frontend/src/hooks/useAnonymizerWorkflow.ts +++ b/frontend/src/hooks/useAnonymizerWorkflow.ts @@ -113,6 +113,7 @@ export function useAnonymizerWorkflow() { const invalidatingSelection = useSelectionInvalidation( { setSelectedColumns: setCsvSelectedColumns, toggleColumn: toggleCsvColumn, updateColumnStrategy: updateCsvColumnStrategy }, clearArtifacts, + () => busy !== 'idle', ) function updateSetting(key: K, value: AppSettings[K]) { diff --git a/frontend/src/hooks/usePasteDataWorkflow.ts b/frontend/src/hooks/usePasteDataWorkflow.ts index 989c161..fb1ca2b 100644 --- a/frontend/src/hooks/usePasteDataWorkflow.ts +++ b/frontend/src/hooks/usePasteDataWorkflow.ts @@ -221,6 +221,13 @@ export function usePasteDataWorkflow({ setResult(null) setPreview(null) } + const previousTokenizationKey = useRef(activeTokenizationKey) + + useEffect(() => { + if (previousTokenizationKey.current === activeTokenizationKey) return + previousTokenizationKey.current = activeTokenizationKey + clearOutput() + }, [activeTokenizationKey, setCopyStatus]) const invalidatingSelection = useSelectionInvalidation( selection, diff --git a/frontend/src/hooks/usePreviewWorkflow.ts b/frontend/src/hooks/usePreviewWorkflow.ts index 8a2879a..1acc166 100644 --- a/frontend/src/hooks/usePreviewWorkflow.ts +++ b/frontend/src/hooks/usePreviewWorkflow.ts @@ -1,3 +1,4 @@ +import { useEffect, useRef } from 'react' import type { Dispatch, SetStateAction } from 'react' import { firstPreflightBlocker, preflightAnonymization, previewAnonymization } from '../tauri' import type { @@ -44,6 +45,16 @@ export function usePreviewWorkflow( const localAiReady = localAi.ready const selectedUsesTokenization = selectionUsesTokenization(selectedColumns) const selectedTokenizationKey = selectedUsesTokenization ? tokenizationKey : null + const operationSequence = useRef(0) + const selectedColumnsFingerprint = selectedColumns.join(',') + + useEffect(() => { + operationSequence.current += 1 + if (busy === 'preview') setBusy('idle') + }, [inputPath, selectedColumnsFingerprint, tokenizationKey, preparedAnalysis]) + useEffect(() => () => { + operationSequence.current += 1 + }, []) const canPreview = Boolean( hasColumns && @@ -72,6 +83,7 @@ export function usePreviewWorkflow( return } + const sequence = ++operationSequence.current setBusy('preview') setError(null) try { @@ -88,6 +100,7 @@ export function usePreviewWorkflow( localAi: localAiRequest, preparedAnalysis, }) + if (sequence !== operationSequence.current) return const blocker = firstPreflightBlocker(preflight) if (blocker) { setPreview(null) @@ -104,12 +117,13 @@ export function usePreviewWorkflow( preparedAnalysis, tokenizationKey: tokenizationKeyForPreview, }) + if (sequence !== operationSequence.current) return setPreview(nextPreview) setResult(null) } catch (caught) { - setError(messageFrom(caught)) + if (sequence === operationSequence.current) setError(messageFrom(caught)) } finally { - setBusy('idle') + if (sequence === operationSequence.current) setBusy('idle') } } diff --git a/frontend/src/hooks/useQuickGenerateWorkflow.ts b/frontend/src/hooks/useQuickGenerateWorkflow.ts index 392421d..cf19c4d 100644 --- a/frontend/src/hooks/useQuickGenerateWorkflow.ts +++ b/frontend/src/hooks/useQuickGenerateWorkflow.ts @@ -1,4 +1,4 @@ -import { useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { generateQuickValues } from '../tauri' import type { AnonymizationStrategy, DataType, QuickTransformData } from '../types' import { messageFrom } from '../utils/errors' @@ -50,6 +50,13 @@ export function useQuickGenerateWorkflow({ setResult(null) setCopyStatus(null) } + const previousTokenizationKey = useRef(activeTokenizationKey) + + useEffect(() => { + if (previousTokenizationKey.current === activeTokenizationKey) return + previousTokenizationKey.current = activeTokenizationKey + clearOutput() + }, [activeTokenizationKey, setCopyStatus]) function setDataType(nextDataType: DataType) { setDataTypeState(nextDataType) diff --git a/package.json b/package.json index d9ced14..d241c59 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "csv-anonymizer", - "version": "1.0.92", + "version": "1.0.93", "description": "Desktop CSV anonymizer for local-first data transformation workflows.", "desktopName": "csv-anonymizer.desktop", "homepage": "https://github.com/ddv1982/csv-data-anonymizer", diff --git a/src-tauri/src/commands/job_commands.rs b/src-tauri/src/commands/job_commands.rs index dd63afb..9e18937 100644 --- a/src-tauri/src/commands/job_commands.rs +++ b/src-tauri/src/commands/job_commands.rs @@ -52,6 +52,13 @@ pub async fn start_anonymize_job( "Sample row count", ) .map_err(CommandError::invalid_input)?; + // Parse secrets before admission so malformed input never occupies the single-run lease. + let tokenization_key = request + .tokenization_key + .as_deref() + .map(csv_anonymizer_core::TokenizationKey::parse_hex) + .transpose() + .map_err(|error| CommandError::invalid_input(error.to_string()))?; // Refuse a busy app before asking the user anything. `authorize_or_confirm_output_file` // can open a blocking native dialog, and running it first walked a user through // confirming a destination only to be told afterwards that another job holds the only @@ -97,12 +104,6 @@ pub async fn start_anonymize_job( .prepared_analysis .as_ref() .map(snapshot_detection_summary); - let tokenization_key = request - .tokenization_key - .as_deref() - .map(csv_anonymizer_core::TokenizationKey::parse_hex) - .transpose() - .map_err(|error| CommandError::invalid_input(error.to_string()))?; let _job_handle = tauri::async_runtime::spawn_blocking(move || { let result = catch_unwind(AssertUnwindSafe(|| { @@ -334,4 +335,23 @@ mod tests { ); assert_eq!(repeat.state, AnonymizeJobState::Failed); } + + #[test] + fn malformed_tokenization_key_can_be_retried_without_consuming_lease() { + let jobs = AnonymizeJobStore::default(); + let malformed = csv_anonymizer_core::TokenizationKey::parse_hex("not-hex"); + assert!(malformed.is_err()); + + // Parsing happens before command admission; a valid retry can therefore acquire the + // otherwise-unused lease instead of being rejected as a concurrent run. + let valid = "11".repeat(32); + assert!(csv_anonymizer_core::TokenizationKey::parse_hex(&valid).is_ok()); + let job = jobs + .create_job_for_output(None) + .expect("valid retry admitted"); + assert_eq!( + job.snapshot().expect("status").state, + AnonymizeJobState::Running + ); + } } diff --git a/src-tauri/src/local_ai/candidate_detector.rs b/src-tauri/src/local_ai/candidate_detector.rs index e10e326..cc22352 100644 --- a/src-tauri/src/local_ai/candidate_detector.rs +++ b/src-tauri/src/local_ai/candidate_detector.rs @@ -77,6 +77,8 @@ pub fn local_candidate_detector(model: &str) -> Result, request: LocalAiRequest) { let result = tauri::async_runtime::block_on(download_model(job.clone(), request.model_name())); match result { - Ok(()) if job.should_cancel() => job.finish_canceled(), - Ok(()) => job.finish_success(), + Ok(true) if job.should_cancel() => job.finish_canceled(), + Ok(true) => job.finish_success(), + Ok(false) if job.should_cancel() => job.finish_canceled(), + Ok(false) => job.finish_error(INCOMPLETE_DOWNLOAD_ERROR.to_string()), Err(_) if job.should_cancel() => job.finish_canceled(), Err(error) => job.finish_error(error), } } -async fn download_model(job: Arc, model: String) -> Result<(), String> { +async fn download_model(job: Arc, model: String) -> Result { let client = download_client()?; let mut response = client .post(format!("{DEFAULT_OLLAMA_ENDPOINT}/api/pull")) @@ -186,36 +188,40 @@ async fn download_model(job: Arc, model: String) -> Result<( .map_err(|error| format!("Could not read Ollama download progress: {error}"))? { if job.should_cancel() { - return Ok(()); + return Ok(false); } pending.extend_from_slice(&chunk); while let Some(newline_index) = pending.iter().position(|byte| *byte == b'\n') { let line = pending.drain(..=newline_index).collect::>(); - process_download_line(&job, &line)?; + if process_download_line(&job, &line)? { + return Ok(true); + } } } - if !pending.is_empty() { - process_download_line(&job, &pending)?; + if !pending.is_empty() && process_download_line(&job, &pending)? { + return Ok(true); } - Ok(()) + Ok(false) } -fn process_download_line(job: &LocalAiDownloadJob, line: &[u8]) -> Result<(), String> { +fn process_download_line(job: &LocalAiDownloadJob, line: &[u8]) -> Result { let line = String::from_utf8_lossy(line); let line = line.trim(); if line.is_empty() { - return Ok(()); + return Ok(false); } let progress = serde_json::from_str::(line) .map_err(|error| format!("Ollama returned invalid download progress: {error}"))?; if let Some(error) = progress.error { return Err(format!("Ollama model download failed: {error}")); } + let completed = progress.status.as_deref() == Some("success"); job.report_progress(progress); - Ok(()) + Ok(completed) } +const INCOMPLETE_DOWNLOAD_ERROR: &str = "Ollama ended the model download before reporting success."; #[cfg(test)] mod tests { @@ -346,23 +352,41 @@ mod tests { } #[test] - fn download_progress_line_updates_status() { + fn download_progress_line_updates_status_without_marking_completion() { let store = LocalAiDownloadStore::default(); let job = store.create_job("gemma3:4b".to_string()).unwrap(); - process_download_line( + let completed = process_download_line( &job, br#"{"status":"pulling manifest","completed":4,"total":10} "#, ) .unwrap(); + assert!(!completed); let status = job.snapshot().unwrap(); assert_eq!(status.status_message, "pulling manifest"); assert_eq!(status.completed_bytes, Some(4)); assert_eq!(status.total_bytes, Some(10)); } + #[test] + fn empty_or_nonterminal_eof_does_not_count_as_success() { + let store = LocalAiDownloadStore::default(); + let job = store.create_job("gemma3:4b".to_string()).unwrap(); + + assert!(!process_download_line(&job, b"").unwrap()); + assert!(!process_download_line(&job, br#"{"status":"pulling layers"}"#).unwrap()); + } + + #[test] + fn explicit_ollama_success_record_marks_completion() { + let store = LocalAiDownloadStore::default(); + let job = store.create_job("gemma3:4b".to_string()).unwrap(); + + assert!(process_download_line(&job, br#"{"status":"success"}"#).unwrap()); + } + #[test] fn download_progress_line_reports_ollama_error() { let store = LocalAiDownloadStore::default(); diff --git a/src-tauri/src/local_ai/mod.rs b/src-tauri/src/local_ai/mod.rs index 035d178..d1444c3 100644 --- a/src-tauri/src/local_ai/mod.rs +++ b/src-tauri/src/local_ai/mod.rs @@ -33,6 +33,8 @@ pub fn open_setup_url() -> Result<(), String> { fn client() -> Result { BlockingClient::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) .timeout(Duration::from_secs(120)) .connect_timeout(Duration::from_secs(2)) .build() @@ -41,6 +43,8 @@ fn client() -> Result { fn download_client() -> Result { AsyncClient::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) .timeout(OLLAMA_DOWNLOAD_TIMEOUT) .read_timeout(OLLAMA_DOWNLOAD_READ_TIMEOUT) .connect_timeout(Duration::from_secs(2)) diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 6717cee..38b7d77 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -71,5 +71,5 @@ "productName": "CSV Anonymizer", "mainBinaryName": "csv-anonymizer", "identifier": "io.github.ddv1982.csv-data-anonymizer", - "version": "1.0.92" + "version": "1.0.93" }