Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# Changelog

## v1.0.93 - 2026-09-08

- Close anonymization disclosure paths in pasted text and XML, including complete PEM redaction, character-reference transformation, and honest residual-audit evidence.
- Revalidate Smart preview replacements against full-run values, retain output handles through publication, and prevent malformed keys from consuming the only anonymization job slot.
- Harden Local AI transport and download completion, invalidate stale frontend artifacts, correct Label report counts, and update the audited Browserslist dependency.

## v1.0.92 - 2026-08-19

- Stop the per-column Action dropdowns from clipping their text: a short control box inherited the body line-height, so the line box overflowed the select and cut the glyphs.
Expand Down
6 changes: 3 additions & 3 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ members = [
resolver = "3"

[workspace.package]
version = "1.0.92"
version = "1.0.93"
edition = "2024"
license = "MIT"
authors = ["Douwe de Vries <douwe.de.vries.82@gmail.com>"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
<category>Utility</category>
</categories>
<releases>
<release version="1.0.93" date="2026-09-08" />
<release version="1.0.92" date="2026-08-19" />
<release version="1.0.91" date="2026-08-18" />
<release version="1.0.90" date="2026-08-18" />
Expand Down
32 changes: 18 additions & 14 deletions crates/csv-anonymizer-core/src/csv_io.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use crate::error::{AnonymizerError, Result, csv_error};
use crate::file_ops::replace_file_atomically;
use crate::file_ops::replace_file_atomically_with_handle;
use crate::process_control::{check_canceled, report_progress};
use crate::sampling::SpreadSampler;
use crate::strategies::{TransformState, transform_row_with_state};
Expand Down Expand Up @@ -509,23 +509,28 @@ pub(crate) fn process_file_with_control_and_overwrite(
) -> Result<ProcessResult> {
validate_file(input_path)?;
let start_time = Instant::now();
let mut result = replace_file_atomically(output_path, overwrite, |temporary_output_path| {
process_file_to_temporary_output(
input_path,
temporary_output_path,
columns,
options,
control,
start_time,
)
})?;
let mut result = replace_file_atomically_with_handle(
output_path,
overwrite,
|temporary_output_path, temporary_output_file| {
process_file_to_temporary_output(
input_path,
temporary_output_path,
temporary_output_file,
columns,
options,
control,
start_time,
)
},
)?;
result.output_path = output_path.to_path_buf();
Ok(result)
}

fn process_file_to_temporary_output(
input_path: &Path,
temporary_output_path: &Path,
temporary_output_file: &mut std::fs::File,
columns: &[ColumnMetadata],
options: ProcessOptions<'_>,
control: Option<&mut ProcessControl<'_>>,
Expand All @@ -538,8 +543,7 @@ fn process_file_to_temporary_output(
.map_err(csv_error)?;
let mut writer = WriterBuilder::new()
.has_headers(false)
.from_path(temporary_output_path)
.map_err(csv_error)?;
.from_writer(&mut *temporary_output_file);

process_csv_reader_to_writer(
&mut reader,
Expand Down
50 changes: 45 additions & 5 deletions crates/csv-anonymizer-core/src/detection/spans.rs
Original file line number Diff line number Diff line change
Expand Up @@ -250,18 +250,33 @@ fn push_secret_spans<'a>(content: &'a str, candidates: &mut Vec<PrivacySpan<'a>>
}
}

for regex_match in private_key_marker_pattern().find_iter(content) {
for begin in private_key_marker_pattern().find_iter(content) {
// Redact the complete PEM object, not just its BEGIN line. If a log truncates
// before the END marker, conservatively treat the remainder as key material:
// publishing an unterminated fragment can still disclose the private key.
let label = begin
.as_str()
.strip_prefix("-----BEGIN ")
.and_then(|label| label.strip_suffix("-----"));
let end = label
.and_then(|label| {
let marker = format!("-----END {label}-----");
content[begin.end()..]
.find(&marker)
.map(|offset| begin.end() + offset + marker.len())
})
.unwrap_or(content.len());
candidates.push(PrivacySpan {
field_name: "secret",
kind: PrivacyFindingKind::CredentialOrSecret,
data_type: DataType::String,
start: regex_match.start(),
end: regex_match.end(),
value: regex_match.as_str(),
start: begin.start(),
end,
value: &content[begin.start()..end],
confidence: Confidence::High,
score: 99,
detector: "pattern:private-key",
reason: "Private key marker pattern.",
reason: "Private key PEM block, including delimiter and payload.",
priority: 2,
});
}
Expand Down Expand Up @@ -594,6 +609,31 @@ mod tests {
));
}

#[test]
fn private_key_spans_cover_complete_and_unterminated_pem_blocks() {
let complete = "prefix\n-----BEGIN RSA PRIVATE KEY-----\nabc123\n-----END RSA PRIVATE KEY-----\nsuffix";
let spans = collect_privacy_spans(complete);
let span = spans
.iter()
.find(|span| span.detector == "pattern:private-key")
.expect("PEM block should be detected");
assert_eq!(
span.value,
"-----BEGIN RSA PRIVATE KEY-----\nabc123\n-----END RSA PRIVATE KEY-----"
);
assert_eq!(&complete[..span.start], "prefix\n");
assert_eq!(&complete[span.end..], "\nsuffix");

let truncated = "-----BEGIN PRIVATE KEY-----\nsecret payload";
let span = collect_privacy_spans(truncated)
.into_iter()
.find(|span| span.detector == "pattern:private-key")
.expect("unterminated PEM block should be detected");
assert_eq!(span.start, 0);
assert_eq!(span.end, truncated.len());
assert_eq!(span.value, truncated);
}

fn phone_span_confidence(content: &str) -> Option<Confidence> {
collect_privacy_spans(content)
.iter()
Expand Down
10 changes: 6 additions & 4 deletions crates/csv-anonymizer-core/src/direct_input/documents.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,13 +85,13 @@ fn transform_value_document(
provider: Option<&mut dyn SmartReplacementProvider>,
tokenization_key: Option<&crate::TokenizationKey>,
) -> Result<(Value, PasteTransformData)> {
let start_time = Instant::now();
let (analysis, coverage) =
analyze_value_document_with_coverage(format, &value, input.sample_row_count)?;
let metadata = select_columns(&analysis.columns, &input.columns, &input.controls)?;
let selected_by_path = selected_columns_by_source(&metadata);
let smart_replacements =
prepare_value_smart_replacements(&value, format, &metadata, &input, provider)?;
let start_time = Instant::now();
let mut state = TransformState::with_smart_replacements_if_active(smart_replacements)
.with_tokenization_key(tokenization_key.cloned());
let mut row_indices = HashMap::new();
Expand All @@ -106,13 +106,15 @@ fn transform_value_document(
transform_json_value(&mut value, &mut Vec::new(), &mut context);

let row_count = infer_value_row_count(&value);
// The output is serialized by the caller, which knows whether this document is
// JSON or YAML; everything else about the run is settled here.
let mut report = state.report();
// Scalar document traversal does not run the row-level residual audit; mark that
// distinction explicitly so an empty fingerprint intersection is not a verified pass.
report.residual_audit_incomplete = true;
let result = paste_transform_data(
String::new(),
row_count,
&metadata,
state.report(),
report,
coverage,
start_time,
);
Expand Down
106 changes: 106 additions & 0 deletions crates/csv-anonymizer-core/src/direct_input/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -908,6 +908,112 @@ fn transforms_selected_xml_cdata() {
assert!(result.output.contains("<![CDATA["));
}

#[test]
fn transforms_xml_text_cdata_and_references_as_one_scalar() {
let input = r#"<users><user><name>Ada&#32;<![CDATA[Lovelace]]>&amp;</name><note>keep</note></user></users>"#;
let analysis = analyze_paste_data(PasteAnalyzeParams {
content: input.to_string(),
format: PasteDataFormat::Xml,
sample_row_count: 10,
})
.unwrap();
let name = analysis
.columns
.iter()
.find(|column| column.name == "users.user.name")
.unwrap();
let result = transform_paste_data(PasteTransformParams {
content: input.to_string(),
format: PasteDataFormat::Xml,
columns: vec![name.index],
controls: vec![ColumnControl {
column_index: name.index,
type_override: Some(DataType::FullName),
strategy: AnonymizationStrategy::Redact,
}],
sample_row_count: 100,
preview_smart_replacements: Vec::new(),
})
.unwrap();
let mut reader = quick_xml::Reader::from_str(&result.output);
loop {
if matches!(reader.read_event().unwrap(), quick_xml::events::Event::Eof) {
break;
}
}
assert!(result.output.contains("<note>keep</note>"));
}

#[test]
fn smart_xml_replacements_use_combined_text_cdata_and_references() {
let input = r#"<users><user><name>Ada&#32;<![CDATA[Lovelace]]>&amp;</name></user></users>"#;
let analysis = analyze_paste_data(PasteAnalyzeParams {
content: input.to_string(),
format: PasteDataFormat::Xml,
sample_row_count: 10,
})
.unwrap();
let name = analysis
.columns
.iter()
.find(|column| column.name == "users.user.name")
.unwrap();
let mut provider = RecordingSmartProvider::default();

let result = transform_paste_data_with_smart_provider(
PasteTransformParams {
content: input.to_string(),
format: PasteDataFormat::Xml,
columns: vec![name.index],
controls: vec![ColumnControl {
column_index: name.index,
type_override: Some(DataType::FullName),
strategy: AnonymizationStrategy::LocalAi,
}],
sample_row_count: 100,
preview_smart_replacements: Vec::new(),
},
Some(&mut provider),
)
.unwrap();

assert_eq!(provider.requests, vec![vec!["Ada Lovelace&".to_string()]]);
assert!(result.output.contains("Generated Person 1"));
}

#[test]
fn scalar_document_marks_unmeasured_residual_audit_as_review() {
let input = r#"{"first":"alice@example.com","second":"alice@example.com"}"#;
let analysis = analyze_paste_data(PasteAnalyzeParams {
content: input.to_string(),
format: PasteDataFormat::Json,
sample_row_count: 10,
})
.unwrap();
let selected = analysis
.columns
.iter()
.map(|column| column.index)
.collect::<Vec<_>>();
let result = transform_paste_data(PasteTransformParams {
content: input.to_string(),
format: PasteDataFormat::Json,
columns: selected,
controls: Vec::new(),
sample_row_count: 100,
preview_smart_replacements: Vec::new(),
})
.unwrap();
let audit = result
.privacy_report
.evidence
.iter()
.find(|item| item.id == "broad-residual-value-audit")
.expect("scalar report should include residual audit evidence");
assert_eq!(audit.status, crate::types::ReleaseEvidenceStatus::Review);
assert!(audit.detail.contains("was not run"));
}

#[test]
fn json_paths_distinguish_literal_dotted_keys_from_nested_keys() {
let input = r#"{
Expand Down
11 changes: 8 additions & 3 deletions crates/csv-anonymizer-core/src/direct_input/text.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,14 @@ pub(super) fn replay_text_candidate_evidence(
provider,
tokenization_key,
)?;
let row_count = input.content.split('\n').count();
let mut report = replay.report;
report.residual_audit_incomplete = true;
let row_count = input.content.lines().count().max(1);
Ok(paste_transform_data(
replay.output,
row_count,
&replay.columns,
replay.report,
report,
DetectionCoverage::values(row_count, row_count),
start_time,
))
Expand Down Expand Up @@ -415,11 +417,14 @@ pub(super) fn transform_text_with_smart_provider(
}
output.push_str(&input.content[last_end..]);

let mut report = state.report();
// Span replacement is scalar-oriented and does not drive the row-level residual audit.
report.residual_audit_incomplete = true;
Ok(paste_transform_data(
output,
analysis.row_count,
&metadata,
state.report(),
report,
coverage,
start_time,
))
Expand Down
Loading
Loading