From 85883c44f229b3006244a1a93aafe84889aced6b Mon Sep 17 00:00:00 2001 From: Ranjeet Singh Date: Tue, 13 Oct 2026 14:55:08 +0530 Subject: [PATCH] MLE-29419: Hash expression values in URI templates when writing documents --- docs/copy.md | 5 +++ docs/import/common-import-features.md | 13 +++++++ docs/import/incremental-write.md | 13 +++++++ .../flux/api/WriteDocumentsOptions.java | 8 ++++ .../marklogic/flux/impl/copy/CopyCommand.java | 15 +++++++- .../impl/importdata/WriteDocumentParams.java | 13 +++++++ .../api/AggregateJsonFilesImporterTest.java | 37 +++++++++++++++++++ .../flux/impl/copy/CopyOptionsTest.java | 13 +++++++ .../importdata/ImportFilesOptionsTest.java | 5 +++ 9 files changed, 121 insertions(+), 1 deletion(-) diff --git a/docs/copy.md b/docs/copy.md index d7cd2611..553773c3 100644 --- a/docs/copy.md +++ b/docs/copy.md @@ -88,6 +88,11 @@ writing documents. Similar to the output connection options, each option for con is prefixed with `output`. For example, to specify collections for the documents, `--output-collections` is used instead of `--collections`. +Similarly, to construct document URIs via a template, use `--output-uri-template`. As of Flux 2.2.0, you can use +`--output-uri-template-hash-values` to hash each resolved expression value in the template using SHA-256 (lowercase hex) +before constructing the document URI. You can also use `--output-uri-template-warn-on-missing-field` to log a warning and +replace missing expressions with a UUID instead of failing. + ## Building a RAG data pipeline [Retrieval-augmented generation](https://www.progress.com/marklogic/solutions/generative-ai), or RAG, with MarkLogic depends on preparing data so that the most relevant diff --git a/docs/import/common-import-features.md b/docs/import/common-import-features.md index 8a8f8a07..afd765a8 100644 --- a/docs/import/common-import-features.md +++ b/docs/import/common-import-features.md @@ -27,6 +27,7 @@ command reads. The following command line options offer further control over eac | `--uri-replace` | Comma-delimited list of regular expressions and replacement values, with each replacement value surrounded by single quotes. | | `--uri-template` | Template for each URI containing one or more column names. | | `--uri-template-warn-on-missing-field` | Added in 2.0.0; causes a warning if the template specified by `--uri-template` has an expression that returns null for a document. The expression will instead be replaced with the string "UNRESOLVED-" followed by a random UUID. | +| `--uri-template-hash-values` | Added in 2.2.0; causes each expression value in the template specified by `--uri-template` to be hashed using SHA-256 (lowercase hex) before constructing the document URI. | ### Replacing URI contents @@ -78,6 +79,18 @@ As of Flux 2.0.0, you can instead use the `--uri-template-warn-on-missing-field` and replace the expression with "UNRESOLVED-" followed by a random UUID. This option can be useful when you are not certain that your URI template will work for every document you intend to write to MarkLogic. +Starting in Flux 2.2.0, you can use the `--uri-template-hash-values` option to hash each resolved expression value +using SHA-256 (lowercase hex) before constructing the document URI: + + --uri-template "/employee/{id}/{last_name}.json" --uri-template-hash-values + +When specified, every expression in the template is individually replaced by its 64-character lowercase hex digest. +This is useful for preventing sensitive values (such as employee IDs or national IDs) from appearing verbatim in +document URIs, and for generating consistent, repeatable URIs from tabular datasets that lack file paths (which is +especially valuable when combined with [incremental write](incremental-write.md)). Note that this unsalted hashing +provides pseudonymization, not complete anonymization, as low-entropy inputs could potentially be recovered via dictionary +or brute-force attacks. + The following techniques can assist you with writing a URI template: 1. Run the import command with `--limit 1` to write a single JSON document to MarkLogic. You can then see the JSON diff --git a/docs/import/incremental-write.md b/docs/import/incremental-write.md index 4b84ccdc..990f9b31 100644 --- a/docs/import/incremental-write.md +++ b/docs/import/incremental-write.md @@ -197,3 +197,16 @@ To disable this behavior, include the `--incremental-write-disable-json-canonica know that the JSON documents you write to MarkLogic do not require any canonicalization, e.g. the order of keys in each document is guaranteed to be the same, and thus you can avoid the cost of canonicalizing each JSON document. +## Generating deterministic URIs with URI templates + +Incremental write relies on consistent, deterministic URIs across repeated import runs so that existing documents in MarkLogic can be matched against incoming records. While file-based imports naturally produce consistent URIs based on file paths, tabular data sources (such as JDBC, CSV, or JSON lines) often lack inherent file paths. + +In such cases, you can combine `--incremental-write` with `--uri-template` and `--uri-template-hash-values` (added in Flux 2.2.0). Hashing expression values in the template produces consistent, deterministic URIs while preventing sensitive primary keys (such as SSNs or account numbers) from appearing verbatim in document URIs: + + --incremental-write \ + --uri-template "/employee/{id}.json" \ + --uri-template-hash-values + +Note that unsalted SHA-256 hashing provides pseudonymization rather than full anonymization, as low-entropy values could potentially be recovered via dictionary or brute-force attacks. + + diff --git a/flux-cli/src/main/java/com/marklogic/flux/api/WriteDocumentsOptions.java b/flux-cli/src/main/java/com/marklogic/flux/api/WriteDocumentsOptions.java index 3e92bc51..b8a338ea 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/api/WriteDocumentsOptions.java +++ b/flux-cli/src/main/java/com/marklogic/flux/api/WriteDocumentsOptions.java @@ -62,6 +62,14 @@ public interface WriteDocumentsOptions { */ T uriTemplateWarnOnMissingField(); + /** + * Causes Flux to hash each resolved expression value using SHA-256 before substituting it into the URI template. + * + * @return this options instance + * @since 2.2.0 + */ + T uriTemplateHashValues(); + /** * @since 1.3.0 */ diff --git a/flux-cli/src/main/java/com/marklogic/flux/impl/copy/CopyCommand.java b/flux-cli/src/main/java/com/marklogic/flux/impl/copy/CopyCommand.java index 14ebd748..b63fcf8b 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/impl/copy/CopyCommand.java +++ b/flux-cli/src/main/java/com/marklogic/flux/impl/copy/CopyCommand.java @@ -176,6 +176,12 @@ public static class CopyWriteDocumentsParams implements CopyWriteDocumentsOption ) private boolean uriTemplateWarnOnMissingField; + @CommandLine.Option( + names = "--output-uri-template-hash-values", + description = "Hashes each resolved expression value in the URI template using SHA-256 before constructing the document URI." + ) + private boolean uriTemplateHashValues; + @CommandLine.Option( names = {"--output-doc-metadata"}, description = "Specify one or more metadata values to be added to each document; e.g. --output-doc-metadata param=value ." @@ -252,7 +258,8 @@ protected Map makeOptions() { Options.WRITE_URI_REPLACE, uriReplace, Options.WRITE_URI_SUFFIX, uriSuffix, Options.WRITE_URI_TEMPLATE, uriTemplate, - Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD, uriTemplateWarnOnMissingField ? "true" : null + Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD, uriTemplateWarnOnMissingField ? "true" : null, + Options.WRITE_URI_TEMPLATE_HASH_VALUES, uriTemplateHashValues ? "true" : null ); if (additionalWriteOptions != null) { @@ -394,6 +401,12 @@ public CopyWriteDocumentsOptions uriTemplateWarnOnMissingField() { return this; } + @Override + public CopyWriteDocumentsOptions uriTemplateHashValues() { + this.uriTemplateHashValues = true; + return this; + } + @Override public CopyWriteDocumentsOptions metadataValues(Map metadataValues) { this.metadataValues = metadataValues; diff --git a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/WriteDocumentParams.java b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/WriteDocumentParams.java index 70fa492f..8b72d789 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/WriteDocumentParams.java +++ b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/WriteDocumentParams.java @@ -145,6 +145,12 @@ public class WriteDocumentParams implements Wri ) private boolean uriTemplateWarnOnMissingField; + @CommandLine.Option( + names = "--uri-template-hash-values", + description = "Hashes each resolved expression value in the URI template using SHA-256 before constructing the document URI." + ) + private boolean uriTemplateHashValues; + @CommandLine.Option( names = {"--doc-metadata"}, description = "Specify one or more metadata values to be added to each document; e.g. --doc-metadata param=value ." @@ -198,6 +204,7 @@ public Map makeOptions() { Options.WRITE_URI_SUFFIX, uriSuffix, Options.WRITE_URI_TEMPLATE, uriTemplate, Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD, uriTemplateWarnOnMissingField ? "true" : null, + Options.WRITE_URI_TEMPLATE_HASH_VALUES, uriTemplateHashValues ? "true" : null, Options.STREAM_FILES, streaming ? "true" : null ); @@ -375,6 +382,12 @@ public T uriTemplateWarnOnMissingField() { return (T) this; } + @Override + public T uriTemplateHashValues() { + this.uriTemplateHashValues = true; + return (T) this; + } + public void setStreaming(boolean streaming) { this.streaming = streaming; } diff --git a/flux-cli/src/test/java/com/marklogic/flux/api/AggregateJsonFilesImporterTest.java b/flux-cli/src/test/java/com/marklogic/flux/api/AggregateJsonFilesImporterTest.java index f193442d..31ad2a63 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/api/AggregateJsonFilesImporterTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/api/AggregateJsonFilesImporterTest.java @@ -7,6 +7,10 @@ import com.marklogic.flux.AbstractTest; import org.junit.jupiter.api.Test; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -85,4 +89,37 @@ void jsonLinesRaw() { assertCollectionSize("json-lines", 2); } + + @Test + void uriTemplateHashValues() { + Flux.importAggregateJsonFiles() + .connectionString(makeConnectionString()) + .from(options -> options + .paths("src/test/resources/delimited-files/custom-delimiter-json.txt") + .jsonLines() + .additionalOptions(Map.of("lineSep", ":\n"))) + .to(options -> options + .permissionsString(DEFAULT_PERMISSIONS) + .collections("custom-delimiter-hashed") + .uriTemplate("/custom-delimiter/{firstName}.json") + .uriTemplateHashValues()) + .execute(); + + assertCollectionSize("custom-delimiter-hashed", 3); + Stream.of("firstName-1", "firstName-2", "firstName-3").forEach(name -> { + byte[] hash = sha256Hex(name); + String expectedUri = String.format("/custom-delimiter/%s.json", HexFormat.of().formatHex(hash)); + JsonNode doc = readJsonDocument(expectedUri); + assertEquals(name, doc.get("firstName").asText()); + assertTrue(doc.has("lastName")); + }); + } + + private static byte[] sha256Hex(String value) { + try { + return MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } } diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyOptionsTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyOptionsTest.java index b1f3590c..62fc178d 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyOptionsTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyOptionsTest.java @@ -15,6 +15,7 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; class CopyOptionsTest extends AbstractOptionsTest { @@ -82,6 +83,7 @@ void allWriteParams() { "--output-uri-suffix", ".xml", "--output-uri-template", "/{example}.xml", "--output-uri-template-warn-on-missing-field", + "--output-uri-template-hash-values", "--splitter-text", "--embedder", "doesnt-matter", "--embedder-prompt", "Hello: ", @@ -121,6 +123,7 @@ void allWriteParams() { Options.WRITE_URI_SUFFIX, ".xml", Options.WRITE_URI_TEMPLATE, "/{example}.xml", Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD, "true", + Options.WRITE_URI_TEMPLATE_HASH_VALUES, "true", Options.WRITE_SPLITTER_TEXT, "true", Options.WRITE_EMBEDDER_MODEL_FUNCTION_CLASS_NAME, "doesnt-matter", Options.WRITE_EMBEDDER_PROMPT, "Hello: ", @@ -287,4 +290,14 @@ void systemProperties() { systemProps.keySet().forEach(System::clearProperty); } } + + @Test + void outputUriTemplateHashValuesNotSet() { + CopyCommand command = (CopyCommand) getCommand("copy", + "--connection-string", "test:test@host:8000" + ); + Map options = command.writeParams.makeOptions(); + assertFalse(options.containsKey(Options.WRITE_URI_TEMPLATE_HASH_VALUES), + "WRITE_URI_TEMPLATE_HASH_VALUES must not be present when --output-uri-template-hash-values is not specified."); + } } diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java index 3988f94c..fdf05699 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java @@ -42,6 +42,7 @@ void test() { "--uri-suffix", ".suffix", "--uri-template", "/test/{value}.json", "--uri-template-warn-on-missing-field", + "--uri-template-hash-values", "--streaming", "--extract-text", "--extracted-text-document-type", "XML", @@ -91,6 +92,7 @@ void test() { Options.WRITE_URI_SUFFIX, ".suffix", Options.WRITE_URI_TEMPLATE, "/test/{value}.json", Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD, "true", + Options.WRITE_URI_TEMPLATE_HASH_VALUES, "true", Options.STREAM_FILES, "true", Options.WRITE_EXTRACTED_TEXT, "true", Options.WRITE_EXTRACTED_TEXT_DOCUMENT_TYPE, "XML", @@ -221,6 +223,9 @@ void emptyJsonPointerPath() { assertFalse(options.containsKey(Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD), "As a boolean option, this should not be present when the option is not set so that the feature " + "defaults to false, as of Flux 2.0.0."); + assertFalse(options.containsKey(Options.WRITE_URI_TEMPLATE_HASH_VALUES), + "As a boolean option, this should not be present when the option is not set so that the feature " + + "defaults to false, as of Flux 2.2.0."); } @Test