MLE-29419: Hash expression values in URI templates when writing documents

docs/copy.md

@@ -88,6 +88,11 @@ writing documents. Similar to the output connection options, each option for con
 is prefixed with `output`. For example, to specify collections for the documents, `--output-collections` is used instead
 of `--collections`.
 
+Similarly, to construct document URIs via a template, use `--output-uri-template`. As of Flux 2.2.0, you can use 
+`--output-uri-template-hash-values` to hash each resolved expression value in the template using SHA-256 (lowercase hex) 
+before constructing the document URI. You can also use `--output-uri-template-warn-on-missing-field` to log a warning and 
+replace missing expressions with a UUID instead of failing.
+
 ## Building a RAG data pipeline
 
 [Retrieval-augmented generation](https://www.progress.com/marklogic/solutions/generative-ai), or RAG, with MarkLogic depends on preparing data so that the most relevant

docs/import/common-import-features.md

@@ -27,6 +27,7 @@ command reads. The following command line options offer further control over eac
 | `--uri-replace` | Comma-delimited list of regular expressions and replacement values, with each replacement value surrounded by single quotes. |
 | `--uri-template` | Template for each URI containing one or more column names. |
 | `--uri-template-warn-on-missing-field` | Added in 2.0.0; causes a warning if the template specified by `--uri-template` has an expression that returns null for a document. The expression will instead be replaced with the string "UNRESOLVED-" followed by a random UUID. |
+| `--uri-template-hash-values` | Added in 2.2.0; causes each expression value in the template specified by `--uri-template` to be hashed using SHA-256 (lowercase hex) before constructing the document URI. |
 
 ### Replacing URI contents
 
@@ -78,6 +79,18 @@ As of Flux 2.0.0, you can instead use the `--uri-template-warn-on-missing-field`
 and replace the expression with "UNRESOLVED-" followed by a random UUID. This option can be useful when you are not
 certain that your URI template will work for every document you intend to write to MarkLogic.
 
+Starting in Flux 2.2.0, you can use the `--uri-template-hash-values` option to hash each resolved expression value 
+using SHA-256 (lowercase hex) before constructing the document URI:
+
+    --uri-template "/employee/{id}/{last_name}.json" --uri-template-hash-values
+
+When specified, every expression in the template is individually replaced by its 64-character lowercase hex digest. 
+This is useful for preventing sensitive values (such as employee IDs or national IDs) from appearing verbatim in 
+document URIs, and for generating consistent, repeatable URIs from tabular datasets that lack file paths (which is 
+especially valuable when combined with [incremental write](incremental-write.md)). Note that this unsalted hashing 
+provides pseudonymization, not complete anonymization, as low-entropy inputs could potentially be recovered via dictionary 
+or brute-force attacks.
+
 The following techniques can assist you with writing a URI template:
 
 1. Run the import command with `--limit 1` to write a single JSON document to MarkLogic. You can then see the JSON 

docs/import/incremental-write.md

@@ -197,3 +197,16 @@ To disable this behavior, include the `--incremental-write-disable-json-canonica
 know that the JSON documents you write to MarkLogic do not require any canonicalization, e.g. the order of keys in 
 each document is guaranteed to be the same, and thus you can avoid the cost of canonicalizing each JSON document.
 
+## Generating deterministic URIs with URI templates
+
+Incremental write relies on consistent, deterministic URIs across repeated import runs so that existing documents in MarkLogic can be matched against incoming records. While file-based imports naturally produce consistent URIs based on file paths, tabular data sources (such as JDBC, CSV, or JSON lines) often lack inherent file paths.
+
+In such cases, you can combine `--incremental-write` with `--uri-template` and `--uri-template-hash-values` (added in Flux 2.2.0). Hashing expression values in the template produces consistent, deterministic URIs while preventing sensitive primary keys (such as SSNs or account numbers) from appearing verbatim in document URIs:
+
+    --incremental-write \
+    --uri-template "/employee/{id}.json" \
+    --uri-template-hash-values
+
+Note that unsalted SHA-256 hashing provides pseudonymization rather than full anonymization, as low-entropy values could potentially be recovered via dictionary or brute-force attacks.
+
+

flux-cli/src/main/java/com/marklogic/flux/api/WriteDocumentsOptions.java

@@ -62,6 +62,14 @@ public interface WriteDocumentsOptions<T extends WriteDocumentsOptions> {
      */
     T uriTemplateWarnOnMissingField();
 
+    /**
+     * Causes Flux to hash each resolved expression value using SHA-256 before substituting it into the URI template.
+     *
+     * @return this options instance
+     * @since 2.2.0
+     */
+    T uriTemplateHashValues();
+
     /**
      * @since 1.3.0
      */

flux-cli/src/main/java/com/marklogic/flux/impl/copy/CopyCommand.java

@@ -176,6 +176,12 @@ public static class CopyWriteDocumentsParams implements CopyWriteDocumentsOption
         )
         private boolean uriTemplateWarnOnMissingField;
 
+        @CommandLine.Option(
+            names = "--output-uri-template-hash-values",
+            description = "Hashes each resolved expression value in the URI template using SHA-256 before constructing the document URI."
+        )
+        private boolean uriTemplateHashValues;
+
         @CommandLine.Option(
             names = {"--output-doc-metadata"},
             description = "Specify one or more metadata values to be added to each document; e.g. --output-doc-metadata param=value ."
@@ -252,7 +258,8 @@ protected Map<String, String> makeOptions() {
                 Options.WRITE_URI_REPLACE, uriReplace,
                 Options.WRITE_URI_SUFFIX, uriSuffix,
                 Options.WRITE_URI_TEMPLATE, uriTemplate,
-                Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD, uriTemplateWarnOnMissingField ? "true" : null
+                Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD, uriTemplateWarnOnMissingField ? "true" : null,
+                Options.WRITE_URI_TEMPLATE_HASH_VALUES, uriTemplateHashValues ? "true" : null
             );
 
             if (additionalWriteOptions != null) {
@@ -394,6 +401,12 @@ public CopyWriteDocumentsOptions uriTemplateWarnOnMissingField() {
             return this;
         }
 
+        @Override
+        public CopyWriteDocumentsOptions uriTemplateHashValues() {
+            this.uriTemplateHashValues = true;
+            return this;
+        }
+
         @Override
         public CopyWriteDocumentsOptions metadataValues(Map<String, String> metadataValues) {
             this.metadataValues = metadataValues;

flux-cli/src/main/java/com/marklogic/flux/impl/importdata/WriteDocumentParams.java

@@ -145,6 +145,12 @@ public class WriteDocumentParams<T extends WriteDocumentsOptions> implements Wri
     )
     private boolean uriTemplateWarnOnMissingField;
 
+    @CommandLine.Option(
+        names = "--uri-template-hash-values",
+        description = "Hashes each resolved expression value in the URI template using SHA-256 before constructing the document URI."
+    )
+    private boolean uriTemplateHashValues;
+
     @CommandLine.Option(
         names = {"--doc-metadata"},
         description = "Specify one or more metadata values to be added to each document; e.g. --doc-metadata param=value ."
@@ -198,6 +204,7 @@ public Map<String, String> makeOptions() {
             Options.WRITE_URI_SUFFIX, uriSuffix,
             Options.WRITE_URI_TEMPLATE, uriTemplate,
             Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD, uriTemplateWarnOnMissingField ? "true" : null,
+            Options.WRITE_URI_TEMPLATE_HASH_VALUES, uriTemplateHashValues ? "true" : null,
             Options.STREAM_FILES, streaming ? "true" : null
         );
 
@@ -375,6 +382,12 @@ public T uriTemplateWarnOnMissingField() {
         return (T) this;
     }
 
+    @Override
+    public T uriTemplateHashValues() {
+        this.uriTemplateHashValues = true;
+        return (T) this;
+    }
+
     public void setStreaming(boolean streaming) {
         this.streaming = streaming;
     }

flux-cli/src/test/java/com/marklogic/flux/api/AggregateJsonFilesImporterTest.java

@@ -7,6 +7,10 @@
 import com.marklogic.flux.AbstractTest;
 import org.junit.jupiter.api.Test;
 
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.HexFormat;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -85,4 +89,37 @@ void jsonLinesRaw() {
 
         assertCollectionSize("json-lines", 2);
     }
+
+    @Test
+    void uriTemplateHashValues() {
+        Flux.importAggregateJsonFiles()
+            .connectionString(makeConnectionString())
+            .from(options -> options
+                .paths("src/test/resources/delimited-files/custom-delimiter-json.txt")
+                .jsonLines()
+                .additionalOptions(Map.of("lineSep", ":\n")))
+            .to(options -> options
+                .permissionsString(DEFAULT_PERMISSIONS)
+                .collections("custom-delimiter-hashed")
+                .uriTemplate("/custom-delimiter/{firstName}.json")
+                .uriTemplateHashValues())
+            .execute();
+
+        assertCollectionSize("custom-delimiter-hashed", 3);
+        Stream.of("firstName-1", "firstName-2", "firstName-3").forEach(name -> {
+            byte[] hash = sha256Hex(name);
+            String expectedUri = String.format("/custom-delimiter/%s.json", HexFormat.of().formatHex(hash));
+            JsonNode doc = readJsonDocument(expectedUri);
+            assertEquals(name, doc.get("firstName").asText());
+            assertTrue(doc.has("lastName"));
+        });
+    }
+
+    private static byte[] sha256Hex(String value) {
+        try {
+            return MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8));
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }

flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyOptionsTest.java

@@ -15,6 +15,7 @@
 import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 class CopyOptionsTest extends AbstractOptionsTest {
@@ -82,6 +83,7 @@ void allWriteParams() {
             "--output-uri-suffix", ".xml",
             "--output-uri-template", "/{example}.xml",
             "--output-uri-template-warn-on-missing-field",
+            "--output-uri-template-hash-values",
             "--splitter-text",
             "--embedder", "doesnt-matter",
             "--embedder-prompt", "Hello: ",
@@ -121,6 +123,7 @@ void allWriteParams() {
             Options.WRITE_URI_SUFFIX, ".xml",
             Options.WRITE_URI_TEMPLATE, "/{example}.xml",
             Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD, "true",
+            Options.WRITE_URI_TEMPLATE_HASH_VALUES, "true",
             Options.WRITE_SPLITTER_TEXT, "true",
             Options.WRITE_EMBEDDER_MODEL_FUNCTION_CLASS_NAME, "doesnt-matter",
             Options.WRITE_EMBEDDER_PROMPT, "Hello: ",
@@ -287,4 +290,14 @@ void systemProperties() {
             systemProps.keySet().forEach(System::clearProperty);
         }
     }
+
+    @Test
+    void outputUriTemplateHashValuesNotSet() {
+        CopyCommand command = (CopyCommand) getCommand("copy",
+            "--connection-string", "test:test@host:8000"
+        );
+        Map<String, String> options = command.writeParams.makeOptions();
+        assertFalse(options.containsKey(Options.WRITE_URI_TEMPLATE_HASH_VALUES),
+            "WRITE_URI_TEMPLATE_HASH_VALUES must not be present when --output-uri-template-hash-values is not specified.");
+    }
 }

flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java

@@ -42,6 +42,7 @@ void test() {
             "--uri-suffix", ".suffix",
             "--uri-template", "/test/{value}.json",
             "--uri-template-warn-on-missing-field",
+            "--uri-template-hash-values",
             "--streaming",
             "--extract-text",
             "--extracted-text-document-type", "XML",
@@ -91,6 +92,7 @@ void test() {
             Options.WRITE_URI_SUFFIX, ".suffix",
             Options.WRITE_URI_TEMPLATE, "/test/{value}.json",
             Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD, "true",
+            Options.WRITE_URI_TEMPLATE_HASH_VALUES, "true",
             Options.STREAM_FILES, "true",
             Options.WRITE_EXTRACTED_TEXT, "true",
             Options.WRITE_EXTRACTED_TEXT_DOCUMENT_TYPE, "XML",
@@ -221,6 +223,9 @@ void emptyJsonPointerPath() {
         assertFalse(options.containsKey(Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD),
             "As a boolean option, this should not be present when the option is not set so that the feature " +
                 "defaults to false, as of Flux 2.0.0.");
+        assertFalse(options.containsKey(Options.WRITE_URI_TEMPLATE_HASH_VALUES),
+            "As a boolean option, this should not be present when the option is not set so that the feature " +
+                "defaults to false, as of Flux 2.2.0.");
     }
 
     @Test