diff --git a/docs/import/splitting.md b/docs/import/splitting.md index 0ae2680d..c275a525 100644 --- a/docs/import/splitting.md +++ b/docs/import/splitting.md @@ -382,14 +382,35 @@ The above options will result in documents having URIs of `/chunk/(UUID).json`. #### Controlling document metadata -Each sidecar document defaults to the same permissions as its source document and no collections assigned to it. -You can assign different permissions via the following, where each "role" is the name of a MarkLogic and each +By default, each sidecar document has no collections assigned to it and defaults to the permissions specified via +`--permissions` (or your user default permissions if `--permissions` is omitted). Sidecar documents do not inherit +collections or permissions from their source document by default. + +You can explicitly assign permissions to each sidecar document via the following, where each "role" is the name of a MarkLogic role and each "capability" is one of "read", "update", "insert", or "execute": --splitter-sidecar-permissions role,capability,role,capability,etc. -You can assign collections to each sidecar document via the following, which accepts a comma-delimited sequence of +You can explicitly assign collections to each sidecar document via the following, which accepts a comma-delimited sequence of collection names: --splitter-sidecar-collections collection1,collection2,etc +To have sidecar chunk documents inherit the collections of their source document, include: + + --splitter-sidecar-inherit-collections + +When `--splitter-sidecar-inherit-collections` is enabled, each sidecar document will receive the collections of its source document. If `--splitter-sidecar-collections` is also specified, the resulting collections will be the deduplicated union of the inherited collections and the explicitly specified collections. If the source document has no collections, only the explicit collections (if any) are applied. + +To have sidecar chunk documents inherit the permissions of their source document, include: + + --splitter-sidecar-inherit-permissions + +When `--splitter-sidecar-inherit-permissions` is enabled, each sidecar document will receive the permissions of its source document. If `--splitter-sidecar-permissions` is also specified, the resulting permissions will be the union of the inherited permissions and the explicitly specified permissions. If the source document has no permissions, only the explicit permissions (if any) are applied. + +Inherited collections and permissions are those of the source document as it is written to MarkLogic. If you +specify `--collections` or `--permissions` (or `--output-collections` or `--output-permissions` for the `copy` +command), those values replace the collections and permissions read from the source and are therefore the values +inherited by each sidecar document. Inheriting permissions is primarily useful when the source documents have their +own permissions, such as when using the `copy` command or importing archive files. + diff --git a/flux-cli/src/main/java/com/marklogic/flux/api/SplitterOptions.java b/flux-cli/src/main/java/com/marklogic/flux/api/SplitterOptions.java index 57b5627f..73f483d0 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/api/SplitterOptions.java +++ b/flux-cli/src/main/java/com/marklogic/flux/api/SplitterOptions.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.api; @@ -36,8 +36,18 @@ public interface SplitterOptions { SplitterOptions outputCollections(String... collections); + /** + * @since 2.2.0 + */ + SplitterOptions outputInheritCollections(boolean inheritCollections); + SplitterOptions outputPermissionsString(String rolesAndCapabilities); + /** + * @since 2.2.0 + */ + SplitterOptions outputInheritPermissions(boolean inheritPermissions); + SplitterOptions outputRootName(String rootName); SplitterOptions outputUriPrefix(String uriPrefix); diff --git a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/SplitterParams.java b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/SplitterParams.java index 619ccb9b..234bc182 100644 --- a/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/SplitterParams.java +++ b/flux-cli/src/main/java/com/marklogic/flux/impl/importdata/SplitterParams.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.flux.impl.importdata; @@ -96,12 +96,24 @@ public class SplitterParams implements SplitterOptions { ) private String collections; + @CommandLine.Option( + names = "--splitter-sidecar-inherit-collections", + description = "Whether sidecar chunk documents should inherit the collections of their source document." + ) + private boolean inheritCollections; + @CommandLine.Option( names = "--splitter-sidecar-permissions", description = "Comma-delimited sequence of MarkLogic role names and capabilities to add to each chunk document - e.g. role1,read,role2,update,role3,execute." ) private String permissions; + @CommandLine.Option( + names = "--splitter-sidecar-inherit-permissions", + description = "Whether sidecar chunk documents should inherit the permissions of their source document." + ) + private boolean inheritPermissions; + @CommandLine.Option( names = "--splitter-sidecar-root-name", description = "Name of a root field to add to each JSON sidecar document, or name of the root element for each XML sidecar document." @@ -143,7 +155,9 @@ public Map makeOptions() { Options.WRITE_SPLITTER_SIDECAR_DOCUMENT_TYPE, documentType != null ? documentType.name() : null, Options.WRITE_SPLITTER_SIDECAR_COLLECTIONS, collections, + Options.WRITE_SPLITTER_SIDECAR_INHERIT_COLLECTIONS, inheritCollections ? "true" : null, Options.WRITE_SPLITTER_SIDECAR_PERMISSIONS, permissions, + Options.WRITE_SPLITTER_SIDECAR_INHERIT_PERMISSIONS, inheritPermissions ? "true" : null, Options.WRITE_SPLITTER_SIDECAR_ROOT_NAME, rootName, Options.WRITE_SPLITTER_SIDECAR_URI_PREFIX, uriPrefix, Options.WRITE_SPLITTER_SIDECAR_URI_SUFFIX, uriSuffix @@ -252,12 +266,24 @@ public SplitterOptions outputCollections(String... collections) { return this; } + @Override + public SplitterOptions outputInheritCollections(boolean inheritCollections) { + this.inheritCollections = inheritCollections; + return this; + } + @Override public SplitterOptions outputPermissionsString(String rolesAndCapabilities) { this.permissions = rolesAndCapabilities; return this; } + @Override + public SplitterOptions outputInheritPermissions(boolean inheritPermissions) { + this.inheritPermissions = inheritPermissions; + return this; + } + @Override public SplitterOptions outputRootName(String rootName) { this.rootName = rootName; diff --git a/flux-cli/src/test/java/com/marklogic/flux/api/SplitterOptionsTest.java b/flux-cli/src/test/java/com/marklogic/flux/api/SplitterOptionsTest.java new file mode 100644 index 00000000..2c1af868 --- /dev/null +++ b/flux-cli/src/test/java/com/marklogic/flux/api/SplitterOptionsTest.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.flux.api; + +import com.marklogic.flux.impl.AbstractOptionsTest; +import com.marklogic.flux.impl.importdata.SplitterParams; +import com.marklogic.spark.Options; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicReference; + +class SplitterOptionsTest extends AbstractOptionsTest { + + @Test + void inheritMetadata() { + AtomicReference reference = new AtomicReference<>(); + + Flux.importGenericFiles() + .to(options -> options.splitter(splitterOptions -> { + splitterOptions + .jsonPointers("/text") + .outputInheritCollections(true) + .outputInheritPermissions(true); + reference.set(splitterOptions); + })); + + SplitterParams params = (SplitterParams) reference.get(); + assertOptions(params.makeOptions(), + Options.WRITE_SPLITTER_JSON_POINTERS, "/text", + Options.WRITE_SPLITTER_SIDECAR_INHERIT_COLLECTIONS, "true", + Options.WRITE_SPLITTER_SIDECAR_INHERIT_PERMISSIONS, "true" + ); + } +} diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyOptionsTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyOptionsTest.java index b1f3590c..34f0be47 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyOptionsTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyOptionsTest.java @@ -83,6 +83,8 @@ void allWriteParams() { "--output-uri-template", "/{example}.xml", "--output-uri-template-warn-on-missing-field", "--splitter-text", + "--splitter-sidecar-inherit-collections", + "--splitter-sidecar-inherit-permissions", "--embedder", "doesnt-matter", "--embedder-prompt", "Hello: ", "--embedder-prop", "e1=v1", @@ -122,6 +124,8 @@ void allWriteParams() { Options.WRITE_URI_TEMPLATE, "/{example}.xml", Options.WRITE_URI_TEMPLATE_WARN_ON_MISSING_FIELD, "true", Options.WRITE_SPLITTER_TEXT, "true", + Options.WRITE_SPLITTER_SIDECAR_INHERIT_COLLECTIONS, "true", + Options.WRITE_SPLITTER_SIDECAR_INHERIT_PERMISSIONS, "true", Options.WRITE_EMBEDDER_MODEL_FUNCTION_CLASS_NAME, "doesnt-matter", Options.WRITE_EMBEDDER_PROMPT, "Hello: ", Options.WRITE_EMBEDDER_MODEL_FUNCTION_OPTION_PREFIX + "e1", "v1", diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyTest.java index e5d02046..4e0b8cb1 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/copy/CopyTest.java @@ -5,10 +5,12 @@ import com.marklogic.client.DatabaseClient; import com.marklogic.client.ext.helper.ClientHelper; +import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.SearchHandle; import com.marklogic.client.query.QueryManager; import com.marklogic.client.query.StructuredQueryDefinition; import com.marklogic.flux.AbstractTest; +import com.marklogic.junit5.PermissionsTester; import org.junit.jupiter.api.Test; import picocli.CommandLine; @@ -41,6 +43,35 @@ void sameDatabase() { assertDirectoryCount("/copied/", 15); } + @Test + void splitWithInheritedCollectionsAndPermissions() { + run( + "copy", + "--uris", "/author/author1.json", + "--connection-string", makeConnectionString(), + "--output-uri-prefix", "/copied", + // Replaces the source 'test-data' collection, which is preserved between tests and would affect other tests. + "--output-collections", "author-copies", + "--splitter-json-pointer", "/LastName", + "--splitter-sidecar-collections", "chunks,author-copies", + "--splitter-sidecar-permissions", "flux-test-role,update,qconsole-user,read", + "--splitter-sidecar-inherit-collections", + "--splitter-sidecar-inherit-permissions" + ); + + String chunkUri = "/copied/author/author1.json-chunks-1.json"; + DocumentMetadataHandle metadata = getDatabaseClient().newDocumentManager().readMetadata(chunkUri, new DocumentMetadataHandle()); + assertEquals(2, metadata.getCollections().size(), "Expecting the inherited 'author-copies' collection " + + "plus the explicit 'chunks' collection."); + assertTrue(metadata.getCollections().containsAll(List.of("author-copies", "chunks"))); + + PermissionsTester tester = readDocumentPermissions(chunkUri); + tester.assertReadPermissionExists("flux-test-role"); + tester.assertUpdatePermissionExists("flux-test-role"); + tester.assertUpdatePermissionExists("admin"); + tester.assertReadPermissionExists("qconsole-user"); + } + @Test void sameConnectionStringButDifferentOutputDatabase() { runCopyToOtherDatabase(otherDbName -> { diff --git a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java index 3988f94c..fe46509b 100644 --- a/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java +++ b/flux-cli/src/test/java/com/marklogic/flux/impl/importdata/ImportFilesOptionsTest.java @@ -124,7 +124,9 @@ void splitterOptions() { "--splitter-sidecar-max-chunks", "10", "--splitter-sidecar-document-type", "xml", "--splitter-sidecar-collections", "c1,c2", + "--splitter-sidecar-inherit-collections", "--splitter-sidecar-permissions", "role1,read,role2,update", + "--splitter-sidecar-inherit-permissions", "--splitter-sidecar-root-name", "root1", "--splitter-sidecar-uri-prefix", "/prefix", "--splitter-sidecar-uri-suffix", ".json", @@ -143,7 +145,9 @@ void splitterOptions() { Options.WRITE_SPLITTER_SIDECAR_MAX_CHUNKS, "10", Options.WRITE_SPLITTER_SIDECAR_DOCUMENT_TYPE, "XML", Options.WRITE_SPLITTER_SIDECAR_COLLECTIONS, "c1,c2", + Options.WRITE_SPLITTER_SIDECAR_INHERIT_COLLECTIONS, "true", Options.WRITE_SPLITTER_SIDECAR_PERMISSIONS, "role1,read,role2,update", + Options.WRITE_SPLITTER_SIDECAR_INHERIT_PERMISSIONS, "true", Options.WRITE_SPLITTER_SIDECAR_ROOT_NAME, "root1", Options.WRITE_SPLITTER_SIDECAR_URI_PREFIX, "/prefix", Options.WRITE_SPLITTER_SIDECAR_URI_SUFFIX, ".json",