diff --git a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/cache/DataDomainCache.kt b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/cache/DataDomainCache.kt index 7d3276b418..0aa4cd6374 100644 --- a/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/cache/DataDomainCache.kt +++ b/package-toolkit/runtime/src/main/kotlin/com/atlan/pkg/cache/DataDomainCache.kt @@ -11,7 +11,7 @@ import com.atlan.pkg.PackageContext import com.atlan.pkg.Utils import com.atlan.pkg.serde.cell.DataDomainXformer -class DataDomainCache( +open class DataDomainCache( val ctx: PackageContext<*>, ) : AssetCache(ctx, "domain") { private val logger = Utils.getLogger(this.javaClass.name) @@ -25,10 +25,28 @@ class DataDomainCache( /** {@inheritDoc} */ override fun lookupById(id: String?) { - val result = lookupById(id, 0, ctx.client.maxNetworkRetries) - if (result != null) cache(result.guid, getIdentityForAsset(result), result) + val result = id?.let { fetchById(it) } + if (result != null) { + // getIdentityForAsset reads the parent's identity from the cache only. On a cache + // that was not preloaded, a sub-domain fetched before its parent would otherwise be + // cached under just its own name instead of its full path, so fetch (and cache) the + // missing parent first. This recurses up the hierarchy, one level per ancestor. + val parentGuid = result.parentDomain?.guid + if (parentGuid != null && getIdentity(parentGuid, true) == null) { + getByGuid(parentGuid) + } + cache(result.guid, getIdentityForAsset(result), result) + } } + /** + * Retrieve a single data domain from Atlan by its GUID. + * + * @param guid unique identifier (GUID) of the data domain + * @return the data domain, or null if it could not be found + */ + protected open fun fetchById(guid: String): DataDomain? = lookupById(guid, 0, ctx.client.maxNetworkRetries) + /** {@inheritDoc} */ private fun lookupById( guid: String?, diff --git a/package-toolkit/runtime/src/test/kotlin/cache/DataDomainCacheSubDomainTest.kt b/package-toolkit/runtime/src/test/kotlin/cache/DataDomainCacheSubDomainTest.kt new file mode 100644 index 0000000000..0c00e86be9 --- /dev/null +++ b/package-toolkit/runtime/src/test/kotlin/cache/DataDomainCacheSubDomainTest.kt @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2023 Atlan Pte. Ltd. */ +package cache + +import com.atlan.mock.MockAtlanTenant +import com.atlan.mock.MockConfig +import com.atlan.model.assets.DataDomain +import com.atlan.model.enums.AtlanStatus +import com.atlan.pkg.PackageContext +import com.atlan.pkg.Utils +import com.atlan.pkg.cache.DataDomainCache +import com.atlan.pkg.serde.cell.DataDomainXformer.DATA_DOMAIN_DELIMITER +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * A sub-domain looked up by GUID on a cold (not preloaded) cache must still resolve to its + * full path. If the parent is read cache-only, the sub-domain's identity collapses to just its + * own name -- and an import, whose preloaded cache keys domains by full path, can then no + * longer find that domain. + */ +class DataDomainCacheSubDomainTest { + companion object { + init { + MockAtlanTenant.initializeClient() + } + + private val config = Utils.parseConfig("{}", "{}") + private val ctx = PackageContext(config, MockAtlanTenant.client, true) + + private const val GRANDPARENT_GUID = "0b9f5a52-0000-4000-8000-000000000001" + private const val PARENT_GUID = "0b9f5a52-0000-4000-8000-000000000002" + private const val CHILD_GUID = "0b9f5a52-0000-4000-8000-000000000003" + } + + /** Serves domains from memory instead of searching a tenant. */ + private class InMemoryDomainCache( + ctx: PackageContext<*>, + domains: List, + ) : DataDomainCache(ctx) { + private val byGuid = domains.associateBy { it.guid } + + override fun fetchById(guid: String): DataDomain? = byGuid[guid] + } + + private fun domain( + guid: String, + name: String, + parentGuid: String? = null, + ): DataDomain { + val builder = + DataDomain + ._internal() + .guid(guid) + .qualifiedName("default/domain/$name/super") + .name(name) + .status(AtlanStatus.ACTIVE) + if (parentGuid != null) builder.parentDomain(DataDomain.refByGuid(parentGuid)) + return builder.build() + } + + @Test + fun subDomainOnColdCacheResolvesToFullPath() { + val cache = + InMemoryDomainCache( + ctx, + listOf( + domain(PARENT_GUID, "Parent"), + domain(CHILD_GUID, "Child", PARENT_GUID), + ), + ) + // Same two calls DataDomainXformer.encodeFromGuid makes. + cache.getByGuid(CHILD_GUID) + assertEquals("Parent${DATA_DOMAIN_DELIMITER}Child", cache.getIdentity(CHILD_GUID)) + } + + @Test + fun nestedSubDomainOnColdCacheResolvesEveryAncestor() { + val cache = + InMemoryDomainCache( + ctx, + listOf( + domain(GRANDPARENT_GUID, "Grandparent"), + domain(PARENT_GUID, "Parent", GRANDPARENT_GUID), + domain(CHILD_GUID, "Child", PARENT_GUID), + ), + ) + cache.getByGuid(CHILD_GUID) + assertEquals( + "Grandparent${DATA_DOMAIN_DELIMITER}Parent${DATA_DOMAIN_DELIMITER}Child", + cache.getIdentity(CHILD_GUID), + ) + } +} diff --git a/samples/packages/asset-export-basic/src/main/kotlin/AssetExporter.kt b/samples/packages/asset-export-basic/src/main/kotlin/AssetExporter.kt index f6cecacff2..2d6714c188 100644 --- a/samples/packages/asset-export-basic/src/main/kotlin/AssetExporter.kt +++ b/samples/packages/asset-export-basic/src/main/kotlin/AssetExporter.kt @@ -55,6 +55,14 @@ class AssetExporter( CSVWriter(filename).use { csv -> csv.writeHeader(headerNames) val start = System.currentTimeMillis() + + // Preload the domain cache up-front so identities resolve in level-order + // (parents-before-children). Assets are streamed in parallel, so without this + // a sub-domain can be looked up before its parent is cached and get exported + // as just its own name rather than its full path -- which the import side + // then cannot resolve. + ctx.dataDomainCache.preload() + csv.streamAssets(assets.stream(true), this, assets.count(), batchSize, logger) logger.info { "Total time taken: ${System.currentTimeMillis() - start} ms" } }