From e585980dec541952539135d9b1f170988fd3fdff Mon Sep 17 00:00:00 2001 From: Tobias Bocanegra Date: Fri, 4 Sep 2026 15:12:57 +0200 Subject: [PATCH] feat(storage): add Azure Blob Storage backend Adds @adobe/helix-shared-storage-azure, an AbstractStorageBackend implementation over @azure/storage-blob, mirroring the plug-in shape of helix-shared-storage-s3 (StorageAzure subclass + one-line import swap, HLX_AZURE_STORAGE_CONNECTION_STRING / HLX_AZURE_STORAGE_ACCOUNT_NAME + HLX_AZURE_STORAGE_ACCOUNT_KEY env-driven default factory). Also handles two migration quirks seen when moving content from S3 to Azure: falling back to a content_encoding metadata property when the native Content-Encoding header is empty, and transparently substituting `-`/`_` in metadata keys since Azure metadata names can't contain `-`. --- package-lock.json | 400 ++++++++++++++- .../.mocha-multi.json | 6 + .../helix-shared-storage-azure/.npmignore | 9 + .../helix-shared-storage-azure/.nycrc.json | 10 + .../helix-shared-storage-azure/.releaserc.cjs | 1 + .../helix-shared-storage-azure/LICENSE.txt | 264 ++++++++++ packages/helix-shared-storage-azure/README.md | 30 ++ .../helix-shared-storage-azure/package.json | 42 ++ .../src/AzureBackend.js | 424 ++++++++++++++++ .../src/StorageAzure.js | 39 ++ .../src/createDefaultBackendFactory.js | 94 ++++ .../helix-shared-storage-azure/src/index.js | 18 + .../test/StorageAzure.test.js | 44 ++ .../test/createDefaultBackendFactory.test.js | 48 ++ .../test/setup-env.js | 24 + .../test/storage.test.js | 465 ++++++++++++++++++ .../helix-shared-storage-azure/test/utils.js | 79 +++ 17 files changed, 1996 insertions(+), 1 deletion(-) create mode 100644 packages/helix-shared-storage-azure/.mocha-multi.json create mode 100644 packages/helix-shared-storage-azure/.npmignore create mode 100644 packages/helix-shared-storage-azure/.nycrc.json create mode 120000 packages/helix-shared-storage-azure/.releaserc.cjs create mode 100644 packages/helix-shared-storage-azure/LICENSE.txt create mode 100644 packages/helix-shared-storage-azure/README.md create mode 100644 packages/helix-shared-storage-azure/package.json create mode 100644 packages/helix-shared-storage-azure/src/AzureBackend.js create mode 100644 packages/helix-shared-storage-azure/src/StorageAzure.js create mode 100644 packages/helix-shared-storage-azure/src/createDefaultBackendFactory.js create mode 100644 packages/helix-shared-storage-azure/src/index.js create mode 100644 packages/helix-shared-storage-azure/test/StorageAzure.test.js create mode 100644 packages/helix-shared-storage-azure/test/createDefaultBackendFactory.test.js create mode 100644 packages/helix-shared-storage-azure/test/setup-env.js create mode 100644 packages/helix-shared-storage-azure/test/storage.test.js create mode 100644 packages/helix-shared-storage-azure/test/utils.js diff --git a/package-lock.json b/package-lock.json index 4743d0fd0..924e401d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -160,6 +160,10 @@ "resolved": "packages/helix-shared-storage", "link": true }, + "node_modules/@adobe/helix-shared-storage-azure": { + "resolved": "packages/helix-shared-storage-azure", + "link": true + }, "node_modules/@adobe/helix-shared-storage-s3": { "resolved": "packages/helix-shared-storage-s3", "link": true @@ -532,6 +536,208 @@ "node": ">=18.0.0" } }, + "node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.11.0.tgz", + "integrity": "sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.11.1.tgz", + "integrity": "sha512-2QygG2F76ZpMP2eMztiJvAiFMu71M9rDeU7vO/QKg5Css7MgM4frUOslFjhVjRhbGaCNPtz/S8M6y46/fFKVuQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-http-compat": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.5.0.tgz", + "integrity": "sha512-BoSmXPx2er1Ai+wKlDvj29jIQespCNBwEmKyZVHO2kEFsWbGjAjwMCGzug3DJM5/QYIV3vej0S1zcU5bq9fa8w==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "@azure/core-client": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0" + } + }, + "node_modules/@azure/core-lro": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", + "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-paging": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.7.0.tgz", + "integrity": "sha512-7GEAoIsaoBr6KELNRb8nypowCqvk8dnCHFCYg4XD4lOQGY2GqjQg5IhkRjyBFRO18CGSMq05PaNqSOE9GQro3g==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.25.0.tgz", + "integrity": "sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", + "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.14.0.tgz", + "integrity": "sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-xml": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.6.0.tgz", + "integrity": "sha512-e7lX/dk//F6Qf7BB6PTY4+p2yuOQtyOeHGyapYHNwqSp2OnYpwQt49A/Nin2XmKBQ69pwagR4k/lQBq8lbHQkA==", + "license": "MIT", + "dependencies": { + "fast-xml-parser": "^5.5.9", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.4.0.tgz", + "integrity": "sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==", + "license": "MIT", + "dependencies": { + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-blob": { + "version": "12.33.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.33.0.tgz", + "integrity": "sha512-2SX8oP8PyblUcAFZSg39c8Ls+tFjavM6sBeV+qpw33mRzRhI/5hrFJmJ/x0H9xx5l6ECPvgSP8uPxqTeVbHNIA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.2.0", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.5", + "@azure/logger": "^1.1.4", + "@azure/storage-common": "^12.4.1", + "events": "^3.0.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-common": { + "version": "12.5.0", + "resolved": "https://registry.npmjs.org/@azure/storage-common/-/storage-common-12.5.0.tgz", + "integrity": "sha512-bttzuhQiCIwrkzjPDA+AtAR7dg19L/CC6ztcqJ5LfvWpXuys9mHp0UQ0udYnoUvv9SCT9KTR5kqFvFr0e6k0lQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-http-compat": "^2.4.0", + "@azure/core-rest-pipeline": "^1.24.0", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.1.4", + "events": "^3.3.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -988,6 +1194,18 @@ "@tybys/wasm-util": "^0.9.0" } }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2210,6 +2428,55 @@ "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", "license": "MIT" }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.9.tgz", + "integrity": "sha512-edSdeAqkdxBVzA1yL1LrLCml1YjyCVvPMtMqJpbF+6K609tHe8V6sQUzFQSGcYNhcuhOceZtjvN32+mpIth30A==", + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@typespec/ts-http-runtime/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@typespec/ts-http-runtime/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@typespec/ts-http-runtime/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.4.0.tgz", @@ -2395,6 +2662,18 @@ "dev": true, "license": "MIT" }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -4993,6 +5272,15 @@ "node": ">= 0.6" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/execa": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/execa/-/execa-10.0.1.tgz", @@ -5183,6 +5471,45 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.1.tgz", + "integrity": "sha512-TBw6K/fxoQGGjCmZDw9w/ZwP3uDcnTM4YH/g+PFRWr8sbe5idXtxNN6vITh4+1ruCZaho6uBFurElsA7F0zzgw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.2", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastestsmallesttextencoderdecoder": { "version": "1.0.22", "resolved": "https://registry.npmjs.org/fastestsmallesttextencoderdecoder/-/fastestsmallesttextencoderdecoder-1.0.22.tgz", @@ -7005,6 +7332,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/is-weakmap": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz", @@ -11837,6 +12176,21 @@ "node": ">=8" } }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -14088,6 +14442,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.2.tgz", + "integrity": "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, "node_modules/super-regex": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/super-regex/-/super-regex-1.1.0.tgz", @@ -15308,6 +15677,21 @@ "dev": true, "license": "ISC" }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/xml2js": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", @@ -15966,9 +16350,23 @@ "xml2js": "^0.6.2" } }, + "packages/helix-shared-storage-azure": { + "name": "@adobe/helix-shared-storage-azure", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "@adobe/helix-shared-process-queue": "^3.0.4", + "@adobe/helix-shared-storage": "^3.0.0", + "@azure/storage-blob": "^12.28.0" + }, + "devDependencies": { + "nock": "14.0.17", + "xml2js": "^0.6.2" + } + }, "packages/helix-shared-storage-s3": { "name": "@adobe/helix-shared-storage-s3", - "version": "1.0.0", + "version": "1.1.1", "license": "Apache-2.0", "dependencies": { "@adobe/fetch": "^4.1.2", diff --git a/packages/helix-shared-storage-azure/.mocha-multi.json b/packages/helix-shared-storage-azure/.mocha-multi.json new file mode 100644 index 000000000..aa2be2a23 --- /dev/null +++ b/packages/helix-shared-storage-azure/.mocha-multi.json @@ -0,0 +1,6 @@ +{ + "reporterEnabled": "spec,xunit", + "xunitReporterOptions": { + "output": "junit/test-results.xml" + } +} diff --git a/packages/helix-shared-storage-azure/.npmignore b/packages/helix-shared-storage-azure/.npmignore new file mode 100644 index 000000000..868317d21 --- /dev/null +++ b/packages/helix-shared-storage-azure/.npmignore @@ -0,0 +1,9 @@ +coverage/ +node_modules/ +junit/ +test/ +docs/ +logs/ +test-results.xml +renovate.json +.* diff --git a/packages/helix-shared-storage-azure/.nycrc.json b/packages/helix-shared-storage-azure/.nycrc.json new file mode 100644 index 000000000..f6bb44c26 --- /dev/null +++ b/packages/helix-shared-storage-azure/.nycrc.json @@ -0,0 +1,10 @@ +{ + "reporter": [ + "lcov", + "text" + ], + "check-coverage": true, + "lines": 100, + "branches": 100, + "statements": 100 +} diff --git a/packages/helix-shared-storage-azure/.releaserc.cjs b/packages/helix-shared-storage-azure/.releaserc.cjs new file mode 120000 index 000000000..7ab453681 --- /dev/null +++ b/packages/helix-shared-storage-azure/.releaserc.cjs @@ -0,0 +1 @@ +../../.releaserc.cjs \ No newline at end of file diff --git a/packages/helix-shared-storage-azure/LICENSE.txt b/packages/helix-shared-storage-azure/LICENSE.txt new file mode 100644 index 000000000..883ab098f --- /dev/null +++ b/packages/helix-shared-storage-azure/LICENSE.txt @@ -0,0 +1,264 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +APACHE JACKRABBIT SUBCOMPONENTS + +Apache Jackrabbit includes parts with separate copyright notices and license +terms. Your use of these subcomponents is subject to the terms and conditions +of the following licenses: + + XPath 2.0/XQuery 1.0 Parser: + http://www.w3.org/2002/11/xquery-xpath-applets/xgrammar.zip + + Copyright (C) 2002 World Wide Web Consortium, (Massachusetts Institute of + Technology, European Research Consortium for Informatics and Mathematics, + Keio University). All Rights Reserved. + + This work is distributed under the W3C(R) Software License in the hope + that it will be useful, but WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + W3C(R) SOFTWARE NOTICE AND LICENSE + http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231 + + This work (and included software, documentation such as READMEs, or + other related items) is being provided by the copyright holders under + the following license. By obtaining, using and/or copying this work, + you (the licensee) agree that you have read, understood, and will comply + with the following terms and conditions. + + Permission to copy, modify, and distribute this software and its + documentation, with or without modification, for any purpose and + without fee or royalty is hereby granted, provided that you include + the following on ALL copies of the software and documentation or + portions thereof, including modifications: + + 1. The full text of this NOTICE in a location viewable to users + of the redistributed or derivative work. + + 2. Any pre-existing intellectual property disclaimers, notices, + or terms and conditions. If none exist, the W3C Software Short + Notice should be included (hypertext is preferred, text is + permitted) within the body of any redistributed or derivative code. + + 3. Notice of any changes or modifications to the files, including + the date changes were made. (We recommend you provide URIs to the + location from which the code is derived.) + + THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT + HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR FITNESS + FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE OR + DOCUMENTATION WILL NOT INFRINGE ANY THIRD PARTY PATENTS, COPYRIGHTS, + TRADEMARKS OR OTHER RIGHTS. + + COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL + OR CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE SOFTWARE OR + DOCUMENTATION. + + The name and trademarks of copyright holders may NOT be used in + advertising or publicity pertaining to the software without specific, + written prior permission. Title to copyright in this software and + any associated documentation will at all times remain with + copyright holders. diff --git a/packages/helix-shared-storage-azure/README.md b/packages/helix-shared-storage-azure/README.md new file mode 100644 index 000000000..2b142db03 --- /dev/null +++ b/packages/helix-shared-storage-azure/README.md @@ -0,0 +1,30 @@ +# @adobe/helix-shared-storage-azure + +An Azure Blob Storage `StorageBackend` implementation for [`@adobe/helix-shared-storage`](../helix-shared-storage). See [`AbstractStorageBackend`](../helix-shared-storage/src/AbstractStorageBackend.js) for the backend convention this package implements. + +## Usage + +Existing consumers of `@adobe/helix-shared-storage`'s `Storage.fromContext()` migrate with a one-line import change: + +```diff +- import { Storage } from '@adobe/helix-shared-storage'; ++ import { StorageAzure as Storage } from '@adobe/helix-shared-storage-azure'; +``` + +No other call site changes are required — `Storage.fromContext(context)`, `storage.contentBus()`, etc. all keep working exactly as before. + +The default backend factory authenticates against Azure Blob Storage from environment variables, preferring a connection string when set: + +- `HLX_AZURE_STORAGE_CONNECTION_STRING`, or +- `HLX_AZURE_STORAGE_ACCOUNT_NAME` + `HLX_AZURE_STORAGE_ACCOUNT_KEY` + +Consumers constructing `Storage` directly (bypassing `fromContext`) pass an explicit `backendFactory`: + +```js +import { Storage } from '@adobe/helix-shared-storage'; +import { createDefaultBackendFactory } from '@adobe/helix-shared-storage-azure'; + +const storage = new Storage({ + backendFactory: createDefaultBackendFactory(process.env), +}); +``` diff --git a/packages/helix-shared-storage-azure/package.json b/packages/helix-shared-storage-azure/package.json new file mode 100644 index 000000000..02cc0be6b --- /dev/null +++ b/packages/helix-shared-storage-azure/package.json @@ -0,0 +1,42 @@ +{ + "name": "@adobe/helix-shared-storage-azure", + "version": "1.0.0", + "description": "Shared modules of the Helix Project - Azure Blob Storage backend for @adobe/helix-shared-storage", + "main": "src/index.js", + "type": "module", + "scripts": { + "test": "c8 mocha", + "lint": "eslint .", + "clean": "rm -rf package-lock.json node_modules" + }, + "mocha": { + "reporter": "mocha-multi-reporters", + "reporter-options": "configFile=.mocha-multi.json", + "require": [ + "test/setup-env.js", + "mocha-suppress-logs" + ] + }, + "repository": { + "type": "git", + "url": "https://github.com/adobe/helix-shared.git" + }, + "author": "", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/adobe/helix-shared/issues" + }, + "homepage": "https://github.com/adobe/helix-shared#readme", + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "nock": "14.0.17", + "xml2js": "^0.6.2" + }, + "dependencies": { + "@adobe/helix-shared-process-queue": "^3.0.4", + "@adobe/helix-shared-storage": "^3.0.0", + "@azure/storage-blob": "^12.28.0" + } +} diff --git a/packages/helix-shared-storage-azure/src/AzureBackend.js b/packages/helix-shared-storage-azure/src/AzureBackend.js new file mode 100644 index 000000000..bc24ebb57 --- /dev/null +++ b/packages/helix-shared-storage-azure/src/AzureBackend.js @@ -0,0 +1,424 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-disable no-param-reassign */ +import { promisify } from 'util'; +import { buffer } from 'node:stream/consumers'; +import zlib from 'zlib'; +import processQueue from '@adobe/helix-shared-process-queue'; +import { AbstractStorageBackend } from '@adobe/helix-shared-storage'; + +const gunzip = promisify(zlib.gunzip); + +/** + * Maximum number of blobs deleted concurrently in a bulk `remove()` call. Azure has no + * S3-style bulk-delete API, so each key needs its own request; this bounds concurrency + * instead of chunk size. + */ +const REMOVE_CONCURRENCY = 8; + +/** + * `SYSTEM_META_FIELD_NAMES` (the common, lowerCamelCase system-property names shared across + * the `StorageBackend` API), mapped to the corresponding Azure `BlobHTTPHeaders` field. Drives + * `head()`'s/`put()`'s system-field mapping. Azure has no native equivalent of S3's `Expires` + * header, so `expires` is intentionally left unmapped (dropped) rather than faked via custom + * metadata. + */ +const SYSTEM_META_FIELDS = { + contentType: 'blobContentType', + contentEncoding: 'blobContentEncoding', + cacheControl: 'blobCacheControl', + contentDisposition: 'blobContentDisposition', + contentLanguage: 'blobContentLanguage', +}; + +/** + * Returns the last segment of a key, treating an optional trailing `/` as a folder separator. + * + * @param {string} key + * @returns {string} + */ +function basename(key) { + const trimmed = key.endsWith('/') ? key.slice(0, -1) : key; + const slash = trimmed.lastIndexOf('/'); + return slash >= 0 ? trimmed.substring(slash + 1) : trimmed; +} + +/** + * Azure metadata property names must be valid C# identifiers and cannot contain `-`. Since + * none of our metadata keys ever legitimately contain a literal `_`, `-` is transparently and + * losslessly substituted with `_` on write, and reversed on read, so callers never need to + * know about this Azure-specific restriction (this is also what blobs migrated from S3, whose + * metadata keys freely used `-`, ended up doing). + * + * @param {Object.} [metadata] + * @returns {Object.} + */ +function toAzureMetadata(metadata = {}) { + return Object.fromEntries( + Object.entries(metadata).map(([key, value]) => [key.replace(/-/g, '_'), value]), + ); +} + +/** + * The inverse of {@link toAzureMetadata}, applied to metadata read back from Azure. + * + * @param {Object.} [metadata] + * @returns {Object.} + */ +function fromAzureMetadata(metadata = {}) { + return Object.fromEntries( + Object.entries(metadata).map(([key, value]) => [key.replace(/_/g, '-'), value]), + ); +} + +/** + * Maps a downloaded blob's HTTP headers into the common, lowerCamelCase `CommonObjectMeta` + * field names, shared by `get()`'s `meta` output and `head()`'s return value. Falls back to a + * `content_encoding` custom metadata property when the native `Content-Encoding` header is + * empty — blobs migrated from S3 don't always have the native header set, but the migration + * preserves the original encoding under this metadata key. + * + * @param {object} raw a blob download or getProperties response + * @returns {object} + */ +function commonFieldsFromRaw(raw) { + const result = {}; + Object.entries(SYSTEM_META_FIELDS).forEach(([common]) => { + if (raw[common] !== undefined) { + result[common] = raw[common]; + } + }); + if (!result.contentEncoding && raw.metadata?.content_encoding) { + result.contentEncoding = raw.metadata.content_encoding; + } + return result; +} + +/** + * @typedef {Object} AzureBackendOptions + * @property {import('@azure/storage-blob').ContainerClient} client + * @property {string} [name] backend family tag used for error tagging when mirrored + * @property {string} bucketName + * @property {Console} [log] + */ + +/** + * Azure Blob Storage {@link import('@adobe/helix-shared-storage').StorageBackend} + * implementation. A single instance wraps one Azure `ContainerClient` bound to one + * container (Azure's equivalent of a bucket). + * + * @implements {import('@adobe/helix-shared-storage').StorageBackend} + */ +export class AzureBackend extends AbstractStorageBackend { + /** + * @param {AzureBackendOptions} opts + */ + constructor({ + client, name = 'Azure', bucketName, log = console, + }) { + super(); + this._client = client; + this._name = name; + this._bucketName = bucketName; + this._log = log; + } + + /** @type {string} */ + get name() { + return this._name; + } + + /** @type {string} */ + get bucketName() { + return this._bucketName; + } + + /** @type {import('@azure/storage-blob').ContainerClient} */ + get client() { + return this._client; + } + + /** + * @param {string} key already-sanitized object key + * @param {Record} [meta] output object that receives the object's custom + * metadata plus its recognized system fields (using the same common, lowerCamelCase names + * as `head()`, e.g. `contentType`), and `etag`/`lastModified` + * @returns {Promise} + */ + async get(key, meta = null) { + const blobClient = this._client.getBlobClient(key); + try { + const result = await blobClient.download(); + this._log.info(`object downloaded from: ${this._bucketName}/${key}`); + + const buf = await buffer(result.readableStreamBody); + const commonFields = commonFieldsFromRaw(result); + if (meta) { + Object.assign(meta, fromAzureMetadata(result.metadata)); + meta.etag = result.etag; + meta.lastModified = result.lastModified; + Object.assign(meta, commonFields); + } + if (commonFields.contentEncoding === 'gzip') { + return await gunzip(buf); + } + return buf; + } catch (e) { + /* c8 ignore next 3 */ + if (e.statusCode !== 404) { + throw e; + } + return null; + } + } + + /** + * @param {string} key already-sanitized object key + * @param {Record} [headOpts] extra fields merged into the underlying + * `getProperties()` call + * @returns {Promise} + */ + async head(key, headOpts = {}) { + const blobClient = this._client.getBlobClient(key); + try { + const raw = await blobClient.getProperties(headOpts); + this._log.info(`Object metadata downloaded from: ${this._bucketName}/${key}`); + return { + etag: raw.etag, + contentLength: raw.contentLength, + lastModified: raw.lastModified, + metadata: fromAzureMetadata(raw.metadata), + ...commonFieldsFromRaw(raw), + raw, + }; + } catch (e) { + /* c8 ignore next 3 */ + if (e.statusCode !== 404) { + throw e; + } + return null; + } + } + + /** + * @param {string} key already-sanitized object key + * @param {Buffer|string} body data to store + * @param {import('@adobe/helix-shared-storage').PutOptions} [opts] + * @returns {Promise} + */ + async put(key, body, opts = {}) { + const data = typeof body === 'string' ? Buffer.from(body) : body; + const blockBlobClient = this._client.getBlockBlobClient(key); + const blobHTTPHeaders = {}; + Object.entries(SYSTEM_META_FIELDS).forEach(([common, azure]) => { + blobHTTPHeaders[azure] = opts[common]; + }); + const raw = await blockBlobClient.upload(data, data.length, { + blobHTTPHeaders, + metadata: toAzureMetadata(opts.metadata), + }); + this._log.info(`object uploaded to: ${this._bucketName}/${key}`); + return { + etag: raw.etag, contentType: opts.contentType, raw, + }; + } + + /** + * Replace an object's user metadata (and any recognized system properties within `meta`). + * Azure has no S3-style single self-copy trick for this, so metadata and HTTP headers are + * updated via two separate, parallel calls — `setMetadata()` and `setHTTPHeaders()` — each of + * which fully replaces its respective property set, matching `putMeta()`'s + * "fully replaces existing metadata" contract. + * + * @param {string} path already-sanitized object key + * @param {Object.} meta new metadata (fully replaces existing metadata) + * @returns {Promise} + */ + async putMeta(path, meta = {}) { + const blobClient = this._client.getBlobClient(path); + const customMetadata = {}; + const blobHTTPHeaders = {}; + Object.entries(meta).forEach(([key, value]) => { + const azureField = SYSTEM_META_FIELDS[key]; + if (azureField) { + blobHTTPHeaders[azureField] = value; + } else { + customMetadata[key] = value; + } + }); + const [metadataRaw, headersRaw] = await Promise.all([ + blobClient.setMetadata(toAzureMetadata(customMetadata)), + blobClient.setHTTPHeaders(blobHTTPHeaders), + ]); + this._log.info(`Metadata updated for: ${this._bucketName}/${path}`); + return { raw: { metadata: metadataRaw, headers: headersRaw } }; + } + + /** + * @param {string} src already-sanitized source key + * @param {string} dst already-sanitized destination key + * @param {import('@adobe/helix-shared-storage').CopyOptions} [opts] + * @returns {Promise} + * @throws an error with `status: 404` if the source object does not exist + */ + async copy(src, dst, opts = {}) { + const srcBlobClient = this._client.getBlobClient(src); + const dstBlobClient = this._client.getBlobClient(dst); + + // Unlike S3, Azure's `beginCopyFromURL` always copies the source's HTTP headers verbatim + // and only lets the request override `metadata` (S3's `COPY` vs. `REPLACE` directive maps + // onto omitting vs. passing `metadata` here). There is no way to override system HTTP + // headers (contentType, etc.) as part of the copy call itself, so a `REPLACE` directive + // needs a follow-up `setHTTPHeaders()` call against the destination once the copy lands. + const copyOpts = { ...opts.copyOpts }; + if (opts.metadataDirective === 'REPLACE') { + copyOpts.metadata = opts.metadata ?? {}; + } + + try { + const poller = await dstBlobClient.beginCopyFromURL(srcBlobClient.url, copyOpts); + let raw = await poller.pollUntilDone(); + if (opts.metadataDirective === 'REPLACE') { + const blobHTTPHeaders = {}; + Object.entries(SYSTEM_META_FIELDS).forEach(([common, azure]) => { + blobHTTPHeaders[azure] = opts[common]; + }); + raw = await dstBlobClient.setHTTPHeaders(blobHTTPHeaders); + } + this._log.info(`object copied from ${this._bucketName}/${src} to: ${this._bucketName}/${dst}`); + return { etag: raw.etag, raw }; + } catch (e) { + /* c8 ignore next 3 */ + if (e.statusCode !== 404) { + throw e; + } + const e2 = new Error(`source does not exist: ${this._bucketName}/${src}`); + e2.status = 404; + throw e2; + } + } + + /** + * @param {string|string[]} pathOrPaths single already-sanitized key, or array of them + * @param {import('@adobe/helix-shared-storage').RemoveOptions} [opts] + * @returns {Promise| + * import('@adobe/helix-shared-storage').BulkRemoveResult>} + */ + async remove(pathOrPaths, opts = {}) { + const { stopOnError = false } = opts; + const bucket = this._bucketName; + const log = this._log; + + if (Array.isArray(pathOrPaths)) { + const result = { Deleted: [], Errors: [] }; + await processQueue(pathOrPaths, async (key) => { + try { + const raw = await this._client.getBlobClient(key).deleteIfExists(); + result.Deleted.push({ Key: key, raw }); + } catch (e) { + log.warn(`error while deleting ${bucket}/${key}: ${e.message} (${e.statusCode})`); + result.Errors.push({ Key: key, message: e.message }); + if (stopOnError) { + const msg = `removing ${key} from bucket ${bucket} failed: ${e.message}`; + log.error(msg); + const e2 = new Error(msg); + e2.status = e.statusCode; + throw e2; + } + } + }, REMOVE_CONCURRENCY); + log.info(`deleted ${result.Deleted.length} files (${result.Errors.length} errors)`); + return result; + } + + try { + const raw = await this._client.getBlobClient(pathOrPaths).deleteIfExists(); + log.info(`object deleted: ${bucket}/${pathOrPaths}`); + return raw; + } catch (e) { + const msg = `removing ${bucket}/${pathOrPaths} from storage failed: ${e.message}`; + log.error(msg); + const e2 = new Error(msg); + e2.status = e.statusCode; + throw e2; + } + } + + /** + * @param {string} prefix already-sanitized key prefix to list under + * @param {import('@adobe/helix-shared-storage').BackendListOptions} [opts] + * @returns {Promise} + */ + async list(prefix, opts = {}) { + const { shallow = false, maxItems = Number.POSITIVE_INFINITY } = opts; + + const objects = []; + const iter = shallow + ? this._client.listBlobsByHierarchy('/', { prefix }) + : this._client.listBlobsFlat({ prefix }); + // eslint-disable-next-line no-restricted-syntax + for await (const item of iter) { + if (objects.length >= maxItems) { + break; + } + if (item.kind === 'prefix') { + objects.push({ key: item.name, name: basename(item.name), isFolder: true }); + } else { + objects.push({ + key: item.name, + name: basename(item.name), + isFolder: item.name.endsWith('/'), + lastModified: item.properties.lastModified, + contentLength: item.properties.contentLength, + contentType: item.properties.contentType, + }); + } + } + return { prefix, objects, continuationToken: undefined }; + } + + /** + * @param {string} prefix already-sanitized key prefix to browse + * @param {import('@adobe/helix-shared-storage').BrowseOptions} [opts] + * @returns {Promise} + */ + async browse(prefix, opts = {}) { + const { continuationToken, maxItems } = opts; + + const iter = this._client + .listBlobsByHierarchy('/', { prefix }) + .byPage({ continuationToken, maxPageSize: maxItems }); + const { value } = await iter.next(); + + const objects = []; + value.segment.blobPrefixes.forEach(({ name }) => { + objects.push({ key: name, name: basename(name), isFolder: true }); + }); + value.segment.blobItems.forEach((item) => { + objects.push({ + key: item.name, + name: basename(item.name), + isFolder: false, + lastModified: item.properties.lastModified, + contentLength: item.properties.contentLength, + contentType: item.properties.contentType, + }); + }); + + return { + prefix, + objects, + continuationToken: value.continuationToken || undefined, + }; + } +} diff --git a/packages/helix-shared-storage-azure/src/StorageAzure.js b/packages/helix-shared-storage-azure/src/StorageAzure.js new file mode 100644 index 000000000..855146c08 --- /dev/null +++ b/packages/helix-shared-storage-azure/src/StorageAzure.js @@ -0,0 +1,39 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { Storage } from '@adobe/helix-shared-storage'; +import { createDefaultBackendFactory } from './createDefaultBackendFactory.js'; + +/** + * `Storage` subclass pre-wired with the default Azure Blob Storage `backendFactory`, so + * existing consumers of `Storage.fromContext()` migrate with a one-line import change: + * + * ```diff + * - import { Storage } from '@adobe/helix-shared-storage'; + * + import { StorageAzure as Storage } from '@adobe/helix-shared-storage-azure'; + * ``` + * + * No other call site changes are required. + */ +export class StorageAzure extends Storage { + /** + * @param {import('@adobe/helix-shared-storage').StorageContext} context + * @param {Partial} [opts] + * @returns {StorageAzure} + */ + static fromContext(context, opts = {}) { + return super.fromContext(context, { + backendFactory: createDefaultBackendFactory(context.env, { log: context.log }), + ...opts, + }); + } +} diff --git a/packages/helix-shared-storage-azure/src/createDefaultBackendFactory.js b/packages/helix-shared-storage-azure/src/createDefaultBackendFactory.js new file mode 100644 index 000000000..593f69dcf --- /dev/null +++ b/packages/helix-shared-storage-azure/src/createDefaultBackendFactory.js @@ -0,0 +1,94 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +import { + BlobServiceClient, + StorageSharedKeyCredential, +} from '@azure/storage-blob'; +import { AzureBackend } from './AzureBackend.js'; + +/** + * @typedef {Object} CreateDefaultBackendFactoryOptions + * @property {Console} [log] + */ + +/** + * @typedef {Object} BackendFactoryOpts + * @property {string} [connectionString] + * @property {string} [accountName] + * @property {string} [accountKey] + */ + +/** + * Parses the env vars consumed by `createBackendFactory()` into a plain opts object. + * + * @param {Record} [env] environment variables (e.g. `context.env`) + * @returns {BackendFactoryOpts} + */ +function parseBackendFactoryEnvOpts(env = {}) { + const { + HLX_AZURE_STORAGE_CONNECTION_STRING: connectionString, + HLX_AZURE_STORAGE_ACCOUNT_NAME: accountName, + HLX_AZURE_STORAGE_ACCOUNT_KEY: accountKey, + } = env; + + return { connectionString, accountName, accountKey }; +} + +/** + * Builds the `backendFactory` for `Storage`: Azure Blob Storage as the backend, from an + * already-parsed {@link BackendFactoryOpts}. Use this directly when you need to override + * individual values; use `createDefaultBackendFactory()` to build straight from `env`. + * + * A connection string, when given, takes precedence over account name/key credentials. + * + * @param {BackendFactoryOpts} opts + * @param {CreateDefaultBackendFactoryOptions} [factoryOpts] + * @returns {function(string): import('@adobe/helix-shared-storage').StorageBackend} + */ +export function createBackendFactory({ + connectionString, + accountName, + accountKey, +}, { log = console } = {}) { + let blobServiceClient; + if (connectionString) { + log.debug('Creating BlobServiceClient from connection string'); + blobServiceClient = BlobServiceClient.fromConnectionString(connectionString); + } else { + log.debug('Creating BlobServiceClient from account name/key'); + blobServiceClient = new BlobServiceClient( + `https://${accountName}.blob.core.windows.net`, + new StorageSharedKeyCredential(accountName, accountKey), + ); + } + + return (bucketId) => new AzureBackend({ + client: blobServiceClient.getContainerClient(bucketId), + name: 'Azure', + bucketName: bucketId, + log, + }); +} + +/** + * Builds the default `backendFactory` for `Storage`: Azure Blob Storage, authenticated from + * `env`, preferring `HLX_AZURE_STORAGE_CONNECTION_STRING` over + * `HLX_AZURE_STORAGE_ACCOUNT_NAME`/`HLX_AZURE_STORAGE_ACCOUNT_KEY` when both are set. + * + * @param {Record} [env] environment variables (e.g. `context.env`) + * @param {CreateDefaultBackendFactoryOptions} [opts] + * @returns {function(string): import('@adobe/helix-shared-storage').StorageBackend} + */ +export function createDefaultBackendFactory(env = {}, opts = {}) { + return createBackendFactory(parseBackendFactoryEnvOpts(env), opts); +} diff --git a/packages/helix-shared-storage-azure/src/index.js b/packages/helix-shared-storage-azure/src/index.js new file mode 100644 index 000000000..e30ccc8b7 --- /dev/null +++ b/packages/helix-shared-storage-azure/src/index.js @@ -0,0 +1,18 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +export { AzureBackend } from './AzureBackend.js'; +export { + createDefaultBackendFactory, + createBackendFactory, +} from './createDefaultBackendFactory.js'; +export { StorageAzure } from './StorageAzure.js'; diff --git a/packages/helix-shared-storage-azure/test/StorageAzure.test.js b/packages/helix-shared-storage-azure/test/StorageAzure.test.js new file mode 100644 index 000000000..8251a1f0d --- /dev/null +++ b/packages/helix-shared-storage-azure/test/StorageAzure.test.js @@ -0,0 +1,44 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ +import assert from 'assert'; +import { Storage } from '@adobe/helix-shared-storage'; +import { StorageAzure } from '../src/StorageAzure.js'; + +const env = { + HLX_AZURE_STORAGE_ACCOUNT_NAME: 'fake', + HLX_AZURE_STORAGE_ACCOUNT_KEY: 'ZmFrZQ==', +}; + +describe('StorageAzure', () => { + it('fromContext() caches an instance that is both StorageAzure and Storage', () => { + const context = { env, log: console, attributes: {} }; + const stor = StorageAzure.fromContext(context); + assert.ok(stor instanceof StorageAzure); + assert.ok(stor instanceof Storage); + assert.strictEqual(context.attributes.storage, stor); + }); + + it('fromContext() returns the same cached instance on repeated calls', () => { + const context = { env, log: console, attributes: {} }; + const first = StorageAzure.fromContext(context); + const second = StorageAzure.fromContext(context); + assert.strictEqual(first, second); + }); + + it('forwards opts without requiring the caller to pass backendFactory directly', () => { + const context = { env, log: console, attributes: {} }; + const stor = StorageAzure.fromContext(context, { bucketNames: JSON.stringify({ code: 'my-code-bus' }) }); + assert.strictEqual(stor.codeBus().bucket, 'my-code-bus'); + }); +}); diff --git a/packages/helix-shared-storage-azure/test/createDefaultBackendFactory.test.js b/packages/helix-shared-storage-azure/test/createDefaultBackendFactory.test.js new file mode 100644 index 000000000..139da4002 --- /dev/null +++ b/packages/helix-shared-storage-azure/test/createDefaultBackendFactory.test.js @@ -0,0 +1,48 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ +import assert from 'assert'; +import { createDefaultBackendFactory } from '../src/createDefaultBackendFactory.js'; +import { AzureBackend } from '../src/AzureBackend.js'; + +describe('createDefaultBackendFactory()', () => { + it('builds a BlobServiceClient from a connection string', () => { + const factory = createDefaultBackendFactory({ + HLX_AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=https;AccountName=fake;AccountKey=ZmFrZQ==;EndpointSuffix=core.windows.net', + }, { log: console }); + const backend = factory('helix-code-bus'); + assert.ok(backend instanceof AzureBackend); + assert.strictEqual(backend.name, 'Azure'); + assert.strictEqual(backend.bucketName, 'helix-code-bus'); + }); + + it('prefers the connection string when both credential styles are set', () => { + const factory = createDefaultBackendFactory({ + HLX_AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=https;AccountName=fake;AccountKey=ZmFrZQ==;EndpointSuffix=core.windows.net', + HLX_AZURE_STORAGE_ACCOUNT_NAME: 'other', + HLX_AZURE_STORAGE_ACCOUNT_KEY: 'ZmFrZQ==', + }, { log: console }); + const backend = factory('helix-code-bus'); + assert.strictEqual(backend.client.accountName, 'fake'); + }); + + it('falls back to account name/key when no connection string is set', () => { + const factory = createDefaultBackendFactory({ + HLX_AZURE_STORAGE_ACCOUNT_NAME: 'fake', + HLX_AZURE_STORAGE_ACCOUNT_KEY: 'ZmFrZQ==', + }, { log: console }); + const backend = factory('helix-code-bus'); + assert.ok(backend instanceof AzureBackend); + assert.strictEqual(backend.client.accountName, 'fake'); + }); +}); diff --git a/packages/helix-shared-storage-azure/test/setup-env.js b/packages/helix-shared-storage-azure/test/setup-env.js new file mode 100644 index 000000000..3385334cf --- /dev/null +++ b/packages/helix-shared-storage-azure/test/setup-env.js @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +import { resolve } from 'path'; +import { fileURLToPath } from 'url'; + +// eslint-disable-next-line no-console +console.log('Forcing HTTP/1.1 for @adobe/fetch'); +process.env.HELIX_FETCH_FORCE_HTTP1 = 'true'; +process.env.HELIX_ONEDRIVE_LOCAL_AUTH_CACHE = 'true'; +process.env.HELIX_ONEDRIVE_NO_SHARE_LINK_CACHE = 'true'; + +// eslint-disable-next-line no-underscore-dangle +global.__rootdir = resolve(fileURLToPath(import.meta.url), '..', '..'); +// eslint-disable-next-line no-underscore-dangle +global.__testdir = resolve(fileURLToPath(import.meta.url), '..'); diff --git a/packages/helix-shared-storage-azure/test/storage.test.js b/packages/helix-shared-storage-azure/test/storage.test.js new file mode 100644 index 000000000..de2926bcd --- /dev/null +++ b/packages/helix-shared-storage-azure/test/storage.test.js @@ -0,0 +1,465 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/* eslint-env mocha */ +import assert from 'assert'; +import { promisify } from 'util'; +import zlib from 'zlib'; +import { BlobServiceClient, StorageSharedKeyCredential } from '@azure/storage-blob'; +import { Storage } from '@adobe/helix-shared-storage'; +import { Nock } from './utils.js'; +import { StorageAzure } from '../src/StorageAzure.js'; +import { AzureBackend } from '../src/AzureBackend.js'; + +const gzip = promisify(zlib.gzip); + +const HLX_AZURE_STORAGE_ACCOUNT_NAME = 'fake'; +const HLX_AZURE_STORAGE_ACCOUNT_KEY = Buffer.from('fake-key').toString('base64'); + +const BASE_URL = `https://${HLX_AZURE_STORAGE_ACCOUNT_NAME}.blob.core.windows.net`; + +/** + * Builds an `AzureBackend` wired directly against fake, static test credentials, bound to + * container `helix-code-bus`. + */ +function buildTestBackend() { + const client = new BlobServiceClient( + BASE_URL, + new StorageSharedKeyCredential(HLX_AZURE_STORAGE_ACCOUNT_NAME, HLX_AZURE_STORAGE_ACCOUNT_KEY), + { retryOptions: { maxTries: 1 } }, + ); + return new AzureBackend({ + client: client.getContainerClient('helix-code-bus'), + name: 'Azure', + bucketName: 'helix-code-bus', + log: console, + }); +} + +function listResponseXml({ blobs = [], prefixes = [], nextMarker = '' } = {}) { + const blobsXml = blobs.map(({ + name, contentLength = 0, contentType = 'application/octet-stream', + }) => ` + + ${name} + + Mon, 01 Jan 2024 00:00:00 GMT + "0x8D1234567890ABC" + ${contentLength} + ${contentType} + BlockBlob + + `).join(''); + const prefixesXml = prefixes.map((name) => ` + + ${name} + `).join(''); + return ` + + + + 5000 + / + ${prefixesXml}${blobsXml} + ${nextMarker} +`; +} + +describe('AzureBackend storage test', () => { + let nock; + let backend; + + beforeEach(() => { + nock = new Nock().env({ + HLX_AZURE_STORAGE_ACCOUNT_NAME, + HLX_AZURE_STORAGE_ACCOUNT_KEY, + }); + backend = buildTestBackend(); + }); + + afterEach(() => { + nock.done(); + }); + + it('creates a storage from context, wired to a code-bus', () => { + const ctx = { + env: { HLX_AZURE_STORAGE_ACCOUNT_NAME, HLX_AZURE_STORAGE_ACCOUNT_KEY }, + log: console, + attributes: {}, + }; + const stor = StorageAzure.fromContext(ctx); + assert.ok(stor instanceof StorageAzure); + assert.ok(stor instanceof Storage); + assert.strictEqual(stor.codeBus().bucket, 'helix-code-bus'); + assert.ok(stor.codeBus().client); + }); + + it('can get an object through the bucket facade', async () => { + const stor = new Storage({ backendFactory: () => backend }); + nock(BASE_URL) + .get('/helix-code-bus/foo') + .reply(200, 'hello, world.', { 'content-length': '13', etag: '"abc"' }); + const ret = await stor.codeBus().get('/foo'); + assert.strictEqual(ret.toString(), 'hello, world.'); + }); + + it('get compressed object and populates the meta object', async () => { + const body = await gzip('hello, world.'); + nock(BASE_URL) + .get('/helix-code-bus/foo') + .reply(200, body, { + 'content-type': 'text/plain', + 'content-encoding': 'gzip', + 'cache-control': 'no-store', + etag: '"499455814df5a2001612b12b575bfaaa"', + 'last-modified': 'Mon, 03 Nov 2025 07:08:10 GMT', + 'x-ms-meta-x_source_location': 'github', + 'content-length': String(body.length), + }); + const meta = {}; + const ret = await backend.get('foo', meta); + assert.strictEqual(ret.toString(), 'hello, world.'); + assert.strictEqual(meta.contentType, 'text/plain'); + assert.strictEqual(meta.contentEncoding, 'gzip'); + assert.strictEqual(meta.cacheControl, 'no-store'); + assert.strictEqual(meta['x-source-location'], 'github'); + assert.strictEqual(meta.etag, '"499455814df5a2001612b12b575bfaaa"'); + }); + + it('get falls back to the content_encoding metadata property when the native header is empty', async () => { + const body = await gzip('hello, world.'); + nock(BASE_URL) + .get('/helix-code-bus/foo') + .reply(200, body, { + 'content-type': 'text/plain', + etag: '"abc"', + 'x-ms-meta-content_encoding': 'gzip', + 'content-length': String(body.length), + }); + const meta = {}; + const ret = await backend.get('foo', meta); + assert.strictEqual(ret.toString(), 'hello, world.'); + assert.strictEqual(meta.contentEncoding, 'gzip'); + }); + + it('head falls back to the content_encoding metadata property when the native header is empty', async () => { + nock(BASE_URL) + .head('/helix-code-bus/foo') + .reply(200, '', { + 'content-type': 'text/plain', + etag: '"abc"', + 'x-ms-meta-content_encoding': 'gzip', + }); + const meta = await backend.head('foo'); + assert.strictEqual(meta.contentEncoding, 'gzip'); + }); + + it('get returns null for not found', async () => { + nock(BASE_URL) + .get('/helix-code-bus/foo') + .reply(404, 'BlobNotFound', { + 'content-type': 'application/xml', + }); + const ret = await backend.get('foo'); + assert.strictEqual(ret, null); + }); + + it('get throws error', async () => { + nock(BASE_URL) + .get('/helix-code-bus/foo') + .reply(500, 'InternalError', { + 'content-type': 'application/xml', + }); + await assert.rejects(backend.get('foo')); + }); + + it('can get metadata of an object', async () => { + nock(BASE_URL) + .head('/helix-code-bus/foo') + .reply(200, '', { + 'content-type': 'text/html', + 'content-length': '123', + etag: '"abc"', + 'last-modified': 'Mon, 03 Nov 2025 07:08:10 GMT', + 'x-ms-meta-foo': 'bar', + }); + const meta = await backend.head('foo'); + assert.strictEqual(meta.contentType, 'text/html'); + assert.strictEqual(meta.contentLength, 123); + assert.strictEqual(meta.etag, '"abc"'); + assert.deepStrictEqual(meta.metadata, { foo: 'bar' }); + }); + + it('head returns null for not found', async () => { + nock(BASE_URL) + .head('/helix-code-bus/foo') + .reply(404); + const meta = await backend.head('foo'); + assert.strictEqual(meta, null); + }); + + it('head throws error', async () => { + nock(BASE_URL) + .head('/helix-code-bus/foo') + .reply(500); + await assert.rejects(backend.head('foo')); + }); + + it('can put an object (string body)', async () => { + nock(BASE_URL) + .put('/helix-code-bus/foo') + .reply(201, '', { etag: '"abc"' }); + const raw = await backend.put('foo', 'hello, world.', { contentType: 'text/plain' }); + assert.strictEqual(raw.etag, '"abc"'); + assert.strictEqual(raw.contentType, 'text/plain'); + }); + + it('can put an object (buffer body)', async () => { + nock(BASE_URL) + .put('/helix-code-bus/foo') + .reply(201, '', { etag: '"abc"' }); + const raw = await backend.put('foo', Buffer.from('hello, world.')); + assert.strictEqual(raw.etag, '"abc"'); + }); + + it('put substitutes `-` with `_` in metadata keys (Azure does not allow `-`)', async () => { + nock(BASE_URL) + .put('/helix-code-bus/foo') + .matchHeader('x-ms-meta-x_last_modified_by', 'ksexton@adobe.com') + .reply(201, '', { etag: '"abc"' }); + await backend.put('foo', 'hello, world.', { metadata: { 'x-last-modified-by': 'ksexton@adobe.com' } }); + }); + + it('putMeta substitutes `-` with `_` in metadata keys (Azure does not allow `-`)', async () => { + nock(BASE_URL) + .put('/helix-code-bus/foo') + .query({ comp: 'metadata' }) + .matchHeader('x-ms-meta-x_last_modified_by', 'ksexton@adobe.com') + .reply(200, '', { etag: '"abc"' }); + nock(BASE_URL) + .put('/helix-code-bus/foo') + .query({ comp: 'properties' }) + .reply(200, '', { etag: '"abc"' }); + await backend.putMeta('foo', { 'x-last-modified-by': 'ksexton@adobe.com' }); + }); + + it('head restores `-` from `_` in metadata keys read back from Azure', async () => { + nock(BASE_URL) + .head('/helix-code-bus/foo') + .reply(200, '', { + etag: '"abc"', + 'x-ms-meta-x_last_modified_by': 'ksexton@adobe.com', + 'x-ms-meta-x_source_last_modified': 'Wed, 05 Aug 2026 21:54:45 GMT', + }); + const meta = await backend.head('foo'); + assert.strictEqual(meta.metadata['x-last-modified-by'], 'ksexton@adobe.com'); + assert.strictEqual(meta.metadata['x-source-last-modified'], 'Wed, 05 Aug 2026 21:54:45 GMT'); + }); + + it('can replace metadata', async () => { + nock(BASE_URL) + .put('/helix-code-bus/foo') + .query({ comp: 'metadata' }) + .reply(200, '', { etag: '"abc"' }); + nock(BASE_URL) + .put('/helix-code-bus/foo') + .query({ comp: 'properties' }) + .reply(200, '', { etag: '"abc"' }); + const raw = await backend.putMeta('foo', { contentType: 'text/plain', myKey: 'myValue' }); + assert.ok(raw.raw.metadata); + assert.ok(raw.raw.headers); + }); + + it('can copy an object (COPY directive, source preserved)', async () => { + nock(BASE_URL) + .put('/helix-code-bus/dst') + .reply(202, '', { + etag: '"abc"', + 'x-ms-copy-id': 'copy-1', + 'x-ms-copy-status': 'success', + }); + const raw = await backend.copy('src', 'dst'); + assert.strictEqual(raw.etag, '"abc"'); + }); + + it('can copy an object (REPLACE directive, follow-up setHTTPHeaders)', async () => { + nock(BASE_URL) + .put('/helix-code-bus/dst') + .reply(202, '', { + etag: '"abc"', + 'x-ms-copy-id': 'copy-1', + 'x-ms-copy-status': 'success', + }); + nock(BASE_URL) + .put('/helix-code-bus/dst') + .query({ comp: 'properties' }) + .reply(200, '', { etag: '"def"' }); + const raw = await backend.copy('src', 'dst', { + metadataDirective: 'REPLACE', + contentType: 'text/plain', + metadata: { foo: 'bar' }, + }); + assert.strictEqual(raw.etag, '"def"'); + }); + + it('can copy an object (REPLACE directive, no explicit metadata)', async () => { + nock(BASE_URL) + .put('/helix-code-bus/dst') + .reply(202, '', { + etag: '"abc"', + 'x-ms-copy-id': 'copy-1', + 'x-ms-copy-status': 'success', + }); + nock(BASE_URL) + .put('/helix-code-bus/dst') + .query({ comp: 'properties' }) + .reply(200, '', { etag: '"def"' }); + const raw = await backend.copy('src', 'dst', { metadataDirective: 'REPLACE' }); + assert.strictEqual(raw.etag, '"def"'); + }); + + it('copy throws 404 when source is missing', async () => { + nock(BASE_URL) + .put('/helix-code-bus/dst') + .reply(404, 'BlobNotFound', { + 'content-type': 'application/xml', + }); + await assert.rejects( + backend.copy('src', 'dst'), + (e) => e.status === 404, + ); + }); + + it('can remove a single object', async () => { + nock(BASE_URL) + .delete('/helix-code-bus/foo') + .reply(202); + await backend.remove('foo'); + }); + + it('remove of a single object throws on error', async () => { + nock(BASE_URL) + .delete('/helix-code-bus/foo') + .reply(500); + await assert.rejects(backend.remove('foo')); + }); + + it('can remove multiple objects', async () => { + nock(BASE_URL) + .delete('/helix-code-bus/foo1') + .reply(202) + .delete('/helix-code-bus/foo2') + .reply(202); + const result = await backend.remove(['foo1', 'foo2']); + assert.strictEqual(result.Deleted.length, 2); + assert.strictEqual(result.Errors.length, 0); + }); + + it('collects errors when removing multiple objects without stopOnError', async () => { + nock(BASE_URL) + .delete('/helix-code-bus/foo1') + .reply(202) + .delete('/helix-code-bus/foo2') + .reply(500); + const result = await backend.remove(['foo1', 'foo2']); + assert.strictEqual(result.Deleted.length, 1); + assert.strictEqual(result.Errors.length, 1); + }); + + it('stops removing multiple objects on first error with stopOnError', async () => { + nock(BASE_URL) + .delete('/helix-code-bus/foo1') + .reply(500); + await assert.rejects( + backend.remove(['foo1'], { stopOnError: true }), + ); + }); + + it('can list objects at the bucket root (no slash in key)', async () => { + nock(BASE_URL) + .get('/helix-code-bus') + .query((q) => q.restype === 'container' && q.comp === 'list') + .reply(200, listResponseXml({ + blobs: [{ name: 'root.html', contentLength: 123, contentType: 'text/html' }], + }), { 'content-type': 'application/xml' }); + const { objects } = await backend.list(''); + assert.strictEqual(objects[0].key, 'root.html'); + assert.strictEqual(objects[0].name, 'root.html'); + }); + + it('can list objects deeply', async () => { + nock(BASE_URL) + .get('/helix-code-bus') + .query((q) => q.restype === 'container' && q.comp === 'list') + .reply(200, listResponseXml({ + blobs: [{ name: 'foo/bar.html', contentLength: 123, contentType: 'text/html' }], + }), { 'content-type': 'application/xml' }); + const { objects } = await backend.list('foo/'); + assert.strictEqual(objects.length, 1); + assert.strictEqual(objects[0].key, 'foo/bar.html'); + assert.strictEqual(objects[0].name, 'bar.html'); + assert.strictEqual(objects[0].isFolder, false); + }); + + it('can list objects shallowly, including folders', async () => { + nock(BASE_URL) + .get('/helix-code-bus') + .query((q) => q.restype === 'container' && q.comp === 'list' && q.delimiter === '/') + .reply(200, listResponseXml({ + prefixes: ['foo/sub/'], + blobs: [{ name: 'foo/bar.html', contentLength: 123, contentType: 'text/html' }], + }), { 'content-type': 'application/xml' }); + const { objects } = await backend.list('foo/', { shallow: true }); + const folder = objects.find((o) => o.isFolder); + assert.strictEqual(folder.key, 'foo/sub/'); + assert.strictEqual(folder.name, 'sub'); + }); + + it('list stops at maxItems', async () => { + nock(BASE_URL) + .get('/helix-code-bus') + .query((q) => q.restype === 'container' && q.comp === 'list') + .reply(200, listResponseXml({ + blobs: [ + { name: 'foo/a.html' }, + { name: 'foo/b.html' }, + ], + }), { 'content-type': 'application/xml' }); + const { objects } = await backend.list('foo/', { maxItems: 1 }); + assert.strictEqual(objects.length, 1); + }); + + it('can browse a single page with a continuation token', async () => { + nock(BASE_URL) + .get('/helix-code-bus') + .query((q) => q.restype === 'container' && q.comp === 'list' && q.delimiter === '/') + .reply(200, listResponseXml({ + prefixes: ['foo/sub/'], + blobs: [{ name: 'foo/bar.html', contentLength: 123, contentType: 'text/html' }], + nextMarker: 'next-page-token', + }), { 'content-type': 'application/xml' }); + const { objects, continuationToken } = await backend.browse('foo/'); + assert.strictEqual(objects.length, 2); + assert.strictEqual(continuationToken, 'next-page-token'); + }); + + it('browse returns no continuation token on the last page', async () => { + nock(BASE_URL) + .get('/helix-code-bus') + .query((q) => q.restype === 'container' && q.comp === 'list' && q.delimiter === '/') + .reply(200, listResponseXml({ + blobs: [{ name: 'foo/bar.html', contentLength: 123, contentType: 'text/html' }], + }), { 'content-type': 'application/xml' }); + const { continuationToken } = await backend.browse('foo/'); + assert.strictEqual(continuationToken, undefined); + }); +}); diff --git a/packages/helix-shared-storage-azure/test/utils.js b/packages/helix-shared-storage-azure/test/utils.js new file mode 100644 index 000000000..6c5bab349 --- /dev/null +++ b/packages/helix-shared-storage-azure/test/utils.js @@ -0,0 +1,79 @@ +/* + * Copyright 2020 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +import assert from 'assert'; +// eslint-disable-next-line import/no-extraneous-dependencies +import nock from 'nock'; + +/** + * @constructor + */ +export function Nock() { + /** @type {Record} */ + let savedEnv; + + function noMatchHandler(req) { + unmatched.push(req); + } + + /** + * @param {string} url + * @returns {nock.Scope} + */ + function nocker(url) { + let scope = scopes[url]; + if (!scope) { + scope = nock(url); + scopes[url] = scope; + } + if (!unmatched) { + unmatched = []; + nock.emitter.on('no match', noMatchHandler); + } + nock.disableNetConnect(); + return scope; + } + + nocker.env = (overrides = {}) => { + savedEnv = { ...process.env }; + Object.assign(process.env, { + AWS_REGION: 'us-east-1', + AWS_ACCESS_KEY_ID: 'dummy-id', + AWS_SECRET_ACCESS_KEY: 'dummy-key', + ...overrides, + }); + return nocker; + }; + + nocker.done = () => { + if (savedEnv) { + process.env = savedEnv; + } + + if (unmatched) { + assert.deepStrictEqual(unmatched.map((req) => req.options || req), []); + nock.emitter.off('no match', noMatchHandler); + } + try { + Object.values(scopes).forEach((s) => s.done()); + } finally { + nock.cleanAll(); + } + }; + + return nocker; +}