diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 817ff0277547..ff6e33e63b70 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -1103,7 +1103,8 @@ "filename": "sdk/storage/azure-storage-blob-cryptography/**", "words": [ "akek", - "azstorage" + "azstorage", + "misordered" ] }, { diff --git a/sdk/storage/azure-storage-blob-cryptography/CHANGELOG.md b/sdk/storage/azure-storage-blob-cryptography/CHANGELOG.md index aec4c4c9388d..fad240253c77 100644 --- a/sdk/storage/azure-storage-blob-cryptography/CHANGELOG.md +++ b/sdk/storage/azure-storage-blob-cryptography/CHANGELOG.md @@ -7,6 +7,7 @@ ### Breaking Changes ### Bugs Fixed +- Fixed a bug where client-side encryption 2.0 could not detect a rearrangement of otherwise-untampered authenticated regions in blob content. This is now detected and an exception is thrown. For data recovery purposes, this behavior can be reverted by setting the `AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS` environment variable (or the `Azure.Storage.CseV2AllowMisorderedAuthRegions` system property) to `true`. - Fixed an issue where the client-side encryption (v2) region nonce counter was truncated to 32 bits, which could cause GCM nonce reuse for blobs exceeding 2^32 authenticated regions. The full 64-bit region index is now used so every region receives a unique nonce. Blobs with at most 2^31 authenticated regions remain byte-for-byte compatible; diff --git a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/BlobDecryptionPolicy.java b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/BlobDecryptionPolicy.java index 80ce271d246d..c814998a642a 100644 --- a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/BlobDecryptionPolicy.java +++ b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/BlobDecryptionPolicy.java @@ -109,8 +109,14 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN boolean padding = hasPadding(responseHeaders, encryptionData, encryptedRange); + // Use the operation-scoped validator if one was installed (e.g. downloadContentWithResponse and + // ranged downloads always set it up). Sharing it means a reliable-download ranged resume of this + // same operation - which re-enters the pipeline through the range branch with the same context - + // enforces one nonce scheme across the initial and resumed portions. + CseV2NonceOrderValidator nonceValidator = CseV2NonceOrderValidator.fromContext(context); + Flux plainTextData = this.decryptBlob(httpResponse.getBody(), encryptedRange, padding, - encryptionData, httpResponse.getRequest().getUrl()); + encryptionData, httpResponse.getRequest().getUrl(), nonceValidator); return Mono.just(new BlobDecryptionPolicy.DecryptedResponse(httpResponse, plainTextData)); } else { @@ -126,6 +132,11 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN EncryptionData encryptionData = (EncryptionData) context.getData(CryptographyConstants.ENCRYPTION_DATA_KEY).get(); + // Shared across every chunk of this download so the CSEv2 nonce scheme is enforced consistently across the + // whole operation (partitioned downloadToFile / chunked openInputStream issue one ranged request per + // chunk). Set up once per operation alongside the encryption data. + CseV2NonceOrderValidator nonceValidator = CseV2NonceOrderValidator.fromContext(context); + EncryptedBlobRange encryptedRange = EncryptedBlobRange.getEncryptedBlobRangeFromHeader(initialRangeHeader, encryptionData); if (context.getHttpRequest().getHeaders().getValue(RANGE_HEADER) != null) { @@ -151,7 +162,7 @@ public Mono process(HttpPipelineCallContext context, HttpPipelineN boolean padding = hasPadding(httpResponse.getHeaders(), encryptionData, encryptedRange); Flux plainTextData = this.decryptBlob(httpResponse.getBody(), encryptedRange, padding, - encryptionData, httpResponse.getRequest().getUrl()); + encryptionData, httpResponse.getRequest().getUrl(), nonceValidator); return new DecryptedResponse(httpResponse, plainTextData); } else { @@ -207,10 +218,13 @@ private boolean isDownloadResponse(HttpResponse httpResponse) { * @param encryptedBlobRange A {@link EncryptedBlobRange} indicating the range to decrypt * @param padding Boolean indicating if the padding mode should be set or not. * @param encryptionData The {@link EncryptionData} + * @param nonceValidator The per-download-operation validator that enforces a single CSEv2 nonce scheme across all + * chunks. May be {@code null} (e.g. a full-blob single-shot download), in which case decryption uses a fresh + * per-call validator. * @return A Flux ByteBuffer that has been decrypted */ Flux decryptBlob(Flux encryptedFlux, EncryptedBlobRange encryptedBlobRange, boolean padding, - EncryptionData encryptionData, URL requestUri) { + EncryptionData encryptionData, URL requestUri, CseV2NonceOrderValidator nonceValidator) { String uriToLog = requestUri.getHost() + requestUri.getPath(); // The number of bytes we have put into the Cipher so far. @@ -218,7 +232,7 @@ Flux decryptBlob(Flux encryptedFlux, EncryptedBlobRange // The number of bytes that have been sent to the downstream so far. AtomicLong totalOutputBytes = new AtomicLong(0); - Decryptor decryptor = Decryptor.getDecryptor(keyResolver, keyWrapper, encryptionData); + Decryptor decryptor = Decryptor.getDecryptor(keyResolver, keyWrapper, encryptionData, nonceValidator); Flux dataToTrim = decryptor.getKeyEncryptionKey() .flatMapMany( key -> decryptor.decrypt(encryptedFlux, encryptedBlobRange, padding, uriToLog, totalInputBytes, key)); diff --git a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/CryptographyConstants.java b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/CryptographyConstants.java index 30763ad782c1..92a4b22ad4e4 100644 --- a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/CryptographyConstants.java +++ b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/CryptographyConstants.java @@ -33,6 +33,12 @@ final class CryptographyConstants { static final String ENCRYPTION_DATA_KEY = "encryptiondata"; + /** + * Context key under which a per-download-operation {@link CseV2NonceOrderValidator} is shared, so that the CSEv2 + * nonce scheme is enforced consistently across every (possibly concurrent) chunk of a single download. + */ + static final String GCM_NONCE_VALIDATOR_KEY = "gcmNonceValidator"; + static final HttpHeaderName ENCRYPTION_METADATA_HEADER = HttpHeaderName.fromString(Constants.HeaderConstants.X_MS_META + "-" + ENCRYPTION_DATA_KEY); @@ -55,6 +61,18 @@ final class CryptographyConstants { static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocate(0); + /** + * System property name that, when set to {@code true}, disables detection of reordered client-side encryption v2 + * authenticated regions. Intended for data recovery only. + */ + static final String ALLOW_MISORDERED_REGIONS_PROPERTY = "Azure.Storage.CseV2AllowMisorderedAuthRegions"; + + /** + * Environment variable name that, when set to {@code true}, disables detection of reordered client-side encryption + * v2 authenticated regions. Intended for data recovery only. + */ + static final String ALLOW_MISORDERED_REGIONS_ENV_VAR = "AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS"; + private CryptographyConstants() { } diff --git a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/CseV2NonceOrderValidator.java b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/CseV2NonceOrderValidator.java new file mode 100644 index 000000000000..75bd155f1037 --- /dev/null +++ b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/CseV2NonceOrderValidator.java @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.specialized.cryptography; + +import com.azure.core.http.HttpPipelineCallContext; +import com.azure.core.util.CoreUtils; +import com.azure.core.util.logging.ClientLogger; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.concurrent.atomic.AtomicReference; + +import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.ALLOW_MISORDERED_REGIONS_ENV_VAR; +import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.ALLOW_MISORDERED_REGIONS_PROPERTY; +import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.GCM_NONCE_VALIDATOR_KEY; +import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.NONCE_LENGTH; + +/** + * Detects rearrangement of client-side encryption v2 authenticated regions by validating each region's nonce against + * the value expected for its sequential position. + *

+ * Each CSEv2 region is encrypted under a unique, sequential nonce derived from the region's index. Because the nonce is + * stored alongside the ciphertext, individual regions of otherwise untampered ciphertext can be rearranged without + * invalidating any single region's GCM tag, silently corrupting the decrypted plaintext. Validating that each region's + * nonce matches its position detects this. + *

+ * CSEv2 is cross-SDK interoperable and each Azure Storage SDK encodes the region counter into the nonce differently + * (see {@link NonceScheme}). Decryption itself is unaffected (it uses the inline nonce), but reorder validation must + * reconstruct the expected nonce, which requires knowing the encoder's scheme. This validator begins by considering all + * known schemes and intersects the set of schemes still consistent with every region seen so far, collapsing to a + * single scheme once enough regions have been read. Once collapsed, subsequent regions are checked lock-free against + * the resolved scheme. + *

+ * A single instance is intended to be shared across an entire logical download operation (which may span multiple + * concurrent HTTP range requests, e.g. a parallel {@code downloadToFile} or a chunked {@code openInputStream}). Sharing + * the instance means the scheme is enforced consistently across the whole download rather than being re-established per + * chunk; otherwise, because the schemes share a value space, a region relocated across a chunk boundary to a colliding + * position could pass validation. The intersection is performed atomically and is order-independent, so it is safe + * under concurrent region processing. + */ +final class CseV2NonceOrderValidator { + private static final ClientLogger LOGGER = new ClientLogger(CseV2NonceOrderValidator.class); + + private final boolean validationEnabled; + private final AtomicReference> candidateSchemes + = new AtomicReference<>(EnumSet.allOf(NonceScheme.class)); + + CseV2NonceOrderValidator() { + // Read the data-recovery bypass switch once per download operation. + this.validationEnabled = !cseV2AllowMisorderedAuthRegions(); + } + + /** + * Retrieves the operation-scoped validator installed in the pipeline context, if any. A single instance is shared + * across every chunk of a download operation so the nonce scheme is enforced consistently (see the class-level + * documentation). + * + * @param context The pipeline call context for the current request. + * @return The shared validator, or {@code null} if none was installed (e.g. a full-blob single-shot download). + */ + static CseV2NonceOrderValidator fromContext(HttpPipelineCallContext context) { + return (CseV2NonceOrderValidator) context.getData(GCM_NONCE_VALIDATOR_KEY).orElse(null); + } + + /** + * Validates that the nonce of an authenticated region is consistent with the region occupying its sequential + * position, under a single recognized cross-SDK nonce scheme enforced across the whole download. A region whose + * nonce matches none of the schemes for its position - or that is inconsistent with the scheme established by + * earlier regions - indicates a reorder or other tampering. + * + * @param actualNonce The nonce read from the downloaded region. + * @param nonceLength The length of the nonce. + * @param region The 0-based index of this region's position in the blob. + * @throws RuntimeException If the region is invalid or its integrity cannot be verified. + */ + void validateRegion(byte[] actualNonce, int nonceLength, long region) { + if (!validationEnabled) { + return; + } + + // CSEv2 uses a fixed 12-byte nonce. If the metadata advertises any other length, positional validation is + // impossible, so fail closed rather than silently skipping the integrity check (leaving reorders undetected). + if (nonceLength != NONCE_LENGTH || actualNonce.length != nonceLength) { + throw LOGGER.logExceptionAsError(new IllegalStateException( + "Cannot verify the authenticated-region order of client-side encrypted (v2) content because its nonce " + + "length is invalid (expected " + NONCE_LENGTH + "). " + recoveryInstruction())); + } + + // Fast path: once the scheme has collapsed to a single encoding, verify directly without mutating shared state. + EnumSet current = candidateSchemes.get(); + if (current.size() == 1) { + NonceScheme locked = current.iterator().next(); + if (!Arrays.equals(locked.expectedNonce(region, nonceLength), actualNonce)) { + throw LOGGER.logExceptionAsError(reorderException()); + } + return; + } + + // Determine which schemes are consistent with this region at its position. + EnumSet matchesHere = EnumSet.noneOf(NonceScheme.class); + for (NonceScheme scheme : NonceScheme.values()) { + if (Arrays.equals(scheme.expectedNonce(region, nonceLength), actualNonce)) { + matchesHere.add(scheme); + } + } + + if (matchesHere.isEmpty()) { + throw LOGGER.logExceptionAsError(reorderException()); + } + + // Intersect the shared candidate set with the schemes matching this region. Atomic and order-independent, so it + // remains correct even if regions are processed concurrently. A valid blob keeps its own scheme in the set at + // every region; a reorder that looks valid under a different scheme for a single region is caught here because + // it is inconsistent with the scheme the rest of the download uses. + EnumSet remaining = candidateSchemes.updateAndGet(existing -> { + EnumSet next = EnumSet.copyOf(existing); + next.retainAll(matchesHere); + return next; + }); + if (remaining.isEmpty()) { + throw LOGGER.logExceptionAsError(reorderException()); + } + } + + private static RuntimeException reorderException() { + return new IllegalStateException( + "Encountered an out-of-order authenticated region while decrypting client-side encrypted (v2) content. " + + "This may indicate that the blob's authenticated regions have been rearranged or otherwise tampered " + + "with. " + recoveryInstruction()); + } + + private static String recoveryInstruction() { + return "To recover data from an affected blob, set the \"" + ALLOW_MISORDERED_REGIONS_ENV_VAR + + "\" environment variable (or the \"" + ALLOW_MISORDERED_REGIONS_PROPERTY + + "\" system property) to \"true\"."; + } + + /** + * Whether detection of reordered client-side encryption v2 authenticated regions should be disabled. + *

+ * This is a data-recovery escape hatch, read live from a system property or environment variable. When enabled, the + * client will not throw when it encounters authenticated regions that appear to have been rearranged, allowing + * (potentially tampered) plaintext to be recovered. + *

+ * {@link com.azure.core.util.Configuration} is intentionally not used here because the global configuration caches + * the first value it reads for a given name, which would prevent the switch from being honored if it is set after + * the first read. + * + * @return {@code true} if reordered authenticated regions should be allowed, {@code false} otherwise. + */ + private static boolean cseV2AllowMisorderedAuthRegions() { + String value = System.getProperty(ALLOW_MISORDERED_REGIONS_PROPERTY); + if (CoreUtils.isNullOrEmpty(value)) { + value = System.getenv(ALLOW_MISORDERED_REGIONS_ENV_VAR); + } + return Boolean.parseBoolean(value); + } + + /** + * The recognized ways Azure Storage client-side encryption v2 SDKs encode a region's sequential counter into its + * GCM nonce. Decryption uses the inline nonce and is unaffected by these differences, but reorder detection must + * reconstruct the expected nonce, which requires knowing the encoder's scheme. The complete set of SDKs that + * produce CSEv2 content is .NET, Java, and Python. + */ + private enum NonceScheme { + /** + * Java: the region index written as an 8-byte big-endian value in the first 8 bytes, remaining bytes zero. + * 0-based. See {@link EncryptorV2}. + */ + JAVA { + @Override + byte[] expectedNonce(long regionIndex, int nonceLength) { + byte[] nonce = new byte[nonceLength]; + for (int i = 0; i < Long.BYTES; i++) { + nonce[i] = (byte) (regionIndex >>> (Long.SIZE - Byte.SIZE - Byte.SIZE * i)); + } + return nonce; + } + }, + + /** + * Python: the region index encoded as a big-endian integer across the whole nonce (counter in the low bytes). + * 0-based. See azure-storage-blob {@code encrypt_data_v2}. + */ + PYTHON { + @Override + byte[] expectedNonce(long regionIndex, int nonceLength) { + byte[] nonce = new byte[nonceLength]; + for (int i = 0; i < Long.BYTES; i++) { + nonce[nonceLength - 1 - i] = (byte) (regionIndex >>> (Byte.SIZE * i)); + } + return nonce; + } + }, + + /** + * .NET: four zero bytes followed by an 8-byte little-endian counter. 1-based (the first region uses counter 1). + * See Azure.Storage.Common {@code GcmAuthenticatedCryptographicTransform}. + */ + DOTNET { + @Override + byte[] expectedNonce(long regionIndex, int nonceLength) { + byte[] nonce = new byte[nonceLength]; + long counter = regionIndex + 1; + for (int i = 0; i < Long.BYTES; i++) { + nonce[(nonceLength - Long.BYTES) + i] = (byte) (counter >>> (Byte.SIZE * i)); + } + return nonce; + } + }; + + /** + * Produces the nonce this scheme assigns to the given region index. + * + * @param regionIndex The 0-based region index. + * @param nonceLength The nonce length (12 for CSEv2). Must be at least {@link Long#BYTES}. + * @return The expected nonce bytes. + */ + abstract byte[] expectedNonce(long regionIndex, int nonceLength); + } +} diff --git a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/Decryptor.java b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/Decryptor.java index aed6ee717a35..f8bbc1755d86 100644 --- a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/Decryptor.java +++ b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/Decryptor.java @@ -89,7 +89,7 @@ abstract Flux decrypt(Flux encryptedFlux, EncryptedBlobR boolean padding, String requestUri, AtomicLong totalInputBytes, byte[] contentEncryptionKey); static Decryptor getDecryptor(AsyncKeyEncryptionKeyResolver keyResolver, AsyncKeyEncryptionKey keyWrapper, - EncryptionData encryptionData) { + EncryptionData encryptionData, CseV2NonceOrderValidator nonceValidator) { if (encryptionData == null) { return new NoOpDecryptor(keyResolver, keyWrapper, null); } @@ -99,7 +99,7 @@ static Decryptor getDecryptor(AsyncKeyEncryptionKeyResolver keyResolver, AsyncKe case ENCRYPTION_PROTOCOL_V2: case ENCRYPTION_PROTOCOL_V2_1: - return new DecryptorV2(keyResolver, keyWrapper, encryptionData); + return new DecryptorV2(keyResolver, keyWrapper, encryptionData, nonceValidator); default: throw LOGGER.logExceptionAsError(new IllegalStateException( diff --git a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/DecryptorV2.java b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/DecryptorV2.java index 5129e800a5fb..ab49faa701a5 100644 --- a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/DecryptorV2.java +++ b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/DecryptorV2.java @@ -35,9 +35,16 @@ class DecryptorV2 extends Decryptor { private static final ClientLogger LOGGER = new ClientLogger(DecryptorV2.class); + /* + * Shared across every chunk of a single download operation so that the CSEv2 nonce scheme is enforced consistently + * across the whole download (see CseV2NonceOrderValidator). + */ + private final CseV2NonceOrderValidator nonceValidator; + protected DecryptorV2(AsyncKeyEncryptionKeyResolver keyResolver, AsyncKeyEncryptionKey keyWrapper, - EncryptionData encryptionData) { + EncryptionData encryptionData, CseV2NonceOrderValidator nonceValidator) { super(keyResolver, keyWrapper, encryptionData); + this.nonceValidator = nonceValidator != null ? nonceValidator : new CseV2NonceOrderValidator(); } @Override @@ -49,11 +56,28 @@ Flux decrypt(Flux encryptedFlux, EncryptedBlobRange encr BufferStagingArea stagingArea = new BufferStagingArea(authenticatedRegionDataLength + TAG_LENGTH + nonceLength, authenticatedRegionDataLength + TAG_LENGTH + nonceLength); + /* + * Each CSEv2 region is encrypted under a unique, sequential nonce derived from the region's index. + * Because the nonce is stored alongside the ciphertext, individual regions of otherwise + * untampered ciphertext can be rearranged without invalidating any single region's authentication tag, + * silently corrupting the decrypted plaintext. The shared nonceValidator asserts that each region's nonce + * matches the value expected for its sequential position, under a single cross-SDK nonce scheme enforced across + * the whole download operation. The download always begins on a region boundary, so the first region's index is + * derived from the requested range (0 for a full-blob download). + */ + final long initialRegion = authenticatedRegionDataLength == 0 + ? 0 + : encryptedBlobRange.getOriginalRange().getOffset() / authenticatedRegionDataLength; + return encryptedFlux.flatMapSequential(stagingArea::write, 1, 1) .concatWith(Flux.defer(stagingArea::flush)) - .flatMapSequential(aggregator -> { + .index() + .flatMapSequential(indexedAggregator -> { // Get the IV out of the beginning of the aggregator - byte[] gmcIv = aggregator.getFirstNBytes(nonceLength); + byte[] gmcIv = indexedAggregator.getT2().getFirstNBytes(nonceLength); + + long region = initialRegion + indexedAggregator.getT1(); + nonceValidator.validateRegion(gmcIv, nonceLength, region); Cipher gmcCipher; try { @@ -63,8 +87,8 @@ Flux decrypt(Flux encryptedFlux, EncryptedBlobRange encr } ByteBuffer decryptedRegion = ByteBuffer.allocate(authenticatedRegionDataLength); - return aggregator.asFlux().map(buffer -> { - // Write into the preallocated buffer and always return this buffer. + return indexedAggregator.getT2().asFlux().map(buffer -> { + // Write into the pre-allocated buffer and always return this buffer. try { gmcCipher.update(buffer, decryptedRegion); } catch (ShortBufferException e) { diff --git a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlobAsyncClient.java b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlobAsyncClient.java index b3bc72db6624..e93d985e788b 100644 --- a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlobAsyncClient.java @@ -68,6 +68,7 @@ import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.AGENT_METADATA_KEY; import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.AGENT_METADATA_VALUE; import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.ENCRYPTION_DATA_KEY; +import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.GCM_NONCE_VALIDATOR_KEY; /** * This class provides a client side encryption client that contains generic blob operations for Azure Storage Blobs. @@ -764,6 +765,7 @@ private Mono populateRequestConditionsAndContext(BlobRequestConditions re = EncryptionData.getAndValidateEncryptionData(encryptionDataKey, requiresEncryption); result = result.contextWrite(context -> context.put(ENCRYPTION_DATA_KEY, encryptionData)) + .contextWrite(context -> context.put(GCM_NONCE_VALIDATOR_KEY, new CseV2NonceOrderValidator())) .contextWrite(context -> context.put(Constants.ADJUSTED_BLOB_LENGTH_KEY, EncryptedBlobLength .computeAdjustedBlobLength(encryptionData, response.getValue().getBlobSize()))); } diff --git a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlobClient.java b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlobClient.java index e475aabf3342..38132fd72bba 100644 --- a/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlobClient.java +++ b/sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlobClient.java @@ -52,6 +52,7 @@ import java.util.Set; import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.ENCRYPTION_DATA_KEY; +import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.GCM_NONCE_VALIDATOR_KEY; /** * This class provides a client side encryption client that contains generic blob operations for Azure Storage Blobs. @@ -498,6 +499,7 @@ private Context populateRequestConditionsAndContext(BlobRequestConditions reques encryptedBlobAsyncClient.isEncryptionRequired()); context = context.addData(ENCRYPTION_DATA_KEY, encryptionData) + .addData(GCM_NONCE_VALIDATOR_KEY, new CseV2NonceOrderValidator()) .addData(Constants.ADJUSTED_BLOB_LENGTH_KEY, EncryptedBlobLength.computeAdjustedBlobLength(encryptionData, initialProperties.getBlobSize())); } diff --git a/sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/DecryptionTests.java b/sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/DecryptionTests.java index 0f50e3e04924..7e391bd5b67e 100644 --- a/sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/DecryptionTests.java +++ b/sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/DecryptionTests.java @@ -60,7 +60,7 @@ public void decryption(int testCase) throws InvalidKeyException, IOException { StepVerifier .create(FluxUtil.collectBytesInByteBufferStream( blobDecryptionPolicy.decryptBlob(flow, new EncryptedBlobRange(blobRange, encryptionData), true, - encryptionData, new URL("http://www.foo.com/path")))) + encryptionData, new URL("http://www.foo.com/path"), null))) .assertNext(bytes -> assertByteBuffersEqual(desiredOutput, ByteBuffer.wrap(bytes))) .verifyComplete(); } diff --git a/sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/DecryptorV2ReorderTests.java b/sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/DecryptorV2ReorderTests.java new file mode 100644 index 000000000000..01db29f74983 --- /dev/null +++ b/sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/DecryptorV2ReorderTests.java @@ -0,0 +1,462 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.specialized.cryptography; + +import com.azure.storage.blob.models.BlobRange; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.api.parallel.Isolated; +import reactor.core.publisher.Flux; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicLong; + +import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.AES; +import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.AES_GCM_NO_PADDING; +import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.ENCRYPTION_PROTOCOL_V2; +import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.NONCE_LENGTH; +import static com.azure.storage.blob.specialized.cryptography.CryptographyConstants.TAG_LENGTH; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for detection of reordered client-side encryption v2 authenticated regions. These exercise + * {@link EncryptorV2} and {@link DecryptorV2} directly and do not require a storage account. + *

+ * The compatibility switch is a process-global system property, so this class is isolated and run on a single thread to + * avoid interfering with (or being interfered with by) tests running in parallel. + */ +@Isolated +@Execution(ExecutionMode.SAME_THREAD) +public class DecryptorV2ReorderTests { + private static final int REGION_DATA_LENGTH = 1024; + private static final int REGION_TOTAL_LENGTH = NONCE_LENGTH + REGION_DATA_LENGTH + TAG_LENGTH; + private static final int REGION_COUNT = 4; + // A region index (2^32) beyond the 32-bit range, used to confirm the full 64-bit region counter validates + // correctly past the point where an int-based counter would have wrapped. + private static final long LARGE_REGION_INDEX = 1L << 32; + private static final Random RANDOM = new Random(); + + @AfterEach + public void clearSwitch() { + System.clearProperty(CryptographyConstants.ALLOW_MISORDERED_REGIONS_PROPERTY); + } + + @Test + public void unmodifiedContentDecryptsSuccessfully() { + byte[] cek = randomBytes(32); + byte[] plaintext = randomBytes(REGION_DATA_LENGTH * REGION_COUNT); + byte[] ciphertext = encrypt(cek, plaintext); + + assertArrayEquals(plaintext, decrypt(cek, ciphertext, 0)); + } + + @Test + public void detectsRegionReorder() { + byte[] cek = randomBytes(32); + byte[] plaintext = randomBytes(REGION_DATA_LENGTH * REGION_COUNT); + byte[] ciphertext = encrypt(cek, plaintext); + + // Swap two otherwise-untampered authenticated regions. + swapRegions(ciphertext, 2, 3, REGION_TOTAL_LENGTH); + + IllegalStateException e = assertThrows(IllegalStateException.class, () -> decrypt(cek, ciphertext, 0)); + assertTrue(e.getMessage().contains("out-of-order"), e.getMessage()); + } + + @Test + public void detectsRegionReorderOnRangedDownload() { + byte[] cek = randomBytes(32); + byte[] plaintext = randomBytes(REGION_DATA_LENGTH * REGION_COUNT); + byte[] ciphertext = encrypt(cek, plaintext); + + // Simulate a ranged download that begins at region 1 by dropping the first region's ciphertext. + byte[] ranged = Arrays.copyOfRange(ciphertext, REGION_TOTAL_LENGTH, ciphertext.length); + + // A ranged download starting at the correct region decrypts successfully (no false positive). + byte[] expected = Arrays.copyOfRange(plaintext, REGION_DATA_LENGTH, plaintext.length); + assertArrayEquals(expected, decrypt(cek, ranged, REGION_DATA_LENGTH)); + + // Reordering within the ranged content is still detected, with the region index offset by the range. + swapRegions(ranged, 1, 2, REGION_TOTAL_LENGTH); + assertThrows(IllegalStateException.class, () -> decrypt(cek, ranged, REGION_DATA_LENGTH)); + } + + @Test + public void failsClosedWhenNonceLengthIsInvalid() { + // CSEv2 nonces are always 12 bytes. Metadata advertising any other length makes positional validation + // impossible, so the validator must fail closed rather than silently skip the integrity check. + CseV2NonceOrderValidator validator = new CseV2NonceOrderValidator(); + byte[] shortNonce = new byte[Long.BYTES]; + + assertThrows(IllegalStateException.class, () -> validator.validateRegion(shortNonce, shortNonce.length, 0)); + } + + @Test + public void recoverySwitchBypassesInvalidNonceLength() { + // The data-recovery switch disables all validation, so it short-circuits before the nonce-length check. + System.setProperty(CryptographyConstants.ALLOW_MISORDERED_REGIONS_PROPERTY, "true"); + CseV2NonceOrderValidator validator = new CseV2NonceOrderValidator(); + byte[] shortNonce = new byte[Long.BYTES]; + + assertDoesNotThrow(() -> validator.validateRegion(shortNonce, shortNonce.length, 0)); + } + + @Test + public void compatSwitchAllowsReorder() { + System.setProperty(CryptographyConstants.ALLOW_MISORDERED_REGIONS_PROPERTY, "true"); + + byte[] cek = randomBytes(32); + byte[] plaintext = randomBytes(REGION_DATA_LENGTH * REGION_COUNT); + byte[] ciphertext = encrypt(cek, plaintext); + swapRegions(ciphertext, 2, 3, REGION_TOTAL_LENGTH); + + // With the switch enabled, decryption does not throw and recovers plaintext in the received order. + byte[] recovered = decrypt(cek, ciphertext, 0); + + byte[] expected = plaintext.clone(); + swapRegions(expected, 2, 3, REGION_DATA_LENGTH); + assertArrayEquals(expected, recovered); + } + + @Test + public void decryptsRegionsAcrossIntegerMaxValueBoundary() { + // The region index is encoded as a full 64-bit big-endian value, so regions whose indices cross the + // Integer.MAX_VALUE (2^31) boundary - reachable at ~32 GiB with the minimum 16-byte region size - must still + // validate and decrypt. + byte[] cek = randomBytes(32); + long firstRegion = Integer.MAX_VALUE; + byte[] region0Plaintext = randomBytes(REGION_DATA_LENGTH); + byte[] region1Plaintext = randomBytes(REGION_DATA_LENGTH); + + byte[] ciphertext = concat(encryptRegionAt(cek, firstRegion, region0Plaintext), + encryptRegionAt(cek, firstRegion + 1, region1Plaintext)); + + long offset = firstRegion * REGION_DATA_LENGTH; + byte[] recovered = decrypt(cek, ciphertext, offset); + + assertArrayEquals(concat(region0Plaintext, region1Plaintext), recovered); + } + + @Test + public void detectsReorderAcrossIntegerMaxValueBoundary() { + // Detection must still fire across the int boundary: place a region whose nonce belongs to a different index + // than expected and confirm it is rejected. + byte[] cek = randomBytes(32); + long firstRegion = Integer.MAX_VALUE; + + byte[] ciphertext = concat(encryptRegionAt(cek, firstRegion, randomBytes(REGION_DATA_LENGTH)), + // Expected index here is firstRegion + 1, but this region is nonced for firstRegion + 2. + encryptRegionAt(cek, firstRegion + 2, randomBytes(REGION_DATA_LENGTH))); + + long offset = firstRegion * REGION_DATA_LENGTH; + assertThrows(IllegalStateException.class, () -> decrypt(cek, ciphertext, offset)); + } + + @Test + public void decryptsRegionJustBelowIntWrapBoundary() { + // Region 2^32 - 1 is the last region representable in 32 bits; with the full 64-bit counter it validates and + // decrypts normally. + byte[] cek = randomBytes(32); + long region = LARGE_REGION_INDEX - 1; + byte[] plaintext = randomBytes(REGION_DATA_LENGTH); + + byte[] ciphertext = encryptRegionAt(cek, region, plaintext); + byte[] recovered = decrypt(cek, ciphertext, region * REGION_DATA_LENGTH); + + assertArrayEquals(plaintext, recovered); + } + + @Test + public void decryptsRegionAtIntWrapBoundary() { + // Region 2^32 is where an int-based counter would have wrapped and reused region 0's nonce. With the full + // 64-bit counter its nonce is distinct, so the region validates and decrypts normally. + byte[] cek = randomBytes(32); + long region = LARGE_REGION_INDEX; + byte[] plaintext = randomBytes(REGION_DATA_LENGTH); + byte[] ciphertext = encryptRegionAt(cek, region, plaintext); + + byte[] recovered = decrypt(cek, ciphertext, region * REGION_DATA_LENGTH); + assertArrayEquals(plaintext, recovered); + } + + // ---- Cross-SDK interoperability: Java must still decrypt (and detect reorders in) blobs written by other SDKs, + // which encode the region counter into the nonce differently. ---- + + @Test + public void pythonEncodedBlobDecryptsWithoutFalseReorder() { + byte[] cek = randomBytes(32); + byte[][] plaintext = { + randomBytes(REGION_DATA_LENGTH), + randomBytes(REGION_DATA_LENGTH), + randomBytes(REGION_DATA_LENGTH), + randomBytes(REGION_DATA_LENGTH) }; + byte[] ciphertext = buildBlob(cek, DecryptorV2ReorderTests::pythonNonce, plaintext); + + assertArrayEquals(flatten(plaintext), decrypt(cek, ciphertext, 0)); + } + + @Test + public void dotnetEncodedBlobDecryptsWithoutFalseReorder() { + byte[] cek = randomBytes(32); + byte[][] plaintext = { + randomBytes(REGION_DATA_LENGTH), + randomBytes(REGION_DATA_LENGTH), + randomBytes(REGION_DATA_LENGTH), + randomBytes(REGION_DATA_LENGTH) }; + byte[] ciphertext = buildBlob(cek, DecryptorV2ReorderTests::dotnetNonce, plaintext); + + assertArrayEquals(flatten(plaintext), decrypt(cek, ciphertext, 0)); + } + + @Test + public void detectsReorderInPythonEncodedBlob() { + byte[] cek = randomBytes(32); + byte[][] plaintext = { + randomBytes(REGION_DATA_LENGTH), + randomBytes(REGION_DATA_LENGTH), + randomBytes(REGION_DATA_LENGTH), + randomBytes(REGION_DATA_LENGTH) }; + byte[] ciphertext = buildBlob(cek, DecryptorV2ReorderTests::pythonNonce, plaintext); + swapRegions(ciphertext, 2, 3, REGION_TOTAL_LENGTH); + + assertThrows(IllegalStateException.class, () -> decrypt(cek, ciphertext, 0)); + } + + @Test + public void detectsReorderInDotnetEncodedBlob() { + byte[] cek = randomBytes(32); + byte[][] plaintext = { + randomBytes(REGION_DATA_LENGTH), + randomBytes(REGION_DATA_LENGTH), + randomBytes(REGION_DATA_LENGTH), + randomBytes(REGION_DATA_LENGTH) }; + byte[] ciphertext = buildBlob(cek, DecryptorV2ReorderTests::dotnetNonce, plaintext); + swapRegions(ciphertext, 1, 2, REGION_TOTAL_LENGTH); + + assertThrows(IllegalStateException.class, () -> decrypt(cek, ciphertext, 0)); + } + + @Test + public void detectsSchemeSwitchMidBlob() { + // A blob whose first region uses one SDK's nonce scheme and a later region uses another's cannot be a single + // valid blob; the scheme is locked after the first region, so the switch is detected. + byte[] cek = randomBytes(32); + byte[] region0 = encryptRegionWithNonce(cek, dotnetNonce(0), randomBytes(REGION_DATA_LENGTH)); + byte[] region1 = encryptRegionWithNonce(cek, javaNonce(1), randomBytes(REGION_DATA_LENGTH)); + + assertThrows(IllegalStateException.class, () -> decrypt(cek, concat(region0, region1), 0)); + } + + @Test + public void rejectsMixedNonceEncodingsAcrossCollidingValueSpace() { + // The supported SDK nonce encodings share a value space, so accepting them independently per region would + // weaken reorder detection. For example the Java nonce for region 1 is byte-identical to the .NET nonce for + // region 16,777,215, so a region could be moved across that boundary and still pass a naive per-region union + // check. Detection must instead lock onto a single encoding and enforce it for every region. (Mirrors the + // Python SDK's test_decrypt_rejects_mixed_nonce_encodings.) + assertArrayEquals(javaNonce(1), dotnetNonce(16_777_215)); + + byte[] cek = randomBytes(32); + // Region 0 uses the Java/Python encoding (all zeros); region 1 uses the .NET encoding. A per-region union + // check would accept both, but single-encoding enforcement rejects the mix. + byte[] region0 = encryptRegionWithNonce(cek, javaNonce(0), randomBytes(REGION_DATA_LENGTH)); + byte[] region1 = encryptRegionWithNonce(cek, dotnetNonce(1), randomBytes(REGION_DATA_LENGTH)); + + assertThrows(IllegalStateException.class, () -> decrypt(cek, concat(region0, region1), 0)); + } + + @Test + public void singleRegionRangedDownloadCannotDetectCollisionSubstitution() { + // KNOWN LIMITATION (documented in DecryptorV2): a download containing only one region cannot cross-check + // regions to establish which SDK's nonce scheme the blob uses. Because the schemes share a value space + // (javaNonce(1) == dotnetNonce(16,777,215)), a lone region's nonce is valid for its position under more than + // one scheme. Here a Java-encoded region 1 is served for a single-region ranged read of region 16,777,215; it + // matches the .NET scheme for that position, so it is accepted rather than flagged. A full or multi-region + // download anchors the scheme from earlier regions and DOES detect such substitutions (see + // rejectsMixedNonceEncodingsAcrossCollidingValueSpace). This matches the other SDKs' cross-SDK detection. + assertArrayEquals(javaNonce(1), dotnetNonce(16_777_215)); + + byte[] cek = randomBytes(32); + byte[] region1Plaintext = randomBytes(REGION_DATA_LENGTH); + byte[] substituted = encryptRegionWithNonce(cek, javaNonce(1), region1Plaintext); + + long collidingOffset = 16_777_215L * REGION_DATA_LENGTH; + byte[] recovered = decrypt(cek, substituted, collidingOffset); + + // The substitution is not detected on a single-region ranged download; the region still decrypts under its + // inline nonce. (A multi-region download would have anchored the scheme and rejected this.) + assertArrayEquals(region1Plaintext, recovered); + } + + @Test + public void sharedValidatorEnforcesSchemeAcrossChunks() { + // A download operation may span multiple decrypt() calls (parallel downloadToFile / chunked openInputStream). + // A shared validator must intersect the candidate encodings across chunks; otherwise the encoding could change + // at a chunk boundary and, at an encoding collision, let a relocated region pass. Mirrors the Python SDK's + // test_nonce_validator_enforces_single_encoding_across_chunks. + // + // This is validated at the unit level (one validator instance driven across two decrypt() calls standing in for + // two chunks) because an end-to-end client test of this specific bypass is impractical: the smallest cross- + // scheme nonce collision is javaNonce(1) == dotnetNonce(16,777,215), so it only manifests at region index + // 16,777,215 - i.e. a blob with ~16.7M regions. + assertArrayEquals(javaNonce(1), dotnetNonce(16_777_215)); + + byte[] cek = randomBytes(32); + + // A region relocated to the colliding .NET index while carrying Java's region-1 nonce. + byte[] relocated = encryptRegionWithNonce(cek, javaNonce(1), randomBytes(REGION_DATA_LENGTH)); + long collidingOffset = 16_777_215L * REGION_DATA_LENGTH; + + // Regression guard: a fresh per-chunk validator sees only this region, whose sole consistent encoding is .NET, + // so it accepts the relocation. This is exactly the bypass that sharing one validator across chunks prevents - + // a regression that created one validator per chunk would let this through. + assertDoesNotThrow(() -> decrypt(cek, relocated, collidingOffset, new CseV2NonceOrderValidator())); + + // Shared validator: an earlier chunk of two Java-encoded regions resolves the shared encoding to Java... + CseV2NonceOrderValidator shared = new CseV2NonceOrderValidator(); + byte[] chunk1 = concat(encryptRegionWithNonce(cek, javaNonce(0), randomBytes(REGION_DATA_LENGTH)), + encryptRegionWithNonce(cek, javaNonce(1), randomBytes(REGION_DATA_LENGTH))); + decrypt(cek, chunk1, 0, shared); + + // ...so the same relocated region in a later chunk is now rejected. + assertThrows(IllegalStateException.class, () -> decrypt(cek, relocated, collidingOffset, shared)); + } + + private static byte[] encrypt(byte[] cek, byte[] plaintext) { + SecretKey key = new SecretKeySpec(cek, AES); + EncryptorV2 encryptor = new EncryptorV2(key, + new BlobClientSideEncryptionOptions().setAuthenticatedRegionDataLengthInBytes(REGION_DATA_LENGTH), + ENCRYPTION_PROTOCOL_V2); + List buffers = encryptor.encrypt(Flux.just(ByteBuffer.wrap(plaintext))).collectList().block(); + return toBytes(buffers); + } + + /** + * Encrypts a single region using the Java SDK nonce scheme (region index written big-endian as a full 64-bit + * value). Produces {@code nonce || ciphertext || tag}. Used to craft ciphertext for arbitrary (very large) region + * indices without materializing all preceding regions. + */ + private static byte[] encryptRegionAt(byte[] cek, long regionIndex, byte[] plaintext) { + return encryptRegionWithNonce(cek, javaNonce(regionIndex), plaintext); + } + + private static byte[] encryptRegionWithNonce(byte[] cek, byte[] nonce, byte[] plaintext) { + try { + Cipher cipher = Cipher.getInstance(AES_GCM_NO_PADDING); + cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(cek, AES), new GCMParameterSpec(TAG_LENGTH * 8, nonce)); + return concat(nonce, cipher.doFinal(plaintext)); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + } + + /** Builds a multi-region CSEv2 blob whose region i uses the nonce produced by {@code nonceForRegion}. */ + private static byte[] buildBlob(byte[] cek, java.util.function.LongFunction nonceForRegion, + byte[][] regionPlaintext) { + byte[] result = new byte[0]; + for (int i = 0; i < regionPlaintext.length; i++) { + result = concat(result, encryptRegionWithNonce(cek, nonceForRegion.apply(i), regionPlaintext[i])); + } + return result; + } + + // Independent implementations of each SDK's nonce scheme, serving as cross-SDK test vectors. + + private static byte[] javaNonce(long regionIndex) { + // Full 64-bit big-endian region index (matches EncryptorV2 after the nonce-counter widening fix). + return ByteBuffer.allocate(NONCE_LENGTH).putLong(regionIndex).array(); + } + + private static byte[] pythonNonce(long regionIndex) { + // 12-byte big-endian integer (counter in the low bytes), 0-based. + byte[] nonce = new byte[NONCE_LENGTH]; + for (int i = 0; i < Long.BYTES; i++) { + nonce[NONCE_LENGTH - 1 - i] = (byte) (regionIndex >>> (Byte.SIZE * i)); + } + return nonce; + } + + private static byte[] dotnetNonce(long regionIndex) { + // Four zero bytes then an 8-byte little-endian counter, 1-based. + byte[] nonce = new byte[NONCE_LENGTH]; + long counter = regionIndex + 1; + for (int i = 0; i < Long.BYTES; i++) { + nonce[(NONCE_LENGTH - Long.BYTES) + i] = (byte) (counter >>> (Byte.SIZE * i)); + } + return nonce; + } + + private static byte[] flatten(byte[][] arrays) { + byte[] result = new byte[0]; + for (byte[] a : arrays) { + result = concat(result, a); + } + return result; + } + + private static byte[] decrypt(byte[] cek, byte[] ciphertext, long offset) { + return decrypt(cek, ciphertext, offset, new CseV2NonceOrderValidator()); + } + + private static byte[] decrypt(byte[] cek, byte[] ciphertext, long offset, CseV2NonceOrderValidator validator) { + EncryptionData encryptionData = new EncryptionData() + .setEncryptionAgent(new EncryptionAgent(ENCRYPTION_PROTOCOL_V2, EncryptionAlgorithm.AES_GCM_256)) + .setEncryptedRegionInfo(new EncryptedRegionInfo(REGION_DATA_LENGTH, NONCE_LENGTH)); + DecryptorV2 decryptor = new DecryptorV2(null, null, encryptionData, validator); + EncryptedBlobRange range = new EncryptedBlobRange(new BlobRange(offset), encryptionData); + + List out + = decryptor.decrypt(Flux.just(ByteBuffer.wrap(ciphertext)), range, false, "uri", new AtomicLong(0), cek) + .collectList() + .block(); + return toBytes(out); + } + + private static byte[] toBytes(List buffers) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (ByteBuffer buffer : buffers) { + ByteBuffer duplicate = buffer.duplicate(); + byte[] arr = new byte[duplicate.remaining()]; + duplicate.get(arr); + out.write(arr, 0, arr.length); + } + return out.toByteArray(); + } + + private static void swapRegions(byte[] buffer, int leftRegion, int rightRegion, int regionLength) { + int leftOffset = leftRegion * regionLength; + int rightOffset = rightRegion * regionLength; + for (int i = 0; i < regionLength; i++) { + byte temp = buffer[leftOffset + i]; + buffer[leftOffset + i] = buffer[rightOffset + i]; + buffer[rightOffset + i] = temp; + } + } + + private static byte[] randomBytes(int length) { + byte[] bytes = new byte[length]; + RANDOM.nextBytes(bytes); + return bytes; + } + + private static byte[] concat(byte[] left, byte[] right) { + byte[] result = Arrays.copyOf(left, left.length + right.length); + System.arraycopy(right, 0, result, left.length, right.length); + return result; + } +} diff --git a/sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlockBlobApiTests.java b/sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlockBlobApiTests.java index 9e0ebac5b644..aa673f8467d7 100644 --- a/sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlockBlobApiTests.java +++ b/sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlockBlobApiTests.java @@ -380,6 +380,123 @@ public void testForCrossPlat() { encryptClient.upload(BinaryData.fromBytes(data), true); } + // Cross-SDK interoperability: Java must decrypt CSE v2 blobs written by other SDKs, whose per-region GCM nonces + // encode the region counter differently than Java's. This exercises a container of Python-produced blobs. Requires + // a container coordinated between languages; run manually. Verification/reorder-detection of the nonce schemes is + // covered without a live account by DecryptorV2ReorderTests. + @Disabled + @Test + public void crossPlatDecryptPythonV2() { + // The Python azure-storage-blob KeyWrapper("key1") test helper wraps the CEK with A256KW using this fixed KEK. + byte[] kek = fromHex("bea4114b9e4a07da664683ad2bad76412043e8bc90a4117d47c30fd4b4196d11"); + AsyncKeyEncryptionKey key = new A256KwKey("key1", kek); + + String containerName = "cross-sdk-cse-v2"; + String endpoint = ENV.getPrimaryAccount().getBlobEndpoint(); + + // Reference plaintext uploaded alongside the encrypted blob. + BlobClient plaintextClient = new BlobClientBuilder().endpoint(endpoint) + .containerName(containerName) + .blobName("python_plaintext") + .credential(ENV.getPrimaryAccount().getCredential()) + .buildClient(); + byte[] expected = plaintextClient.downloadContent().toBytes(); + + // Decrypt the Python-encrypted blob (Python nonce scheme) and confirm it round-trips without a false reorder. + EncryptedBlobClient decryptionClient = new EncryptedBlobClientBuilder(EncryptionVersion.V2).endpoint(endpoint) + .containerName(containerName) + .blobName("python_encrypted") + .key(key, "A256KW") + .credential(ENV.getPrimaryAccount().getCredential()) + .buildEncryptedBlobClient(); + ByteArrayOutputStream decrypted = new ByteArrayOutputStream(); + decryptionClient.downloadStream(decrypted); + + assertArraysEqual(expected, decrypted.toByteArray()); + } + + // Cross-SDK interoperability: Java must decrypt CSE v2 blobs written by other SDKs, whose per-region GCM nonces + // encode the region counter differently than Java's. This exercises a container of .NET-produced blobs. Requires + // a container coordinated between languages; run manually. Verification/reorder-detection of the nonce schemes is + // covered without a live account by DecryptorV2ReorderTests. + @Disabled + @Test + public void crossPlatDecryptDotnetV2() { + // A256KW (RFC 3394) key wrap with the fixed KEK coordinated across the .NET/Python/Java cross-SDK tests. .NET's + // test helper stores the key under the id "local:key1" (Python uses "key1"), so the id must match here. + byte[] kek = fromHex("bea4114b9e4a07da664683ad2bad76412043e8bc90a4117d47c30fd4b4196d11"); + AsyncKeyEncryptionKey key = new A256KwKey("local:key1", kek); + + String containerName = "cross-sdk-cse-v2"; + String endpoint = ENV.getPrimaryAccount().getBlobEndpoint(); + + // Reference plaintext uploaded alongside the encrypted blob. + BlobClient plaintextClient = new BlobClientBuilder().endpoint(endpoint) + .containerName(containerName) + .blobName("dotnet_plaintext") + .credential(ENV.getPrimaryAccount().getCredential()) + .buildClient(); + byte[] expected = plaintextClient.downloadContent().toBytes(); + + // Decrypt the .NET-encrypted blob (.NET nonce scheme) and confirm it round-trips without a false reorder. + EncryptedBlobClient decryptionClient = new EncryptedBlobClientBuilder(EncryptionVersion.V2).endpoint(endpoint) + .containerName(containerName) + .blobName("dotnet_encrypted") + .key(key, "A256KW") + .credential(ENV.getPrimaryAccount().getCredential()) + .buildEncryptedBlobClient(); + ByteArrayOutputStream decrypted = new ByteArrayOutputStream(); + decryptionClient.downloadStream(decrypted); + + assertArraysEqual(expected, decrypted.toByteArray()); + } + + private static byte[] fromHex(String hex) { + byte[] out = new byte[hex.length() / 2]; + for (int i = 0; i < out.length; i++) { + out[i] = (byte) Integer.parseInt(hex.substring(i * 2, i * 2 + 2), 16); + } + return out; + } + + // AES key-wrap (A256KW / RFC 3394) key encryption key, matching the wrapping used by other SDKs' test helpers. + static final class A256KwKey implements AsyncKeyEncryptionKey { + private final String keyId; + private final javax.crypto.SecretKey kek; + + A256KwKey(String keyId, byte[] kekBytes) { + this.keyId = keyId; + this.kek = new SecretKeySpec(kekBytes, "AES"); + } + + @Override + public Mono getKeyId() { + return Mono.just(keyId); + } + + @Override + public Mono wrapKey(String algorithm, byte[] key) { + try { + Cipher cipher = Cipher.getInstance("AESWrap"); + cipher.init(Cipher.WRAP_MODE, kek); + return Mono.just(cipher.wrap(new SecretKeySpec(key, "AES"))); + } catch (GeneralSecurityException e) { + return Mono.error(e); + } + } + + @Override + public Mono unwrapKey(String algorithm, byte[] encryptedKey) { + try { + Cipher cipher = Cipher.getInstance("AESWrap"); + cipher.init(Cipher.UNWRAP_MODE, kek); + return Mono.just(cipher.unwrap(encryptedKey, "AES", Cipher.SECRET_KEY).getEncoded()); + } catch (GeneralSecurityException e) { + return Mono.error(e); + } + } + } + static class NoOpKey implements AsyncKeyEncryptionKey { @Override public Mono getKeyId() { @@ -599,6 +716,50 @@ public void encryptionV2DowngradeAttack() { assertThrows(Exception.class, () -> bec.downloadStream(new ByteArrayOutputStream())); } + @LiveOnly + @Test + public void encryptionV2DetectRegionReorder() { + // Region must be large enough for at least a few regions; use the minimum region size to keep the blob small. + int regionDataLength = 16; + int regionCount = 4; + int regionTotalLength = NONCE_LENGTH + regionDataLength + TAG_LENGTH; + byte[] plaintext = getRandomByteArray(regionDataLength * regionCount); + + String blobName = generateBlobName(); + EncryptedBlobClient encryptedClient = new EncryptedBlobClient(getEncryptedClientBuilder(fakeKey, null, + ENV.getPrimaryAccount().getCredential(), cc.getBlobContainerUrl(), EncryptionVersion.V2_1) + .blobName(blobName) + .clientSideEncryptionOptions( + new BlobClientSideEncryptionOptions().setAuthenticatedRegionDataLengthInBytes(regionDataLength)) + .buildEncryptedBlobAsyncClient()); + + // Upload with encryption. + encryptedClient.upload(BinaryData.fromBytes(plaintext)); + + // Download the raw ciphertext (bypassing decryption) and preserve its encryption metadata. + BlobClient plainClient = cc.getBlobClient(blobName); + byte[] ciphertext = plainClient.downloadContent().toBytes(); + Map metadata = plainClient.getProperties().getMetadata(); + + // Swap two otherwise-untampered authenticated regions and re-upload the tampered ciphertext. + swapRegions(ciphertext, 2, 3, regionTotalLength); + plainClient.upload(BinaryData.fromBytes(ciphertext), true); + plainClient.setMetadata(metadata); + + // The reordering must now be detected on download. + assertThrows(Exception.class, () -> encryptedClient.downloadStream(new ByteArrayOutputStream())); + } + + private static void swapRegions(byte[] buffer, int leftRegion, int rightRegion, int regionLength) { + int leftOffset = leftRegion * regionLength; + int rightOffset = rightRegion * regionLength; + for (int i = 0; i < regionLength; i++) { + byte temp = buffer[leftOffset + i]; + buffer[leftOffset + i] = buffer[rightOffset + i]; + buffer[rightOffset + i] = temp; + } + } + @Test public void downloadUnencryptedData() { // Create client