From 227c4ddf0878ec87b8bfe1d3746604401fef02fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Fri, 5 Jun 2026 18:09:18 +0200 Subject: [PATCH 1/3] Reformat with prettier Issue: CLDSRV-888 --- lib/api/apiUtils/object/getReplicationInfo.js | 25 +- lib/metadata/acl.js | 88 +- tests/unit/api/apiUtils/getReplicationInfo.js | 315 +++--- tests/unit/api/objectReplicationMD.js | 926 +++++++++--------- 4 files changed, 719 insertions(+), 635 deletions(-) diff --git a/lib/api/apiUtils/object/getReplicationInfo.js b/lib/api/apiUtils/object/getReplicationInfo.js index b7e4afb25d..f9f8b92403 100644 --- a/lib/api/apiUtils/object/getReplicationInfo.js +++ b/lib/api/apiUtils/object/getReplicationInfo.js @@ -1,5 +1,4 @@ -const { isServiceAccount, getServiceAccountProperties } = - require('../authorization/permissionChecks'); +const { isServiceAccount, getServiceAccountProperties } = require('../authorization/permissionChecks'); const { replicationBackends } = require('arsenal').constants; function _getBackend(objectMD, site) { @@ -23,15 +22,13 @@ function _getStorageClasses(s3config, rule) { const { replicationEndpoints } = s3config; // If no storage class, use the given default endpoint or the sole endpoint if (replicationEndpoints.length > 0) { - const endPoint = - replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0]; + const endPoint = replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0]; return [endPoint.site]; } return undefined; } -function _getReplicationInfo(s3config, rule, replicationConfig, content, operationType, - objectMD, bucketMD) { +function _getReplicationInfo(s3config, rule, replicationConfig, content, operationType, objectMD, bucketMD) { const storageTypes = []; const backends = []; const storageClasses = _getStorageClasses(s3config, rule); @@ -39,9 +36,7 @@ function _getReplicationInfo(s3config, rule, replicationConfig, content, operati return undefined; } storageClasses.forEach(storageClass => { - const storageClassName = - storageClass.endsWith(':preferred_read') ? - storageClass.split(':')[0] : storageClass; + const storageClassName = storageClass.endsWith(':preferred_read') ? storageClass.split(':')[0] : storageClass; // TODO CLDSRV-646: for consistency, should we look at replicationEndpoints instead, like // `_getStorageClasses()` ? const location = s3config.locationConstraints[storageClassName]; @@ -80,8 +75,7 @@ function _getReplicationInfo(s3config, rule, replicationConfig, content, operati * @param {AuthInfo} [authInfo] - authentication info of object owner * @return {undefined} */ -function getReplicationInfo( - s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) { +function getReplicationInfo(s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) { const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA']; const config = bucketMD.getReplicationConfiguration(); @@ -106,17 +100,14 @@ function getReplicationInfo( if (!authInfo || !isServiceAccount(authInfo.getCanonicalID())) { doReplicate = true; } else { - const serviceAccountProps = getServiceAccountProperties( - authInfo.getCanonicalID()); + const serviceAccountProps = getServiceAccountProperties(authInfo.getCanonicalID()); doReplicate = serviceAccountProps.canReplicate; } if (doReplicate) { - const rule = config.rules.find( - rule => (objKey.startsWith(rule.prefix) && rule.enabled)); + const rule = config.rules.find(rule => objKey.startsWith(rule.prefix) && rule.enabled); if (rule) { // TODO CLDSRV-646 : should "merge" the replicationInfo for different rules - return _getReplicationInfo( - s3config, rule, config, content, operationType, objectMD, bucketMD); + return _getReplicationInfo(s3config, rule, config, content, operationType, objectMD, bucketMD); } } } diff --git a/lib/metadata/acl.js b/lib/metadata/acl.js index f48ab7aa42..961b95d79f 100644 --- a/lib/metadata/acl.js +++ b/lib/metadata/acl.js @@ -31,14 +31,16 @@ const acl = { * contain the same number of elements, and all elements from one * grant are incuded in the other grant */ - return oldAcl[grant].length === newAcl[grant].length - && oldAcl[grant].every(value => newAcl[grant].includes(value)); + return ( + oldAcl[grant].length === newAcl[grant].length && oldAcl[grant].every(value => newAcl[grant].includes(value)) + ); }, addObjectACL(bucket, objectKey, objectMD, addACLParams, params, log, cb) { log.trace('updating object acl in metadata'); - const isAclUnchanged = Object.keys(objectMD.acl).length === Object.keys(addACLParams).length - && Object.keys(objectMD.acl).every(grant => this._aclGrantDidNotChange(grant, objectMD.acl, addACLParams)); + const isAclUnchanged = + Object.keys(objectMD.acl).length === Object.keys(addACLParams).length && + Object.keys(objectMD.acl).every(grant => this._aclGrantDidNotChange(grant, objectMD.acl, addACLParams)); if (!isAclUnchanged) { /* eslint-disable no-param-reassign */ objectMD.acl = addACLParams; @@ -77,14 +79,22 @@ const acl = { }; let validCannedACL = []; if (resourceType === 'bucket') { - validCannedACL = - ['private', 'public-read', 'public-read-write', - 'authenticated-read', 'log-delivery-write']; + validCannedACL = [ + 'private', + 'public-read', + 'public-read-write', + 'authenticated-read', + 'log-delivery-write', + ]; } else if (resourceType === 'object') { - validCannedACL = - ['private', 'public-read', 'public-read-write', - 'authenticated-read', 'bucket-owner-read', - 'bucket-owner-full-control']; + validCannedACL = [ + 'private', + 'public-read', + 'public-read-write', + 'authenticated-read', + 'bucket-owner-read', + 'bucket-owner-full-control', + ]; } // parse canned acl @@ -98,45 +108,34 @@ const acl = { } // parse grant headers - const grantReadHeader = - aclUtils.parseGrant(headers['x-amz-grant-read'], 'READ'); + const grantReadHeader = aclUtils.parseGrant(headers['x-amz-grant-read'], 'READ'); let grantWriteHeader = []; if (resourceType === 'bucket') { - grantWriteHeader = aclUtils - .parseGrant(headers['x-amz-grant-write'], 'WRITE'); + grantWriteHeader = aclUtils.parseGrant(headers['x-amz-grant-write'], 'WRITE'); } - const grantReadACPHeader = aclUtils - .parseGrant(headers['x-amz-grant-read-acp'], 'READ_ACP'); - const grantWriteACPHeader = aclUtils - .parseGrant(headers['x-amz-grant-write-acp'], 'WRITE_ACP'); - const grantFullControlHeader = aclUtils - .parseGrant(headers['x-amz-grant-full-control'], 'FULL_CONTROL'); - const allGrantHeaders = - [].concat(grantReadHeader, grantWriteHeader, - grantReadACPHeader, grantWriteACPHeader, - grantFullControlHeader).filter(item => item !== undefined); + const grantReadACPHeader = aclUtils.parseGrant(headers['x-amz-grant-read-acp'], 'READ_ACP'); + const grantWriteACPHeader = aclUtils.parseGrant(headers['x-amz-grant-write-acp'], 'WRITE_ACP'); + const grantFullControlHeader = aclUtils.parseGrant(headers['x-amz-grant-full-control'], 'FULL_CONTROL'); + const allGrantHeaders = [] + .concat(grantReadHeader, grantWriteHeader, grantReadACPHeader, grantWriteACPHeader, grantFullControlHeader) + .filter(item => item !== undefined); if (allGrantHeaders.length === 0) { return cb(null, currentResourceACL); } - const usersIdentifiedByEmail = allGrantHeaders - .filter(it => it && it.userIDType.toLowerCase() === 'emailaddress'); - const usersIdentifiedByGroup = allGrantHeaders - .filter(item => item && item.userIDType.toLowerCase() === 'uri'); + const usersIdentifiedByEmail = allGrantHeaders.filter( + it => it && it.userIDType.toLowerCase() === 'emailaddress', + ); + const usersIdentifiedByGroup = allGrantHeaders.filter(item => item && item.userIDType.toLowerCase() === 'uri'); const justEmails = usersIdentifiedByEmail.map(item => item.identifier); - const validGroups = [ - constants.allAuthedUsersId, - constants.publicId, - constants.logId, - ]; + const validGroups = [constants.allAuthedUsersId, constants.publicId, constants.logId]; for (let i = 0; i < usersIdentifiedByGroup.length; i++) { if (validGroups.indexOf(usersIdentifiedByGroup[i].identifier) < 0) { return cb(errors.InvalidArgument); } } - const usersIdentifiedByID = allGrantHeaders - .filter(item => item && item.userIDType.toLowerCase() === 'id'); + const usersIdentifiedByID = allGrantHeaders.filter(item => item && item.userIDType.toLowerCase() === 'id'); // TODO: Consider whether want to verify with Vault // whether canonicalID is associated with existing // account before adding to ACL @@ -148,22 +147,22 @@ const acl = { if (err) { return cb(err); } - const reconstructedUsersIdentifiedByEmail = aclUtils. - reconstructUsersIdentifiedByEmail(results, - usersIdentifiedByEmail); + const reconstructedUsersIdentifiedByEmail = aclUtils.reconstructUsersIdentifiedByEmail( + results, + usersIdentifiedByEmail, + ); const allUsers = [].concat( reconstructedUsersIdentifiedByEmail, usersIdentifiedByGroup, - usersIdentifiedByID); - const revisedACL = - aclUtils.sortHeaderGrants(allUsers, resourceACL); + usersIdentifiedByID, + ); + const revisedACL = aclUtils.sortHeaderGrants(allUsers, resourceACL); return cb(null, revisedACL); }); } else { // If don't have to look up canonicalID's just sort grants // and add to bucket - const revisedACL = aclUtils - .sortHeaderGrants(allGrantHeaders, resourceACL); + const revisedACL = aclUtils.sortHeaderGrants(allGrantHeaders, resourceACL); return cb(null, revisedACL); } return undefined; @@ -171,4 +170,3 @@ const acl = { }; module.exports = acl; - diff --git a/tests/unit/api/apiUtils/getReplicationInfo.js b/tests/unit/api/apiUtils/getReplicationInfo.js index d8bec4c1e4..9134905bb4 100644 --- a/tests/unit/api/apiUtils/getReplicationInfo.js +++ b/tests/unit/api/apiUtils/getReplicationInfo.js @@ -2,15 +2,25 @@ const assert = require('assert'); const BucketInfo = require('arsenal').models.BucketInfo; const AuthInfo = require('arsenal').auth.AuthInfo; -const getReplicationInfo = - require('../../../../lib/api/apiUtils/object/getReplicationInfo'); +const getReplicationInfo = require('../../../../lib/api/apiUtils/object/getReplicationInfo'); function _getObjectReplicationInfo(s3config, replicationConfig) { const bucketInfo = new BucketInfo( - 'testbucket', 'someCanonicalId', 'accountDisplayName', + 'testbucket', + 'someCanonicalId', + 'accountDisplayName', new Date().toJSON(), - null, null, null, null, null, null, null, null, null, - replicationConfig); + null, + null, + null, + null, + null, + null, + null, + null, + null, + replicationConfig, + ); return getReplicationInfo(s3config, 'fookey', bucketInfo, true, 123, null, null); } @@ -36,39 +46,46 @@ const TEST_CONFIG = { azureStorageAccountName: 'fakeaccountname', azureStorageAccessKey: 'Fake00Key001', bucketMatch: true, - azureContainerName: 's3test' - } + azureContainerName: 's3test', + }, }, }, - replicationEndpoints: [{ - site: 'zenko', - servers: ['127.0.0.1:8000'], - default: true, - }, { - site: 'us-east-2', - type: 'aws_s3', - }], + replicationEndpoints: [ + { + site: 'zenko', + servers: ['127.0.0.1:8000'], + default: true, + }, + { + site: 'us-east-2', + type: 'aws_s3', + }, + ], }; describe('getReplicationInfo helper', () => { it('should get replication info when rules are enabled', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], destination: 'tosomewhere', }; const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'awsbackend', @@ -81,11 +98,13 @@ describe('getReplicationInfo helper', () => { it('should not get replication info when rules are disabled', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: false, - storageClass: 'awsbackend', - }], + rules: [ + { + prefix: '', + enabled: false, + storageClass: 'awsbackend', + }, + ], destination: 'tosomewhere', }; const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); @@ -95,21 +114,25 @@ describe('getReplicationInfo helper', () => { it('should get replication info with single cloud target', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], destination: 'tosomewhere', }; const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'awsbackend', @@ -122,25 +145,30 @@ describe('getReplicationInfo helper', () => { it('should get replication info with multiple cloud targets', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend,azurebackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend,azurebackend', + }, + ], destination: 'tosomewhere', }; const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }, { - site: 'azurebackend', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + { + site: 'azurebackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'awsbackend,azurebackend', @@ -150,30 +178,34 @@ describe('getReplicationInfo helper', () => { }); }); - it('should get replication info with multiple cloud targets and ' + - 'preferred read location', () => { + it('should get replication info with multiple cloud targets and ' + 'preferred read location', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend:preferred_read,azurebackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend:preferred_read,azurebackend', + }, + ], destination: 'tosomewhere', preferredReadLocation: 'awsbackend', }; const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }, { - site: 'azurebackend', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + { + site: 'azurebackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'awsbackend:preferred_read,azurebackend', @@ -183,61 +215,84 @@ describe('getReplicationInfo helper', () => { }); }); - it('should not get replication info when service account type ' + - 'cannot trigger replication', () => { + it('should not get replication info when service account type ' + 'cannot trigger replication', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], destination: 'tosomewhere', }; const bucketInfo = new BucketInfo( - 'testbucket', 'abcdef/lifecycle', 'Lifecycle Service Account', + 'testbucket', + 'abcdef/lifecycle', + 'Lifecycle Service Account', new Date().toJSON(), - null, null, null, null, null, null, null, null, null, - replicationConfig); + null, + null, + null, + null, + null, + null, + null, + null, + null, + replicationConfig, + ); const authInfo = new AuthInfo({ canonicalID: 'abcdef/lifecycle', accountDisplayName: 'Lifecycle Service Account', }); - const replicationInfo = getReplicationInfo(TEST_CONFIG, - 'fookey', bucketInfo, true, 123, null, null, authInfo); + const replicationInfo = getReplicationInfo(TEST_CONFIG, 'fookey', bucketInfo, true, 123, null, null, authInfo); assert.deepStrictEqual(replicationInfo, undefined); }); - it('should get replication info when service account type can ' + - 'trigger replication', () => { + it('should get replication info when service account type can ' + 'trigger replication', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], destination: 'tosomewhere', }; const bucketInfo = new BucketInfo( - 'testbucket', 'abcdef/md-ingestion', + 'testbucket', + 'abcdef/md-ingestion', 'Metadata Ingestion Service Account', new Date().toJSON(), - null, null, null, null, null, null, null, null, null, - replicationConfig); + null, + null, + null, + null, + null, + null, + null, + null, + null, + replicationConfig, + ); const authInfo = new AuthInfo({ canonicalID: 'abcdef/md-ingestion', accountDisplayName: 'Metadata Ingestion Service Account', }); - const replicationInfo = getReplicationInfo(TEST_CONFIG, - 'fookey', bucketInfo, true, 123, null, null, authInfo); + const replicationInfo = getReplicationInfo(TEST_CONFIG, 'fookey', bucketInfo, true, 123, null, null, authInfo); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'awsbackend', @@ -250,20 +305,24 @@ describe('getReplicationInfo helper', () => { it('should get replication info with default StorageClass when rules are enabled', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2', - rules: [{ - prefix: '', - enabled: true, - }], + rules: [ + { + prefix: '', + enabled: true, + }, + ], destination: 'tosomewhere', }; const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'zenko', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'zenko', @@ -276,26 +335,29 @@ describe('getReplicationInfo helper', () => { it('should return undefined with specified StorageClass mode if no replication endpoint is configured', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [{ - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }], + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], destination: 'tosomewhere', }; const configWithNoReplicationEndpoint = { locationConstraints: TEST_CONFIG.locationConstraints, replicationEndpoints: [], }; - const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, - replicationConfig); + const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, replicationConfig); assert.deepStrictEqual(replicationInfo, { status: 'PENDING', - backends: [{ - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }], + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], content: ['METADATA'], destination: 'tosomewhere', storageClass: 'awsbackend', @@ -308,18 +370,19 @@ describe('getReplicationInfo helper', () => { it('should return undefined with default StorageClass if no replication endpoint is configured', () => { const replicationConfig = { role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2', - rules: [{ - prefix: '', - enabled: true, - }], + rules: [ + { + prefix: '', + enabled: true, + }, + ], destination: 'tosomewhere', }; const configWithNoReplicationEndpoint = { locationConstraints: TEST_CONFIG.locationConstraints, replicationEndpoints: [], }; - const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, - replicationConfig); + const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, replicationConfig); assert.deepStrictEqual(replicationInfo, undefined); }); }); diff --git a/tests/unit/api/objectReplicationMD.js b/tests/unit/api/objectReplicationMD.js index 48451b43ce..fa228cf3cb 100644 --- a/tests/unit/api/objectReplicationMD.js +++ b/tests/unit/api/objectReplicationMD.js @@ -4,16 +4,14 @@ const crypto = require('crypto'); const BucketInfo = require('arsenal').models.BucketInfo; -const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = - require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const constants = require('../../../constants'); const { metadata } = require('arsenal').storage.metadata.inMemory.metadata; const DummyRequest = require('../DummyRequest'); const { objectDelete } = require('../../../lib/api/objectDelete'); const objectPut = require('../../../lib/api/objectPut'); const objectCopy = require('../../../lib/api/objectCopy'); -const completeMultipartUpload = - require('../../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); const objectPutACL = require('../../../lib/api/objectPutACL'); const objectPutTagging = require('../../../lib/api/objectPutTagging'); const objectDeleteTagging = require('../../../lib/api/objectDeleteTagging'); @@ -55,19 +53,20 @@ const objectACLReq = { // Get an object request with the given key. function getObjectPutReq(key, hasContent) { const bodyContent = hasContent ? 'body content' : ''; - return new DummyRequest({ - bucketName, - namespace, - objectKey: key, - headers: {}, - url: `/${bucketName}/${key}`, - }, Buffer.from(bodyContent, 'utf8')); + return new DummyRequest( + { + bucketName, + namespace, + objectKey: key, + headers: {}, + url: `/${bucketName}/${key}`, + }, + Buffer.from(bodyContent, 'utf8'), + ); } -const taggingPutReq = new TaggingConfigTester() - .createObjectTaggingRequest('PUT', bucketName, keyA); -const taggingDeleteReq = new TaggingConfigTester() - .createObjectTaggingRequest('DELETE', bucketName, keyA); +const taggingPutReq = new TaggingConfigTester().createObjectTaggingRequest('PUT', bucketName, keyA); +const taggingDeleteReq = new TaggingConfigTester().createObjectTaggingRequest('DELETE', bucketName, keyA); const emptyReplicationMD = { status: '', @@ -99,35 +98,34 @@ function checkObjectReplicationInfo(key, expected) { // Put the object key and check the replication information. function putObjectAndCheckMD(key, expected, cb) { - return objectPut(authInfo, getObjectPutReq(key, true), undefined, log, - err => { - if (err) { - return cb(err); - } - checkObjectReplicationInfo(key, expected); - return cb(); - }); + return objectPut(authInfo, getObjectPutReq(key, true), undefined, log, err => { + if (err) { + return cb(err); + } + checkObjectReplicationInfo(key, expected); + return cb(); + }); } // Create the bucket in metadata. function createBucket() { - metadata - .buckets.set(bucketName, new BucketInfo(bucketName, ownerID, '', '')); - metadata.keyMaps.set(bucketName, new Map); + metadata.buckets.set(bucketName, new BucketInfo(bucketName, ownerID, '', '')); + metadata.keyMaps.set(bucketName, new Map()); } // Create the bucket in metadata with versioning and a replication config. function createBucketWithReplication(hasStorageClass) { createBucket(); const config = { - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', destination: 'arn:aws:s3:::source-bucket', - rules: [{ - prefix: keyA, - enabled: true, - id: 'test-id', - }], + rules: [ + { + prefix: keyA, + enabled: true, + id: 'test-id', + }, + ], }; if (hasStorageClass) { config.rules[0].storageClass = storageClassType; @@ -140,22 +138,21 @@ function createBucketWithReplication(hasStorageClass) { // Create the shadow bucket in metadata for MPUs with a recent model number. function createShadowBucket(key, uploadId) { - const overviewKey = `overview${constants.splitter}` + - `${key}${constants.splitter}${uploadId}`; - metadata.buckets - .set(mpuShadowBucket, new BucketInfo(mpuShadowBucket, ownerID, '', '')); - // Set modelVersion to use the most recent splitter. + const overviewKey = `overview${constants.splitter}` + `${key}${constants.splitter}${uploadId}`; + metadata.buckets.set(mpuShadowBucket, new BucketInfo(mpuShadowBucket, ownerID, '', '')); + // Set modelVersion to use the most recent splitter. Object.assign(metadata.buckets.get(mpuShadowBucket), { _mdBucketModelVersion: 5, }); - metadata.keyMaps.set(mpuShadowBucket, new Map); - metadata.keyMaps.get(mpuShadowBucket).set(overviewKey, new Map); + metadata.keyMaps.set(mpuShadowBucket, new Map()); + metadata.keyMaps.get(mpuShadowBucket).set(overviewKey, new Map()); Object.assign(metadata.keyMaps.get(mpuShadowBucket).get(overviewKey), { id: uploadId, eventualStorageBucket: bucketName, initiator: { DisplayName: 'accessKey1displayName', - ID: ownerID }, + ID: ownerID, + }, key, uploadId, }); @@ -170,24 +167,26 @@ function putMPU(key, body, cb) { const calculatedHash = md5Hash.digest('hex'); const partKey = `${uploadId}${constants.splitter}00001`; const obj = { - partLocations: [{ - key: 1, - dataStoreName: 'scality-internal-mem', - dataStoreETag: `1:${calculatedHash}`, - }], + partLocations: [ + { + key: 1, + dataStoreName: 'scality-internal-mem', + dataStoreETag: `1:${calculatedHash}`, + }, + ], key: partKey, }; obj['content-md5'] = calculatedHash; obj['content-length'] = body.length; - metadata.keyMaps.get(mpuShadowBucket).set(partKey, new Map); + metadata.keyMaps.get(mpuShadowBucket).set(partKey, new Map()); const partMap = metadata.keyMaps.get(mpuShadowBucket).get(partKey); Object.assign(partMap, obj); const postBody = '' + - '' + - '1' + - `"${calculatedHash}"` + - '' + + '' + + '1' + + `"${calculatedHash}"` + + '' + ''; const req = { bucketName, @@ -217,8 +216,7 @@ function copyObject(sourceObjectKey, copyObjectKey, hasContent, cb) { headers: {}, url: `/${bucketName}/${sourceObjectKey}`, }); - return objectCopy(authInfo, req, bucketName, sourceObjectKey, undefined, - log, cb); + return objectCopy(authInfo, req, bucketName, sourceObjectKey, undefined, log, cb); }); } @@ -230,26 +228,33 @@ describe('Replication object MD without bucket replication config', () => { afterEach(() => cleanup()); - it('should not update object metadata', done => - putObjectAndCheckMD(keyA, emptyReplicationMD, done)); + it('should not update object metadata', done => putObjectAndCheckMD(keyA, emptyReplicationMD, done)); it('should not update object metadata if putting object ACL', done => - async.series([ - next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), - next => objectPutACL(authInfo, objectACLReq, log, next), - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); - return done(); - })); + async.series( + [ + next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), + next => objectPutACL(authInfo, objectACLReq, log, next), + ], + err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); + return done(); + }, + )); describe('Object tagging', () => { - beforeEach(done => async.series([ - next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), - next => objectPutTagging(authInfo, taggingPutReq, log, next), - ], err => done(err))); + beforeEach(done => + async.series( + [ + next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), + next => objectPutTagging(authInfo, taggingPutReq, log, next), + ], + err => done(err), + ), + ); it('should not update object metadata if putting tag', done => { checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); @@ -257,18 +262,20 @@ describe('Replication object MD without bucket replication config', () => { }); it('should not update object metadata if deleting tag', done => - async.series([ - // Put a new version to update replication MD content array. - next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), - next => objectDeleteTagging(authInfo, taggingDeleteReq, log, - next), - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); - return done(); - })); + async.series( + [ + // Put a new version to update replication MD content array. + next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), + next => objectDeleteTagging(authInfo, taggingDeleteReq, log, next), + ], + err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); + return done(); + }, + )); it('should not update object metadata if completing MPU', done => putMPU(keyA, 'content', err => { @@ -291,430 +298,455 @@ describe('Replication object MD without bucket replication config', () => { }); [true, false].forEach(hasStorageClass => { - describe('Replication object MD with bucket replication config ' + - `${hasStorageClass ? 'with' : 'without'} storage class`, () => { - const replicationMD = { - status: 'PENDING', - backends: [{ - site: 'zenko', + describe( + 'Replication object MD with bucket replication config ' + + `${hasStorageClass ? 'with' : 'without'} storage class`, + () => { + const replicationMD = { status: 'PENDING', + backends: [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], + content: ['DATA', 'METADATA'], + destination: bucketARN, + storageClass: 'zenko', + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + storageType: '', dataStoreVersionId: '', - }], - content: ['DATA', 'METADATA'], - destination: bucketARN, - storageClass: 'zenko', - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', - storageType: '', - dataStoreVersionId: '', - isNFS: undefined, - }; - const newReplicationMD = hasStorageClass ? Object.assign(replicationMD, - { storageClass: storageClassType }) : replicationMD; - const replicateMetadataOnly = Object.assign({}, newReplicationMD, - { content: ['METADATA'] }); - - beforeEach(() => { - cleanup(); - createBucketWithReplication(hasStorageClass); - }); - - afterEach(() => { - cleanup(); - delete config.locationConstraints['zenko']; - }); - - it('should update metadata when replication config prefix matches ' + - 'an object key', done => - putObjectAndCheckMD(keyA, newReplicationMD, done)); - - it('should update metadata when replication config prefix matches ' + - 'the start of an object key', done => - putObjectAndCheckMD(`${keyA}abc`, newReplicationMD, done)); - - it('should not update metadata when replication config prefix does ' + - 'not match the start of an object key', done => - putObjectAndCheckMD(`abc${keyA}`, emptyReplicationMD, done)); - - it('should not update metadata when replication config prefix does ' + - 'not apply', done => - putObjectAndCheckMD(keyB, emptyReplicationMD, done)); - - it("should update status to 'PENDING' if putting a new version", done => - putObjectAndCheckMD(keyA, newReplicationMD, err => { - if (err) { - return done(err); - } - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - // Update metadata to a status after replication has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - return putObjectAndCheckMD(keyA, newReplicationMD, done); - })); - - it("should update status to 'PENDING' and content to '['METADATA']' " + - 'if putting 0 byte object', done => - objectPut(authInfo, getObjectPutReq(keyA, false), undefined, log, - err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - })); - - it('should update metadata if putting object ACL and CRR replication', done => { - // Set 'zenko' as a typical CRR location (i.e. no type) - config.locationConstraints['zenko'] = { - ...config.locationConstraints['zenko'], - type: '', + isNFS: undefined, }; - - async.series([ - next => putObjectAndCheckMD(keyA, newReplicationMD, next), - next => { - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - // Update metadata to a status after replication has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - objectPutACL(authInfo, objectACLReq, log, next); - }, - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); + const newReplicationMD = hasStorageClass + ? Object.assign(replicationMD, { storageClass: storageClassType }) + : replicationMD; + const replicateMetadataOnly = Object.assign({}, newReplicationMD, { content: ['METADATA'] }); + + beforeEach(() => { + cleanup(); + createBucketWithReplication(hasStorageClass); }); - }); - - it('should not update metadata if putting object ACL and cloud replication', done => { - // Set 'zenko' as a typical cloud location (i.e. type) - config.locationConstraints['zenko'] = { - ...config.locationConstraints['zenko'], - type: 'aws_s3', - }; - const replicationMD = { ...newReplicationMD, storageType: 'aws_s3' }; - - let completedReplicationInfo; - async.series([ - next => putObjectAndCheckMD(keyA, replicationMD, next), - next => { - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - // Update metadata to a status after replication has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - completedReplicationInfo = JSON.parse( - JSON.stringify(objectMD.replicationInfo)); - objectPutACL(authInfo, objectACLReq, log, next); - }, - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, completedReplicationInfo); - return done(); + afterEach(() => { + cleanup(); + delete config.locationConstraints['zenko']; }); - }); - - it('should update metadata if putting a delete marker', done => - async.series([ - next => putObjectAndCheckMD(keyA, newReplicationMD, err => { - if (err) { - return next(err); - } - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - // Set metadata to a status after replication has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - return next(); - }), - next => objectDelete(authInfo, deleteReq, log, next), - ], err => { - if (err) { - return done(err); - } - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - assert.strictEqual(objectMD.isDeleteMarker, true); - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - })); - it('should not update metadata if putting a delete marker owned by ' + - 'Lifecycle service account', done => - async.series([ - next => putObjectAndCheckMD(keyA, newReplicationMD, next), - next => objectDelete(authInfoLifecycleService, deleteReq, - log, next), - ], err => { - if (err) { - return done(err); - } - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - assert.strictEqual(objectMD.isDeleteMarker, true); - checkObjectReplicationInfo(keyA, emptyReplicationMD); - return done(); - })); + it('should update metadata when replication config prefix matches ' + 'an object key', done => + putObjectAndCheckMD(keyA, newReplicationMD, done), + ); - describe('Object tagging', () => { - beforeEach(done => async.series([ - next => putObjectAndCheckMD(keyA, newReplicationMD, next), - next => objectPutTagging(authInfo, taggingPutReq, log, next), - ], err => done(err))); + it('should update metadata when replication config prefix matches ' + 'the start of an object key', done => + putObjectAndCheckMD(`${keyA}abc`, newReplicationMD, done), + ); - it("should update status to 'PENDING' and content to " + - "'['METADATA']'if putting tag", done => { - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - }); + it( + 'should not update metadata when replication config prefix does ' + + 'not match the start of an object key', + done => putObjectAndCheckMD(`abc${keyA}`, emptyReplicationMD, done), + ); - it("should update status to 'PENDING' and content to " + - "'['METADATA']' if deleting tag", done => - async.series([ - // Put a new version to update replication MD content array. - next => putObjectAndCheckMD(keyA, newReplicationMD, next), - next => objectDeleteTagging(authInfo, taggingDeleteReq, log, - next), - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - })); - }); + it('should not update metadata when replication config prefix does ' + 'not apply', done => + putObjectAndCheckMD(keyB, emptyReplicationMD, done), + ); - describe('Complete MPU', () => { - it("should update status to 'PENDING' and content to " + - "'['DATA, METADATA']' if completing MPU", done => - putMPU(keyA, 'content', err => { + it("should update status to 'PENDING' if putting a new version", done => + putObjectAndCheckMD(keyA, newReplicationMD, err => { if (err) { return done(err); } - checkObjectReplicationInfo(keyA, newReplicationMD); - return done(); + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Update metadata to a status after replication has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + return putObjectAndCheckMD(keyA, newReplicationMD, done); })); - it("should update status to 'PENDING' and content to " + - "'['METADATA']' if completing MPU with 0 bytes", done => - putMPU(keyA, '', err => { + it("should update status to 'PENDING' and content to '['METADATA']' " + 'if putting 0 byte object', done => + objectPut(authInfo, getObjectPutReq(keyA, false), undefined, log, err => { if (err) { return done(err); } checkObjectReplicationInfo(keyA, replicateMetadataOnly); return done(); - })); - - it('should not update replicationInfo if key does not apply', - done => putMPU(keyB, 'content', err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyB, emptyReplicationMD); - return done(); - })); - }); - - describe('Object copy', () => { - it("should update status to 'PENDING' and content to " + - "'['DATA, METADATA']' if copying object", done => - copyObject(keyB, keyA, true, err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, newReplicationMD); - return done(); - })); + }), + ); - it("should update status to 'PENDING' and content to " + - "'['METADATA']' if copying object with 0 bytes", done => - copyObject(keyB, keyA, false, err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - })); + it('should update metadata if putting object ACL and CRR replication', done => { + // Set 'zenko' as a typical CRR location (i.e. no type) + config.locationConstraints['zenko'] = { + ...config.locationConstraints['zenko'], + type: '', + }; - it('should not update replicationInfo if key does not apply', - done => { - const copyKey = `foo-${keyA}`; - return copyObject(keyB, copyKey, true, err => { + async.series( + [ + next => putObjectAndCheckMD(keyA, newReplicationMD, next), + next => { + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Update metadata to a status after replication has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + objectPutACL(authInfo, objectACLReq, log, next); + }, + ], + err => { if (err) { return done(err); } - checkObjectReplicationInfo(copyKey, emptyReplicationMD); + checkObjectReplicationInfo(keyA, replicateMetadataOnly); return done(); - }); - }); - }); + }, + ); + }); - ['awsbackend', - 'azurebackend', - 'gcpbackend', - 'awsbackend,azurebackend'].forEach(backend => { - const storageTypeMap = { - 'awsbackend': 'aws_s3', - 'azurebackend': 'azure', - 'gcpbackend': 'gcp', - 'awsbackend,azurebackend': 'aws_s3,azure', - }; - const storageType = storageTypeMap[backend]; - const backends = backend.split(',').map(site => ({ - site, - status: 'PENDING', - dataStoreVersionId: '', - })); - describe('Object metadata replicationInfo storageType value', - () => { - const expectedReplicationInfo = { - status: 'PENDING', - backends, - content: ['DATA', 'METADATA'], - destination: 'arn:aws:s3:::destination-bucket', - storageClass: backend, - role: 'arn:aws:iam::account-id:role/resource', - storageType, - dataStoreVersionId: '', - isNFS: undefined, + it('should not update metadata if putting object ACL and cloud replication', done => { + // Set 'zenko' as a typical cloud location (i.e. type) + config.locationConstraints['zenko'] = { + ...config.locationConstraints['zenko'], + type: 'aws_s3', }; - // Expected for a metadata-only replication operation (for - // example, putting object tags). - const expectedReplicationInfoMD = Object.assign({}, - expectedReplicationInfo, { content: ['METADATA'] }); - - beforeEach(() => - // We have already created the bucket, so update the - // replication configuration to include a location - // constraint for the `storageClass`. This results in a - // `storageType` of 'aws_s3', for example. - Object.assign(metadata.buckets.get(bucketName), { - _replicationConfiguration: { - role: 'arn:aws:iam::account-id:role/resource', - destination: 'arn:aws:s3:::destination-bucket', - rules: [{ - prefix: keyA, - enabled: true, - id: 'test-id', - storageClass: backend, - }], - }, - })); + const replicationMD = { ...newReplicationMD, storageType: 'aws_s3' }; - it('should update on a put object request', done => - putObjectAndCheckMD(keyA, expectedReplicationInfo, done)); - - it('should update on a complete MPU object request', done => - putMPU(keyA, 'content', err => { + let completedReplicationInfo; + async.series( + [ + next => putObjectAndCheckMD(keyA, replicationMD, next), + next => { + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Update metadata to a status after replication has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + completedReplicationInfo = JSON.parse(JSON.stringify(objectMD.replicationInfo)); + objectPutACL(authInfo, objectACLReq, log, next); + }, + ], + err => { if (err) { return done(err); } - const expected = - Object.assign({}, expectedReplicationInfo, - { content: ['DATA', 'METADATA', 'MPU'] }); - checkObjectReplicationInfo(keyA, expected); + checkObjectReplicationInfo(keyA, completedReplicationInfo); return done(); - })); + }, + ); + }); - it('should update on a copy object request', done => - copyObject(keyB, keyA, true, err => { + it('should update metadata if putting a delete marker', done => + async.series( + [ + next => + putObjectAndCheckMD(keyA, newReplicationMD, err => { + if (err) { + return next(err); + } + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Set metadata to a status after replication has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + return next(); + }), + next => objectDelete(authInfo, deleteReq, log, next), + ], + err => { if (err) { return done(err); } - checkObjectReplicationInfo(keyA, - expectedReplicationInfo); + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + assert.strictEqual(objectMD.isDeleteMarker, true); + checkObjectReplicationInfo(keyA, replicateMetadataOnly); return done(); - })); - - it('should update on a put object ACL request', done => { - let completedReplicationInfo; - async.series([ - next => putObjectAndCheckMD(keyA, - expectedReplicationInfo, next), - next => { - const objectMD = metadata.keyMaps - .get(bucketName).get(keyA); - // Update metadata to a status after replication - // has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - completedReplicationInfo = JSON.parse( - JSON.stringify(objectMD.replicationInfo)); - objectPutACL(authInfo, objectACLReq, log, next); - }, - ], err => { + }, + )); + + it('should not update metadata if putting a delete marker owned by ' + 'Lifecycle service account', done => + async.series( + [ + next => putObjectAndCheckMD(keyA, newReplicationMD, next), + next => objectDelete(authInfoLifecycleService, deleteReq, log, next), + ], + err => { if (err) { return done(err); } - checkObjectReplicationInfo(keyA, completedReplicationInfo); + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + assert.strictEqual(objectMD.isDeleteMarker, true); + checkObjectReplicationInfo(keyA, emptyReplicationMD); return done(); - }); + }, + ), + ); + + describe('Object tagging', () => { + beforeEach(done => + async.series( + [ + next => putObjectAndCheckMD(keyA, newReplicationMD, next), + next => objectPutTagging(authInfo, taggingPutReq, log, next), + ], + err => done(err), + ), + ); + + it("should update status to 'PENDING' and content to " + "'['METADATA']'if putting tag", done => { + checkObjectReplicationInfo(keyA, replicateMetadataOnly); + return done(); }); - it('should update on a put object tagging request', done => - async.series([ - next => putObjectAndCheckMD(keyA, - expectedReplicationInfo, next), - next => objectPutTagging(authInfo, taggingPutReq, log, - next), - ], err => { + it("should update status to 'PENDING' and content to " + "'['METADATA']' if deleting tag", done => + async.series( + [ + // Put a new version to update replication MD content array. + next => putObjectAndCheckMD(keyA, newReplicationMD, next), + next => objectDeleteTagging(authInfo, taggingDeleteReq, log, next), + ], + err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, replicateMetadataOnly); + return done(); + }, + ), + ); + }); + + describe('Complete MPU', () => { + it( + "should update status to 'PENDING' and content to " + "'['DATA, METADATA']' if completing MPU", + done => + putMPU(keyA, 'content', err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, newReplicationMD); + return done(); + }), + ); + + it( + "should update status to 'PENDING' and content to " + + "'['METADATA']' if completing MPU with 0 bytes", + done => + putMPU(keyA, '', err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, replicateMetadataOnly); + return done(); + }), + ); + + it('should not update replicationInfo if key does not apply', done => + putMPU(keyB, 'content', err => { if (err) { return done(err); } - const expected = Object.assign({}, - expectedReplicationInfo, - { content: ['METADATA', 'PUT_TAGGING'] }); - checkObjectReplicationInfo(keyA, expected); + checkObjectReplicationInfo(keyB, emptyReplicationMD); return done(); })); + }); - it('should update on a delete tagging request', done => - async.series([ - next => putObjectAndCheckMD(keyA, - expectedReplicationInfo, next), - next => objectDeleteTagging(authInfo, taggingDeleteReq, - log, next), - ], err => { + describe('Object copy', () => { + it( + "should update status to 'PENDING' and content to " + "'['DATA, METADATA']' if copying object", + done => + copyObject(keyB, keyA, true, err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, newReplicationMD); + return done(); + }), + ); + + it( + "should update status to 'PENDING' and content to " + + "'['METADATA']' if copying object with 0 bytes", + done => + copyObject(keyB, keyA, false, err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, replicateMetadataOnly); + return done(); + }), + ); + + it('should not update replicationInfo if key does not apply', done => { + const copyKey = `foo-${keyA}`; + return copyObject(keyB, copyKey, true, err => { if (err) { return done(err); } - const expected = Object.assign({}, - expectedReplicationInfo, - { content: ['METADATA', 'DELETE_TAGGING'] }); - checkObjectReplicationInfo(keyA, expected); + checkObjectReplicationInfo(copyKey, emptyReplicationMD); return done(); - })); + }); + }); + }); - it('should update when putting a delete marker', done => - async.series([ - next => putObjectAndCheckMD(keyA, - expectedReplicationInfo, err => { + ['awsbackend', 'azurebackend', 'gcpbackend', 'awsbackend,azurebackend'].forEach(backend => { + const storageTypeMap = { + awsbackend: 'aws_s3', + azurebackend: 'azure', + gcpbackend: 'gcp', + 'awsbackend,azurebackend': 'aws_s3,azure', + }; + const storageType = storageTypeMap[backend]; + const backends = backend.split(',').map(site => ({ + site, + status: 'PENDING', + dataStoreVersionId: '', + })); + describe('Object metadata replicationInfo storageType value', () => { + const expectedReplicationInfo = { + status: 'PENDING', + backends, + content: ['DATA', 'METADATA'], + destination: 'arn:aws:s3:::destination-bucket', + storageClass: backend, + role: 'arn:aws:iam::account-id:role/resource', + storageType, + dataStoreVersionId: '', + isNFS: undefined, + }; + + // Expected for a metadata-only replication operation (for + // example, putting object tags). + const expectedReplicationInfoMD = Object.assign({}, expectedReplicationInfo, { + content: ['METADATA'], + }); + + beforeEach(() => + // We have already created the bucket, so update the + // replication configuration to include a location + // constraint for the `storageClass`. This results in a + // `storageType` of 'aws_s3', for example. + Object.assign(metadata.buckets.get(bucketName), { + _replicationConfiguration: { + role: 'arn:aws:iam::account-id:role/resource', + destination: 'arn:aws:s3:::destination-bucket', + rules: [ + { + prefix: keyA, + enabled: true, + id: 'test-id', + storageClass: backend, + }, + ], + }, + }), + ); + + it('should update on a put object request', done => + putObjectAndCheckMD(keyA, expectedReplicationInfo, done)); + + it('should update on a complete MPU object request', done => + putMPU(keyA, 'content', err => { + if (err) { + return done(err); + } + const expected = Object.assign({}, expectedReplicationInfo, { + content: ['DATA', 'METADATA', 'MPU'], + }); + checkObjectReplicationInfo(keyA, expected); + return done(); + })); + + it('should update on a copy object request', done => + copyObject(keyB, keyA, true, err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, expectedReplicationInfo); + return done(); + })); + + it('should update on a put object ACL request', done => { + let completedReplicationInfo; + async.series( + [ + next => putObjectAndCheckMD(keyA, expectedReplicationInfo, next), + next => { + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Update metadata to a status after replication + // has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + completedReplicationInfo = JSON.parse(JSON.stringify(objectMD.replicationInfo)); + objectPutACL(authInfo, objectACLReq, log, next); + }, + ], + err => { if (err) { - return next(err); + return done(err); } - // Update metadata to a status indicating that - // replication has occurred for the object. - metadata - .keyMaps - .get(bucketName) - .get(keyA) - .replicationInfo - .status = 'COMPLETED'; - return next(); - }), - next => objectDelete(authInfo, deleteReq, log, next), - ], err => { - if (err) { - return done(err); - } - // Is it, in fact, a delete marker? - assert(metadata - .keyMaps - .get(bucketName) - .get(keyA) - .isDeleteMarker); - checkObjectReplicationInfo(keyA, - expectedReplicationInfoMD); - return done(); - })); + checkObjectReplicationInfo(keyA, completedReplicationInfo); + return done(); + }, + ); + }); + + it('should update on a put object tagging request', done => + async.series( + [ + next => putObjectAndCheckMD(keyA, expectedReplicationInfo, next), + next => objectPutTagging(authInfo, taggingPutReq, log, next), + ], + err => { + if (err) { + return done(err); + } + const expected = Object.assign({}, expectedReplicationInfo, { + content: ['METADATA', 'PUT_TAGGING'], + }); + checkObjectReplicationInfo(keyA, expected); + return done(); + }, + )); + + it('should update on a delete tagging request', done => + async.series( + [ + next => putObjectAndCheckMD(keyA, expectedReplicationInfo, next), + next => objectDeleteTagging(authInfo, taggingDeleteReq, log, next), + ], + err => { + if (err) { + return done(err); + } + const expected = Object.assign({}, expectedReplicationInfo, { + content: ['METADATA', 'DELETE_TAGGING'], + }); + checkObjectReplicationInfo(keyA, expected); + return done(); + }, + )); + + it('should update when putting a delete marker', done => + async.series( + [ + next => + putObjectAndCheckMD(keyA, expectedReplicationInfo, err => { + if (err) { + return next(err); + } + // Update metadata to a status indicating that + // replication has occurred for the object. + metadata.keyMaps.get(bucketName).get(keyA).replicationInfo.status = 'COMPLETED'; + return next(); + }), + next => objectDelete(authInfo, deleteReq, log, next), + ], + err => { + if (err) { + return done(err); + } + // Is it, in fact, a delete marker? + assert(metadata.keyMaps.get(bucketName).get(keyA).isDeleteMarker); + checkObjectReplicationInfo(keyA, expectedReplicationInfoMD); + return done(); + }, + )); + }); }); - }); - }); + }, + ); }); From 8ed4687ffa69354c75b2fed330c21b5af09802c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Fri, 5 Jun 2026 18:10:08 +0200 Subject: [PATCH 2/3] Build multi-destination replicationInfo per backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match all enabled rules with the object's prefix (highest-priority wins on site collision) and stamp destination/role per CRR backend. Cloud backends get a bare entry — their bucket and credentials live in the location config. ACL replication uses per-backend `role` as the CRR marker so cloud backends keep their status. Legacy V1 and comma-separated `StorageClass` remain supported on read. Issue: CLDSRV-888 --- lib/api/apiUtils/object/getReplicationInfo.js | 163 ++-- lib/metadata/acl.js | 27 +- tests/unit/api/apiUtils/getReplicationInfo.js | 824 +++++++++++------- tests/unit/api/objectReplicationMD.js | 203 ++++- 4 files changed, 783 insertions(+), 434 deletions(-) diff --git a/lib/api/apiUtils/object/getReplicationInfo.js b/lib/api/apiUtils/object/getReplicationInfo.js index f9f8b92403..4349483abe 100644 --- a/lib/api/apiUtils/object/getReplicationInfo.js +++ b/lib/api/apiUtils/object/getReplicationInfo.js @@ -1,68 +1,61 @@ const { isServiceAccount, getServiceAccountProperties } = require('../authorization/permissionChecks'); -const { replicationBackends } = require('arsenal').constants; +const { constants, models } = require('arsenal'); -function _getBackend(objectMD, site) { - const backends = objectMD ? objectMD.replicationInfo.backends : []; - const backend = backends.find(o => o.site === site); - // If the backend already exists, just update the status. - if (backend) { - return Object.assign({}, backend, { status: 'PENDING' }); - } - return { - site, - status: 'PENDING', - dataStoreVersionId: '', - }; -} +const { replicationBackends } = constants; +const { ReplicationConfiguration } = models; -function _getStorageClasses(s3config, rule) { - if (rule.storageClass) { - return rule.storageClass.split(','); - } - const { replicationEndpoints } = s3config; - // If no storage class, use the given default endpoint or the sole endpoint - if (replicationEndpoints.length > 0) { - const endPoint = replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0]; - return [endPoint.site]; - } - return undefined; +/** + * Apply the default replication endpoint as a fallback storageClass + * for rules that don't specify one. Returns a new rules array; rules + * without a resolvable storageClass are dropped so they never reach + * the backend resolver. + */ +function _withDefaultStorageClass(rules, s3config) { + const { replicationEndpoints = [] } = s3config; + const fallback = replicationEndpoints.find(e => e.default)?.site ?? replicationEndpoints[0]?.site; + return rules + .map(rule => { + if (rule.storageClass) { + return rule; + } + if (!fallback) { + return null; + } + return { ...rule, storageClass: fallback }; + }) + .filter(Boolean); } -function _getReplicationInfo(s3config, rule, replicationConfig, content, operationType, objectMD, bucketMD) { - const storageTypes = []; - const backends = []; - const storageClasses = _getStorageClasses(s3config, rule); - if (!storageClasses) { - return undefined; +/** + * Check whether the authenticated user is allowed to trigger replication. + * Internal service accounts (e.g. Lifecycle) are not allowed unless their + * account properties explicitly permit it (e.g. MD ingestion). + * @param {AuthInfo} [authInfo] - authentication info of the request issuer + * @return {boolean} true if the user can trigger replication + */ +function _canUserReplicate(authInfo) { + if (!authInfo) { + return true; } - storageClasses.forEach(storageClass => { - const storageClassName = storageClass.endsWith(':preferred_read') ? storageClass.split(':')[0] : storageClass; - // TODO CLDSRV-646: for consistency, should we look at replicationEndpoints instead, like - // `_getStorageClasses()` ? - const location = s3config.locationConstraints[storageClassName]; - if (location && replicationBackends[location.type]) { - storageTypes.push(location.type); - } - backends.push(_getBackend(objectMD, storageClassName)); - }); - if (storageTypes.length > 0 && operationType) { - content.push(operationType); + const canonicalId = authInfo.getCanonicalID(); + if (!isServiceAccount(canonicalId)) { + return true; } - return { - status: 'PENDING', - backends, - content, - destination: replicationConfig.destination, - storageClass: storageClasses.join(','), - role: replicationConfig.role, - storageType: storageTypes.join(','), - isNFS: bucketMD.isNFS(), - }; + const props = getServiceAccountProperties(canonicalId); + return !!props?.canReplicate; } /** * Get the object replicationInfo to replicate data and metadata, or only - * metadata if the operation only changes metadata or the object is 0 bytes + * metadata if the operation only changes metadata or the object is 0 bytes. + * + * The rule-matching / dedup / per-backend stamping logic lives in + * arsenal's `ReplicationConfiguration.resolveBackends`. This function + * is the cloudserver-specific shell: it enforces the service-account + * gate, supplies a default storageClass from `replicationEndpoints`, + * decides the `content` array based on the operation kind, and + * stitches the result into a `replicationInfo` envelope. + * * @param {object} s3config - Cloudserver configuration object * @param {object} s3config.locationConstraints - Configured map of location constraints * @param {object[]} s3config.replicationEndpoints - Configured replication endpoints @@ -73,45 +66,41 @@ function _getReplicationInfo(s3config, rule, replicationConfig, content, operati * @param {string} operationType - The type of operation to replicate * @param {object} objectMD - The object metadata * @param {AuthInfo} [authInfo] - authentication info of object owner - * @return {undefined} + * @return {object|undefined} */ function getReplicationInfo(s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) { - const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA']; const config = bucketMD.getReplicationConfiguration(); + if (!config || !_canUserReplicate(authInfo)) { + return undefined; + } - // Do not replicate object in the following cases: - // - // - bucket does not have a replication configuration - // - // - replication configuration does not apply to the object - // (i.e. no rule matches object prefix) - // - // - replication configuration applies to the object (i.e. a rule matches - // object prefix) but the status is disabled - // - // - object owner is an internal service account like Lifecycle, - // unless the account properties explicitly allow it to - // replicate like MD ingestion (because we do not want to - // replicate objects created from actions triggered by internal - // services, by design) + const isCloud = site => !!replicationBackends[s3config.locationConstraints[site]?.type]; + const rules = _withDefaultStorageClass(config.rules || [], s3config); + const backends = ReplicationConfiguration.resolveBackends( + { ...config, rules }, + objKey, + isCloud, + objectMD?.replicationInfo?.backends, + ); - if (config) { - let doReplicate = false; - if (!authInfo || !isServiceAccount(authInfo.getCanonicalID())) { - doReplicate = true; - } else { - const serviceAccountProps = getServiceAccountProperties(authInfo.getCanonicalID()); - doReplicate = serviceAccountProps.canReplicate; - } - if (doReplicate) { - const rule = config.rules.find(rule => objKey.startsWith(rule.prefix) && rule.enabled); - if (rule) { - // TODO CLDSRV-646 : should "merge" the replicationInfo for different rules - return _getReplicationInfo(s3config, rule, config, content, operationType, objectMD, bucketMD); - } - } + if (backends.length === 0) { + return undefined; } - return undefined; + + const hasCloudBackend = backends.some(b => isCloud(b.site)); + + const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA']; + if (hasCloudBackend && operationType) { + content.push(operationType); + } + + return { + status: 'PENDING', + backends, + content, + role: ReplicationConfiguration.resolveSourceRole(config.role), + isNFS: bucketMD.isNFS(), + }; } module.exports = getReplicationInfo; diff --git a/lib/metadata/acl.js b/lib/metadata/acl.js index 961b95d79f..f801816fdb 100644 --- a/lib/metadata/acl.js +++ b/lib/metadata/acl.js @@ -46,16 +46,31 @@ const acl = { objectMD.acl = addACLParams; objectMD.originOp = 's3:ObjectAcl:Put'; - // Use storageType to determine if replication update is needed, as it is set only for - // "cloud" locations. This ensures that we reset replication when CRR is used, but not - // when multi-backend replication (i.e. Zenko) is used. - // TODO: this should be refactored to properly update the replication info, accounting - // for multiple rules and resetting the status only if needed CLDSRV-646 + // Rebuild replication info from the current bucket config to + // pick up any new destinations. A "role" on the entry means + // backbeat uses IAM role-based auth on the destination + // (CRR). No role means credentials live in the location + // configuration (cloud, including scality-to-scality via + // stored S3 creds), and ACL replication is not supported — + // preserve the existing status instead of resetting to + // PENDING. + const hasDestRole = b => !!b.role; + const replicationInfo = getReplicationInfo(config, objectKey, bucket, true); - if (replicationInfo && !replicationInfo.storageType) { + if (replicationInfo && replicationInfo.backends.some(hasDestRole)) { + const backends = replicationInfo.backends.map(b => { + if (hasDestRole(b)) { + return b; + } + + const existing = objectMD.replicationInfo.backends.find(e => e.site === b.site); + return existing || b; + }); + objectMD.replicationInfo = { ...objectMD.replicationInfo, ...replicationInfo, + backends, }; } diff --git a/tests/unit/api/apiUtils/getReplicationInfo.js b/tests/unit/api/apiUtils/getReplicationInfo.js index 9134905bb4..07c4d680dc 100644 --- a/tests/unit/api/apiUtils/getReplicationInfo.js +++ b/tests/unit/api/apiUtils/getReplicationInfo.js @@ -4,7 +4,7 @@ const BucketInfo = require('arsenal').models.BucketInfo; const AuthInfo = require('arsenal').auth.AuthInfo; const getReplicationInfo = require('../../../../lib/api/apiUtils/object/getReplicationInfo'); -function _getObjectReplicationInfo(s3config, replicationConfig) { +function _getObjectReplicationInfo(s3config, replicationConfig, key, objectMD) { const bucketInfo = new BucketInfo( 'testbucket', 'someCanonicalId', @@ -21,7 +21,7 @@ function _getObjectReplicationInfo(s3config, replicationConfig) { null, replicationConfig, ); - return getReplicationInfo(s3config, 'fookey', bucketInfo, true, 123, null, null); + return getReplicationInfo(s3config, key || 'fookey', bucketInfo, true, 123, null, objectMD || null); } const TEST_CONFIG = { @@ -49,6 +49,9 @@ const TEST_CONFIG = { azureContainerName: 's3test', }, }, + 'crr-site': { + objectId: 'crr-site', + }, }, replicationEndpoints: [ { @@ -63,326 +66,541 @@ const TEST_CONFIG = { ], }; +const TWO_PART_ROLE = 'arn:aws:iam::root:role/src-role,arn:aws:iam::root:role/dst-role'; + describe('getReplicationInfo helper', () => { - it('should get replication info when rules are enabled', () => { - const replicationConfig = { - role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [ - { - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }, - ], - destination: 'tosomewhere', - }; - const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); - assert.deepStrictEqual(replicationInfo, { - status: 'PENDING', - backends: [ - { - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA'], - destination: 'tosomewhere', - storageClass: 'awsbackend', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'aws_s3', - isNFS: undefined, + describe('V1 format (single rule match)', () => { + it('should get replication info when rules are enabled', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], + destination: 'tosomewhere', + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); + assert.deepStrictEqual(replicationInfo, { + status: 'PENDING', + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], + content: ['METADATA'], + role: 'arn:aws:iam::root:role/src-role', + isNFS: undefined, + }); }); - }); - it('should not get replication info when rules are disabled', () => { - const replicationConfig = { - role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [ - { - prefix: '', - enabled: false, - storageClass: 'awsbackend', - }, - ], - destination: 'tosomewhere', - }; - const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); - assert.deepStrictEqual(replicationInfo, undefined); - }); + it('should not get replication info when rules are disabled', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [ + { + prefix: '', + enabled: false, + storageClass: 'awsbackend', + }, + ], + destination: 'tosomewhere', + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); + assert.deepStrictEqual(replicationInfo, undefined); + }); - it('should get replication info with single cloud target', () => { - const replicationConfig = { - role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [ - { - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }, - ], - destination: 'tosomewhere', - }; - const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); - assert.deepStrictEqual(replicationInfo, { - status: 'PENDING', - backends: [ - { - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA'], - destination: 'tosomewhere', - storageClass: 'awsbackend', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'aws_s3', - isNFS: undefined, + it('should match all V1 rules with overlapping prefixes', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + { + prefix: '', + enabled: true, + storageClass: 'azurebackend', + }, + ], + destination: 'tosomewhere', + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); + assert.strictEqual(replicationInfo.backends.length, 2); + assert.deepStrictEqual(replicationInfo.backends.map(b => b.site).sort(), ['awsbackend', 'azurebackend']); }); - }); - it('should get replication info with multiple cloud targets', () => { - const replicationConfig = { - role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [ - { - prefix: '', - enabled: true, - storageClass: 'awsbackend,azurebackend', - }, - ], - destination: 'tosomewhere', - }; - const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); - assert.deepStrictEqual(replicationInfo, { - status: 'PENDING', - backends: [ - { - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }, - { - site: 'azurebackend', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA'], - destination: 'tosomewhere', - storageClass: 'awsbackend,azurebackend', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'aws_s3,azure', - isNFS: undefined, + it('should get replication info with multiple cloud targets (legacy comma-separated)', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend,azurebackend', + }, + ], + destination: 'tosomewhere', + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); + assert.deepStrictEqual(replicationInfo, { + status: 'PENDING', + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + { + site: 'azurebackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], + content: ['METADATA'], + role: 'arn:aws:iam::root:role/src-role', + isNFS: undefined, + }); }); - }); - it('should get replication info with multiple cloud targets and ' + 'preferred read location', () => { - const replicationConfig = { - role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [ - { - prefix: '', - enabled: true, - storageClass: 'awsbackend:preferred_read,azurebackend', - }, - ], - destination: 'tosomewhere', - preferredReadLocation: 'awsbackend', - }; - const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); - assert.deepStrictEqual(replicationInfo, { - status: 'PENDING', - backends: [ - { - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }, - { - site: 'azurebackend', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA'], - destination: 'tosomewhere', - storageClass: 'awsbackend:preferred_read,azurebackend', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'aws_s3,azure', - isNFS: undefined, + it('should get replication info with multiple cloud targets and ' + 'preferred read location', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend:preferred_read,azurebackend', + }, + ], + destination: 'tosomewhere', + preferredReadLocation: 'awsbackend', + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); + assert.deepStrictEqual(replicationInfo, { + status: 'PENDING', + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + { + site: 'azurebackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], + content: ['METADATA'], + role: 'arn:aws:iam::root:role/src-role', + isNFS: undefined, + }); }); - }); - it('should not get replication info when service account type ' + 'cannot trigger replication', () => { - const replicationConfig = { - role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [ - { - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }, - ], - destination: 'tosomewhere', - }; - const bucketInfo = new BucketInfo( - 'testbucket', - 'abcdef/lifecycle', - 'Lifecycle Service Account', - new Date().toJSON(), - null, - null, - null, - null, - null, - null, - null, - null, - null, - replicationConfig, - ); - const authInfo = new AuthInfo({ - canonicalID: 'abcdef/lifecycle', - accountDisplayName: 'Lifecycle Service Account', + it('should not get replication info when service account type ' + 'cannot trigger replication', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], + destination: 'tosomewhere', + }; + const bucketInfo = new BucketInfo( + 'testbucket', + 'abcdef/lifecycle', + 'Lifecycle Service Account', + new Date().toJSON(), + null, + null, + null, + null, + null, + null, + null, + null, + null, + replicationConfig, + ); + const authInfo = new AuthInfo({ + canonicalID: 'abcdef/lifecycle', + accountDisplayName: 'Lifecycle Service Account', + }); + const replicationInfo = getReplicationInfo( + TEST_CONFIG, + 'fookey', + bucketInfo, + true, + 123, + null, + null, + authInfo, + ); + assert.deepStrictEqual(replicationInfo, undefined); }); - const replicationInfo = getReplicationInfo(TEST_CONFIG, 'fookey', bucketInfo, true, 123, null, null, authInfo); - assert.deepStrictEqual(replicationInfo, undefined); - }); - it('should get replication info when service account type can ' + 'trigger replication', () => { - const replicationConfig = { - role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [ - { - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }, - ], - destination: 'tosomewhere', - }; - const bucketInfo = new BucketInfo( - 'testbucket', - 'abcdef/md-ingestion', - 'Metadata Ingestion Service Account', - new Date().toJSON(), - null, - null, - null, - null, - null, - null, - null, - null, - null, - replicationConfig, - ); - const authInfo = new AuthInfo({ - canonicalID: 'abcdef/md-ingestion', - accountDisplayName: 'Metadata Ingestion Service Account', + it('should get replication info when service account type can ' + 'trigger replication', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], + destination: 'tosomewhere', + }; + const bucketInfo = new BucketInfo( + 'testbucket', + 'abcdef/md-ingestion', + 'Metadata Ingestion Service Account', + new Date().toJSON(), + null, + null, + null, + null, + null, + null, + null, + null, + null, + replicationConfig, + ); + const authInfo = new AuthInfo({ + canonicalID: 'abcdef/md-ingestion', + accountDisplayName: 'Metadata Ingestion Service Account', + }); + const replicationInfo = getReplicationInfo( + TEST_CONFIG, + 'fookey', + bucketInfo, + true, + 123, + null, + null, + authInfo, + ); + assert.deepStrictEqual(replicationInfo, { + status: 'PENDING', + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], + content: ['METADATA'], + role: 'arn:aws:iam::root:role/src-role', + isNFS: undefined, + }); }); - const replicationInfo = getReplicationInfo(TEST_CONFIG, 'fookey', bucketInfo, true, 123, null, null, authInfo); - assert.deepStrictEqual(replicationInfo, { - status: 'PENDING', - backends: [ - { - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA'], - destination: 'tosomewhere', - storageClass: 'awsbackend', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'aws_s3', - isNFS: undefined, + + it('should fall back to default StorageClass and resolve as a CRR backend', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [ + { + prefix: '', + enabled: true, + }, + ], + destination: 'tosomewhere', + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); + assert.deepStrictEqual(replicationInfo, { + status: 'PENDING', + backends: [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: '', + destination: 'tosomewhere', + role: 'arn:aws:iam::root:role/dst-role', + }, + ], + content: ['METADATA'], + role: 'arn:aws:iam::root:role/src-role', + isNFS: undefined, + }); }); - }); - it('should get replication info with default StorageClass when rules are enabled', () => { - const replicationConfig = { - role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2', - rules: [ - { - prefix: '', - enabled: true, - }, - ], - destination: 'tosomewhere', - }; - const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig); - assert.deepStrictEqual(replicationInfo, { - status: 'PENDING', - backends: [ - { - site: 'zenko', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA'], - destination: 'tosomewhere', - storageClass: 'zenko', - role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2', - storageType: '', - isNFS: undefined, + it('should return replication info with cloud backend even when no replication endpoint is configured', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [ + { + prefix: '', + enabled: true, + storageClass: 'awsbackend', + }, + ], + destination: 'tosomewhere', + }; + const configWithNoReplicationEndpoint = { + locationConstraints: TEST_CONFIG.locationConstraints, + replicationEndpoints: [], + }; + const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, replicationConfig); + assert.deepStrictEqual(replicationInfo, { + status: 'PENDING', + backends: [ + { + site: 'awsbackend', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], + content: ['METADATA'], + role: 'arn:aws:iam::root:role/src-role', + isNFS: undefined, + }); }); - }); - it('should return undefined with specified StorageClass mode if no replication endpoint is configured', () => { - const replicationConfig = { - role: 'arn:aws:iam::root:role/s3-replication-role', - rules: [ - { - prefix: '', - enabled: true, - storageClass: 'awsbackend', - }, - ], - destination: 'tosomewhere', - }; - const configWithNoReplicationEndpoint = { - locationConstraints: TEST_CONFIG.locationConstraints, - replicationEndpoints: [], - }; - const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, replicationConfig); - assert.deepStrictEqual(replicationInfo, { - status: 'PENDING', - backends: [ - { - site: 'awsbackend', - status: 'PENDING', - dataStoreVersionId: '', - }, - ], - content: ['METADATA'], - destination: 'tosomewhere', - storageClass: 'awsbackend', - role: 'arn:aws:iam::root:role/s3-replication-role', - storageType: 'aws_s3', - isNFS: undefined, + it('should return undefined with default StorageClass if no replication endpoint is configured', () => { + const replicationConfig = { + role: TWO_PART_ROLE, + rules: [ + { + prefix: '', + enabled: true, + }, + ], + destination: 'tosomewhere', + }; + const configWithNoReplicationEndpoint = { + locationConstraints: TEST_CONFIG.locationConstraints, + replicationEndpoints: [], + }; + const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, replicationConfig); + assert.deepStrictEqual(replicationInfo, undefined); }); }); - it('should return undefined with default StorageClass if no replication endpoint is configured', () => { - const replicationConfig = { - role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2', - rules: [ - { - prefix: '', - enabled: true, - }, - ], - destination: 'tosomewhere', - }; - const configWithNoReplicationEndpoint = { - locationConstraints: TEST_CONFIG.locationConstraints, - replicationEndpoints: [], - }; - const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, replicationConfig); - assert.deepStrictEqual(replicationInfo, undefined); + // --- V2 Format Tests (multi-rule matching) --- + describe('V2 format (multi-rule matching)', () => { + const V2_ROLE = 'arn:aws:iam::123456:role/src-role,arn:aws:iam::111111:role/dst-role'; + + it('should match all rules with overlapping prefixes', () => { + const replicationConfig = { + role: V2_ROLE, + rules: [ + { + prefix: '', + enabled: true, + priority: 1, + storageClass: 'awsbackend', + destination: 'arn:aws:s3:::bucket-a', + }, + { + prefix: 'docs', + enabled: true, + priority: 2, + storageClass: 'azurebackend', + destination: 'arn:aws:s3:::bucket-b', + }, + ], + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig, 'docs/report.pdf'); + assert.strictEqual(replicationInfo.status, 'PENDING'); + assert.strictEqual(replicationInfo.backends.length, 2); + const awsBackend = replicationInfo.backends.find(b => b.site === 'awsbackend'); + const azureBackend = replicationInfo.backends.find(b => b.site === 'azurebackend'); + assert.ok(awsBackend); + assert.ok(azureBackend); + // Cloud backends do not carry per-backend destination/role + // or storageType (location type is resolved from config). + assert.strictEqual(awsBackend.destination, undefined); + assert.strictEqual(awsBackend.role, undefined); + assert.strictEqual(awsBackend.storageType, undefined); + assert.strictEqual(azureBackend.destination, undefined); + assert.strictEqual(azureBackend.role, undefined); + assert.strictEqual(azureBackend.storageType, undefined); + }); + + it('should only match rules whose prefix matches the object key', () => { + const replicationConfig = { + role: V2_ROLE, + rules: [ + { + prefix: '', + enabled: true, + priority: 1, + storageClass: 'awsbackend', + destination: 'arn:aws:s3:::bucket-a', + }, + { + prefix: 'logs', + enabled: true, + priority: 2, + storageClass: 'azurebackend', + destination: 'arn:aws:s3:::bucket-b', + }, + ], + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig, 'docs/report.pdf'); + assert.strictEqual(replicationInfo.backends.length, 1); + assert.strictEqual(replicationInfo.backends[0].site, 'awsbackend'); + }); + + it('should keep two CRR backends for same site but different destinations', () => { + const replicationConfig = { + role: V2_ROLE, + rules: [ + { + prefix: '', + enabled: true, + priority: 1, + storageClass: 'crr-site', + destination: 'arn:aws:s3:::bucket-a', + }, + { + prefix: 'docs', + enabled: true, + priority: 5, + storageClass: 'crr-site', + destination: 'arn:aws:s3:::bucket-b', + account: '222222', + }, + ], + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig, 'docs/report.pdf'); + assert.strictEqual(replicationInfo.backends.length, 2); + const buckets = replicationInfo.backends.map(b => b.destination).sort(); + assert.deepStrictEqual(buckets, ['arn:aws:s3:::bucket-a', 'arn:aws:s3:::bucket-b']); + }); + + it('should dedup CRR rules with same (site, destination, role)', () => { + const replicationConfig = { + role: V2_ROLE, + rules: [ + { + prefix: '', + enabled: true, + priority: 1, + storageClass: 'crr-site', + destination: 'arn:aws:s3:::bucket-a', + account: '222222', + }, + { + prefix: 'docs', + enabled: true, + priority: 5, + storageClass: 'crr-site', + destination: 'arn:aws:s3:::bucket-a', + account: '222222', + }, + ], + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig, 'docs/report.pdf'); + assert.strictEqual(replicationInfo.backends.length, 1); + assert.strictEqual(replicationInfo.backends[0].destination, 'arn:aws:s3:::bucket-a'); + assert.strictEqual(replicationInfo.backends[0].role, 'arn:aws:iam::222222:role/dst-role'); + }); + + it('should skip disabled rules', () => { + const replicationConfig = { + role: V2_ROLE, + rules: [ + { + prefix: '', + enabled: true, + priority: 1, + storageClass: 'awsbackend', + destination: 'arn:aws:s3:::bucket-a', + }, + { + prefix: '', + enabled: false, + priority: 2, + storageClass: 'azurebackend', + destination: 'arn:aws:s3:::bucket-b', + }, + ], + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig, 'docs/report.pdf'); + assert.strictEqual(replicationInfo.backends.length, 1); + assert.strictEqual(replicationInfo.backends[0].site, 'awsbackend'); + }); + + it('should return undefined when no V2 rules match', () => { + const replicationConfig = { + role: V2_ROLE, + rules: [ + { + prefix: 'logs/', + enabled: true, + priority: 1, + storageClass: 'awsbackend', + destination: 'arn:aws:s3:::bucket-a', + }, + ], + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig, 'docs/report.pdf'); + assert.strictEqual(replicationInfo, undefined); + }); + + it('should set top-level role to the source role only', () => { + const replicationConfig = { + role: V2_ROLE, + rules: [ + { + prefix: '', + enabled: true, + priority: 1, + storageClass: 'awsbackend', + destination: 'arn:aws:s3:::bucket-a', + }, + ], + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig, 'fookey'); + assert.strictEqual(replicationInfo.role, 'arn:aws:iam::123456:role/src-role'); + }); + + it('should handle mixed CRR and cloud backends', () => { + const replicationConfig = { + role: V2_ROLE, + rules: [ + { + prefix: '', + enabled: true, + priority: 1, + storageClass: 'crr-site', + destination: 'arn:aws:s3:::bucket-a', + account: '222222', + }, + { + prefix: '', + enabled: true, + priority: 2, + storageClass: 'awsbackend', + destination: 'arn:aws:s3:::bucket-b', + }, + ], + }; + const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig, 'fookey'); + assert.strictEqual(replicationInfo.backends.length, 2); + const crrBackend = replicationInfo.backends.find(b => b.site === 'crr-site'); + const cloudBackend = replicationInfo.backends.find(b => b.site === 'awsbackend'); + assert.ok(crrBackend); + assert.ok(cloudBackend); + // CRR backend has destination/role + assert.strictEqual(crrBackend.destination, 'arn:aws:s3:::bucket-a'); + assert.strictEqual(crrBackend.role, 'arn:aws:iam::222222:role/dst-role'); + assert.strictEqual(crrBackend.storageType, undefined); + // Cloud backend has neither (location type is resolved from config) + assert.strictEqual(cloudBackend.destination, undefined); + assert.strictEqual(cloudBackend.role, undefined); + assert.strictEqual(cloudBackend.storageType, undefined); + }); }); }); diff --git a/tests/unit/api/objectReplicationMD.js b/tests/unit/api/objectReplicationMD.js index fa228cf3cb..3977bb57f2 100644 --- a/tests/unit/api/objectReplicationMD.js +++ b/tests/unit/api/objectReplicationMD.js @@ -77,23 +77,16 @@ const emptyReplicationMD = { role: '', storageType: '', dataStoreVersionId: '', - isNFS: undefined, -}; -const expectedEmptyReplicationMD = { - status: '', - backends: [], - content: [], - destination: '', - storageClass: '', - role: '', - storageType: '', - dataStoreVersionId: '', }; // Check that the object key has the expected replication information. +// Normalizes via JSON round-trip to drop undefined-valued keys so that +// expectations don't need to know whether the MD path went through a +// metadata read (which JSON-serializes and drops undefined fields). function checkObjectReplicationInfo(key, expected) { const objectMD = metadata.keyMaps.get(bucketName).get(key); - assert.deepStrictEqual(objectMD.replicationInfo, expected); + const actual = JSON.parse(JSON.stringify(objectMD.replicationInfo)); + assert.deepStrictEqual(actual, expected); } // Put the object key and check the replication information. @@ -240,7 +233,7 @@ describe('Replication object MD without bucket replication config', () => { if (err) { return done(err); } - checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); + checkObjectReplicationInfo(keyA, emptyReplicationMD); return done(); }, )); @@ -257,7 +250,7 @@ describe('Replication object MD without bucket replication config', () => { ); it('should not update object metadata if putting tag', done => { - checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); + checkObjectReplicationInfo(keyA, emptyReplicationMD); return done(); }); @@ -272,7 +265,7 @@ describe('Replication object MD without bucket replication config', () => { if (err) { return done(err); } - checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); + checkObjectReplicationInfo(keyA, emptyReplicationMD); return done(); }, )); @@ -282,7 +275,7 @@ describe('Replication object MD without bucket replication config', () => { if (err) { return done(err); } - checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); + checkObjectReplicationInfo(keyA, emptyReplicationMD); return done(); })); @@ -309,19 +302,17 @@ describe('Replication object MD without bucket replication config', () => { site: 'zenko', status: 'PENDING', dataStoreVersionId: '', + destination: bucketARN, + role: 'arn:aws:iam::account-id:role/dest-resource', }, ], content: ['DATA', 'METADATA'], - destination: bucketARN, - storageClass: 'zenko', - role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + storageClass: '', + role: 'arn:aws:iam::account-id:role/src-resource', storageType: '', dataStoreVersionId: '', - isNFS: undefined, }; - const newReplicationMD = hasStorageClass - ? Object.assign(replicationMD, { storageClass: storageClassType }) - : replicationMD; + const newReplicationMD = replicationMD; const replicateMetadataOnly = Object.assign({}, newReplicationMD, { content: ['METADATA'] }); beforeEach(() => { @@ -407,7 +398,16 @@ describe('Replication object MD without bucket replication config', () => { type: 'aws_s3', }; - const replicationMD = { ...newReplicationMD, storageType: 'aws_s3' }; + const replicationMD = { + ...newReplicationMD, + backends: [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], + }; let completedReplicationInfo; async.series( @@ -584,29 +584,20 @@ describe('Replication object MD without bucket replication config', () => { }); ['awsbackend', 'azurebackend', 'gcpbackend', 'awsbackend,azurebackend'].forEach(backend => { - const storageTypeMap = { - awsbackend: 'aws_s3', - azurebackend: 'azure', - gcpbackend: 'gcp', - 'awsbackend,azurebackend': 'aws_s3,azure', - }; - const storageType = storageTypeMap[backend]; const backends = backend.split(',').map(site => ({ site, status: 'PENDING', dataStoreVersionId: '', })); - describe('Object metadata replicationInfo storageType value', () => { + describe('Object metadata replicationInfo for cloud backends', () => { const expectedReplicationInfo = { status: 'PENDING', backends, content: ['DATA', 'METADATA'], - destination: 'arn:aws:s3:::destination-bucket', - storageClass: backend, + storageClass: '', role: 'arn:aws:iam::account-id:role/resource', - storageType, + storageType: '', dataStoreVersionId: '', - isNFS: undefined, }; // Expected for a metadata-only replication operation (for @@ -618,8 +609,7 @@ describe('Replication object MD without bucket replication config', () => { beforeEach(() => // We have already created the bucket, so update the // replication configuration to include a location - // constraint for the `storageClass`. This results in a - // `storageType` of 'aws_s3', for example. + // constraint for the storage class. Object.assign(metadata.buckets.get(bucketName), { _replicationConfiguration: { role: 'arn:aws:iam::account-id:role/resource', @@ -750,3 +740,140 @@ describe('Replication object MD without bucket replication config', () => { }, ); }); + +describe('Replication object MD with CRR and cloud destinations on the same object', () => { + const crrSite = 'crr-site'; + const cloudSite = 'awsbackend'; + const crrRule = { + id: 'rule-crr', + prefix: keyA, + enabled: true, + priority: 1, + storageClass: crrSite, + destination: 'arn:aws:s3:::crr-bucket', + }; + const cloudRule = { + id: 'rule-cloud', + prefix: keyA, + enabled: true, + priority: 2, + storageClass: cloudSite, + destination: 'arn:aws:s3:::aws-bucket', + }; + + function setupBucket(rules) { + cleanup(); + createBucket(); + config.locationConstraints[crrSite] = { type: '' }; + Object.assign(metadata.buckets.get(bucketName), { + _versioningConfiguration: { status: 'Enabled' }, + _replicationConfiguration: { + role: 'arn:aws:iam::account-id:role/src-role,' + 'arn:aws:iam::account-id:role/dst-role', + rules, + }, + }); + } + + function completeAllBackends() { + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + objectMD.replicationInfo.status = 'COMPLETED'; + objectMD.replicationInfo.backends.forEach(b => { + // eslint-disable-next-line no-param-reassign + b.status = 'COMPLETED'; + }); + } + + afterEach(() => { + cleanup(); + delete config.locationConstraints[crrSite]; + }); + + it( + 'should reset only the CRR backend to PENDING on putObjectACL, ' + + 'preserving the completed status of the cloud backend', + done => { + setupBucket([crrRule, cloudRule]); + async.series( + [ + next => objectPut(authInfo, getObjectPutReq(keyA, true), undefined, log, next), + next => { + completeAllBackends(); + return objectPutACL(authInfo, objectACLReq, log, next); + }, + ], + err => { + if (err) { + return done(err); + } + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + const crrBackend = objectMD.replicationInfo.backends.find(b => b.site === crrSite); + const cloudBackend = objectMD.replicationInfo.backends.find(b => b.site === cloudSite); + // CRR backend is re-kicked: status reset to PENDING with the + // resolved destination role stamped on the entry. + assert.strictEqual(crrBackend.status, 'PENDING'); + assert.strictEqual(crrBackend.role, 'arn:aws:iam::account-id:role/dst-role'); + assert.strictEqual(crrBackend.destination, 'arn:aws:s3:::crr-bucket'); + // Cloud backend is left alone: no ACL replication for cloud, + // and no resolved role/destination on the entry. + assert.strictEqual(cloudBackend.status, 'COMPLETED'); + assert.strictEqual(cloudBackend.role, undefined); + assert.strictEqual(cloudBackend.destination, undefined); + return done(); + }, + ); + }, + ); + + it('should not touch replicationInfo when no CRR backend is present', done => { + setupBucket([cloudRule]); + async.series( + [ + next => objectPut(authInfo, getObjectPutReq(keyA, true), undefined, log, next), + next => { + completeAllBackends(); + return objectPutACL(authInfo, objectACLReq, log, next); + }, + ], + err => { + if (err) { + return done(err); + } + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Status untouched because nothing to ACL-replicate. + assert.strictEqual(objectMD.replicationInfo.status, 'COMPLETED'); + objectMD.replicationInfo.backends.forEach(b => { + assert.strictEqual(b.status, 'COMPLETED'); + }); + return done(); + }, + ); + }); + + it('should add a newly configured CRR destination to backends on ' + 'putObjectACL', done => { + setupBucket([cloudRule]); + async.series( + [ + next => objectPut(authInfo, getObjectPutReq(keyA, true), undefined, log, next), + next => { + completeAllBackends(); + // Operator adds a CRR destination after the object was + // already replicated to cloud. + metadata.buckets.get(bucketName)._replicationConfiguration.rules.unshift(crrRule); + return objectPutACL(authInfo, objectACLReq, log, next); + }, + ], + err => { + if (err) { + return done(err); + } + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + const crrBackend = objectMD.replicationInfo.backends.find(b => b.site === crrSite); + const cloudBackend = objectMD.replicationInfo.backends.find(b => b.site === cloudSite); + assert.ok(crrBackend, 'new CRR backend should be added'); + assert.strictEqual(crrBackend.status, 'PENDING'); + assert.strictEqual(cloudBackend.status, 'COMPLETED'); + return done(); + }, + ); + }); +}); From e413e0b6d86e0071bc6e094649bfc61cfc103e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Fri, 29 May 2026 15:04:47 +0200 Subject: [PATCH 3/3] Bump arsenal dependency Issue: CLDSRV-888 --- package.json | 3 +-- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index c1cd9d2b21..19ffcf9476 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "@opentelemetry/instrumentation-ioredis": "~0.64.0", "@opentelemetry/instrumentation-mongodb": "~0.69.0", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/Arsenal#8.4.5", + "arsenal": "git+https://github.com/scality/Arsenal#c3f3bffde6eeb935e03847df010ac37ac7ba8944", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", @@ -133,7 +133,6 @@ "ft_healthchecks": "cd tests/functional/healthchecks && yarn test", "ft_s3cmd": "cd tests/functional/s3cmd && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", "ft_s3curl": "cd tests/functional/s3curl && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", - "ft_scripts": "cd tests/functional/scripts && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", "ft_util": "cd tests/functional/utilities && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", "ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management ft_util ft_backbeat", "ft_search": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 90000 test/mdSearch --exit", diff --git a/yarn.lock b/yarn.lock index e942b928fd..37da58301a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6576,9 +6576,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.4.5": +"arsenal@git+https://github.com/scality/Arsenal#c3f3bffde6eeb935e03847df010ac37ac7ba8944": version "8.4.5" - resolved "git+https://github.com/scality/Arsenal#594473628d622c6b0a9ae1b270fee27ef99e5f85" + resolved "git+https://github.com/scality/Arsenal#c3f3bffde6eeb935e03847df010ac37ac7ba8944" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0"