Bump microVersionId and add isReplica handling

lib/api/apiUtils/object/bumpMicroVersionId.js

@@ -0,0 +1,27 @@
+const { versioning } = require('arsenal');
+const { config } = require('../../../Config');
+
+/**
+ * Bump objectMD.microVersionId. microVersionId is a generic
+ * metadata-revision marker, not a CRR-specific field, but cascaded CRR
+ * is its only consumer today - so we gate on replicationInfo to avoid
+ * inflating storage on objects that wouldn't use it. The gate can be
+ * widened later if another consumer needs it on every object.
+ * Pass `force = true` to bump unconditionally.
+ *
+ * @param {object} objectMD - object MD POJO or `md.getValue()`
+ * @param {boolean} [force] - bump even without replicationInfo
+ * @return {undefined}
+ */
+function bumpMicroVersionId(objectMD, force) {
+    if (!force && !objectMD?.replicationInfo) {
+        return;
+    }
+
+    const { instanceId, replicationGroupId } = config;
+
+    // eslint-disable-next-line no-param-reassign
+    objectMD.microVersionId = versioning.VersionID.generateVersionId(instanceId, replicationGroupId);
+}
+
+module.exports = bumpMicroVersionId;

lib/api/apiUtils/object/getReplicationInfo.js

@@ -1,5 +1,4 @@
-const { isServiceAccount, getServiceAccountProperties } =
-      require('../authorization/permissionChecks');
+const { isServiceAccount, getServiceAccountProperties } = require('../authorization/permissionChecks');
 const { replicationBackends } = require('arsenal').constants;
 
 function _getBackend(objectMD, site) {
@@ -23,25 +22,21 @@ function _getStorageClasses(s3config, rule) {
     const { replicationEndpoints } = s3config;
     // If no storage class, use the given default endpoint or the sole endpoint
     if (replicationEndpoints.length > 0) {
-        const endPoint =
-            replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0];
+        const endPoint = replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0];
         return [endPoint.site];
     }
     return undefined;
 }
 
-function _getReplicationInfo(s3config, rule, replicationConfig, content, operationType,
-    objectMD, bucketMD) {
+function _getReplicationInfo(s3config, rule, replicationConfig, content, operationType, objectMD, bucketMD) {
     const storageTypes = [];
     const backends = [];
     const storageClasses = _getStorageClasses(s3config, rule);
     if (!storageClasses) {
         return undefined;
     }
     storageClasses.forEach(storageClass => {
-        const storageClassName =
-              storageClass.endsWith(':preferred_read') ?
-              storageClass.split(':')[0] : storageClass;
+        const storageClassName = storageClass.endsWith(':preferred_read') ? storageClass.split(':')[0] : storageClass;
         // TODO CLDSRV-646: for consistency, should we look at replicationEndpoints instead, like
         // `_getStorageClasses()` ?
         const location = s3config.locationConstraints[storageClassName];
@@ -62,6 +57,7 @@ function _getReplicationInfo(s3config, rule, replicationConfig, content, operati
         role: replicationConfig.role,
         storageType: storageTypes.join(','),
         isNFS: bucketMD.isNFS(),
+        isReplica: false,
     };
 }
 
@@ -80,8 +76,7 @@ function _getReplicationInfo(s3config, rule, replicationConfig, content, operati
  * @param {AuthInfo} [authInfo] - authentication info of object owner
  * @return {undefined}
  */
-function getReplicationInfo(
-    s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) {
+function getReplicationInfo(s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) {
     const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA'];
     const config = bucketMD.getReplicationConfiguration();
 
@@ -106,17 +101,14 @@ function getReplicationInfo(
         if (!authInfo || !isServiceAccount(authInfo.getCanonicalID())) {
             doReplicate = true;
         } else {
-            const serviceAccountProps = getServiceAccountProperties(
-                authInfo.getCanonicalID());
+            const serviceAccountProps = getServiceAccountProperties(authInfo.getCanonicalID());
             doReplicate = serviceAccountProps.canReplicate;
         }
         if (doReplicate) {
-            const rule = config.rules.find(
-                rule => (objKey.startsWith(rule.prefix) && rule.enabled));
+            const rule = config.rules.find(rule => objKey.startsWith(rule.prefix) && rule.enabled);
             if (rule) {
                 // TODO CLDSRV-646 : should "merge" the replicationInfo for different rules
-                return _getReplicationInfo(
-                    s3config, rule, config, content, operationType, objectMD, bucketMD);
+                return _getReplicationInfo(s3config, rule, config, content, operationType, objectMD, bucketMD);
             }
         }
     }

lib/api/apiUtils/object/versioning.js

@@ -10,8 +10,7 @@
 const versionIdUtils = versioning.VersionID;
 // Use Arsenal function to generate a version ID used internally by metadata
 // for null versions that are created before bucket versioning is configured
-const nonVersionedObjId =
-    versionIdUtils.getInfVid(config.replicationGroupId);
+const nonVersionedObjId = versionIdUtils.getInfVid(config.replicationGroupId);
 
 /** decodeVID - decode the version id
  * @param {string} versionId - version ID
@@ -101,25 +100,24 @@
     nullVersionMD.originOp = 's3:StoreNullVersion';
     metadata.putObjectMD(bucketName, objKey, nullVersionMD, { versionId }, log, err => {
         if (err) {
-            log.debug('error from metadata storing null version as new version',
-            { error: err });
+            log.debug('error from metadata storing null version as new version', { error: err });
         }
-        
+
         cb(err);
     });
 }
 
 /** check existence and get location of null version data for deletion
-* @param {string} bucketName - name of bucket
-* @param {string} objKey - name of object key
-* @param {object} options - metadata options for getting object MD
-* @param {string} options.versionId - version to get from metadata
-* @param {object} mst - info about the master version
-* @param {string} mst.versionId - the master version's version id
-* @param {RequestLogger} log - logger instanceof
-* @param {function} cb - callback
-* @return {undefined} - and call callback with (err, dataToDelete)
-*/
+ * @param {string} bucketName - name of bucket
+ * @param {string} objKey - name of object key
+ * @param {object} options - metadata options for getting object MD
+ * @param {string} options.versionId - version to get from metadata
+ * @param {object} mst - info about the master version
+ * @param {string} mst.versionId - the master version's version id
+ * @param {RequestLogger} log - logger instanceof
+ * @param {function} cb - callback
+ * @return {undefined} - and call callback with (err, dataToDelete)
+ */
 function _prepareNullVersionDeletion(bucketName, objKey, options, mst, log, cb) {
     const nullOptions = {};
     if (!options.deleteData) {
@@ -135,38 +133,40 @@
         // PUT via this option
         nullOptions.deleteNullKey = true;
     }
-    return metadata.getObjectMD(bucketName, objKey, options, log,
-        (err, versionMD) => {
-            if (err) {
-                // the null key may not exist, hence it's a normal
-                // situation to have a NoSuchKey error, in which case
-                // there is nothing to delete
-                if (err.is.NoSuchKey) {
-                    log.debug('null version does not exist', {
-                        method: '_prepareNullVersionDeletion',
-                    });
-                } else {
-                    log.warn('could not get null version metadata', {
-                        error: err,
-                        method: '_prepareNullVersionDeletion',
-                    });
-                }
-                return cb(err);
-            }
-            if (versionMD.location) {
-                const dataToDelete = Array.isArray(versionMD.location) ?
-                      versionMD.location : [versionMD.location];
-                nullOptions.dataToDelete = dataToDelete;
+    return metadata.getObjectMD(bucketName, objKey, options, log, (err, versionMD) => {
+        if (err) {
+            // the null key may not exist, hence it's a normal
+            // situation to have a NoSuchKey error, in which case
+            // there is nothing to delete
+            if (err.is.NoSuchKey) {
+                log.debug('null version does not exist', {
+                    method: '_prepareNullVersionDeletion',
+                });
+            } else {
+                log.warn('could not get null version metadata', {
+                    error: err,
+                    method: '_prepareNullVersionDeletion',
+                });
             }
-            return cb(null, nullOptions);
-        });
+            return cb(err);
+        }
+        if (versionMD.location) {
+            const dataToDelete = Array.isArray(versionMD.location) ? versionMD.location : [versionMD.location];
+            nullOptions.dataToDelete = dataToDelete;
+        }
+        return cb(null, nullOptions);
+    });
 }
 
 function _deleteNullVersionMD(bucketName, objKey, options, log, cb) {
     return metadata.deleteObjectMD(bucketName, objKey, options, log, err => {
         if (err) {
-            log.warn('metadata error deleting null versioned key',
-                     { bucketName, objKey, error: err, method: '_deleteNullVersionMD' });
+            log.warn('metadata error deleting null versioned key', {
+                bucketName,
+                objKey,
+                error: err,
+                method: '_deleteNullVersionMD',
+            });
         }
         return cb(err);
     });
@@ -193,7 +193,7 @@
      version key, if needed
  */
 function processVersioningState(mst, vstat, nullVersionCompatMode) {
-    const versioningSuspended = (vstat === 'Suspended');
+    const versioningSuspended = vstat === 'Suspended';
     const masterIsNull = mst.exists && (mst.isNull || !mst.versionId);
 
     if (versioningSuspended) {
@@ -244,7 +244,7 @@
     if (masterIsNull) {
         // if master is a null version or a non-versioned key,
         // copy it to a new null key
-        const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId;
+        const nullVersionId = mst.isNull && mst.versionId ? mst.versionId : nonVersionedObjId;
         if (nullVersionCompatMode) {
             options.extraMD = {
                 nullVersionId,
@@ -311,8 +311,7 @@
     };
 
     if (objMD.location) {
-        mst.objLocation = Array.isArray(objMD.location) ?
-            objMD.location : [objMD.location];
+        mst.objLocation = Array.isArray(objMD.location) ? objMD.location : [objMD.location];
     }
 
     return mst;
@@ -332,61 +331,67 @@
  * options.versioning - (true/undefined) metadata instruction to create new ver
  * options.isNull - (true/undefined) whether new version is null or not
  */
-function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
-    log, callback) {
+function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD, log, callback) {
     const mst = getMasterState(objMD);
     const vCfg = bucketMD.getVersioningConfiguration();
 
     if (!vCfg) {
         const options = { dataToDelete: mst.objLocation };
         return process.nextTick(callback, null, options);
     }
 
-    const { options, nullVersionId, delOptions } =
-          processVersioningState(mst, vCfg.Status, config.nullVersionCompatMode);
-
-    return async.series([
-        function storeNullVersionMD(next) {
-            if (!nullVersionId) {
-                return process.nextTick(next);
-            }
+    const { options, nullVersionId, delOptions } = processVersioningState(
+        mst,
+        vCfg.Status,
+        config.nullVersionCompatMode,
+    );
+
+    return async.series(
+        [
+            function storeNullVersionMD(next) {
+                if (!nullVersionId) {
+                    return process.nextTick(next);
+                }
 
-            options.nullVersionId = nullVersionId;
-            return _storeNullVersionMD(bucketName, objectKey, nullVersionId, objMD, log, next);
-        },
-        function prepareNullVersionDeletion(next) {
-            if (!delOptions) {
-                return process.nextTick(next);
-            }
-            return _prepareNullVersionDeletion(
-                bucketName, objectKey, delOptions, mst, log,
-                (err, nullOptions) => {
+                options.nullVersionId = nullVersionId;
+                return _storeNullVersionMD(bucketName, objectKey, nullVersionId, objMD, log, next);
+            },
+            function prepareNullVersionDeletion(next) {
+                if (!delOptions) {
+                    return process.nextTick(next);
+                }
+                return _prepareNullVersionDeletion(bucketName, objectKey, delOptions, mst, log, (err, nullOptions) => {
                     if (err) {
                         return next(err);
                     }
                     Object.assign(options, nullOptions);
                     return next();
                 });
-        },
-        function deleteNullVersionMD(next) {
-            if (delOptions &&
-                delOptions.versionId &&
-                delOptions.versionId !== 'null') {
-                // backward-compat: delete old null versioned key
-                return _deleteNullVersionMD(
-                    bucketName, objectKey, { versionId: delOptions.versionId, overheadField }, log, next);
+            },
+            function deleteNullVersionMD(next) {
+                if (delOptions && delOptions.versionId && delOptions.versionId !== 'null') {
+                    // backward-compat: delete old null versioned key
+                    return _deleteNullVersionMD(
+                        bucketName,
+                        objectKey,
+                        { versionId: delOptions.versionId, overheadField },
+                        log,
+                        next,
+                    );
+                }
+                return process.nextTick(next);
+            },
+        ],
+        err => {
+            // it's possible there was a prior request that deleted the
+            // null version, so proceed with putting a new version
+            if (err && err.is.NoSuchKey) {
+                return callback(null, options);
             }
-            return process.nextTick(next);
+            return callback(err, options);
         },
-    ], err => {
-        // it's possible there was a prior request that deleted the
-        // null version, so proceed with putting a new version
-        if (err && err.is.NoSuchKey) {
-            return callback(null, options);
-        }
-        return callback(err, options);
-    });
+    );
 }
 
 /** Return options to pass to Metadata layer for version-specific
  * operations with the given requested version ID
@@ -545,7 +550,7 @@
         restoreRequestedAt: objMD.archive?.restoreRequestedAt,
         restoreRequestedDays: objMD.archive?.restoreRequestedDays,
         restoreCompletedAt: new Date(now),
-        restoreWillExpireAt: new Date(now + (days * scaledMsPerDay)),
+        restoreWillExpireAt: new Date(now + days * scaledMsPerDay),
     };
 
     /* eslint-enable no-param-reassign */