INTEGRATION [PR#1016 > development/8.1] bugfix: S3C-2899 pass vFormat to listing params

lib/algos/list/MPU.js

@@ -1,7 +1,10 @@
 'use strict'; // eslint-disable-line strict
 
+const errors = require('../../errors');
 const { inc, checkLimit, FILTER_END, FILTER_ACCEPT } = require('./tools');
 const DEFAULT_MAX_KEYS = 1000;
+const VSConst = require('../../versioning/constants').VersioningConstants;
+const { BucketVersioningKeyFormat } = VSConst;
 
 function numberDefault(num, defaultNum) {
     const parsedNum = Number.parseInt(num, 10);
@@ -17,10 +20,12 @@ class MultipartUploads {
      *  Init and check parameters
      *  @param {Object} params - The parameters you sent to DBD
      *  @param {RequestLogger} logger - The logger of the request
+     *  @param {String} [vFormat] - versioning key format
      *  @return {undefined}
      */
-    constructor(params, logger) {
+    constructor(params, logger, vFormat) {
         this.params = params;
+        this.vFormat = vFormat || BucketVersioningKeyFormat.v0;
         this.CommonPrefixes = [];
         this.Uploads = [];
         this.IsTruncated = false;
@@ -101,6 +106,13 @@ class MultipartUploads {
         }
     }
 
+    _getObjectKey(obj) {
+        if (this.vFormat === BucketVersioningKeyFormat.v0) {
+            return obj.key;
+        }
+        throw errors.NotImplemented;
+    }
+
     /**
      *  This function applies filter on each element
      *  @param {String} obj - The key and value of the element
@@ -113,7 +125,7 @@ class MultipartUploads {
             this.IsTruncated = this.maxKeys > 0;
             return FILTER_END;
         }
-        const key = obj.key;
+        const key = this._getObjectKey(obj);
         const value = obj.value;
         if (this.delimiter) {
             const mpuPrefixSlice = `overview${this.splitter}`.length;

lib/algos/list/delimiter.js

@@ -1,19 +1,10 @@
 'use strict'; // eslint-disable-line strict
 
+const errors = require('../../errors');
 const Extension = require('./Extension').default;
 const { inc, FILTER_END, FILTER_ACCEPT, FILTER_SKIP } = require('./tools');
-
-/**
- * Find the next delimiter in the path
- *
- * @param {string} key             - path of the object
- * @param {string} delimiter       - string to find
- * @param {number} index           - index to start at
- * @return {number} delimiterIndex - returns -1 in case no delimiter is found
- */
-function nextDelimiter(key, delimiter, index) {
-    return key.indexOf(delimiter, index);
-}
+const VSConst = require('../../versioning/constants').VersioningConstants;
+const { BucketVersioningKeyFormat } = VSConst;
 
 /**
  * Find the common prefix in the path
@@ -62,8 +53,9 @@ class Delimiter extends Extension {
      *                                                   or not
      * @param {RequestLogger} logger                   - The logger of the
      *                                                   request
+     * @param {String} [vFormat]                       - versioning key format
      */
-    constructor(parameters, logger) {
+    constructor(parameters, logger, vFormat) {
         super(parameters, logger);
         // original listing parameters
         this.delimiter = parameters.delimiter;
@@ -76,6 +68,7 @@ class Delimiter extends Extension {
           typeof parameters.alphabeticalOrder !== 'undefined' ?
                                  parameters.alphabeticalOrder : true;
 
+        this.vFormat = vFormat || BucketVersioningKeyFormat.v0;
         // results
         this.CommonPrefixes = [];
         this.Contents = [];
@@ -102,6 +95,13 @@ class Delimiter extends Extension {
     }
 
     genMDParams() {
+        if (this.vFormat === BucketVersioningKeyFormat.v0) {
+            return this.genMDParamsV0();
+        }
+        throw errors.NotImplemented;
+    }
+
+    genMDParamsV0() {
         const params = {};
         if (this.prefix) {
             params.gte = this.prefix;
@@ -150,6 +150,13 @@ class Delimiter extends Extension {
         return FILTER_ACCEPT;
     }
 
+    _getObjectKey(obj) {
+        if (this.vFormat === BucketVersioningKeyFormat.v0) {
+            return obj.key;
+        }
+        throw errors.NotImplemented;
+    }
+
     /**
      *  Filter to apply on each iteration, based on:
      *  - prefix
@@ -162,7 +169,7 @@ class Delimiter extends Extension {
      *  @return {number}          - indicates if iteration should continue
      */
     filter(obj) {
-        const key = obj.key;
+        const key = this._getObjectKey(obj);
         const value = obj.value;
         if ((this.prefix && !key.startsWith(this.prefix))
             || (this.alphabeticalOrder
@@ -172,9 +179,7 @@ class Delimiter extends Extension {
         }
         if (this.delimiter) {
             const baseIndex = this.prefix ? this.prefix.length : 0;
-            const delimiterIndex = nextDelimiter(key,
-                                                 this.delimiter,
-                                                 baseIndex);
+            const delimiterIndex = key.indexOf(this.delimiter, baseIndex);
             if (delimiterIndex === -1) {
                 return this.addContents(key, value);
             }
@@ -211,6 +216,13 @@ class Delimiter extends Extension {
      *                    that it's enough and should move on
      */
     skipping() {
+        if (this.vFormat === BucketVersioningKeyFormat.v0) {
+            return this.skippingV0();
+        }
+        throw errors.NotImplemented;
+    }
+
+    skippingV0() {
         return this[this.nextContinueMarker];
     }
 

lib/algos/list/delimiterMaster.js

@@ -1,8 +1,10 @@
 'use strict'; // eslint-disable-line strict
 
+const errors = require('../../errors');
 const Delimiter = require('./delimiter').Delimiter;
 const Version = require('../../versioning/Version').Version;
 const VSConst = require('../../versioning/constants').VersioningConstants;
+const { BucketVersioningKeyFormat } = VSConst;
 const { FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } = require('./tools');
 
 const VID_SEP = VSConst.VersionId.Separator;
@@ -23,26 +25,23 @@ class DelimiterMaster extends Delimiter {
      * @param {String}  parameters.startAfter - marker per amazon v2 format
      * @param {String}  parameters.continuationToken - obfuscated amazon token
      * @param {RequestLogger} logger          - The logger of the request
+     * @param {String} [vFormat]              - versioning key format
      */
-    constructor(parameters, logger) {
-        super(parameters, logger);
+    constructor(parameters, logger, vFormat) {
+        super(parameters, logger, vFormat);
         // non-PHD master version or a version whose master is a PHD version
         this.prvKey = undefined;
         this.prvPHDKey = undefined;
     }
 
-    /**
-     *  Filter to apply on each iteration, based on:
-     *  - prefix
-     *  - delimiter
-     *  - maxKeys
-     *  The marker is being handled directly by levelDB
-     *  @param {Object} obj       - The key and value of the element
-     *  @param {String} obj.key   - The key of the element
-     *  @param {String} obj.value - The value of the element
-     *  @return {number}          - indicates if iteration should continue
-     */
     filter(obj) {
+        if (this.vFormat === BucketVersioningKeyFormat.v0) {
+            return this.filterV0(obj);
+        }
+        throw errors.NotImplemented;
+    }
+
+    filterV0(obj) {
         let key = obj.key;
         const value = obj.value;
 

lib/algos/list/delimiterVersions.js

@@ -1,16 +1,14 @@
 'use strict'; // eslint-disable-line strict
 
+const errors = require('../../errors');
 const Delimiter = require('./delimiter').Delimiter;
 const Version = require('../../versioning/Version').Version;
 const VSConst = require('../../versioning/constants').VersioningConstants;
 const { inc, FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } =
     require('./tools');
 
 const VID_SEP = VSConst.VersionId.Separator;
-
-function formatVersionKey(key, versionId) {
-    return `${key}${VID_SEP}${versionId}`;
-}
+const { BucketVersioningKeyFormat } = VSConst;
 
 /**
  * Handle object listing with parameters
@@ -25,8 +23,8 @@ function formatVersionKey(key, versionId) {
  * @prop {Number} maxKeys              - number of keys to list
  */
 class DelimiterVersions extends Delimiter {
-    constructor(parameters, logger) {
-        super(parameters, logger);
+    constructor(parameters, logger, vFormat) {
+        super(parameters, logger, vFormat);
         // specific to version listing
         this.keyMarker = parameters.keyMarker;
         this.versionIdMarker = parameters.versionIdMarker;
@@ -39,6 +37,13 @@ class DelimiterVersions extends Delimiter {
     }
 
     genMDParams() {
+        if (this.vFormat === BucketVersioningKeyFormat.v0) {
+            return this.genMDParamsV0();
+        }
+        throw errors.NotImplemented;
+    }
+
+    genMDParamsV0() {
         const params = {};
         if (this.parameters.prefix) {
             params.gte = this.parameters.prefix;
@@ -52,8 +57,8 @@ class DelimiterVersions extends Delimiter {
             if (this.parameters.versionIdMarker) {
                 // versionIdMarker should always come with keyMarker
                 // but may not be the other way around
-                params.gt = formatVersionKey(this.parameters.keyMarker,
-                        this.parameters.versionIdMarker);
+                params.gt = `${this.parameters.keyMarker}` +
+                    `${VID_SEP}${this.parameters.versionIdMarker}`;
             } else {
                 params.gt = inc(this.parameters.keyMarker + VID_SEP);
             }
@@ -98,42 +103,55 @@ class DelimiterVersions extends Delimiter {
      *  @return {number}          - indicates if iteration should continue
      */
     filter(obj) {
-        if (Version.isPHD(obj.value)) {
-            return FILTER_ACCEPT; // trick repd to not increase its streak
+        if (this.vFormat === BucketVersioningKeyFormat.v0) {
+            if (Version.isPHD(obj.value)) {
+                return FILTER_ACCEPT; // trick repd to not increase its streak
+            }
+            return this.filterCommon(obj.key, obj.value);
         }
-        if (this.prefix && !obj.key.startsWith(this.prefix)) {
+        throw errors.NotImplemented;
+    }
+
+    filterCommon(key, value) {
+        if (this.prefix && !key.startsWith(this.prefix)) {
             return FILTER_SKIP;
         }
-        let key = obj.key; // original key
-        let versionId = undefined; // versionId
-        const versionIdIndex = obj.key.indexOf(VID_SEP);
+        let nonversionedKey;
+        let versionId = undefined;
+        const versionIdIndex = key.indexOf(VID_SEP);
         if (versionIdIndex < 0) {
-            this.masterKey = obj.key;
+            nonversionedKey = key;
+            this.masterKey = key;
             this.masterVersionId =
-                Version.from(obj.value).getVersionId() || 'null';
+                Version.from(value).getVersionId() || 'null';
             versionId = this.masterVersionId;
         } else {
-            // eslint-disable-next-line
-            key = obj.key.slice(0, versionIdIndex);
-            // eslint-disable-next-line
-            versionId = obj.key.slice(versionIdIndex + 1);
-            if (this.masterKey === key && this.masterVersionId === versionId) {
+            nonversionedKey = key.slice(0, versionIdIndex);
+            versionId = key.slice(versionIdIndex + 1);
+            if (this.masterKey === nonversionedKey && this.masterVersionId === versionId) {
                 return FILTER_ACCEPT; // trick repd to not increase its streak
             }
             this.masterKey = undefined;
             this.masterVersionId = undefined;
         }
         if (this.delimiter) {
             const baseIndex = this.prefix ? this.prefix.length : 0;
-            const delimiterIndex = key.indexOf(this.delimiter, baseIndex);
+            const delimiterIndex = nonversionedKey.indexOf(this.delimiter, baseIndex);
             if (delimiterIndex >= 0) {
-                return this.addCommonPrefix(key, delimiterIndex);
+                return this.addCommonPrefix(nonversionedKey, delimiterIndex);
             }
         }
-        return this.addContents({ key, value: obj.value, versionId });
+        return this.addContents({ key: nonversionedKey, value, versionId });
     }
 
     skipping() {
+        if (this.vFormat === BucketVersioningKeyFormat.v0) {
+            return this.skippingV0();
+        }
+        throw errors.NotImplemented;
+    }
+
+    skippingV0() {
         if (this.NextMarker) {
             const index = this.NextMarker.lastIndexOf(this.delimiter);
             if (index === this.NextMarker.length - 1) {

package.json

@@ -52,7 +52,7 @@
   "devDependencies": {
     "eslint": "2.13.1",
     "eslint-config-airbnb": "6.2.0",
-    "eslint-config-scality": "scality/Guidelines#71a059ad",
+    "eslint-config-scality": "scality/Guidelines#20dfffc",
     "eslint-plugin-react": "^4.3.0",
     "lolex": "1.5.2",
     "mocha": "5.2.0",

tests/unit/algos/list/delimiterMaster.js

@@ -100,7 +100,7 @@ describe('Delimiter All masters listing algorithm', () => {
         assert.deepStrictEqual(delimiter.result(), EmptyResult);
     });
 
-    it('should skip entries superior to next marker', () => {
+    it('should skip entries inferior to next marker', () => {
         const delimiter = new DelimiterMaster({ marker: 'b' }, fakeLogger);
 
         assert.strictEqual(delimiter.filter({ key: 'a' }), FILTER_SKIP);

yarn.lock

@@ -936,9 +936,9 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/eslint-config-airbnb/-/eslint-config-airbnb-6.2.0.tgz#4a28196aa4617de01b8c914e992a82e5d0886a6e"
   integrity sha1-SigZaqRhfeAbjJFOmSqC5dCIam4=
 
-eslint-config-scality@scality/Guidelines#71a059ad:
+eslint-config-scality@scality/Guidelines#20dfffc:
   version "1.1.0"
-  resolved "https://codeload.github.com/scality/Guidelines/tar.gz/71a059ad3fa0598d5bbb923badda58ccf06cc8a6"
+  resolved "https://codeload.github.com/scality/Guidelines/tar.gz/20dfffcc863bdb3c6bbc93b283e99a33b2fd6136"
   dependencies:
     commander "1.3.2"
     markdownlint "0.0.8"