From e17333b19ee708adf12a7e1274db71474205c6fd Mon Sep 17 00:00:00 2001 From: Alexander Chan Date: Thu, 28 Jun 2018 15:49:46 -0700 Subject: [PATCH] ft: ZENKO-597 account for transient source in TDM --- lib/storage/metadata/MetadataWrapper.js | 1 + .../metadata/mongoclient/DataCounter.js | 23 ++- .../mongoclient/MongoClientInterface.js | 57 +++++++- .../metadata/mongoclient/DataCounter.js | 131 +++++++++++++++++- .../mongoclient/MongoClientInterface.js | 99 ++++++++++--- .../mongoclient/utils/DummyConfigObject.js | 16 +++ 6 files changed, 296 insertions(+), 31 deletions(-) create mode 100644 tests/unit/storage/metadata/mongoclient/utils/DummyConfigObject.js diff --git a/lib/storage/metadata/MetadataWrapper.js b/lib/storage/metadata/MetadataWrapper.js index 7031a7d3f..1bc937572 100644 --- a/lib/storage/metadata/MetadataWrapper.js +++ b/lib/storage/metadata/MetadataWrapper.js @@ -87,6 +87,7 @@ class MetadataWrapper { database: params.mongodb.database, replicationGroupId: params.replicationGroupId, path: params.mongodb.path, + config: params.config, logger, }); this.implName = 'mongoclient'; diff --git a/lib/storage/metadata/mongoclient/DataCounter.js b/lib/storage/metadata/mongoclient/DataCounter.js index 79de0b431..1ba34da24 100644 --- a/lib/storage/metadata/mongoclient/DataCounter.js +++ b/lib/storage/metadata/mongoclient/DataCounter.js @@ -29,6 +29,22 @@ class DataCounter { byLocation: {}, }; this.populated = false; + this.transientList = {}; + } + + /** + * updateTransientList - update data counter list of transient locations + * @param {Object} newLocations - list of locations constraint details + * @return {undefined} + */ + updateTransientList(newLocations) { + if (newLocations && Object.keys(newLocations).length > 0) { + const tempList = {}; + Object.keys(newLocations).forEach(loc => { + tempList[loc] = newLocations[loc].isTransient; + }); + this.transientList = tempList; + } } /** @@ -187,10 +203,11 @@ class DataCounter { * @return {undefined} */ _updateObject(currMD, prevMD, type) { + const transientList = Object.assign({}, this.transientList); if (currMD && prevMD) { // check for changes in replication const { replicationInfo: currLocs, - 'content-length': size } = currMD; + 'content-length': size, dataStoreName } = currMD; const { replicationInfo: prevLocs } = prevMD; const { backends: prevBackends } = prevLocs || {}; const { backends: currBackends } = currLocs || {}; @@ -210,6 +227,10 @@ class DataCounter { } }); } + if (currLocs.status === 'COMPLETED' && + transientList[dataStoreName]) { + this._delLocation(dataStoreName, size, type); + } } } diff --git a/lib/storage/metadata/mongoclient/MongoClientInterface.js b/lib/storage/metadata/mongoclient/MongoClientInterface.js index 91465085a..0f67e618a 100644 --- a/lib/storage/metadata/mongoclient/MongoClientInterface.js +++ b/lib/storage/metadata/mongoclient/MongoClientInterface.js @@ -11,6 +11,7 @@ */ const async = require('async'); +const { EventEmitter } = require('events'); const constants = require('../../../constants'); const errors = require('../../../errors'); @@ -101,7 +102,7 @@ function isUserBucket(bucketName) { class MongoClientInterface { constructor(params) { const { replicaSetHosts, writeConcern, replicaSet, readPreference, path, - database, logger, replicationGroupId } = params; + database, logger, replicationGroupId, config } = params; this.mongoUrl = `mongodb://${replicaSetHosts}/?w=${writeConcern}&` + `replicaSet=${replicaSet}&readPreference=${readPreference}`; this.logger = logger; @@ -112,6 +113,17 @@ class MongoClientInterface { this.database = database; this.lastItemScanTime = null; this.dataCount = new DataCounter(); + if (config && config instanceof EventEmitter) { + this.config = config; + this.config.on('location-constraints-update', () => { + this.dataCount + .updateTransientList(this.config.locationConstraints); + if (this.config.isTest) { + this.config.emit('MongoClientTestDone'); + } + }); + this.dataCount.updateTransientList(this.config.locationConstraints); + } } setup(cb) { @@ -1212,10 +1224,11 @@ class MongoClientInterface { return results; } - _handleMongo(c, filter, log, cb) { + _handleMongo(c, filter, isTransient, log, cb) { const reducedFields = { '_id': 1, 'value.versionId': 1, + 'value.replicationInfo.status': 1, 'value.replicationInfo.backends': 1, 'value.content-length': 1, 'value.dataStoreName': 1, @@ -1252,6 +1265,21 @@ class MongoClientInterface { } }, ]; + const aggCompleted = [ + { $project: { + 'value.dataStoreName': 1, + 'value.content-length': 1, + 'inComplete': { + $eq: ['$value.replicationInfo.status', 'COMPLETED'], + }, + } }, + { $match: { inComplete: true } }, + { $group: { + _id: '$value.dataStoreName', + bytes: { $sum: '$value.content-length' }, + } }, + ]; + return c.aggregate([ { $project: reducedFields }, { $match: filter }, @@ -1259,6 +1287,7 @@ class MongoClientInterface { count: aggCount, data: aggData, repData: aggRepData, + compData: isTransient ? aggCompleted : undefined, } }, ]).toArray((err, res) => { if (err) { @@ -1292,6 +1321,14 @@ class MongoClientInterface { } retResult.data[site] += repDataEntries[site]; }); + if (isTransient && agg.compData) { + const compDataEntries = this._handleEntries(agg.compData); + Object.keys(compDataEntries).forEach(site => { + if (retResult.data[site]) { + retResult.data[site] -= compDataEntries[site]; + } + }); + } return cb(null, retResult); }); } @@ -1304,7 +1341,12 @@ class MongoClientInterface { isVersioned: !!bucketInfo.getVersioningConfiguration(), ownerCanonicalId: bucketInfo.getOwner(), }; - + let isTransient; + if (this.config) { + isTransient = this.config + .getLocationConstraint(retBucketInfo.location) + .isTransient; + } const mstFilter = { '_id': { $regex: /^[^\0]+$/ }, 'value.versionId': { $exists: true }, @@ -1316,9 +1358,12 @@ class MongoClientInterface { }; async.parallel({ - version: done => this._handleMongo(c, verFilter, log, done), - null: done => this._handleMongo(c, nullFilter, log, done), - master: done => this._handleMongo(c, mstFilter, log, done), + version: done => + this._handleMongo(c, verFilter, isTransient, log, done), + null: done => + this._handleMongo(c, nullFilter, isTransient, log, done), + master: done => + this._handleMongo(c, mstFilter, isTransient, log, done), }, (err, res) => { if (err) { return callback(err); diff --git a/tests/unit/storage/metadata/mongoclient/DataCounter.js b/tests/unit/storage/metadata/mongoclient/DataCounter.js index 603647276..f93380ce7 100644 --- a/tests/unit/storage/metadata/mongoclient/DataCounter.js +++ b/tests/unit/storage/metadata/mongoclient/DataCounter.js @@ -76,20 +76,18 @@ const refMultiObj = { }, }; -// eslint-disable-next-line quote-props const singleSite = size => ({ 'content-length': size, - dataStoreName: 'locationOne', - replicationInfo: { + 'dataStoreName': 'locationOne', + 'replicationInfo': { backends: [], }, }); -// eslint-disable-next-line quote-props const multiSite = (size, isComplete) => ({ 'content-length': size, - dataStoreName: 'locationOne', - replicationInfo: { + 'dataStoreName': 'locationOne', + 'replicationInfo': { backends: [{ site: 'locationTwo', status: isComplete ? 'COMPLETED' : 'PENDING', @@ -97,6 +95,17 @@ const multiSite = (size, isComplete) => ({ }, }); +const transientSite = (size, status, backends) => ({ + 'content-length': size, + 'dataStoreName': 'locationOne', + 'replicationInfo': { status, backends }, +}); + +const locationConstraints = { + locationOne: { isTransient: true }, + locationTwo: { isTransient: false }, +}; + const dataCounter = new DataCounter(); describe('DataCounter Class', () => { @@ -112,6 +121,16 @@ describe('DataCounter Class', () => { }); }); +describe('DateCounter::updateTransientList', () => { + afterEach(() => dataCounter.updateTransientList({})); + it('should set transient list', () => { + assert.deepStrictEqual(dataCounter.transientList, {}); + dataCounter.updateTransientList(locationConstraints); + const expectedRes = { locationOne: true, locationTwo: false }; + assert.deepStrictEqual(dataCounter.transientList, expectedRes); + }); +}); + describe('DataCounter::addObject', () => { const tests = [ { @@ -279,6 +298,106 @@ describe('DataCounter::addObject', () => { })); }); +describe('DataCounter, update with transient location', () => { + before(() => dataCounter.updateTransientList(locationConstraints)); + after(() => dataCounter.updateTransientList({})); + + const pCurrMD = transientSite(100, 'PENDING', [ + { site: 'site1', status: 'PENDING' }, + { site: 'site2', status: 'COMPLETED' }, + ]); + const cCurrMD = transientSite(100, 'COMPLETED', [ + { site: 'site1', status: 'COMPLETED' }, + { site: 'site2', status: 'COMPLETED' }, + ]); + const prevMD = transientSite(100, 'PENDING', [ + { site: 'site1', status: 'PENDING' }, + { site: 'site2', status: 'PENDING' }, + ]); + const transientTest = [ + { + it: 'should correctly update DataCounter, ' + + 'version object, replication status = PENDING', + init: refSingleObjVer, + input: [pCurrMD, prevMD, UPDATE_VER], + expectedRes: { + objects: 1, versions: 1, + dataManaged: { + total: { curr: 100, prev: 200 }, + byLocation: { + locationOne: { curr: 100, prev: 100 }, + site2: { curr: 0, prev: 100 }, + }, + }, + }, + }, + { + it: 'should correctly update DataCounter, ' + + 'version object, replication status = COMPLETED', + init: refSingleObjVer, + input: [cCurrMD, prevMD, UPDATE_VER], + expectedRes: { + objects: 1, versions: 1, + dataManaged: { + total: { curr: 100, prev: 200 }, + byLocation: { + locationOne: { curr: 100, prev: 0 }, + site1: { curr: 0, prev: 100 }, + site2: { curr: 0, prev: 100 }, + }, + }, + }, + }, + { + it: 'should correctly update DataCounter, ' + + 'master object, replication status = PENDING', + init: refSingleObjVer, + input: [pCurrMD, prevMD, UPDATE_MST], + expectedRes: { + objects: 1, versions: 1, + dataManaged: { + total: { curr: 200, prev: 100 }, + byLocation: { + locationOne: { curr: 100, prev: 100 }, + site2: { curr: 100, prev: 0 }, + }, + }, + }, + }, + { + it: 'should correctly update DataCounter, ' + + 'master object, replication status = COMPLETED', + init: refSingleObjVer, + input: [cCurrMD, prevMD, UPDATE_MST], + expectedRes: { + objects: 1, versions: 1, + dataManaged: { + total: { curr: 200, prev: 100 }, + byLocation: { + locationOne: { curr: 0, prev: 100 }, + site1: { curr: 100, prev: 0 }, + site2: { curr: 100, prev: 0 }, + }, + }, + }, + }, + ]; + + transientTest.forEach(test => it(test.it, () => { + const { expectedRes, input, init } = test; + dataCounter.set(init); + dataCounter.addObject(...input); + const testResults = dataCounter.results(); + Object.keys(expectedRes).forEach(key => { + if (typeof expectedRes[key] === 'object') { + assert.deepStrictEqual(testResults[key], expectedRes[key]); + } else { + assert.strictEqual(testResults[key], expectedRes[key]); + } + }); + })); +}); + describe('DataCounter::delObject', () => { const tests = [ { diff --git a/tests/unit/storage/metadata/mongoclient/MongoClientInterface.js b/tests/unit/storage/metadata/mongoclient/MongoClientInterface.js index e9a310d44..58e866133 100644 --- a/tests/unit/storage/metadata/mongoclient/MongoClientInterface.js +++ b/tests/unit/storage/metadata/mongoclient/MongoClientInterface.js @@ -3,6 +3,7 @@ const assert = require('assert'); const MongoClientInterface = require( '../../../../../lib/storage/metadata/mongoclient/MongoClientInterface'); const DummyMongoDB = require('./utils/DummyMongoDB'); +const DummyConfigObject = require('./utils/DummyConfigObject'); const DummyRequestLogger = require('./utils/DummyRequestLogger'); const { generateMD } = require('./utils/helper'); @@ -66,6 +67,40 @@ function assertFailureResults(testParams, cb) { }); } +describe('MongoClientInterface, init behavior', () => { + let s3ConfigObj; + const locationConstraints = { + locationOne: { isTransient: true }, + locationTwo: { isTransient: false }, + }; + + beforeEach(() => { + s3ConfigObj = new DummyConfigObject(); + }); + + it('should set DataCounter transientList when declaring a ' + + 'new MongoClientInterface object', () => { + s3ConfigObj.setLocationConstraints(locationConstraints); + const mongoClient = new MongoClientInterface({ config: s3ConfigObj }); + const expectedRes = { locationOne: true, locationTwo: false }; + assert.deepStrictEqual( + mongoClient.dataCount.transientList, expectedRes); + }); + + it('should update DataCounter transientList if location constraints ' + + 'are updated', done => { + const mongoClient = new MongoClientInterface({ config: s3ConfigObj }); + assert.deepStrictEqual(mongoClient.dataCount.transientList, {}); + const expectedRes = { locationOne: true, locationTwo: false }; + s3ConfigObj.once('MongoClientTestDone', () => { + assert.deepStrictEqual( + mongoClient.dataCount.transientList, expectedRes); + return done(); + }); + s3ConfigObj.setLocationConstraints(locationConstraints); + }); +}); + describe('MongoClientInterface::dataCount', () => { describe('MongoClientInterface::putObject', () => { beforeEach(() => { @@ -395,7 +430,7 @@ describe('MongoClientInterface::_handleMongo', () => { const retValues = [new Error('testError')]; mongoTestClient.db.setReturnValues(retValues); const testCollection = mongoTestClient.db.collection('test'); - mongoTestClient._handleMongo(testCollection, {}, log, err => { + mongoTestClient._handleMongo(testCollection, {}, false, log, err => { assert(err, 'Expected error, but got success'); return done(); }); @@ -403,7 +438,8 @@ describe('MongoClientInterface::_handleMongo', () => { it('should return empty object if mongo aggregate has no results', done => { const testCollection = mongoTestClient.db.collection('test'); - mongoTestClient._handleMongo(testCollection, {}, log, (err, res) => { + mongoTestClient._handleMongo(testCollection, {}, false, log, + (err, res) => { assert.ifError(err, `Expected success, but got error ${err}`); assert.deepStrictEqual(res, {}); return done(); @@ -419,29 +455,36 @@ describe('MongoClientInterface::_handleMongo', () => { }]]; mongoTestClient.db.setReturnValues(retValues); const testCollection = mongoTestClient.db.collection('test'); - mongoTestClient._handleMongo(testCollection, {}, log, (err, res) => { + mongoTestClient._handleMongo(testCollection, {}, false, log, + (err, res) => { assert.ifError(err, `Expected success, but got error ${err}`); assert.deepStrictEqual(res, {}); return done(); }); }); - it('should return correct results', done => { - const retValues = [[{ - count: [{ _id: null, count: 100 }], - data: [ - { _id: 'locationone', bytes: 1000 }, - { _id: 'locationtwo', bytes: 1000 }, - ], - repData: [ - { _id: 'awsbackend', bytes: 500 }, - { _id: 'azurebackend', bytes: 500 }, - { _id: 'gcpbackend', bytes: 500 }, - ], - }]]; - mongoTestClient.db.setReturnValues(retValues); + const testRetValues = [[{ + count: [{ _id: null, count: 100 }], + data: [ + { _id: 'locationone', bytes: 1000 }, + { _id: 'locationtwo', bytes: 1000 }, + ], + repData: [ + { _id: 'awsbackend', bytes: 500 }, + { _id: 'azurebackend', bytes: 500 }, + { _id: 'gcpbackend', bytes: 500 }, + ], + compData: [ + { _id: 'locationone', bytes: 500 }, + { _id: 'locationtwo', bytes: 500 }, + ], + }]]; + + it('should return correct results, transient false', done => { + mongoTestClient.db.setReturnValues(testRetValues); const testCollection = mongoTestClient.db.collection('test'); - mongoTestClient._handleMongo(testCollection, {}, log, (err, res) => { + mongoTestClient._handleMongo(testCollection, {}, false, log, + (err, res) => { assert.ifError(err, `Expected success, but got error ${err}`); assert.deepStrictEqual(res, { count: 100, @@ -456,4 +499,24 @@ describe('MongoClientInterface::_handleMongo', () => { return done(); }); }); + + it('should return correct results, transient true', done => { + mongoTestClient.db.setReturnValues(testRetValues); + const testCollection = mongoTestClient.db.collection('test'); + mongoTestClient._handleMongo(testCollection, {}, true, log, + (err, res) => { + assert.ifError(err, `Expected success, but got error ${err}`); + assert.deepStrictEqual(res, { + count: 100, + data: { + locationone: 500, + locationtwo: 500, + awsbackend: 500, + azurebackend: 500, + gcpbackend: 500, + }, + }); + return done(); + }); + }); }); diff --git a/tests/unit/storage/metadata/mongoclient/utils/DummyConfigObject.js b/tests/unit/storage/metadata/mongoclient/utils/DummyConfigObject.js new file mode 100644 index 000000000..50f2089ff --- /dev/null +++ b/tests/unit/storage/metadata/mongoclient/utils/DummyConfigObject.js @@ -0,0 +1,16 @@ +const { EventEmitter } = require('events'); + +class DummyConfigObject extends EventEmitter { + constructor() { + super(); + this.locationConstraints = null; + this.isTest = true; + } + + setLocationConstraints(locationConstraints) { + this.locationConstraints = locationConstraints; + this.emit('location-constraints-update'); + } +} + +module.exports = DummyConfigObject;