move ext. backend related unit tests

eve/main.yml

@@ -31,6 +31,11 @@ stages:
       - ShellCommand:
           name: run lint_md
           command: npm run --silent lint_md
+      - ShellCommand:
+          name: add hostname
+          command: sudo sh -c "echo '127.0.0.1 testrequestbucket.localhost' \
+            >> /etc/hosts"
+
       - ShellCommand:
           name: run test
           command: npm run --silent test

tests/unit/auth/v4/streamingV4/V4Transform.js

@@ -0,0 +1,78 @@
+const assert = require('assert');
+const { Readable } = require('stream');
+
+const V4Transform =
+    require('../../../../../lib/auth/v4/streamingV4/V4Transform');
+const { DummyRequestLogger, DummyAuthConfig } = require('../../../helpers');
+
+const log = new DummyRequestLogger();
+const streamingV4Params = {
+    accessKey: 'accessKey1',
+    signatureFromRequest: '2b8637632a997e06ee7b6c85d7' +
+        '147d2025e8f04d4374f4d7d7320de1618c7509',
+    region: 'us-east-1',
+    scopeDate: '20170516',
+    timestamp: '20170516T204738Z',
+    credentialScope: '20170516/us-east-1/s3/aws4_request',
+};
+const cloudserverConfig = new DummyAuthConfig();
+
+class AuthMe extends Readable {
+    constructor(chunks) {
+        super();
+        this._parts = chunks;
+        this._index = 0;
+    }
+
+    _read() {
+        this.push(this._parts[this._index]);
+        this._index++;
+    }
+}
+
+describe('V4Transform class', () => {
+    it('should authenticate successfully', done => {
+        const v4Transform = new V4Transform(streamingV4Params,
+        cloudserverConfig, log, err => {
+            assert.strictEqual(err, null);
+        });
+        const filler1 = '8;chunk-signature=51d2511f7c6887907dff20474d8db6' +
+        '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n';
+        const filler2 = '0;chunk-signature=c0eac24b7ce72141ec077df9753db' +
+        '4cc8b7991491806689da0395c8bd0231e48\r\n';
+        const chunks = [
+            Buffer.from(filler1),
+            Buffer.from(filler2),
+            null,
+        ];
+        const authMe = new AuthMe(chunks);
+        authMe.pipe(v4Transform);
+        v4Transform.on('finish', () => {
+            done();
+        });
+    });
+
+    it('should ignore data sent after final chunk', done => {
+        const v4Transform = new V4Transform(streamingV4Params,
+        cloudserverConfig, log, err => {
+            assert.strictEqual(err, null);
+            done();
+        });
+        const filler1 = '8;chunk-signature=51d2511f7c6887907dff20474d8db6' +
+        '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n';
+        const filler2 = '0;chunk-signature=c0eac24b7ce72141ec077df9753db' +
+        '4cc8b7991491806689da0395c8bd0231e48\r\n';
+        const filler3 = '\r\n';
+        const chunks = [
+            Buffer.from(filler1),
+            Buffer.from(filler2),
+            Buffer.from(filler3),
+            null,
+        ];
+        const authMe = new AuthMe(chunks);
+        authMe.pipe(v4Transform);
+        v4Transform.on('finish', () => {
+            done();
+        });
+    });
+});

tests/unit/helpers.js

@@ -1,4 +1,5 @@
 'use strict'; // eslint-disable-line strict
+const { EventEmitter } = require('events');
 
 const AuthInfo = require('../../lib/auth/AuthInfo');
 const constants = require('../../lib/constants');
@@ -110,5 +111,39 @@ class DummyRequestLogger {
     }
 }
 
+class DummyAuthConfig extends EventEmitter {
+    constructor() {
+        super();
+
+        this.backends = { auth: 'mem' };
+        this.authData = {
+            accounts:
+            [{ name: 'Bart',
+                email: '[email protected]',
+                arn: 'arn:aws:iam::123456789012:root',
+                canonicalID:
+                    '79a59df900b949e55d96a1e698fbacedf' +
+                    'd6e09d98eacf8f8d5218e7cd47ef2be',
+                shortid: '123456789012',
+                keys: [{ access: 'accessKey1', secret: 'verySecretKey1' }] },
+            { name: 'Lisa',
+                email: '[email protected]',
+                arn: 'arn:aws:iam::123456789013:root',
+                canonicalID:
+                    '79a59df900b949e55d96a1e698fbacedf' +
+                    'd6e09d98eacf8f8d5218e7cd47ef2bf',
+                shortid: '123456789013',
+                keys: [{ access: 'accessKey2', secret: 'verySecretKey2' }] },
+            ],
+        };
+    }
+
+    setAuthDataAccounts(accounts) {
+        this.authData.accounts = accounts;
+        this.emit('authdata-update');
+    }
+}
+
 module.exports = { makeid, timeDiff, makeAuthInfo,
-    createAlteredRequest, zpad, DummyRequestLogger };
+    createAlteredRequest, zpad, DummyRequestLogger,
+    DummyAuthConfig };

tests/unit/storage/data/DummyService.js

@@ -0,0 +1,62 @@
+const uuid = require('uuid/v4');
+
+class DummyService {
+    constructor(config = {}) {
+        this.versioning = config.versioning;
+    }
+    headBucket(params, callback) {
+        return callback();
+    }
+    getBucketVersioning(params, callback) {
+        if (this.versioning) {
+            return callback(null, { Status: 'Enabled' });
+        }
+        return callback(null, {});
+    }
+    headObject(params, callback) {
+        const retObj = {
+            ContentLength: `${1024 * 1024 * 1024}`,
+        };
+        return callback(null, retObj);
+    }
+    completeMultipartUpload(params, callback) {
+        const retObj = {
+            Bucket: params.Bucket,
+            Key: params.Key,
+            ETag: `"${uuid().replace(/-/g, '')}"`,
+            ContentLength: `${1024 * 1024 * 1024}`,
+        };
+        if (this.versioning) {
+            retObj.VersionId = uuid().replace(/-/g, '');
+        }
+        return callback(null, retObj);
+    }
+    upload(params, callback) {
+        this.putObject(params, callback);
+    }
+    putObject(params, callback) {
+        const retObj = {
+            ETag: `"${uuid().replace(/-/g, '')}"`,
+        };
+        if (this.versioning) {
+            retObj.VersionId = uuid().replace(/-/g, '');
+        }
+        return callback(null, retObj);
+    }
+    copyObject(params, callback) {
+        const retObj = {
+            CopyObjectResult: {
+                ETag: `"${uuid().replace(/-/g, '')}"`,
+                LastModified: new Date().toISOString(),
+            },
+            VersionId: null,
+        };
+        if (this.versioning) {
+            retObj.VersionId = uuid().replace(/-/g, '');
+        }
+        return callback(null, retObj);
+    }
+    // To-Do: add tests for other methods
+}
+
+module.exports = DummyService;

tests/unit/storage/data/external/ExternalClients.js

@@ -0,0 +1,107 @@
+const assert = require('assert');
+
+const AwsClient = require('../../../../../lib/storage/data/external/AwsClient');
+const GcpClient = require('../../../../../lib/storage/data/external/GcpClient');
+const AzureClient =
+    require('../../../../../lib/storage/data/external/AzureClient');
+const DummyService = require('../DummyService');
+const { DummyRequestLogger } = require('../../../helpers');
+
+const backendClients = [
+    {
+        Class: AwsClient,
+        name: 'AwsClient',
+        config: {
+            s3Params: {},
+            bucketName: 'awsTestBucketName',
+            dataStoreName: 'awsDataStore',
+            serverSideEncryption: false,
+            type: 'aws',
+        },
+    },
+    {
+        Class: GcpClient,
+        name: 'GcpClient',
+        config: {
+            s3Params: {},
+            bucketName: 'gcpTestBucketName',
+            mpuBucket: 'gcpTestMpuBucketName',
+            dataStoreName: 'gcpDataStore',
+            type: 'gcp',
+        },
+    },
+    {
+        Class: AzureClient,
+        name: 'AzureClient',
+        config: {
+            azureStorageEndpoint: '',
+            azureStorageCredentials: {
+                storageAccountName: 'scality',
+                storageAccessKey: 'Zm9vCg==',
+            },
+            azureContainerName: 'azureTestBucketName',
+            dataStoreName: 'azureDataStore',
+            type: 'azure',
+        },
+    },
+];
+const log = new DummyRequestLogger();
+
+describe('external backend clients', () => {
+    backendClients.forEach(backend => {
+        let testClient;
+
+        before(() => {
+            testClient = new backend.Class(backend.config);
+            testClient._client = new DummyService({ versioning: true });
+        });
+
+        if (backend.config.type !== 'azure') {
+            it(`${backend.name} completeMPU should return correctly ` +
+            'typed mpu results', done => {
+                const jsonList = {
+                    Part: [
+                        {
+                            PartNumber: [1],
+                            ETag: ['testpart0001etag'],
+                        },
+                        {
+                            PartNumber: [2],
+                            ETag: ['testpart0002etag'],
+                        },
+                        {
+                            PartNumber: [3],
+                            ETag: ['testpart0003etag'],
+                        },
+                    ],
+                };
+                const key = 'externalBackendTestKey';
+                const bucketName = 'externalBackendTestBucket';
+                const uploadId = 'externalBackendTestUploadId';
+                testClient.completeMPU(jsonList, null, key,
+                uploadId, bucketName, log, (err, res) => {
+                    assert.strictEqual(typeof res.key, 'string');
+                    assert.strictEqual(typeof res.eTag, 'string');
+                    assert.strictEqual(typeof res.dataStoreVersionId,
+                                       'string');
+                    assert.strictEqual(typeof res.contentLength, 'number');
+                    return done();
+                });
+            });
+        }
+
+        it(`${backend.name} toObjectGetInfo should return correct ` +
+        'objectGetInfo object', () => {
+            const key = 'externalBackendTestKey';
+            const bucketName = 'externalBackendTestBucket';
+            const objectGetInfo = testClient.toObjectGetInfo(key, bucketName);
+            assert.deepStrictEqual(objectGetInfo, {
+                // bucketMatch === false => expect bucket name to be
+                // prefixed to the backend key
+                key: 'externalBackendTestBucket/externalBackendTestKey',
+                dataStoreName: backend.config.dataStoreName,
+            });
+        });
+        // To-Do: test the other external client methods
+    });
+});