diff --git a/lib/file_transfer_agent/gcs_util.js b/lib/file_transfer_agent/gcs_util.js index 2a9cf251d..fc46557ac 100644 --- a/lib/file_transfer_agent/gcs_util.js +++ b/lib/file_transfer_agent/gcs_util.js @@ -69,8 +69,15 @@ function GCSUtil(httpclient, filestream) { } }); - const storage = new Storage({ interceptors_: interceptors }); - + //TODO: SNOW-1789759 hardcoded region will be replaced in the future + const isRegionalUrlEnabled = (stageInfo.region).toLowerCase() === 'me-central2' || stageInfo.useRegionalUrl; + let endPoint = null; + if (stageInfo['endPoint']) { + endPoint = stageInfo['endPoint']; + } else if (isRegionalUrlEnabled) { + endPoint = `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`; + } + const storage = endPoint ? new Storage({ interceptors_: interceptors, apiEndpoint: endPoint }) : new Storage({ interceptors_: interceptors }); client = { gcsToken: gcsToken, gcsClient: storage }; } else { client = null; diff --git a/lib/file_transfer_agent/s3_util.js b/lib/file_transfer_agent/s3_util.js index 088d780b2..de3dfbc67 100644 --- a/lib/file_transfer_agent/s3_util.js +++ b/lib/file_transfer_agent/s3_util.js @@ -54,12 +54,19 @@ function S3Util(connectionConfig, s3, filestream) { this.createClient = function (stageInfo, useAccelerateEndpoint) { const stageCredentials = stageInfo['creds']; const securityToken = stageCredentials['AWS_TOKEN']; + const isRegionalUrlEnabled = stageInfo.useRegionalUrl || stageInfo.useS3RegionalUrl; + // if GS sends us an endpoint, it's likely for FIPS. Use it. let endPoint = null; if (stageInfo['endPoint']) { - endPoint = 'https://' + stageInfo['endPoint']; + endPoint = `https://${stageInfo['endPoint']}`; + } else { + if (stageInfo.region && isRegionalUrlEnabled) { + const domainSuffixForRegionalUrl = (stageInfo.region).toLowerCase().startsWith('cn-') ? 'amazonaws.com.cn' : 'amazonaws.com'; + endPoint = `https://s3.${stageInfo.region}.${domainSuffixForRegionalUrl}`; + } } - + const config = { apiVersion: '2006-03-01', region: stageInfo['region'], @@ -80,7 +87,7 @@ function S3Util(connectionConfig, s3, filestream) { } } if (proxy) { - const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), SNOWFLAKE_S3_DESTINATION); + const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), endPoint || SNOWFLAKE_S3_DESTINATION); config.requestHandler = new NodeHttpHandler({ httpAgent: proxyAgent, httpsAgent: proxyAgent diff --git a/test/unit/file_transfer_agent/gcs_test.js b/test/unit/file_transfer_agent/gcs_test.js index bd2de06e9..6ce0c84d9 100644 --- a/test/unit/file_transfer_agent/gcs_test.js +++ b/test/unit/file_transfer_agent/gcs_test.js @@ -64,6 +64,82 @@ describe('GCS client', function () { GCS = new SnowflakeGCSUtil(httpclient, filestream); }); + describe('GCS client endpoint testing', async function () { + const testCases = [ + { + name: 'when the useRegionalURL is only enabled', + stageInfo: { + endPoint: null, + useRegionalUrl: true, + region: 'mockLocation', + }, + result: 'https://storage.mocklocation.rep.googleapis.com' + }, + { + name: 'when the region is me-central2', + stageInfo: { + endPoint: null, + useRegionalUrl: false, + region: 'me-central2' + }, + result: 'https://storage.me-central2.rep.googleapis.com' + }, + { + name: 'when the region is me-central2 (mixed case)', + stageInfo: { + endPoint: null, + useRegionalUrl: false, + region: 'ME-cEntRal2' + }, + result: 'https://storage.me-central2.rep.googleapis.com' + }, + { + name: 'when the region is me-central2 (uppercase)', + stageInfo: { + endPoint: null, + useRegionalUrl: false, + region: 'ME-CENTRAL2' + }, + result: 'https://storage.me-central2.rep.googleapis.com' + }, + { + name: 'when the endPoint is specified', + stageInfo: { + endPoint: 'https://storage.specialEndPoint.rep.googleapis.com', + useRegionalUrl: false, + region: 'ME-cEntRal1' + }, + result: 'https://storage.specialEndPoint.rep.googleapis.com' + }, + { + name: 'when both the endPoint and the useRegionalUrl are specified', + stageInfo: { + endPoint: 'https://storage.specialEndPoint.rep.googleapis.com', + useRegionalUrl: true, + region: 'ME-cEntRal1' + }, + result: 'https://storage.specialEndPoint.rep.googleapis.com' + }, + { + name: 'when both the endPoint is specified and the region is me-central2', + stageInfo: { + endPoint: 'https://storage.specialEndPoint.rep.googleapis.com', + useRegionalUrl: true, + region: 'ME-CENTRAL2' + }, + result: 'https://storage.specialEndPoint.rep.googleapis.com' + }, + ]; + + testCases.forEach(({ name, stageInfo, result }) => { + it(name, () => { + const client = GCS.createClient({ ...stageInfo, ...meta.stageInfo, creds: { GCS_ACCESS_TOKEN: 'mockToken' } }); + assert.strictEqual(client.gcsClient.apiEndpoint, result); + } ); + + }); + }); + it('extract bucket name and path', async function () { const GCS = new SnowflakeGCSUtil(); diff --git a/test/unit/file_transfer_agent/s3_test.js b/test/unit/file_transfer_agent/s3_test.js index 244a84fbe..64bf0bf07 100644 --- a/test/unit/file_transfer_agent/s3_test.js +++ b/test/unit/file_transfer_agent/s3_test.js @@ -44,7 +44,7 @@ describe('S3 client', function () { before(function () { mock('s3', { - S3: function () { + S3: function (config) { function S3() { this.getObject = function () { function getObject() { @@ -57,6 +57,8 @@ describe('S3 client', function () { return new getObject; }; + + this.config = config; this.putObject = function () { function putObject() { this.then = function (callback) { @@ -82,6 +84,59 @@ describe('S3 client', function () { AWS = new SnowflakeS3Util(noProxyConnectionConfig, s3, filesystem); }); + describe('AWS client endpoint testing', async function () { + const originalStageInfo = meta.stageInfo; + const testCases = [ + { + name: 'when useS3RegionalURL is only enabled', + stageInfo: { + ...originalStageInfo, + useS3RegionalUrl: true, + endPoint: null, + }, + result: null + }, + { + name: 'when useS3RegionalURL and is enabled and domain starts with cn', + stageInfo: { + ...originalStageInfo, + useS3RegionalUrl: true, + endPoint: null, + region: 'cn-mockLocation' + }, + result: 'https://s3.cn-mockLocation.amazonaws.com.cn' + }, + { + name: 'when endPoint is enabled', + stageInfo: { + ...originalStageInfo, + endPoint: 's3.endpoint', + useS3RegionalUrl: false + }, + result: 'https://s3.endpoint' + }, + { + name: 'when both endPoint and useS3PReiongalUrl is valid', + stageInfo: { + ...originalStageInfo, + endPoint: 's3.endpoint', + useS3RegionalUrl: true, + + }, + result: 'https://s3.endpoint' + }, + ]; + + testCases.forEach(({ name, stageInfo, result }) => { + it(name, () => { + const client = AWS.createClient(stageInfo); + assert.strictEqual(client.config.endpoint, result); + } ); + + }); + }); + + it('extract bucket name and path', async function () { let result = extractBucketNameAndPath('sfc-eng-regression/test_sub_dir/'); assert.strictEqual(result.bucketName, 'sfc-eng-regression');