From b529891e617d683cad5efccc803a7d9ee9e6404b Mon Sep 17 00:00:00 2001 From: Juntao Wang Date: Tue, 11 Jun 2024 09:30:59 -0400 Subject: [PATCH] Remove DS pipeline endpoint hack --- .../tests/mocked/pipelines/pipelines.cy.ts | 10 +++---- .../__tests__/utils.spec.ts | 27 ++----------------- .../content/configurePipelinesServer/utils.ts | 7 +---- 3 files changed, 8 insertions(+), 36 deletions(-) diff --git a/frontend/src/__tests__/cypress/cypress/tests/mocked/pipelines/pipelines.cy.ts b/frontend/src/__tests__/cypress/cypress/tests/mocked/pipelines/pipelines.cy.ts index 758d92611d..ae15522b38 100644 --- a/frontend/src/__tests__/cypress/cypress/tests/mocked/pipelines/pipelines.cy.ts +++ b/frontend/src/__tests__/cypress/cypress/tests/mocked/pipelines/pipelines.cy.ts @@ -126,7 +126,7 @@ describe('Pipelines', () => { dspVersion: 'v2', objectStorage: { externalStorage: { - host: 's3.us-east-1.amazonaws.com', + host: 's3.amazonaws.com/', scheme: 'https', bucket: 'sdsd', region: 'us-east-1', @@ -167,7 +167,7 @@ describe('Pipelines', () => { pipelinesGlobal.findConfigurePipelineServerButton().click(); configurePipelineServerModal.findAwsKeyInput().type('test-aws-key'); configurePipelineServerModal.findAwsSecretKeyInput().type('test-secret-key'); - configurePipelineServerModal.findEndpointInput().type('https://s3.amazonaws.com/'); + configurePipelineServerModal.findEndpointInput().type('https://s3.amazonaws.com'); configurePipelineServerModal.findRegionInput().should('have.value', 'us-east-1'); configurePipelineServerModal.findBucketInput().type('test-bucket'); configurePipelineServerModal.findSubmitButton().should('be.enabled'); @@ -202,7 +202,7 @@ describe('Pipelines', () => { dspVersion: 'v2', objectStorage: { externalStorage: { - host: 's3.us-east-1.amazonaws.com', + host: 's3.amazonaws.com', scheme: 'https', bucket: 'test-bucket', region: 'us-east-1', @@ -244,7 +244,7 @@ describe('Pipelines', () => { configurePipelineServerModal.findAwsKeyInput().type('test-aws-key'); configurePipelineServerModal.findAwsSecretKeyInput().type('test-secret-key'); - configurePipelineServerModal.findEndpointInput().type('https://s3.amazonaws.com/'); + configurePipelineServerModal.findEndpointInput().type('https://s3.amazonaws.com'); configurePipelineServerModal.findRegionInput().should('have.value', 'us-east-1'); configurePipelineServerModal.findBucketInput().type('test-bucket'); @@ -302,7 +302,7 @@ describe('Pipelines', () => { dspVersion: 'v2', objectStorage: { externalStorage: { - host: 's3.us-east-1.amazonaws.com', + host: 's3.amazonaws.com', scheme: 'https', bucket: 'test-bucket', region: 'us-east-1', diff --git a/frontend/src/concepts/pipelines/content/configurePipelinesServer/__tests__/utils.spec.ts b/frontend/src/concepts/pipelines/content/configurePipelinesServer/__tests__/utils.spec.ts index 4aed8ca986..82da6d349d 100644 --- a/frontend/src/concepts/pipelines/content/configurePipelinesServer/__tests__/utils.spec.ts +++ b/frontend/src/concepts/pipelines/content/configurePipelinesServer/__tests__/utils.spec.ts @@ -68,7 +68,7 @@ describe('configure pipeline server utils', () => { ]; const spec = createDSPipelineResourceSpec(config, secretsResponse); expect(spec.objectStorage.externalStorage?.scheme).toBe('http'); - expect(spec.objectStorage.externalStorage?.host).toBe('s3.us-east-1.amazonaws.com'); + expect(spec.objectStorage.externalStorage?.host).toBe('s3.amazonaws.com'); }); it('should parse S3 endpoint without scheme', () => { @@ -77,30 +77,7 @@ describe('configure pipeline server utils', () => { config.objectStorage.newValue = [{ key: AwsKeys.S3_ENDPOINT, value: 's3.amazonaws.com' }]; const spec = createDSPipelineResourceSpec(config, secretsResponse); expect(spec.objectStorage.externalStorage?.scheme).toBe('https'); - expect(spec.objectStorage.externalStorage?.host).toBe('s3.us-east-1.amazonaws.com'); - }); - - it('should convert S3 endpoint with region', () => { - const config = createPipelineServerConfig(); - const secretsResponse = createSecretsResponse(); - config.objectStorage.newValue = [ - { key: AwsKeys.S3_ENDPOINT, value: 'http://s3.amazonaws.com' }, - { key: AwsKeys.DEFAULT_REGION, value: 'us-east-2' }, - ]; - const spec = createDSPipelineResourceSpec(config, secretsResponse); - expect(spec.objectStorage.externalStorage?.scheme).toBe('http'); - expect(spec.objectStorage.externalStorage?.host).toBe('s3.us-east-2.amazonaws.com'); - }); - - it('should not convert endpoint when it is not S3', () => { - const config = createPipelineServerConfig(); - const secretsResponse = createSecretsResponse(); - config.objectStorage.newValue = [ - { key: AwsKeys.S3_ENDPOINT, value: 'http://s3.not-amazonaws.com' }, - ]; - const spec = createDSPipelineResourceSpec(config, secretsResponse); - expect(spec.objectStorage.externalStorage?.scheme).toBe('http'); - expect(spec.objectStorage.externalStorage?.host).toBe('s3.not-amazonaws.com'); + expect(spec.objectStorage.externalStorage?.host).toBe('s3.amazonaws.com'); }); it('should include bucket', () => { diff --git a/frontend/src/concepts/pipelines/content/configurePipelinesServer/utils.ts b/frontend/src/concepts/pipelines/content/configurePipelinesServer/utils.ts index 53ac0d89d1..41584d343f 100644 --- a/frontend/src/concepts/pipelines/content/configurePipelinesServer/utils.ts +++ b/frontend/src/concepts/pipelines/content/configurePipelinesServer/utils.ts @@ -108,7 +108,7 @@ export const createDSPipelineResourceSpec = ( dspVersion: 'v2', objectStorage: { externalStorage: { - host: convertAWSHostForRegion(externalStorageHost, externalStorageRegion), + host: externalStorageHost, scheme: externalStorageScheme || 'https', bucket: awsRecord.AWS_S3_BUCKET || '', region: externalStorageRegion, @@ -157,8 +157,3 @@ export const getLabelName = (index: string): string => { const field = PIPELINE_AWS_FIELDS.find((currentField) => currentField.key === index); return field ? field.label : ''; }; - -const convertAWSHostForRegion = (endpoint: string, region: string): string => { - const host = endpoint.replace(/\/$/, '') || ''; - return host === 's3.amazonaws.com' && region !== '' ? `s3.${region}.amazonaws.com` : host; -};