From 21b4f81a507d98e35cb72efc72bb8728ae6501f8 Mon Sep 17 00:00:00 2001 From: Nick Aguilar Date: Mon, 22 Jul 2024 17:18:14 -0700 Subject: [PATCH] Squashed commit of the following: commit 199765a87a153b1aadd3e8598f1ef16de28d938b Merge: 3ea14442f f0e5ac2e4 Author: Nick Aguilar Date: Mon Jul 22 17:16:35 2024 -0700 Merge branch 'main' into fbca-retl-hook commit 3ea14442f05419109d1e4a24ba8f67718ae7a74f Author: Nick Aguilar Date: Mon Jul 22 17:15:11 2024 -0700 Splits off engage and retl AdAccountId settings, fun fact: these cannot share a slug commit f0e5ac2e4c12d23374e7ca203c129aceae7f8007 Author: Joe Ayoub <45374896+joe-ayoub-segment@users.noreply.github.com> Date: Fri Jul 19 16:19:18 2024 +0200 aws s3 bug fixes (#2200) commit 0a1285a1fc6419f3a498e97aafd760db0993c9f1 Author: Nick Aguilar Date: Thu Jul 18 15:08:29 2024 -0700 Generates types commit 8f5f5ed48a05cd2b24d121925fc81bb35033b1b3 Author: Nick Aguilar Date: Thu Jul 18 15:04:14 2024 -0700 WIP on an inexplicably not passing unit test commit fc4f5ed07e908ad29e434a854410aa4351aa3be0 Author: Nick Aguilar Date: Thu Jul 18 14:05:48 2024 -0700 Checks for existence of user selected audience. Saves audience name and ID when returning from hook commit a2a642c395e9f004f65950b09643490da5dbecfa Author: Nick Aguilar Date: Thu Jul 18 12:58:03 2024 -0700 WIP - verifying selected audience exists and saving it's ID as output commit a022704c9668d8ca92ee0651cd64a024f690e78b Author: Nick Aguilar Date: Thu Jul 18 12:47:32 2024 -0700 Removes description param from creating an audience per PRD commit 49add301d15fbaec2c658cfddd3af65a525595d5 Author: Nick Aguilar Date: Thu Jul 18 12:44:07 2024 -0700 Pulls down 200 most recent audiences in dynamic field response commit c2f3e32e477e5df962721b93eac17066ed933284 Author: Nick Aguilar Date: Thu Jul 18 12:35:50 2024 -0700 Removes pagination related code - that work will be saved for later :) commit 6078690694e7041d1020f6b39d03c930fbcd7099 Merge: b831f7b50 c9ecf66fa Author: Nick Aguilar Date: Thu Jul 18 12:31:29 2024 -0700 Merge branch 'main' into fbca-retl-hook commit b831f7b506006ce1e202ea897ce4e7ec583004f2 Author: Nick Aguilar Date: Wed Jul 17 15:14:15 2024 -0700 Sets paging return such that the previous page was the current page commit 365690e16ef5b6ca9576ab1cec6b76925a0b2db8 Author: Nick Aguilar Date: Wed Jul 17 15:09:14 2024 -0700 Can pass dynamicFieldContext & paging into local server request commit 037534c439b66e6d73c1926ca37457c7f9943c4d Author: Nick Aguilar Date: Wed Jul 17 14:37:23 2024 -0700 Fixes build commit 1e9d6168094a3f35787c7cb3fff43903ab9f4dca Author: Nick Aguilar Date: Tue Jul 16 16:55:05 2024 -0700 WIP on a pagination mechanism for dynamic fields commit 151d26cc349530af6201680a36a84a247ad16efe Author: Nick Aguilar Date: Tue Jul 16 16:14:03 2024 -0700 Adds a dynamic audience field, pulls down all customaudiences as choices commit 27e49035faf4ff80640b14654be9e33ca14a93b8 Author: Nick Aguilar Date: Tue Jul 16 15:29:46 2024 -0700 Adds required customer_file_source field commit 60d22911d80df4c98eaaee7a31fbe800ccc0884d Author: Nick Aguilar Date: Tue Jul 16 12:26:51 2024 -0700 WIP - first draft of a FacebookClient class, including an untested createAudience hook for retl commit 5ba6c09e430b11d90554ae6ef12da0353a6fe2ff Author: Nick Aguilar Date: Tue Jul 16 12:05:23 2024 -0700 Introduces shared adAccountId top-level setting and audienceSetting commit 0a20b76db6342aa9718c8a3a60a14a7ef143273b Author: Nick Aguilar Date: Tue Jul 16 12:02:16 2024 -0700 Removes delete action, add action. Introduces single sync action --- .../src/destinations/aws-s3/operations.ts | 78 ++++++++++++------- .../syncAudienceToCSV/generated-types.ts | 2 +- .../aws-s3/syncAudienceToCSV/index.ts | 7 +- .../generated-types.ts | 4 +- .../facebook-custom-audiences/index.ts | 6 +- .../facebook-custom-audiences/sync/index.ts | 9 ++- 6 files changed, 63 insertions(+), 43 deletions(-) diff --git a/packages/destination-actions/src/destinations/aws-s3/operations.ts b/packages/destination-actions/src/destinations/aws-s3/operations.ts index 874e387b64..dd0fcb1417 100644 --- a/packages/destination-actions/src/destinations/aws-s3/operations.ts +++ b/packages/destination-actions/src/destinations/aws-s3/operations.ts @@ -25,60 +25,60 @@ function generateFile(payloads: Payload[], audienceSettings: AudienceSettings): const additionalColumns = payloads[0].additional_identifiers_and_traits_columns ?? [] Object.entries(columnsField).forEach(([_, value]) => { - if (value !== undefined) { + if (![undefined, null, ''].includes(value)) { headers.push(value) } }) additionalColumns.forEach((additionalColumn) => { - headers.push(additionalColumn.value) + if (![undefined, null, ''].includes(additionalColumn.value)) { + headers.push(additionalColumn.value) + } }) const headerString = `${headers.join(audienceSettings.delimiter === 'tab' ? '\t' : audienceSettings.delimiter)}\n` - const rows: string[] = [headerString] payloads.forEach((payload, index, arr) => { const action = payload.propertiesOrTraits[payload.audienceName] const row: string[] = [] - if (headers.includes('audience_name')) { - row.push(enquoteIdentifier(String(payload.audienceName ?? ''))) + if (![undefined, null, ''].includes(columnsField.audience_name)) { + row.push(encodeString(String(payload.audienceName ?? ''))) } - if (headers.includes('audience_id')) { - row.push(enquoteIdentifier(String(payload.audienceId ?? ''))) + if (![undefined, null, ''].includes(columnsField.audience_id)) { + row.push(encodeString(String(payload.audienceId ?? ''))) } - if (headers.includes('audience_action')) { - row.push(enquoteIdentifier(String(action ?? ''))) + if (![undefined, null, ''].includes(columnsField.audience_action)) { + row.push(encodeString(String(action ?? ''))) } - if (headers.includes('email')) { - row.push(enquoteIdentifier(String(payload.email ?? ''))) + if (![undefined, null, ''].includes(columnsField.email)) { + row.push(encodeString(String(payload.email ?? ''))) } - if (headers.includes('user_id')) { - row.push(enquoteIdentifier(String(payload.userId ?? ''))) + if (![undefined, null, ''].includes(columnsField.user_id)) { + row.push(encodeString(String(payload.userId ?? ''))) } - if (headers.includes('anonymous_id')) { - row.push(enquoteIdentifier(String(payload.anonymousId ?? ''))) + if (![undefined, null, ''].includes(columnsField.anonymous_id)) { + row.push(encodeString(String(payload.anonymousId ?? ''))) } - if (headers.includes('timestamp')) { - row.push(enquoteIdentifier(String(payload.timestamp ?? ''))) + if (![undefined, null, ''].includes(columnsField.timestamp)) { + row.push(encodeString(String(payload.timestamp ?? ''))) } - if (headers.includes('message_id')) { - row.push(enquoteIdentifier(String(payload.messageId ?? ''))) + if (![undefined, null, ''].includes(columnsField.message_id)) { + row.push(encodeString(String(payload.messageId ?? ''))) } - if (headers.includes('space_id')) { - row.push(enquoteIdentifier(String(payload.spaceId ?? ''))) + if (![undefined, null, ''].includes(columnsField.space_id)) { + row.push(encodeString(String(payload.spaceId ?? ''))) } - if (headers.includes('integrations_object')) { - row.push(enquoteIdentifier(String(JSON.stringify(payload.integrationsObject) ?? ''))) + if (![undefined, null, ''].includes(columnsField.integrations_object)) { + row.push(encodeString(String(JSON.stringify(payload.integrationsObject) ?? ''))) } - if (headers.includes('properties_or_traits')) { - row.push(enquoteIdentifier(String(JSON.stringify(payload.propertiesOrTraits) ?? ''))) + if (![undefined, null, ''].includes(columnsField.properties_or_traits)) { + row.push(encodeString(String(JSON.stringify(payload.propertiesOrTraits) ?? ''))) } additionalColumns.forEach((additionalColumn) => { - //row.push(enquoteIdentifier(String(JSON.stringify(payload.propertiesOrTraits[additionalColumn.key]) ?? ''))) - row.push(enquoteIdentifier(String(JSON.stringify(additionalColumn.key) ?? ''))) + row.push(encodeString(String(JSON.stringify(payload.propertiesOrTraits[additionalColumn.key]) ?? ''))) }) const isLastRow = arr.length === index + 1 @@ -92,8 +92,28 @@ function generateFile(payloads: Payload[], audienceSettings: AudienceSettings): return rows.join('') } -function enquoteIdentifier(str: string) { +function encodeString(str: string) { return `"${String(str).replace(/"/g, '""')}"` } -export { generateFile, enquoteIdentifier } +function validate(payloads: Payload[], audienceSettings: AudienceSettings) { + const delimiter = audienceSettings.delimiter + const columns = payloads[0].columns + const additionalIdentifierColumns = payloads[0].additional_identifiers_and_traits_columns + + // ensure column names do not contain delimiter + Object.values(columns).forEach((columnName) => { + if (columnName.includes(delimiter)) { + throw new Error(`Column name ${columnName} cannot contain delimiter: ${delimiter}`) + } + }) + + // ensure additional identifier column names do not contain delimiter + additionalIdentifierColumns?.forEach((column) => { + if (column.value.includes(delimiter)) { + throw new Error(`Column name ${column.value} cannot contain delimiter: ${delimiter}`) + } + }) +} + +export { generateFile, validate } diff --git a/packages/destination-actions/src/destinations/aws-s3/syncAudienceToCSV/generated-types.ts b/packages/destination-actions/src/destinations/aws-s3/syncAudienceToCSV/generated-types.ts index 53dbc322b3..3156b2054a 100644 --- a/packages/destination-actions/src/destinations/aws-s3/syncAudienceToCSV/generated-types.ts +++ b/packages/destination-actions/src/destinations/aws-s3/syncAudienceToCSV/generated-types.ts @@ -36,7 +36,7 @@ export interface Payload { /** * Name of column for the unique identifier for the message. */ - messageId?: string + message_id?: string /** * Name of column for the unique identifier for the Segment Engage Space that generated the event. */ diff --git a/packages/destination-actions/src/destinations/aws-s3/syncAudienceToCSV/index.ts b/packages/destination-actions/src/destinations/aws-s3/syncAudienceToCSV/index.ts index 34a3b4d467..75512d3b37 100644 --- a/packages/destination-actions/src/destinations/aws-s3/syncAudienceToCSV/index.ts +++ b/packages/destination-actions/src/destinations/aws-s3/syncAudienceToCSV/index.ts @@ -1,7 +1,7 @@ import type { ActionDefinition } from '@segment/actions-core' import type { Settings, AudienceSettings } from '../generated-types' import type { Payload } from './generated-types' -import { generateFile } from '../operations' +import { generateFile, validate } from '../operations' import { S3CSVClient } from './s3' const action: ActionDefinition = { @@ -52,7 +52,7 @@ const action: ActionDefinition = { description: 'Name of column for timestamp for when the user was added or removed from the Audience', type: 'string' }, - messageId: { + message_id: { label: 'Message ID', description: 'Name of column for the unique identifier for the message.', type: 'string' @@ -241,10 +241,9 @@ const action: ActionDefinition = { } async function processData(payloads: Payload[], settings: Settings, audienceSettings?: AudienceSettings) { + validate(payloads, audienceSettings as AudienceSettings) const fileContent = generateFile(payloads, audienceSettings as AudienceSettings) - const s3Client = new S3CSVClient(settings.s3_aws_region, settings.iam_role_arn, settings.iam_external_id) - await s3Client.uploadS3(settings, audienceSettings as AudienceSettings, fileContent) } diff --git a/packages/destination-actions/src/destinations/facebook-custom-audiences/generated-types.ts b/packages/destination-actions/src/destinations/facebook-custom-audiences/generated-types.ts index 59b2e16987..44bc2e671a 100644 --- a/packages/destination-actions/src/destinations/facebook-custom-audiences/generated-types.ts +++ b/packages/destination-actions/src/destinations/facebook-custom-audiences/generated-types.ts @@ -4,7 +4,7 @@ export interface Settings { /** * Your advertiser account id. Read [more](https://www.facebook.com/business/help/1492627900875762). */ - adAccountId: string + retlAdAccountId: string } // Generated file. DO NOT MODIFY IT BY HAND. @@ -12,7 +12,7 @@ export interface AudienceSettings { /** * Your advertiser account id. Read [more](https://www.facebook.com/business/help/1492627900875762). */ - adAccountId: string + engageAdAccountId: string /** * A brief description about your audience. */ diff --git a/packages/destination-actions/src/destinations/facebook-custom-audiences/index.ts b/packages/destination-actions/src/destinations/facebook-custom-audiences/index.ts index 3d672ec30f..c1c5c7faa4 100644 --- a/packages/destination-actions/src/destinations/facebook-custom-audiences/index.ts +++ b/packages/destination-actions/src/destinations/facebook-custom-audiences/index.ts @@ -16,7 +16,7 @@ const destination: AudienceDestinationDefinition = { authentication: { scheme: 'oauth2', fields: { - adAccountId + retlAdAccountId: adAccountId }, refreshAccessToken: async () => { return { accessToken: 'TODO: Implement this' } @@ -30,7 +30,7 @@ const destination: AudienceDestinationDefinition = { } }, audienceFields: { - adAccountId, + engageAdAccountId: adAccountId, audienceDescription: { type: 'string', label: 'Description', @@ -45,7 +45,7 @@ const destination: AudienceDestinationDefinition = { }, async createAudience(request, createAudienceInput) { const audienceName = createAudienceInput.audienceName - const adAccountId = createAudienceInput.audienceSettings?.adAccountId + const adAccountId = createAudienceInput.audienceSettings?.engageAdAccountId const audienceDescription = createAudienceInput.audienceSettings?.audienceDescription if (!audienceName) { diff --git a/packages/destination-actions/src/destinations/facebook-custom-audiences/sync/index.ts b/packages/destination-actions/src/destinations/facebook-custom-audiences/sync/index.ts index 90101a3c54..a31546b1f6 100644 --- a/packages/destination-actions/src/destinations/facebook-custom-audiences/sync/index.ts +++ b/packages/destination-actions/src/destinations/facebook-custom-audiences/sync/index.ts @@ -10,11 +10,12 @@ const action: ActionDefinition = { hooks: { retlOnMappingSave: { label: 'Select or create an audience in Facebook', - description: 'TODO: Create or select an audience in Facebook.', + description: + 'When saving this mapping, Segment will either create a new audience in Facebook or connect to an existing one. To create a new audience, enter the name of the audience. To connect to an existing audience, select the audience ID from the dropdown.', inputFields: { audienceName: { type: 'string', - label: 'Audience Name', + label: 'Audience Creation Name', description: 'The name of the audience in Facebook.', default: 'TODO: Model Name by default' }, @@ -23,7 +24,7 @@ const action: ActionDefinition = { label: 'Existing Audience ID', description: 'The ID of the audience in Facebook.', dynamic: async (request, { settings }) => { - const fbClient = new FacebookClient(request, settings.adAccountId) + const fbClient = new FacebookClient(request, settings.retlAdAccountId) const { choices, error } = await fbClient.getAllAudiences() if (error) { @@ -51,7 +52,7 @@ const action: ActionDefinition = { } }, performHook: async (request, { settings, hookInputs }) => { - const fbClient = new FacebookClient(request, settings.adAccountId) + const fbClient = new FacebookClient(request, settings.retlAdAccountId) if (hookInputs.existingAudienceId) { const { data, error } = await fbClient.getSingleAudience(hookInputs.existingAudienceId)