From 47094ee3d2dbfa0b432b53e67eb69c2454bb3111 Mon Sep 17 00:00:00 2001 From: Seth Silesky <5115498+silesky@users.noreply.github.com> Date: Mon, 31 Jul 2023 15:36:19 -0500 Subject: [PATCH 1/3] Add setting to prune unmapped categories, and turn off behavior by default. --- .changeset/ten-rabbits-draw.md | 5 +++++ .../consent-tools/src/types/settings.ts | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 .changeset/ten-rabbits-draw.md diff --git a/.changeset/ten-rabbits-draw.md b/.changeset/ten-rabbits-draw.md new file mode 100644 index 000000000..fe04010d0 --- /dev/null +++ b/.changeset/ten-rabbits-draw.md @@ -0,0 +1,5 @@ +--- +'@segment/analytics-consent-tools': patch +--- + +Remove default behavior that prunes unmapped categories from context.contest payload. As such, by default, `allKeys` will no longer be used. Add ability to turn pruning back on via an `pruneUnmappedCategories` setting. diff --git a/packages/consent/consent-tools/src/types/settings.ts b/packages/consent/consent-tools/src/types/settings.ts index 8a08d5552..3800ab9a0 100644 --- a/packages/consent/consent-tools/src/types/settings.ts +++ b/packages/consent/consent-tools/src/types/settings.ts @@ -66,4 +66,24 @@ export interface CreateWrapperSettings { categories: Categories, integrationInfo: Pick ) => boolean + + /** + * Prune consent categories from the `context.consent.categoryPreferences` payload if that category is not mapped to any integration in your Segment.io source. + * This is helpful if you want to save on bytes sent to Segment and do need the complete list of CMP's categories for debugging or other reasons. + * By default, all consent categories returned by `getCategories()` are sent to Segment. + * @default false + * ### Example Behavior + * You have the following categories mappings defined: + * ``` + * FullStory -> 'CAT002', + * Braze -> 'CAT003' + * ``` + * ```ts + * // pruneUnmappedCategories = false (default) + * { CAT0001: true, CAT0002: true, CAT0003: true } + * // pruneUnmappedCategories = true + * { CAT0002: true, CAT0003: true } // pruneUnmappedCategories = true + * ``` + */ + pruneUnmappedCategories?: boolean } From cdd23970dd8c15c5fbe8c4c013a88237c7d5f9b2 Mon Sep 17 00:00:00 2001 From: Seth Silesky <5115498+silesky@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:04:01 -0500 Subject: [PATCH 2/3] wip --- .../domain/__tests__/create-wrapper.test.ts | 183 +++++++++++------- .../src/domain/create-wrapper.ts | 36 +++- 2 files changed, 146 insertions(+), 73 deletions(-) diff --git a/packages/consent/consent-tools/src/domain/__tests__/create-wrapper.test.ts b/packages/consent/consent-tools/src/domain/__tests__/create-wrapper.test.ts index 1a4771dcb..bdd7f5d12 100644 --- a/packages/consent/consent-tools/src/domain/__tests__/create-wrapper.test.ts +++ b/packages/consent/consent-tools/src/domain/__tests__/create-wrapper.test.ts @@ -561,86 +561,139 @@ describe(createWrapper, () => { }) describe('Consent Stamping', () => { - it('should throw an error if there are no configured categories', async () => { - const fn = jest.spyOn(ConsentStamping, 'createConsentStampingMiddleware') - const mockCdnSettings = settingsBuilder - .addActionDestinationSettings({ - creationName: 'Some Other Plugin', + test.each([ + { + getCategories: () => ({ + Something: true, + SomethingElse: false, + }), + returnVal: 'Categories', + }, + { + getCategories: () => + Promise.resolve({ + Something: true, + SomethingElse: false, + }), + returnVal: 'Promise', + }, + ])( + 'should, by default, stamp the event with _all_ consent info if getCategories returns $returnVal', + async ({ getCategories }) => { + const fn = jest.spyOn( + ConsentStamping, + 'createConsentStampingMiddleware' + ) + const mockCdnSettings = settingsBuilder.build() + + wrapTestAnalytics({ + getCategories, + }) + await analytics.load({ + ...DEFAULT_LOAD_SETTINGS, + cdnSettings: mockCdnSettings, }) - .build() - wrapTestAnalytics() - await analytics.load({ - ...DEFAULT_LOAD_SETTINGS, - cdnSettings: mockCdnSettings, + const getCategoriesFn = fn.mock.lastCall[0] + await expect(getCategoriesFn()).resolves.toEqual({ + Something: true, + SomethingElse: false, + }) + } + ) + describe('pruneUnmappedCategories', () => { + it('should throw an error if there are no configured categories', async () => { + const fn = jest.spyOn( + ConsentStamping, + 'createConsentStampingMiddleware' + ) + const mockCdnSettings = settingsBuilder + .addActionDestinationSettings({ + creationName: 'Some Other Plugin', + }) + .build() + + wrapTestAnalytics({ pruneUnmappedCategories: true }) + await analytics.load({ + ...DEFAULT_LOAD_SETTINGS, + cdnSettings: mockCdnSettings, + }) + + const getCategoriesFn = fn.mock.lastCall[0] + await expect(() => + getCategoriesFn() + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"[Validation] Invariant: No consent categories defined in Segment (Received: [])"` + ) }) - const getCategoriesFn = fn.mock.lastCall[0] - await expect(() => - getCategoriesFn() - ).rejects.toThrowErrorMatchingInlineSnapshot( - `"[Validation] Invariant: No consent categories defined in Segment (Received: [])"` - ) - }) + it('should exclude properties that are not configured based on the allCategories array', async () => { + const fn = jest.spyOn( + ConsentStamping, + 'createConsentStampingMiddleware' + ) + const mockCdnSettings = settingsBuilder + .addActionDestinationSettings({ + creationName: 'Some Other Plugin', + ...createConsentSettings(['Foo']), + }) + .build() - it('should exclude properties that are not configured based on the allCategories array', async () => { - const fn = jest.spyOn(ConsentStamping, 'createConsentStampingMiddleware') - const mockCdnSettings = settingsBuilder - .addActionDestinationSettings({ - creationName: 'Some Other Plugin', - ...createConsentSettings(['Foo']), - }) - .build() + ;(mockCdnSettings as any).consentSettings = { + allCategories: ['Foo', 'Bar'], + } - ;(mockCdnSettings as any).consentSettings = { - allCategories: ['Foo', 'Bar'], - } + wrapTestAnalytics({ + pruneUnmappedCategories: true, + getCategories: () => ({ + Foo: true, + Bar: false, + Rand1: false, + Rand2: true, + }), + }) + await analytics.load({ + ...DEFAULT_LOAD_SETTINGS, + cdnSettings: mockCdnSettings, + }) - wrapTestAnalytics({ - getCategories: () => ({ + const getCategoriesFn = fn.mock.lastCall[0] + await expect(getCategoriesFn()).resolves.toEqual({ Foo: true, Bar: false, - Rand1: false, - Rand2: true, - }), - }) - await analytics.load({ - ...DEFAULT_LOAD_SETTINGS, - cdnSettings: mockCdnSettings, + }) }) - const getCategoriesFn = fn.mock.lastCall[0] - await expect(getCategoriesFn()).resolves.toEqual({ - Foo: true, - Bar: false, - }) - }) + it('should exclude properties that are not configured if integrationCategoryMappings are passed', async () => { + const fn = jest.spyOn( + ConsentStamping, + 'createConsentStampingMiddleware' + ) + const mockCdnSettings = settingsBuilder + .addActionDestinationSettings({ + creationName: 'Some Other Plugin', + }) + .build() - it('should exclude properties that are not configured if integrationCategoryMappings are passed', async () => { - const fn = jest.spyOn(ConsentStamping, 'createConsentStampingMiddleware') - const mockCdnSettings = settingsBuilder - .addActionDestinationSettings({ - creationName: 'Some Other Plugin', + wrapTestAnalytics({ + pruneUnmappedCategories: true, + getCategories: () => ({ + Foo: true, + Rand1: true, + Rand2: false, + }), + integrationCategoryMappings: { + 'Some Other Plugin': ['Foo'], + }, + }) + await analytics.load({ + ...DEFAULT_LOAD_SETTINGS, + cdnSettings: mockCdnSettings, }) - .build() - wrapTestAnalytics({ - getCategories: () => ({ - Foo: true, - Rand1: true, - Rand2: false, - }), - integrationCategoryMappings: { - 'Some Other Plugin': ['Foo'], - }, + const getCategoriesFn = fn.mock.lastCall[0] + await expect(getCategoriesFn()).resolves.toEqual({ Foo: true }) }) - await analytics.load({ - ...DEFAULT_LOAD_SETTINGS, - cdnSettings: mockCdnSettings, - }) - - const getCategoriesFn = fn.mock.lastCall[0] - await expect(getCategoriesFn()).resolves.toEqual({ Foo: true }) }) }) }) diff --git a/packages/consent/consent-tools/src/domain/create-wrapper.ts b/packages/consent/consent-tools/src/domain/create-wrapper.ts index 96c52d7b2..8a552594f 100644 --- a/packages/consent/consent-tools/src/domain/create-wrapper.ts +++ b/packages/consent/consent-tools/src/domain/create-wrapper.ts @@ -22,6 +22,7 @@ export const createWrapper: CreateWrapper = (createWrapperOptions) => { shouldLoad, integrationCategoryMappings, shouldEnableIntegration, + pruneUnmappedCategories, } = createWrapperOptions return (analytics) => { @@ -65,13 +66,11 @@ export const createWrapper: CreateWrapper = (createWrapperOptions) => { validateCategories(initialCategories) - const cdnSettingsP = new Promise((resolve) => - analytics.on('initialize', resolve) - ) - - // we don't want to send _every_ category to segment, only the ones that the user has explicitly configured in their integrations - const getFilteredSelectedCategories = async (): Promise => { + const getPrunedCategories = async ( + cdnSettingsP: Promise + ): Promise => { const cdnSettings = await cdnSettingsP + // we don't want to send _every_ category to segment, only the ones that the user has explicitly configured in their integrations let allCategories: string[] // We need to get all the unique categories so we can prune the consent object down to only the categories that are configured // There can be categories that are not included in any integration in the integrations object (e.g. 2 cloud mode categories), which is why we need a special allCategories array @@ -94,14 +93,27 @@ export const createWrapper: CreateWrapper = (createWrapperOptions) => { } const categories = await getCategories() - validateCategories(categories) return pick(categories, allCategories) } + // create getCategories and validate them regardless of whether pruning is turned on or off + const getValidCategoriesForConsentStamping = pipe( + pruneUnmappedCategories + ? getPrunedCategories.bind( + this, + initializeCDNSettingsEventListener(analytics) + ) + : async () => getCategories(), + (categories) => { + validateCategories(categories) + return categories + } + ) + // register listener to stamp all events with latest consent information analytics.addSourceMiddleware( - createConsentStampingMiddleware(getFilteredSelectedCategories) + createConsentStampingMiddleware(getValidCategoriesForConsentStamping) ) const updateCDNSettings: InitOptions['updateCDNSettings'] = ( @@ -205,3 +217,11 @@ const disableIntegrations = ( ) return results } + +const initializeCDNSettingsEventListener = ( + analytics: AnyAnalytics +): Promise => { + return new Promise((resolve) => + analytics.on('initialize', resolve) + ) +} From d7af859bdd7b352dfd8a45884153ca8fb89aa29a Mon Sep 17 00:00:00 2001 From: Seth Silesky <5115498+silesky@users.noreply.github.com> Date: Wed, 2 Aug 2023 13:54:56 -0500 Subject: [PATCH 3/3] wip --- .../domain/__tests__/create-wrapper.test.ts | 40 +++++++++++++++++++ .../src/domain/create-wrapper.ts | 20 ++++------ 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/packages/consent/consent-tools/src/domain/__tests__/create-wrapper.test.ts b/packages/consent/consent-tools/src/domain/__tests__/create-wrapper.test.ts index bdd7f5d12..f7071f3af 100644 --- a/packages/consent/consent-tools/src/domain/__tests__/create-wrapper.test.ts +++ b/packages/consent/consent-tools/src/domain/__tests__/create-wrapper.test.ts @@ -526,6 +526,45 @@ describe(createWrapper, () => { expect(analyticsLoadSpy).not.toBeCalled() }) }) + test.each([ + { + getCategories: () => + ({ + invalidCategory: 'hello', + } as any), + returnVal: 'Categories', + }, + { + getCategories: () => + Promise.resolve({ + invalidCategory: 'hello', + }) as any, + returnVal: 'Promise', + }, + ])( + 'should throw an error if getCategories() returns invalid categories during consent stamping ($returnVal))', + async ({ getCategories }) => { + const fn = jest.spyOn(ConsentStamping, 'createConsentStampingMiddleware') + const mockCdnSettings = settingsBuilder.build() + + wrapTestAnalytics({ + getCategories, + shouldLoad: () => { + // on first load, we should not get an error because this is a valid category setting + return { invalidCategory: true } + }, + }) + await analytics.load({ + ...DEFAULT_LOAD_SETTINGS, + cdnSettings: mockCdnSettings, + }) + + const getCategoriesFn = fn.mock.lastCall[0] + await expect(getCategoriesFn()).rejects.toMatchInlineSnapshot( + `[ValidationError: [Validation] Consent Categories should be {[categoryName: string]: boolean} (Received: {"invalidCategory":"hello"})]` + ) + } + ) describe('shouldEnableIntegration', () => { it('should let user customize the logic that determines whether or not a destination is enabled', async () => { @@ -601,6 +640,7 @@ describe(createWrapper, () => { }) } ) + describe('pruneUnmappedCategories', () => { it('should throw an error if there are no configured categories', async () => { const fn = jest.spyOn( diff --git a/packages/consent/consent-tools/src/domain/create-wrapper.ts b/packages/consent/consent-tools/src/domain/create-wrapper.ts index 8a552594f..7e582ba1b 100644 --- a/packages/consent/consent-tools/src/domain/create-wrapper.ts +++ b/packages/consent/consent-tools/src/domain/create-wrapper.ts @@ -102,14 +102,16 @@ export const createWrapper: CreateWrapper = (createWrapperOptions) => { pruneUnmappedCategories ? getPrunedCategories.bind( this, - initializeCDNSettingsEventListener(analytics) + new Promise((resolve) => + analytics.on('initialize', resolve) + ) ) - : async () => getCategories(), - (categories) => { - validateCategories(categories) + : getCategories, + async (categories) => { + validateCategories(await categories) return categories } - ) + ) as () => Promise // register listener to stamp all events with latest consent information analytics.addSourceMiddleware( @@ -217,11 +219,3 @@ const disableIntegrations = ( ) return results } - -const initializeCDNSettingsEventListener = ( - analytics: AnyAnalytics -): Promise => { - return new Promise((resolve) => - analytics.on('initialize', resolve) - ) -}