Consent: Change default behavior to not prune unmapped categories from payload; put setting behind flag.

.changeset/ten-rabbits-draw.md

@@ -0,0 +1,5 @@
+---
+'@segment/analytics-consent-tools': patch
+---
+
+Remove default behavior that prunes unmapped categories from context.contest payload. As such, by default, `allKeys` will no longer be used. Add ability to turn pruning back on via an `pruneUnmappedCategories` setting.

packages/consent/consent-tools/src/domain/__tests__/create-wrapper.test.ts

@@ -526,6 +526,45 @@ describe(createWrapper, () => {
       expect(analyticsLoadSpy).not.toBeCalled()
     })
   })
+  test.each([
+    {
+      getCategories: () =>
+        ({
+          invalidCategory: 'hello',
+        } as any),
+      returnVal: 'Categories',
+    },
+    {
+      getCategories: () =>
+        Promise.resolve({
+          invalidCategory: 'hello',
+        }) as any,
+      returnVal: 'Promise<Categories>',
+    },
+  ])(
+    'should throw an error if getCategories() returns invalid categories during consent stamping ($returnVal))',
+    async ({ getCategories }) => {
+      const fn = jest.spyOn(ConsentStamping, 'createConsentStampingMiddleware')
+      const mockCdnSettings = settingsBuilder.build()
+
+      wrapTestAnalytics({
+        getCategories,
+        shouldLoad: () => {
+          // on first load, we should not get an error because this is a valid category setting
+          return { invalidCategory: true }
+        },
+      })
+      await analytics.load({
+        ...DEFAULT_LOAD_SETTINGS,
+        cdnSettings: mockCdnSettings,
+      })
+
+      const getCategoriesFn = fn.mock.lastCall[0]
+      await expect(getCategoriesFn()).rejects.toMatchInlineSnapshot(
+        `[ValidationError: [Validation] Consent Categories should be {[categoryName: string]: boolean} (Received: {"invalidCategory":"hello"})]`
+      )
+    }
+  )
 
   describe('shouldEnableIntegration', () => {
     it('should let user customize the logic that determines whether or not a destination is enabled', async () => {
@@ -561,86 +600,140 @@ describe(createWrapper, () => {
   })
 
   describe('Consent Stamping', () => {
-    it('should throw an error if there are no configured categories', async () => {
-      const fn = jest.spyOn(ConsentStamping, 'createConsentStampingMiddleware')
-      const mockCdnSettings = settingsBuilder
-        .addActionDestinationSettings({
-          creationName: 'Some Other Plugin',
+    test.each([
+      {
+        getCategories: () => ({
+          Something: true,
+          SomethingElse: false,
+        }),
+        returnVal: 'Categories',
+      },
+      {
+        getCategories: () =>
+          Promise.resolve({
+            Something: true,
+            SomethingElse: false,
+          }),
+        returnVal: 'Promise<Categories>',
+      },
+    ])(
+      'should, by default, stamp the event with _all_ consent info if getCategories returns $returnVal',
+      async ({ getCategories }) => {
+        const fn = jest.spyOn(
+          ConsentStamping,
+          'createConsentStampingMiddleware'
+        )
+        const mockCdnSettings = settingsBuilder.build()
+
+        wrapTestAnalytics({
+          getCategories,
+        })
+        await analytics.load({
+          ...DEFAULT_LOAD_SETTINGS,
+          cdnSettings: mockCdnSettings,
         })
-        .build()
 
-      wrapTestAnalytics()
-      await analytics.load({
-        ...DEFAULT_LOAD_SETTINGS,
-        cdnSettings: mockCdnSettings,
-      })
+        const getCategoriesFn = fn.mock.lastCall[0]
+        await expect(getCategoriesFn()).resolves.toEqual({
+          Something: true,
+          SomethingElse: false,
+        })
+      }
+    )
 
-      const getCategoriesFn = fn.mock.lastCall[0]
-      await expect(() =>
-        getCategoriesFn()
-      ).rejects.toThrowErrorMatchingInlineSnapshot(
-        `"[Validation] Invariant: No consent categories defined in Segment (Received: [])"`
-      )
-    })
+    describe('pruneUnmappedCategories', () => {
+      it('should throw an error if there are no configured categories', async () => {
+        const fn = jest.spyOn(
+          ConsentStamping,
+          'createConsentStampingMiddleware'
+        )
+        const mockCdnSettings = settingsBuilder
+          .addActionDestinationSettings({
+            creationName: 'Some Other Plugin',
+          })
+          .build()
 
-    it('should exclude properties that are not configured based on the allCategories array', async () => {
-      const fn = jest.spyOn(ConsentStamping, 'createConsentStampingMiddleware')
-      const mockCdnSettings = settingsBuilder
-        .addActionDestinationSettings({
-          creationName: 'Some Other Plugin',
-          ...createConsentSettings(['Foo']),
+        wrapTestAnalytics({ pruneUnmappedCategories: true })
+        await analytics.load({
+          ...DEFAULT_LOAD_SETTINGS,
+          cdnSettings: mockCdnSettings,
         })
-        .build()
 
-      ;(mockCdnSettings as any).consentSettings = {
-        allCategories: ['Foo', 'Bar'],
-      }
+        const getCategoriesFn = fn.mock.lastCall[0]
+        await expect(() =>
+          getCategoriesFn()
+        ).rejects.toThrowErrorMatchingInlineSnapshot(
+          `"[Validation] Invariant: No consent categories defined in Segment (Received: [])"`
+        )
+      })
 
-      wrapTestAnalytics({
-        getCategories: () => ({
+      it('should exclude properties that are not configured based on the allCategories array', async () => {
+        const fn = jest.spyOn(
+          ConsentStamping,
+          'createConsentStampingMiddleware'
+        )
+        const mockCdnSettings = settingsBuilder
+          .addActionDestinationSettings({
+            creationName: 'Some Other Plugin',
+            ...createConsentSettings(['Foo']),
+          })
+          .build()
+
+        ;(mockCdnSettings as any).consentSettings = {
+          allCategories: ['Foo', 'Bar'],
+        }
+
+        wrapTestAnalytics({
+          pruneUnmappedCategories: true,
+          getCategories: () => ({
+            Foo: true,
+            Bar: false,
+            Rand1: false,
+            Rand2: true,
+          }),
+        })
+        await analytics.load({
+          ...DEFAULT_LOAD_SETTINGS,
+          cdnSettings: mockCdnSettings,
+        })
+
+        const getCategoriesFn = fn.mock.lastCall[0]
+        await expect(getCategoriesFn()).resolves.toEqual({
           Foo: true,
           Bar: false,
-          Rand1: false,
-          Rand2: true,
-        }),
-      })
-      await analytics.load({
-        ...DEFAULT_LOAD_SETTINGS,
-        cdnSettings: mockCdnSettings,
+        })
       })
 
-      const getCategoriesFn = fn.mock.lastCall[0]
-      await expect(getCategoriesFn()).resolves.toEqual({
-        Foo: true,
-        Bar: false,
-      })
-    })
+      it('should exclude properties that are not configured if integrationCategoryMappings are passed', async () => {
+        const fn = jest.spyOn(
+          ConsentStamping,
+          'createConsentStampingMiddleware'
+        )
+        const mockCdnSettings = settingsBuilder
+          .addActionDestinationSettings({
+            creationName: 'Some Other Plugin',
+          })
+          .build()
 
-    it('should exclude properties that are not configured if integrationCategoryMappings are passed', async () => {
-      const fn = jest.spyOn(ConsentStamping, 'createConsentStampingMiddleware')
-      const mockCdnSettings = settingsBuilder
-        .addActionDestinationSettings({
-          creationName: 'Some Other Plugin',
+        wrapTestAnalytics({
+          pruneUnmappedCategories: true,
+          getCategories: () => ({
+            Foo: true,
+            Rand1: true,
+            Rand2: false,
+          }),
+          integrationCategoryMappings: {
+            'Some Other Plugin': ['Foo'],
+          },
+        })
+        await analytics.load({
+          ...DEFAULT_LOAD_SETTINGS,
+          cdnSettings: mockCdnSettings,
         })
-        .build()
 
-      wrapTestAnalytics({
-        getCategories: () => ({
-          Foo: true,
-          Rand1: true,
-          Rand2: false,
-        }),
-        integrationCategoryMappings: {
-          'Some Other Plugin': ['Foo'],
-        },
+        const getCategoriesFn = fn.mock.lastCall[0]
+        await expect(getCategoriesFn()).resolves.toEqual({ Foo: true })
       })
-      await analytics.load({
-        ...DEFAULT_LOAD_SETTINGS,
-        cdnSettings: mockCdnSettings,
-      })
-
-      const getCategoriesFn = fn.mock.lastCall[0]
-      await expect(getCategoriesFn()).resolves.toEqual({ Foo: true })
     })
   })
 })

packages/consent/consent-tools/src/domain/create-wrapper.ts

@@ -22,6 +22,7 @@ export const createWrapper: CreateWrapper = (createWrapperOptions) => {
     shouldLoad,
     integrationCategoryMappings,
     shouldEnableIntegration,
+    pruneUnmappedCategories,
   } = createWrapperOptions
 
   return (analytics) => {
@@ -65,13 +66,11 @@ export const createWrapper: CreateWrapper = (createWrapperOptions) => {
 
       validateCategories(initialCategories)
 
-      const cdnSettingsP = new Promise<CDNSettings>((resolve) =>
-        analytics.on('initialize', resolve)
-      )
-
-      // we don't want to send _every_ category to segment, only the ones that the user has explicitly configured in their integrations
-      const getFilteredSelectedCategories = async (): Promise<Categories> => {
+      const getPrunedCategories = async (
+        cdnSettingsP: Promise<CDNSettings>
+      ): Promise<Categories> => {
         const cdnSettings = await cdnSettingsP
+        // we don't want to send _every_ category to segment, only the ones that the user has explicitly configured in their integrations
         let allCategories: string[]
         // We need to get all the unique categories so we can prune the consent object down to only the categories that are configured
         // There can be categories that are not included in any integration in the integrations object (e.g. 2 cloud mode categories), which is why we need a special allCategories array
@@ -94,14 +93,29 @@ export const createWrapper: CreateWrapper = (createWrapperOptions) => {
         }
 
         const categories = await getCategories()
-        validateCategories(categories)
 
         return pick(categories, allCategories)
       }
 
+      // create getCategories and validate them regardless of whether pruning is turned on or off
+      const getValidCategoriesForConsentStamping = pipe(
+        pruneUnmappedCategories
+          ? getPrunedCategories.bind(
+              this,
+              new Promise<CDNSettings>((resolve) =>
+                analytics.on('initialize', resolve)
+              )
+            )
+          : getCategories,
+        async (categories) => {
+          validateCategories(await categories)
+          return categories
+        }
+      ) as () => Promise<Categories>
+
       // register listener to stamp all events with latest consent information
       analytics.addSourceMiddleware(
-        createConsentStampingMiddleware(getFilteredSelectedCategories)
+        createConsentStampingMiddleware(getValidCategoriesForConsentStamping)
       )
 
       const updateCDNSettings: InitOptions['updateCDNSettings'] = (

packages/consent/consent-tools/src/types/settings.ts

@@ -66,4 +66,24 @@ export interface CreateWrapperSettings {
     categories: Categories,
     integrationInfo: Pick<CDNSettingsRemotePlugin, 'creationName'>
   ) => boolean
+
+  /**
+   * Prune consent categories from the `context.consent.categoryPreferences` payload if that category is not mapped to any integration in your Segment.io source.
+   * This is helpful if you want to save on bytes sent to Segment and do need the complete list of CMP's categories for debugging or other reasons.
+   * By default, all consent categories returned by `getCategories()` are sent to Segment.
+   * @default false
+   * ### Example Behavior
+   * You have the following categories mappings defined:
+   * ```
+   * FullStory -> 'CAT002',
+   * Braze -> 'CAT003'
+   * ```
+   * ```ts
+   * // pruneUnmappedCategories = false (default)
+   * { CAT0001: true, CAT0002: true, CAT0003: true }
+   * // pruneUnmappedCategories = true
+   * { CAT0002: true, CAT0003: true  } // pruneUnmappedCategories = true
+   * ```
+   */
+  pruneUnmappedCategories?: boolean
 }