aws-lambda-event-sources': filters option not filtering items

[ETisREAL]

Describe the bug

I am trying to filter the items my dynamodb table streams out in order to avoid unnecessary invocations and cut on costs.

This is my code;

const leaderboardsStreamHandler = new lambda.Function(this, 'leaderboardsStreamHandler', {
            functionName: 'leaderboardsStreamHandler',
            code: lambda.Code.fromAsset(path.resolve('resources/lambdas-code/leaderboardsStreamHandler/code')),
            handler: 'leaderboardsStreamHandler.handler',
            runtime: lambda.Runtime.NODEJS_18_X,
            environment: {
                'TABLE_NAME': props.leaderboardsTable.tableName,
                'STAGE': props.STAGE.toLowerCase()
            },
            events: [
                new lambda_event_sources.DynamoEventSource(props.leaderboardsTable, {
                    startingPosition: lambda.StartingPosition.TRIM_HORIZON,
                    retryAttempts: 2,
                    batchSize: 1,
                    filters: [
                        lambda.FilterCriteria.filter({
                            dynamodb: { Keys: { PK: {S: lambda.FilterRule.beginsWith('PARTICIPANT#')}, SK: {S: lambda.FilterRule.beginsWith('#TOURNAMENT#')} } }
                        })
                    ]
                })
            ],
            logRetention: RetentionDays.ONE_WEEK,
            description: 'Lambda function responsible for creating and updating leaderboards'
        })

Expected Behavior

The event is mapped correctly, so I would expect the items to get filtered on this basis (PK = 'value' and begins_with(SK, 'value')

Current Behavior

Currently no filtering is getting applied. I always receive all the events

Reproduction Steps

Try with the provded code above

Possible Solution

Probaily some miscofiguration or missing functionality in the filtering

Additional Information/Context

No response

CDK CLI Version

2.87

Framework Version

No response

Node.js Version

v18.04

OS

Linux - Ubuntu

Language

Typescript

Language Version

No response

Other information

No response

[pahud]

@ETisREAL

Can you share your cdk synth output?

[ETisREAL]

@pahud Sure, here is the relevant section (if you need it all just let me know, I'l get back ASAP)

 "participantsStreamHandler9F450C21": {
   "Type": "AWS::Lambda::Function",
   "Properties": {
    "Code": {
     "S3Bucket": "cdk-hnb659fds-assets-661765328595-eu-central-1",
     "S3Key": "d72543086c125efe7c12ad7fe6b7258564088739eeb66fcd57d8bbb5ff80bb68.zip"
    },
    "Role": {
     "Fn::GetAtt": [
      "participantsStreamHandlerServiceRole6463E309",
      "Arn"
     ]
    },
    "Description": "Lambda function responsible for inserting participants in the Leaderboards table",
    "Environment": {
     "Variables": {
      "TABLE_NAME": {
       "Fn::ImportValue": "STG-LeaderboardsTableStack:ExportsOutputRefSTGleaderboardsTableC7C289A130FFDA4A"
      },
      "STAGE": "stg"
     }
    },
    "FunctionName": "participantsStreamHandler",
    "Handler": "participantsStreamHandler.handler",
    "Layers": [
     {
      "Ref": "admZipLayerFFD4353E"
     }
    ],
    "Runtime": "nodejs18.x"
   },
   "DependsOn": [
    "participantsStreamHandlerServiceRoleDefaultPolicy8C58998A",
    "participantsStreamHandlerServiceRole6463E309"
   ],
   "Metadata": {
    "aws:cdk:path": "STG-ParticipantsStreamLambdaStack/participantsStreamHandler/Resource",
    "aws:asset:path": "asset.d72543086c125efe7c12ad7fe6b7258564088739eeb66fcd57d8bbb5ff80bb68",
    "aws:asset:is-bundled": false,
    "aws:asset:property": "Code"
   }
  },
  "participantsStreamHandlerDynamoDBEventSourceSTGLeaderboardsTableStackSTGleaderboardsTableB0219B40A93492E4": {
   "Type": "AWS::Lambda::EventSourceMapping",
   "Properties": {
    "FunctionName": {
     "Ref": "participantsStreamHandler9F450C21"
    },
    "BatchSize": 1,
    "EventSourceArn": {
     "Fn::ImportValue": "STG-LeaderboardsTableStack:ExportsOutputFnGetAttSTGleaderboardsTableC7C289A1StreamArn2B3F721C"
    },
    "FilterCriteria": {
     "Filters": [
      {
       "Pattern": "{\"dynamodb\":{\"Keys\":{\"PK\":{\"S\":[\"#LEADERBOARDS\"]},\"SK\":{\"S\":[{\"prefix\":\"LEADERBOARD#\"}]}}}}"
      }
     ]
    },
    "MaximumRetryAttempts": 2,
    "StartingPosition": "TRIM_HORIZON"
   },
   "Metadata": {
    "aws:cdk:path": "STG-ParticipantsStreamLambdaStack/participantsStreamHandler/DynamoDBEventSource:STGLeaderboardsTableStackSTGleaderboardsTableB0219B40/Resource"
   }
  },
  "participantsStreamHandlerLogRetention114235D1": {

[roger-zhangg]

Hey @ETisREAL Thanks for the template, However I can't reproduce this issue in the latest version of CDK.

Template code

import * as cdk from 'aws-cdk-lib';
import { Construct } from 'constructs';
import * as lambda from 'aws-cdk-lib/aws-lambda'
import * as lambda_event_sources from 'aws-cdk-lib/aws-lambda-event-sources'
import { DynamoDbDataSource } from 'aws-cdk-lib/aws-appsync';
import { Table } from 'aws-cdk-lib/aws-dynamodb';
import * as dynamodb from 'aws-cdk-lib/aws-dynamodb';
// import * as sqs from 'aws-cdk-lib/aws-sqs';

export class BuildStack extends cdk.Stack {
  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
    super(scope, id, props);
    const DB = new dynamodb.TableV2(this, "newbkt",{
      partitionKey: {
        name: 'id',
        type: dynamodb.AttributeType.STRING,
      },
      dynamoStream: dynamodb.StreamViewType.OLD_IMAGE,
    }
      );
    const leaderboardsStreamHandler = new lambda.Function(this, 'leaderboardsStreamHandler', {
      functionName: 'leaderboardsStreamHandler',
      code: lambda.Code.fromInline("abc"),
      handler: 'leaderboardsStreamHandler.handler',
      runtime: lambda.Runtime.NODEJS_18_X,
      events: [
          new lambda_event_sources.DynamoEventSource(DB, {
              startingPosition: lambda.StartingPosition.TRIM_HORIZON,
              retryAttempts: 2,
              batchSize: 1,
              filters: [
                  lambda.FilterCriteria.filter({
                      dynamodb: { Keys: { PK: {S: lambda.FilterRule.beginsWith('PARTICIPANT#')}, SK: {S: lambda.FilterRule.beginsWith('#TOURNAMENT#')} } }
                  })
              ]
          })
      ],
      description: 'Lambda function responsible for creating and updating leaderboards'
  })
  }
}

The created Event Source Mapping

leaderboardsStreamHandlerDynamoDBEventSourceBuildStacknewbkt323659D60BC12573:
    Type: AWS::Lambda::EventSourceMapping
    Properties:
      BatchSize: 1
      EventSourceArn:
        Fn::GetAtt:
          - newbkt4C1E3631
          - StreamArn
      FilterCriteria:
        Filters:
          - Pattern: '{"dynamodb":{"Keys":{"PK":{"S":[{"prefix":"PARTICIPANT#"}]},"SK":{"S":[{"prefix":"#TOURNAMENT#"}]}}}}'
      FunctionName:
        Ref: leaderboardsStreamHandlerC0B43229
      MaximumRetryAttempts: 2
      StartingPosition: TRIM_HORIZON
    Metadata:
      aws:cdk:path: BuildStack/leaderboardsStreamHandler/DynamoDBEventSource:BuildStacknewbkt323659D6/Resource

I can see the Filters Pattern is correctly generated.

[ETisREAL]

Hello @roger-zhangg eventually I got it to work. Can't really tell if I changed something somewhere else or if there was a bug in the CDK. Either way, thank you very much for your time :) Much appreciated

[github-actions]

⚠️COMMENT VISIBILITY WARNING⚠️

Comments on closed issues are hard for our team to see.
If you need more assistance, please either tag a team member or open a new issue that references this one.
If you wish to keep having a conversation with other community members under this issue feel free to do so.