From 2fedec73788260d68b15cefd6f4cc8bab8cbec7c Mon Sep 17 00:00:00 2001 From: Alexandros Sigalas Date: Sun, 30 Aug 2020 08:38:04 +0300 Subject: [PATCH] LogTypes API ListAvailableLogTypes (#1427) * Adds lambdamux package to route lambda APIs and generate lambda client code * Add logtypes API * Add logtypes API client * Add logtypes API deployment * mage fmt * Publish LogTypesAPI to AppSync * Fix route handler * Return empty JSON object for empty route results * Split route handler to two functions * Code style * Fix panic when no types are available on DDB * Fix panic * Sort available logtypes as a courtesy * Rename var * mage fmt * Fix duplicate licenses and add comment about stackless error * Add API-side validation support for inputs in lambdamux * Add client side validation * Use lambdamux.Client to flesh out the client code * Use errors.Wrapf for lambdamux client errors * mage fmt * Add comments and handle error stacks * Move logger code related to lambdas to lambdalogger package * Separate bussiness logic from external actions * Add go generate to mage build:api * Add LogTypesAPI.ListAvailableLogTypes to GraphQL * Fix lambda routing and add check for non-pointer structHandlers * Use case-insensitive route name matching by default in lambdamux.Mux * Fix lambdalogger component field * Finalize lambdamux - Removes lambda dependency - Adds multiple routing modes (KeyValue, Keys, PeekKey) - Moves logging middleware to lambdalogger - Adds batch request support - Adds Chain helper to have fallbacks for NotFound errors - Moves lambda client base code to separate path * Finalize lambdamux and apigen Changes in lambdamux - Removes lambda dependency - Adds multiple routing modes (KeyValue, Keys, PeekKey) - Moves logging middleware to lambdalogger - Adds batch request support - Adds Chain helper to have fallbacks for NotFound errors - Move lambdamux under `panther/pkg/x` Changes in apigen - Adds inline models to generated client code - Rename the generator to apigen - Update usage - Move to a separate package at `panther/pkg/x/apigen` * Fix rename gone wrong * Add tests for lambdamux * fix lint --- api/graphql/schema.graphql | 5 + deployments/appsync.yml | 33 ++ deployments/core.yml | 87 ++++++ go.mod | 3 +- go.sum | 15 +- internal/core/logtypesapi/api.go | 43 +++ internal/core/logtypesapi/api_test.go | 31 ++ internal/core/logtypesapi/available.go | 59 ++++ internal/core/logtypesapi/available_test.go | 52 ++++ .../logtypesapi/client/lambdaclient_gen.go | 62 ++++ internal/core/logtypesapi/dynamodb.go | 85 +++++ internal/core/logtypesapi/main/lambda.go | 74 +++++ .../log_processor/common/oplog.go | 4 +- pkg/lambdalogger/lambdalogger.go | 138 ++++++++ pkg/lambdalogger/logger.go | 8 +- pkg/x/apigen/apigen.go | 227 ++++++++++++++ pkg/x/apigen/internal/methods.go | 171 ++++++++++ pkg/x/apigen/internal/models.go | 165 ++++++++++ pkg/x/apigen/internal/models_test.go | 19 ++ pkg/x/apigen/lambdaclient/lambdaclient.go | 94 ++++++ pkg/x/apigen/templates.go | 107 +++++++ pkg/x/lambdamux/batch.go | 140 +++++++++ pkg/x/lambdamux/batch_test.go | 51 +++ pkg/x/lambdamux/demux.go | 114 +++++++ pkg/x/lambdamux/demux_test.go | 128 ++++++++ pkg/x/lambdamux/lambadmux.go | 72 +++++ pkg/x/lambdamux/middleware.go | 79 +++++ pkg/x/lambdamux/mux.go | 234 ++++++++++++++ pkg/x/lambdamux/mux_test.go | 62 ++++ pkg/x/lambdamux/route.go | 294 ++++++++++++++++++ pkg/x/lambdamux/route_test.go | 86 +++++ tools/mage/build_namespace.go | 11 +- web/__generated__/schema.tsx | 22 ++ web/__tests__/__mocks__/builders.generated.ts | 10 + 34 files changed, 2774 insertions(+), 11 deletions(-) create mode 100644 internal/core/logtypesapi/api.go create mode 100644 internal/core/logtypesapi/api_test.go create mode 100644 internal/core/logtypesapi/available.go create mode 100644 internal/core/logtypesapi/available_test.go create mode 100755 internal/core/logtypesapi/client/lambdaclient_gen.go create mode 100644 internal/core/logtypesapi/dynamodb.go create mode 100644 internal/core/logtypesapi/main/lambda.go create mode 100644 pkg/lambdalogger/lambdalogger.go create mode 100644 pkg/x/apigen/apigen.go create mode 100644 pkg/x/apigen/internal/methods.go create mode 100644 pkg/x/apigen/internal/models.go create mode 100644 pkg/x/apigen/internal/models_test.go create mode 100644 pkg/x/apigen/lambdaclient/lambdaclient.go create mode 100644 pkg/x/apigen/templates.go create mode 100644 pkg/x/lambdamux/batch.go create mode 100644 pkg/x/lambdamux/batch_test.go create mode 100644 pkg/x/lambdamux/demux.go create mode 100644 pkg/x/lambdamux/demux_test.go create mode 100644 pkg/x/lambdamux/lambadmux.go create mode 100644 pkg/x/lambdamux/middleware.go create mode 100644 pkg/x/lambdamux/mux.go create mode 100644 pkg/x/lambdamux/mux_test.go create mode 100644 pkg/x/lambdamux/route.go create mode 100644 pkg/x/lambdamux/route_test.go diff --git a/api/graphql/schema.graphql b/api/graphql/schema.graphql index 19548c1d3f..8e16fe0f00 100644 --- a/api/graphql/schema.graphql +++ b/api/graphql/schema.graphql @@ -62,6 +62,7 @@ type Query { policy(input: GetPolicyInput!): PolicyDetails policies(input: ListPoliciesInput): ListPoliciesResponse policiesForResource(input: PoliciesForResourceInput): ListComplianceItemsResponse + listAvailableLogTypes: ListAvailableLogTypesResponse listComplianceIntegrations: [ComplianceIntegration!]! listLogIntegrations: [LogIntegration!]! organizationStats(input: OrganizationStatsInput): OrganizationStatsResponse @@ -1002,3 +1003,7 @@ enum AnalysisTypeEnum { RULE POLICY } + +type ListAvailableLogTypesResponse { + logTypes: [String] +} diff --git a/deployments/appsync.yml b/deployments/appsync.yml index 1dc3541bb4..a315abc0dd 100644 --- a/deployments/appsync.yml +++ b/deployments/appsync.yml @@ -144,6 +144,17 @@ Resources: LambdaConfig: LambdaFunctionArn: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:panther-metrics-api + LogTypesAPILambdaDataSource: + Type: AWS::AppSync::DataSource + DependsOn: GraphQLSchema + Properties: + ApiId: !Ref ApiId + Name: PantherLogTypesAPILambda + Type: AWS_LAMBDA + ServiceRoleArn: !Ref ServiceRole + LambdaConfig: + LambdaFunctionArn: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:panther-logtypes-api + ResourcesAPIHttpDataSource: Type: AWS::AppSync::DataSource DependsOn: GraphQLSchema @@ -1821,3 +1832,25 @@ Resources: #else $util.error($ctx.result.body, "$statusCode", $input) #end + + LogTypesResolver: + Type: AWS::AppSync::Resolver + Properties: + ApiId: !Ref ApiId + TypeName: Query + FieldName: listAvailableLogTypes + DataSourceName: !GetAtt LogTypesAPILambdaDataSource.Name + RequestMappingTemplate: | + { + "version" : "2017-02-28", + "operation": "Invoke", + "payload": $util.toJson({ + "ListAvailableLogTypes": $ctx.args.input + }) + } + ResponseMappingTemplate: | + #if($context.error) + $util.error($context.error.errorMessage, $context.error.errorType, $ctx.args) + #else + $util.toJson($context.result) + #end diff --git a/deployments/core.yml b/deployments/core.yml index ca150954a4..60061e466b 100644 --- a/deployments/core.yml +++ b/deployments/core.yml @@ -147,6 +147,9 @@ Mappings: UsersAPI: Memory: 128 Timeout: 60 + LogTypesAPI: + Memory: 128 + Timeout: 60 Conditions: AttachLayers: !Not [!Equals [!Join ['', !Ref LayerVersionArns], '']] @@ -1208,3 +1211,87 @@ Resources: FunctionName: !Ref MetricsApiFunction FunctionTimeoutSec: !FindInMap [Functions, MetricsAPI, Timeout] ServiceToken: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:panther-cfn-custom-resources + + ##### LogTypes API ##### + LogTypesTable: + Type: AWS::DynamoDB::Table + Properties: + AttributeDefinitions: + - AttributeName: RecordID + AttributeType: S + - AttributeName: RecordKind + AttributeType: S + BillingMode: PAY_PER_REQUEST + KeySchema: + - AttributeName: RecordKind + KeyType: HASH + - AttributeName: RecordID + KeyType: RANGE + PointInTimeRecoverySpecification: # Create periodic table backups + PointInTimeRecoveryEnabled: True + SSESpecification: # Enable server-side encryption + SSEEnabled: True + TableName: panther-logtypes + # + # This ddb table stores settings about log types. + # + + LogTypesTableAlarms: + Type: Custom::DynamoDBAlarms + Properties: + AlarmTopicArn: !Ref AlarmTopicArn + CustomResourceVersion: !Ref CustomResourceVersion + ServiceToken: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:panther-cfn-custom-resources + TableName: !Ref LogTypesTable + + LogTypesAPILogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: /aws/lambda/panther-logtypes-api + RetentionInDays: !Ref CloudWatchLogRetentionDays + + LogTypesAPIMetricFilters: + Type: Custom::LambdaMetricFilters + Properties: + CustomResourceVersion: !Ref CustomResourceVersion + LogGroupName: !Ref LogTypesAPILogGroup + ServiceToken: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:panther-cfn-custom-resources + + LogTypesAPIAlarms: + Type: Custom::LambdaAlarms + Properties: + AlarmTopicArn: !Ref AlarmTopicArn + CustomResourceVersion: !Ref CustomResourceVersion + FunctionMemoryMB: !FindInMap [Functions, LogTypesAPI, Memory] + FunctionName: !Ref LogTypesAPIFunction + FunctionTimeoutSec: !FindInMap [Functions, LogTypesAPI, Timeout] + ServiceToken: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:panther-cfn-custom-resources + + LogTypesAPIFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: ../out/bin/internal/core/logtypesapi/main + Description: Implements logtypes API to manage logtypes. + Environment: + Variables: + DEBUG: !Ref Debug + LOG_TYPES_TABLE_NAME: !Ref LogTypesTable + FunctionName: panther-logtypes-api + # + # This lambda implements logtypes API to manage logtypes. + # + Handler: main + Layers: !If [AttachLayers, !Ref LayerVersionArns, !Ref 'AWS::NoValue'] + MemorySize: !FindInMap [Functions, LogTypesAPI, Memory] + Runtime: go1.x + Timeout: !FindInMap [Functions, LogTypesAPI, Timeout] + Tracing: !If [TracingEnabled, !Ref TracingMode, !Ref 'AWS::NoValue'] + Policies: + - Id: ManageLogTypesTable + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - dynamodb:*Item + - dynamodb:Scan + Resource: !GetAtt LogTypesTable.Arn diff --git a/go.mod b/go.mod index 38019c656a..afa7594dda 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,8 @@ require ( github.com/stretchr/testify v1.6.1 github.com/tidwall/gjson v1.6.0 go.uber.org/zap v1.15.0 - golang.org/x/tools v0.0.0-20200513171743-967c05484029 // indirect + golang.org/x/tools v0.0.0-20200513171743-967c05484029 + gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/go-playground/validator.v9 v9.31.0 gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index 1892bc5041..d0db5b81e8 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -13,8 +14,6 @@ github.com/aws/aws-lambda-go v1.19.1 h1:5iUHbIZ2sG6Yq/J1IN3sWm3+vAB1CWwhI21NffLN github.com/aws/aws-lambda-go v1.19.1/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU= github.com/aws/aws-sdk-go v1.34.11 h1:QF9Gp3vvgIXsp7p5cYS0t7eRkauU3zM2OW4RN6FtYp0= github.com/aws/aws-sdk-go v1.34.11/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/cenkalti/backoff v1.1.0 h1:QnvVp8ikKCDWOsFheytRCoYWYPO/ObCTBGxT19Hc+yE= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -26,6 +25,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= +github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= +github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -126,6 +127,7 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -135,7 +137,6 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/go-syslog v1.0.1 h1:a/ARpnCDr/sX/hVH7dyQVi+COXlEzM4bNIoolOfw99Y= github.com/influxdata/go-syslog/v3 v3.0.0 h1:jichmjSZlYK0VMmlz+k4WeOQd7z745YLsvGMqwtYt4I= github.com/influxdata/go-syslog/v3 v3.0.0/go.mod h1:tulsOp+CecTAYC27u9miMgq21GqXRW6VdKbOG+QSP4Q= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= @@ -157,6 +158,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= @@ -180,6 +182,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= @@ -229,6 +232,7 @@ go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= @@ -241,6 +245,7 @@ golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= @@ -295,8 +300,11 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IV golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -309,4 +317,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/internal/core/logtypesapi/api.go b/internal/core/logtypesapi/api.go new file mode 100644 index 0000000000..556d7f41fb --- /dev/null +++ b/internal/core/logtypesapi/api.go @@ -0,0 +1,43 @@ +package logtypesapi + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" +) + +// Generate a lambda client using genlambdamux +//go:generate go run github.com/panther-labs/panther/pkg/x/apigen -pkg logtypesclient -out ./client/lambdaclient_gen.go + +// API handles the business logic of LogTypesAPI +type API struct { + ExternalAPI ExternalAPI + NativeLogTypes func() []string +} + +// ExternalAPI handles the external actions required for API to be implemented +type ExternalAPI interface { + ListLogTypes(ctx context.Context) ([]string, error) +} + +// Models +// We should list all API models here until we update the generator to produce docs for the models used. +type AvailableLogTypes struct { + LogTypes []string `json:"logTypes"` +} diff --git a/internal/core/logtypesapi/api_test.go b/internal/core/logtypesapi/api_test.go new file mode 100644 index 0000000000..8a21261f4f --- /dev/null +++ b/internal/core/logtypesapi/api_test.go @@ -0,0 +1,31 @@ +package logtypesapi_test + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import "context" + +// TestCase implements logtypes.ExternalAPI +// TODO: Generate test cases with go generate +type TestCase struct { + ListLogTypesOutput []string +} + +func (t *TestCase) ListLogTypes(ctx context.Context) ([]string, error) { + return t.ListLogTypesOutput, nil +} diff --git a/internal/core/logtypesapi/available.go b/internal/core/logtypesapi/available.go new file mode 100644 index 0000000000..2ad83459f6 --- /dev/null +++ b/internal/core/logtypesapi/available.go @@ -0,0 +1,59 @@ +package logtypesapi + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + "sort" + + "go.uber.org/zap" +) + +// ListAvailableLogTypes lists all available log type ids +func (api *API) ListAvailableLogTypes(ctx context.Context) (*AvailableLogTypes, error) { + logTypes, err := api.ExternalAPI.ListLogTypes(ctx) + if err != nil { + return nil, err + } + if api.NativeLogTypes != nil { + native := api.NativeLogTypes() + L(ctx).Debug(`merging native log types with external API`, + zap.Strings(`external`, logTypes), + zap.Strings(`native`, native), + ) + logTypes = appendDistinct(logTypes, native) + } + sort.Strings(logTypes) + return &AvailableLogTypes{ + LogTypes: logTypes, + }, nil +} + +func appendDistinct(dst []string, src []string) []string { +skip: + for _, s := range src { + for _, d := range dst { + if d == s { + continue skip + } + } + dst = append(dst, s) + } + return dst +} diff --git a/internal/core/logtypesapi/available_test.go b/internal/core/logtypesapi/available_test.go new file mode 100644 index 0000000000..1f854cfc92 --- /dev/null +++ b/internal/core/logtypesapi/available_test.go @@ -0,0 +1,52 @@ +package logtypesapi_test + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/panther-labs/panther/internal/core/logtypesapi" +) + +func TestAPI_ListAvailableLogTypes(t *testing.T) { + assert := require.New(t) + ctx := context.Background() + api := logtypesapi.API{ + ExternalAPI: &TestCase{ + ListLogTypesOutput: []string{"foo", "bar", "baz"}, + }, + } + + actual, _ := api.ListAvailableLogTypes(ctx) + assert.Equal(&logtypesapi.AvailableLogTypes{ + LogTypes: []string{"bar", "baz", "foo"}, + }, actual) + + api.NativeLogTypes = func() []string { + return []string{"aaa", "foo"} + } + + actual, _ = api.ListAvailableLogTypes(ctx) + assert.Equal(&logtypesapi.AvailableLogTypes{ + LogTypes: []string{"aaa", "bar", "baz", "foo"}, + }, actual) +} diff --git a/internal/core/logtypesapi/client/lambdaclient_gen.go b/internal/core/logtypesapi/client/lambdaclient_gen.go new file mode 100755 index 0000000000..ba9fc9c960 --- /dev/null +++ b/internal/core/logtypesapi/client/lambdaclient_gen.go @@ -0,0 +1,62 @@ +// Code generated by apigen; DO NOT EDIT. +package logtypesclient + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + + lambdaclient "github.com/panther-labs/panther/pkg/x/apigen/lambdaclient" +) + +// Models + +type LambdaEventPayload struct { + ListAvailableLogTypes *struct{} +} + +type ListAvailableLogTypesOutput struct { + LogTypes string `json:"logTypes"` +} + +// Lambda client + +type LambdaClient struct { + client lambdaclient.Client +} + +func NewLambdaClient(client lambdaclient.Client) *LambdaClient { + if client.Validate == nil { + client.Validate = func(interface{}) error { return nil } + } + return &LambdaClient{ + client: client, + } +} + +func (c *LambdaClient) ListAvailableLogTypes(ctx context.Context) (*ListAvailableLogTypesOutput, error) { + lambdaEvent := LambdaEventPayload{ + ListAvailableLogTypes: &struct{}{}, + } + output := ListAvailableLogTypesOutput{} + if err := c.client.InvokeWithContext(ctx, &lambdaEvent, &output); err != nil { + return nil, err + } + return &output, nil +} diff --git a/internal/core/logtypesapi/dynamodb.go b/internal/core/logtypesapi/dynamodb.go new file mode 100644 index 0000000000..beef354627 --- /dev/null +++ b/internal/core/logtypesapi/dynamodb.go @@ -0,0 +1,85 @@ +package logtypesapi + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" + "go.uber.org/zap" + + "github.com/panther-labs/panther/pkg/lambdalogger" +) + +type ExternalAPIDynamoDB struct { + DB dynamodbiface.DynamoDBAPI + TableName string +} + +var _ ExternalAPI = (*ExternalAPIDynamoDB)(nil) + +var L = lambdalogger.FromContext + +func (s *ExternalAPIDynamoDB) ListLogTypes(ctx context.Context) ([]string, error) { + ddbInput := dynamodb.GetItemInput{ + TableName: aws.String(s.TableName), + ProjectionExpression: aws.String(attrAvailableLogTypes), + Key: mustMarshalMap(&recordKey{ + RecordID: "Status", + RecordKind: recordKindStatus, + }), + } + + ddbOutput, err := s.DB.GetItemWithContext(ctx, &ddbInput) + if err != nil { + L(ctx).Error(`failed to get DynamoDB item`, zap.Error(err)) + return nil, err + } + + item := struct { + AvailableLogTypes []string + }{} + if err := dynamodbattribute.UnmarshalMap(ddbOutput.Item, &item); err != nil { + L(ctx).Error(`failed to unmarshal DynamoDB item`, zap.Error(err)) + return nil, err + } + + return item.AvailableLogTypes, nil +} + +const ( + recordKindStatus = "status" + attrAvailableLogTypes = "AvailableLogTypes" +) + +func mustMarshalMap(val interface{}) map[string]*dynamodb.AttributeValue { + attr, err := dynamodbattribute.MarshalMap(val) + if err != nil { + panic(err) + } + return attr +} + +type recordKey struct { + RecordID string + RecordKind string +} diff --git a/internal/core/logtypesapi/main/lambda.go b/internal/core/logtypesapi/main/lambda.go new file mode 100644 index 0000000000..7ed8bed617 --- /dev/null +++ b/internal/core/logtypesapi/main/lambda.go @@ -0,0 +1,74 @@ +package main + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/kelseyhightower/envconfig" + "gopkg.in/go-playground/validator.v9" + + "github.com/panther-labs/panther/internal/core/logtypesapi" + "github.com/panther-labs/panther/internal/log_analysis/log_processor/registry" + "github.com/panther-labs/panther/pkg/lambdalogger" + "github.com/panther-labs/panther/pkg/x/lambdamux" +) + +var config = struct { + Debug bool + LogTypesTableName string `required:"true" split_words:"true"` +}{} + +func main() { + envconfig.MustProcess("", &config) + + logger := lambdalogger.Config{ + Debug: config.Debug, + Namespace: "api", + Component: "logtypes", + }.MustBuild() + + // Syncing the zap.Logger always results in Lambda errors. Commented code kept as a reminder. + // defer logger.Sync() + + api := &logtypesapi.API{ + ExternalAPI: &logtypesapi.ExternalAPIDynamoDB{ + DB: dynamodb.New(session.Must(session.NewSession())), + TableName: config.LogTypesTableName, + }, + // Use the default registry with all available log types + NativeLogTypes: registry.AvailableLogTypes, + } + + validate := validator.New() + + mux := lambdamux.Mux{ + // use case-insensitive route matching + RouteName: lambdamux.IgnoreCase, + Validate: validate.Struct, + } + + mux.MustHandleMethods(api) + + // Adds logger to lambda context with a Lambda request ID field and debug output + handler := lambdalogger.Wrap(logger, &mux) + + lambda.StartHandler(handler) +} diff --git a/internal/log_analysis/log_processor/common/oplog.go b/internal/log_analysis/log_processor/common/oplog.go index e925c651cf..eae8baeb84 100644 --- a/internal/log_analysis/log_processor/common/oplog.go +++ b/internal/log_analysis/log_processor/common/oplog.go @@ -62,14 +62,14 @@ var ( -- show all sns activity filter namespace="Panther" and component="LogProcessor" - | filter Service='sns' + | filter ExternalAPI='sns' | fields @timestamp, topicArn | sort @timestamp desc | limit 200 -- show all s3 activity filter namespace="Panther" and component="LogProcessor" - | filter Service='s3' + | filter ExternalAPI='s3' | fields @timestamp, bucket, key | sort @timestamp desc | limit 200 diff --git a/pkg/lambdalogger/lambdalogger.go b/pkg/lambdalogger/lambdalogger.go new file mode 100644 index 0000000000..2c4fdf43a5 --- /dev/null +++ b/pkg/lambdalogger/lambdalogger.go @@ -0,0 +1,138 @@ +package lambdalogger + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + + "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-lambda-go/lambdacontext" + "github.com/pkg/errors" + "go.uber.org/zap" +) + +// Well-known fields +const ( + FieldRequestID = "requestId" + FieldApplication = "application" + FieldNamespace = "namespace" + FieldComponent = "component" +) + +type Config struct { + Debug bool + Namespace string + Component string + Options []zap.Option +} + +func (c Config) MustBuild() (logger *zap.Logger) { + logger, err := c.Build() + if err != nil { + panic(errors.WithStack(err)) + } + return +} + +func (c Config) Build() (*zap.Logger, error) { + // We do not use zap.NewDevelopmentConfig() (even for DEBUG) because it disables json logging. + config := zap.NewProductionConfig() + if c.Debug { + config.Level = zap.NewAtomicLevelAt(zap.DebugLevel) + } + config.InitialFields = c.InitialFields() + return config.Build(c.Options...) +} + +func (c *Config) InitialFields() map[string]interface{} { + fields := map[string]interface{}{ + FieldApplication: Application, + } + if c.Namespace != "" { + fields[FieldNamespace] = c.Namespace + } + if c.Component != "" { + fields[FieldComponent] = c.Component + } + return fields +} + +type key struct{} + +var contextKey = &key{} + +func Context(ctx context.Context, logger *zap.Logger) context.Context { + if logger == nil { + logger = nopLogger + } + // Add lambda fields to the logger + logger = withLambdaFieldsFromContext(ctx, logger) + return context.WithValue(ctx, contextKey, logger) +} + +func withLambdaFieldsFromContext(ctx context.Context, logger *zap.Logger) *zap.Logger { + if ctx, ok := lambdacontext.FromContext(ctx); ok { + logger = logger.With( + zap.String(FieldRequestID, ctx.AwsRequestID), + ) + } + return logger +} + +var nopLogger = zap.NewNop() + +func FromContext(ctx context.Context) *zap.Logger { + if logger, ok := ctx.Value(contextKey).(*zap.Logger); ok { + return logger + } + return withLambdaFieldsFromContext(ctx, zap.L()) +} + +type middleware struct { + logger *zap.Logger + debug bool + handler lambda.Handler +} + +func (m *middleware) Invoke(ctx context.Context, payload []byte) (reply []byte, err error) { + logger := m.logger + if m.debug { + defer func() { + logger.Debug(`lambda handler result`, + zap.ByteString("payload", payload), + zap.ByteString("reply", reply), + zap.Error(err), + ) + }() + } + ctx = Context(ctx, logger) + reply, err = m.handler.Invoke(ctx, payload) + return +} +func IsDebug(logger *zap.Logger) bool { + return logger.Core().Enabled(zap.DebugLevel) +} + +func Wrap(logger *zap.Logger, handler lambda.Handler) lambda.Handler { + return &middleware{ + logger: logger, + debug: IsDebug(logger), + handler: handler, + } +} diff --git a/pkg/lambdalogger/logger.go b/pkg/lambdalogger/logger.go index a335a885e7..6d9f38e987 100644 --- a/pkg/lambdalogger/logger.go +++ b/pkg/lambdalogger/logger.go @@ -60,12 +60,12 @@ func ConfigureGlobal( // always tag with requestId and application if initialFields == nil { config.InitialFields = map[string]interface{}{ - "requestId": lc.AwsRequestID, - "application": Application, + FieldRequestID: lc.AwsRequestID, + FieldApplication: Application, } } else { - initialFields["requestId"] = lc.AwsRequestID - initialFields["application"] = Application + initialFields[FieldRequestID] = lc.AwsRequestID + initialFields[FieldApplication] = Application config.InitialFields = initialFields } diff --git a/pkg/x/apigen/apigen.go b/pkg/x/apigen/apigen.go new file mode 100644 index 0000000000..d04930a048 --- /dev/null +++ b/pkg/x/apigen/apigen.go @@ -0,0 +1,227 @@ +package main + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "bytes" + "flag" + "fmt" + "go/format" + "go/types" + "io/ioutil" + "log" + "os" + "path" + "strings" + + "golang.org/x/tools/go/packages" + + "github.com/panther-labs/panther/pkg/x/apigen/internal" +) + +var ( + generatorName string + printUsage = func() { + usage := fmt.Sprintf(`Usage: %s [OPTIONS] [SEARCH]... + +Generate API client code + +ARGS + SEARCH Go package patterns to search for TYPE (defaults to ".") +OPTIONS +`, generatorName) + fmt.Fprint(flag.CommandLine.Output(), usage) + flag.PrintDefaults() + } + + opts = struct { + Filename *string + TargetAPI *string `validate:"required,min=1"` + MethodPrefix *string + PackageName *string + Debug *bool + }{ + Filename: flag.String(`out`, "", "Output file name (defaults to stdout)"), + TargetAPI: flag.String(`target`, "API", "Target API type name (defaults to 'API')"), + MethodPrefix: flag.String(`prefix`, "", "Method name prefix (defaults to no prefix)"), + PackageName: flag.String(`pkg`, "", "Go package name to use (defaults to the package name of TYPE"), + Debug: flag.Bool(`debug`, false, "Print debug output to stderr"), + } + + clientPkgPath = "github.com/panther-labs/panther/pkg/x/apigen/lambdaclient" + clientPkgName = "lambdaclient" + clientPkg = types.NewPackage(clientPkgPath, clientPkgName) +) + +func init() { + // Get the executable name in the system + generatorName = path.Base(os.Args[0]) + flag.Usage = printUsage +} + +func main() { + flag.Parse() + + logOut := ioutil.Discard + if *opts.Debug { + logOut = os.Stderr + } + logger := log.New(logOut, "", log.Lshortfile) + + apiName := *opts.TargetAPI + if apiName == "" { + fmt.Fprintf(flag.CommandLine.Output(), `%s: invalid 'target' option %q +Try '%s -help' for more information +`, apiName, generatorName, generatorName) + os.Exit(1) + } + + // Pass all args as patterns to search + patterns := flag.Args() + if len(patterns) == 0 { + patterns = []string{"."} + } + + pkgConfig := packages.Config{ + //nolint: staticcheck + Mode: packages.LoadSyntax, + Tests: false, + } + + if *opts.Debug { + pkgConfig.Logf = logger.Printf + } + + pkgs, err := packages.Load(&pkgConfig, patterns...) + if err != nil { + log.Fatalln("Failed to load packages", err) + } + index := pkgIndex(pkgs) + apiObj := index.LookupType(apiName) + if apiObj == nil { + log.Fatalf("Failed to find %q in %s", apiName, strings.Join(patterns, ", ")) + } + + apiType, ok := apiObj.Type().(*types.Named) + if !ok { + log.Fatalf("invalid API object %s", apiObj) + } + + methods, err := internal.ParseAPI(*opts.MethodPrefix, apiType) + if err != nil { + logger.Fatal(err) + } + + clientPkg := types.NewPackage(".", *opts.PackageName) + if *opts.PackageName == "" { + clientPkg = apiType.Obj().Pkg() + } + + logger.Printf("Generating lambda client %s.TargetAPI for %s with %d methods", clientPkg.Name(), apiName, len(methods)) + src, err := GenerateClient(clientPkg, apiName, methods) + if err != nil { + logger.Fatal(err) + } + if !*opts.Debug { + src, err = format.Source(src) + if err != nil { + log.Fatal(err) + } + } + if fileName := *opts.Filename; fileName != "" { + if err := os.MkdirAll(path.Dir(fileName), os.ModePerm); err != nil { + log.Fatalln("failed to create directory", err) + } + if err := ioutil.WriteFile(fileName, src, os.ModePerm); err != nil { + log.Fatalln("failed to write", err) + } + return + } + if _, err := os.Stdout.Write(src); err != nil { + log.Fatalln("failed to write", err) + } +} + +type pkgIndex []*packages.Package + +func (pkgs pkgIndex) LookupType(name string) types.Object { + for _, pkg := range pkgs { + if obj := pkg.Types.Scope().Lookup(name); obj != nil { + return obj + } + } + return nil +} + +func (pkgs pkgIndex) Find(name string) *packages.Package { + for _, pkg := range pkgs { + if pkg.Name == name { + return pkg + } + } + return nil +} + +func GenerateClient(pkg *types.Package, apiName string, methods []*Method) ([]byte, error) { + models := internal.NewModels() + if err := models.AddMethods(methods...); err != nil { + return nil, err + } + modelsBuffer := bytes.Buffer{} + models.Write(&modelsBuffer, pkg) + data := struct { + Generator string + PkgName string + API string + Methods []*Method + Models string + Aliases map[string]string + Imports []*types.Package + }{ + Generator: generatorName, + PkgName: pkg.Name(), + API: apiName, + Methods: methods, + Models: modelsBuffer.String(), + Imports: []*types.Package{ + clientPkg, + }, + } + buffer := &bytes.Buffer{} + if err := tplClient.Execute(buffer, data); err != nil { + return nil, err + } + for _, m := range methods { + var err error + switch { + case m.Input != nil && m.Output != nil: + err = tplMethodInputOutput.Execute(buffer, m) + case m.Input != nil: + err = tplMethodInput.Execute(buffer, m) + case m.Output != nil: + err = tplMethodOutput.Execute(buffer, m) + } + if err != nil { + return nil, err + } + } + return buffer.Bytes(), nil +} + +type Method = internal.Method diff --git a/pkg/x/apigen/internal/methods.go b/pkg/x/apigen/internal/methods.go new file mode 100644 index 0000000000..81176b1e0a --- /dev/null +++ b/pkg/x/apigen/internal/methods.go @@ -0,0 +1,171 @@ +package internal + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "fmt" + "go/types" + "log" + "strings" + + "github.com/pkg/errors" + "golang.org/x/tools/go/packages" +) + +var ( + typError = types.Universe.Lookup("error").Type().Underlying().(*types.Interface) + typContext *types.Interface +) + +func init() { + // Load context.Context + pkgConfig := packages.Config{ + //nolint: staticcheck + Mode: packages.LoadSyntax, + } + pkgs, err := packages.Load(&pkgConfig, "context") + if err != nil { + log.Fatalln("Failed to load context package", err) + } + for _, pkg := range pkgs { + if pkg.Name == "context" { + obj := pkg.Types.Scope().Lookup("Context") + if obj == nil { + continue + } + typContext = obj.Type().Underlying().(*types.Interface) + return + } + } + panic("could not resolve context.Context type") +} + +type Method struct { + API string + Input types.Object + Output types.Object + Name string +} + +func ParseAPI(prefix string, api *types.Named) ([]*Method, error) { + var methods []*Method + numMethods := api.NumMethods() + for i := 0; i < numMethods; i++ { + method := api.Method(i) + apiMethod, err := parseMethod(prefix, method) + if err != nil { + return nil, fmt.Errorf(`failed to parse %s.%s method: %s`, api.Obj().Name(), method.Name(), err) + } + if apiMethod == nil { + continue + } + apiMethod.API = api.Obj().Name() + methods = append(methods, apiMethod) + } + return methods, nil +} + +func (m *Method) SetSignature(sig *types.Signature) error { + if sig.Variadic() { + return errors.New(`signature is variadic`) + } + + inputs := sig.Params() + switch numInputs := inputs.Len(); numInputs { + case 0: + case 1: + input := inputs.At(0) + if !isContext(input.Type()) { + m.Input = input + } + case 2: + if in := inputs.At(0); !isContext(in.Type()) { + return fmt.Errorf(`signature param #1 of 2 (%s) is not context.Context`, in.Type()) + } + m.Input = inputs.At(1) + default: + return fmt.Errorf(`too many (%d) params`, numInputs) + } + if m.Input != nil { + if typ := m.Input.Type(); !isPtrToStruct(typ) { + return fmt.Errorf(`param %s is not a pointer to struct`, typ) + } + } + + outputs := sig.Results() + switch numResults := outputs.Len(); numResults { + case 0: + case 1: + output := outputs.At(0) + if !isError(output.Type()) { + m.Output = output + } + case 2: + if out := outputs.At(1); !isError(out.Type()) { + return fmt.Errorf(`result #2 (%s) is not an error`, out.Type()) + } + m.Output = outputs.At(0) + default: + return errors.New(`too many results`) + } + if m.Output != nil { + if typ := m.Output.Type(); !isPtrToStruct(typ) { + return fmt.Errorf(`result %s is not a pointer to struct`, typ) + } + } + return nil +} + +func isPtrToStruct(typ types.Type) bool { + pt, isPointer := typ.(*types.Pointer) + if !isPointer { + return false + } + el := pt.Elem() + if _, isStruct := el.Underlying().(*types.Struct); !isStruct { + return false + } + return true +} + +func parseMethod(prefix string, method *types.Func) (*Method, error) { + if !method.Exported() { + return nil, nil + } + methodName := method.Name() + if !strings.HasPrefix(methodName, prefix) { + return nil, nil + } + m := Method{ + Name: strings.TrimPrefix(methodName, prefix), + } + sig := method.Type().(*types.Signature) + if err := m.SetSignature(sig); err != nil { + return nil, fmt.Errorf(`invalid %s signature %s: %s`, methodName, sig, err) + } + + return &m, nil +} + +func isContext(typ types.Type) bool { + return types.IsInterface(typ) && typ.Underlying().String() == typContext.String() +} +func isError(typ types.Type) bool { + return types.IsInterface(typ) && typ.Underlying().String() == typError.String() +} diff --git a/pkg/x/apigen/internal/models.go b/pkg/x/apigen/internal/models.go new file mode 100644 index 0000000000..aa1a335521 --- /dev/null +++ b/pkg/x/apigen/internal/models.go @@ -0,0 +1,165 @@ +package internal + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "bytes" + "fmt" + "go/types" + "strings" + + "github.com/pkg/errors" +) + +type Models struct { + models map[string]types.Type +} + +func NewModels() *Models { + return &Models{ + models: map[string]types.Type{}, + } +} + +func (m *Models) Write(buf *bytes.Buffer, pkg *types.Package) { + qualifier := types.RelativeTo(pkg) + for name, typ := range m.models { + src := types.TypeString(typ, qualifier) + src = fixTagQuotes(src) + src = fmt.Sprintf(`type %s %s`, name, src) + buf.WriteString(src) + } +} + +func fixTagQuotes(src string) string { + fixed := make([]byte, 0, len(src)) + for i := 0; i < len(src); i++ { + switch c := src[i]; c { + case '"': + fixed = append(fixed, '`') + case '\\': + i++ + fixed = append(fixed, src[i]) + default: + fixed = append(fixed, c) + } + } + return string(fixed) +} + +func (m *Models) AddMethods(methods ...*Method) error { + for _, method := range methods { + if obj := method.Input; obj != nil { + name := withSuffix(method.Name, "Input") + if err := m.AddType(name, obj.Type()); err != nil { + return err + } + } + if obj := method.Output; obj != nil { + name := withSuffix(method.Name, "Output") + if err := m.AddType(name, obj.Type()); err != nil { + return err + } + } + } + return nil +} + +func withSuffix(name, suffix string) string { + if strings.HasSuffix(name, suffix) { + return name + } + return name + suffix +} + +func (m *Models) AddType(name string, typ types.Type) error { + if name == "" { + named, ok := typ.(*types.Named) + if !ok { + return nil + } + name = named.Obj().Name() + } + model := modelType(typ) + if model == nil { + return errors.Errorf("invalid model type %s %s", name, typ) + } + switch typ := model.Underlying().(type) { + case *types.Struct: + m.models[name] = typ + for i := 0; i < typ.NumFields(); i++ { + field := typ.Field(i) + if err := m.AddType("", field.Type()); err != nil { + return errors.WithMessagef(err, "invalid struct field %s", field.Name()) + } + } + case *types.Basic: + return nil + default: + return errors.New(`invalid model type`) + } + return nil +} + +func modelType(typ types.Type) types.Type { + switch typ := typ.Underlying().(type) { + case *types.Pointer: + return modelType(typ.Elem()) + case *types.Struct: + return FlatStruct(typ) + case *types.Array: + return modelType(typ.Elem()) + case *types.Slice: + return modelType(typ.Elem()) + case *types.Basic: + return typ + case *types.Map: + key, ok := typ.Key().Underlying().(*types.Basic) + if !ok { + panic("invalid map key") + } + elem := modelType(typ.Elem()) + return types.NewMap(key, elem) + default: + panic("invalid model type") + } +} + +func FlatStruct(s *types.Struct) *types.Struct { + return types.NewStruct(flattenStruct(nil, nil, s)) +} + +func flattenStruct(fields []*types.Var, tags []string, s *types.Struct) ([]*types.Var, []string) { + for i := 0; i < s.NumFields(); i++ { + field := s.Field(i) + if !field.Exported() { + continue + } + if field.Anonymous() { + if typ, ok := field.Type().Underlying().(*types.Struct); ok { + fields, tags = flattenStruct(fields, tags, typ) + continue + } + } + modelField := types.NewVar(field.Pos(), field.Pkg(), field.Name(), modelType(field.Type())) + fields = append(fields, modelField) + tags = append(tags, s.Tag(i)) + } + return fields, tags +} diff --git a/pkg/x/apigen/internal/models_test.go b/pkg/x/apigen/internal/models_test.go new file mode 100644 index 0000000000..0ad5247615 --- /dev/null +++ b/pkg/x/apigen/internal/models_test.go @@ -0,0 +1,19 @@ +package internal + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ diff --git a/pkg/x/apigen/lambdaclient/lambdaclient.go b/pkg/x/apigen/lambdaclient/lambdaclient.go new file mode 100644 index 0000000000..1504c4d7cf --- /dev/null +++ b/pkg/x/apigen/lambdaclient/lambdaclient.go @@ -0,0 +1,94 @@ +package lambdaclient + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/lambda" + "github.com/aws/aws-sdk-go/service/lambda/lambdaiface" + jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" +) + +// Client is embedded to all clients generated by `genlambdamux`. +// It provides the invocation without the typed methods. +type Client struct { + // LambdaAPI handles the actual invocation + LambdaAPI lambdaiface.LambdaAPI + // LambdaName is the name of the target Lambda function + LambdaName string + // JSON sets the jsoniter.API to use (defaults to jsoniter.ConfigDefault). + JSON jsoniter.API + // Validate enables input validation on the typed inputs of generated clients + Validate func(interface{}) error +} + +// InvokeWithContext invokes the target lambda and handles errors +func (c *Client) InvokeWithContext(ctx context.Context, input, output interface{}) error { + jsonAPI := c.JSON + if jsonAPI == nil { + jsonAPI = jsoniter.ConfigDefault + } + payload, err := jsonAPI.Marshal(input) + if err != nil { + return errors.Wrapf(err, `failed to marshal lambda %q input`, c.LambdaName) + } + lambdaInput := lambda.InvokeInput{ + FunctionName: aws.String(c.LambdaName), + Payload: payload, + } + lambdaOutput, err := c.LambdaAPI.InvokeWithContext(ctx, &lambdaInput) + if err != nil { + return errors.Wrapf(err, `lambda %q invocation failed`, c.LambdaName) + } + if lambdaOutput.FunctionError != nil { + invokeErr := InvokeError{} + if err := jsoniter.Unmarshal(lambdaOutput.Payload, &invokeErr); err != nil { + return errors.Wrapf(err, `failed to unmarshal lambda %q invoke error`, c.LambdaName) + } + return errors.Wrapf(&invokeErr, `lambda %q execution failed`, c.LambdaName) + } + if output == nil { + return nil + } + if err := jsonAPI.Unmarshal(lambdaOutput.Payload, output); err != nil { + return errors.Wrapf(err, `failed to marshal lambda %q response`, c.LambdaName) + } + return nil +} + +// InvokeError is an error that occurred during Lambda execution +type InvokeError struct { + Message string `json:"errorMessage"` + Type string `json:"errorType"` + StackTrace []ErrorStackFrame `json:"stackTrace,omitempty"` +} + +func (e *InvokeError) Error() string { + return e.Message +} + +// ErrorStackFrame is a stack frame from a Lambda that failed with a stack trace +type ErrorStackFrame struct { + Path string `json:"path"` + Line int32 `json:"line"` + Label string `json:"label"` +} diff --git a/pkg/x/apigen/templates.go b/pkg/x/apigen/templates.go new file mode 100644 index 0000000000..e3119acd73 --- /dev/null +++ b/pkg/x/apigen/templates.go @@ -0,0 +1,107 @@ +package main + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "text/template" +) + +var ( + tplClient = template.Must(template.New(`pkg`).Parse(clientTemplate)) + tplMethodInputOutput = template.Must(template.New(`methodInputOutput`).Parse(methodTemplateInputOutput)) + tplMethodInput = template.Must(template.New(`methodInput`).Parse(methodTemplateInput)) + tplMethodOutput = template.Must(template.New(`methodOutput`).Parse(methodTemplateOutput)) +) + +const clientTemplate = `// Code generated by {{ .Generator }}; DO NOT EDIT. +package {{.PkgName}} + +import ( + "context" + {{ range .Imports }}{{ .Name }} {{ printf "%q" .Path }}{{ end }} +) + +// Models + +type LambdaEventPayload struct { +{{- range .Methods }} +{{- if .Input }} + {{ .Name }} *{{ .Name }}Input +{{- else }} + {{ .Name }} *struct{} +{{- end -}} +{{- end }} +} + +{{ .Models }} + +// Lambda client + +type LambdaClient struct { + client lambdaclient.Client +} + +func NewLambdaClient(client lambdaclient.Client) *LambdaClient { + if client.Validate == nil { + client.Validate = func(interface{}) error { return nil } + } + return &LambdaClient{ + client: client, + } +} +` + +const methodTemplateInputOutput = ` +func (c *LambdaClient) {{ .Name }}(ctx context.Context, input *{{.Name}}Input) (*{{.Name}}Output, error) { + if err := c.client.Validate(input); err != nil { + return nil, err + } + lambdaEvent := LambdaEventPayload{ + {{ .Name }}: input, + } + output := {{ .Name }}Output{} + if err := c.client.InvokeWithContext(ctx, &lambdaEvent, &output); err != nil { + return nil, err + } + return &output, nil +} +` +const methodTemplateInput = ` +func (c *LambdaClient) {{ .Name }}(ctx context.Context, input *{{.Name}}Input) error { + if err := c.client.Validate(input); err != nil { + return nil, err + } + lambdaEvent := LambdaEventPayload{ + {{ .Name }}: input, + } + return c.client.InvokeWithContext(ctx, &lambdaEvent, nil) +} +` +const methodTemplateOutput = ` +func (c *LambdaClient) {{ .Name }}(ctx context.Context) (*{{.Name}}Output, error) { + lambdaEvent := LambdaEventPayload{ + {{ .Name }}: &struct{}{}, + } + output := {{ .Name }}Output{} + if err := c.client.InvokeWithContext(ctx, &lambdaEvent, &output); err != nil { + return nil, err + } + return &output, nil +} +` diff --git a/pkg/x/lambdamux/batch.go b/pkg/x/lambdamux/batch.go new file mode 100644 index 0000000000..a6d3bdc110 --- /dev/null +++ b/pkg/x/lambdamux/batch.go @@ -0,0 +1,140 @@ +package lambdamux + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + "sync" + + jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" +) + +type batchJobs struct { + jobs []batchJob + buffer []byte +} + +type batchJob struct { + name string + handler Handler + start, end int +} + +var batchPool = &sync.Pool{ + New: func() interface{} { + return &batchJobs{} + }, +} + +func borrowBatch() *batchJobs { + return batchPool.Get().(*batchJobs) +} +func (b *batchJobs) Recycle() { + if b == nil { + return + } + for i := range b.jobs { + b.jobs[i] = batchJob{} + } + batchPool.Put(b) +} + +func (m *Mux) runBatch(ctx context.Context, b *batchJobs) ([]byte, error) { + // Run the batch + jsonAPI := resolveJSON(m.JSON) + w := jsonAPI.BorrowStream(nil) + defer jsonAPI.ReturnStream(w) + if err := b.Run(ctx, w); err != nil { + return nil, err + } + + // We need to make a copy of the buffer to return it + out := make([]byte, w.Buffered()) + copy(out, w.Buffer()) + return out, nil +} + +func (b *batchJobs) Run(ctx context.Context, w *jsoniter.Stream) error { + w.WriteArrayStart() + for i := range b.jobs { + job := &b.jobs[i] + payload := b.slicePayload(job.start, job.end) + reply, err := job.handler.Invoke(ctx, payload) + if err != nil { + return errors.WithMessagef(err, "batch job %s %d/%d failed", job.name, i, len(b.jobs)) + } + if i != 0 { + w.WriteMore() + } + w.WriteVal(jsoniter.RawMessage(reply)) + } + w.WriteArrayEnd() + return nil +} + +func (b *batchJobs) slicePayload(start, end int) (p []byte) { + if 0 <= start && start < len(b.buffer) { + p = b.buffer[start:] + if 0 <= end && end <= len(p) { + return p[:end] + } + } + return nil +} + +func (b *batchJobs) ReadJobs(mux *Mux, iter *jsoniter.Iterator) error { + buffer := b.buffer[:0] + jobs := b.jobs[:0] + var jobIter *jsoniter.Iterator + defer func() { + if jobIter != nil { + iter.Pool().ReturnIterator(jobIter) + } + }() + for iter.ReadArray() { + payload := iter.SkipAndReturnBytes() + if jobIter == nil { + jobIter = iter.Pool().BorrowIterator(payload) + } else { + jobIter.ResetBytes(payload) + } + jobPayload, name := mux.demux(jobIter, payload) + if err := jobIter.Error; err != nil { + return errors.Wrap(err, `invalid batch JSON payload`) + } + handler, err := mux.Get(name) + if err != nil { + return err + } + start := len(buffer) + buffer = append(buffer, jobPayload...) + jobs = append(jobs, batchJob{ + name: name, + handler: handler, + start: start, + end: len(buffer) - start, + }) + } + *b = batchJobs{ + buffer: buffer, + jobs: jobs, + } + return nil +} diff --git a/pkg/x/lambdamux/batch_test.go b/pkg/x/lambdamux/batch_test.go new file mode 100644 index 0000000000..3a3c9d06c2 --- /dev/null +++ b/pkg/x/lambdamux/batch_test.go @@ -0,0 +1,51 @@ +package lambdamux + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBatch(t *testing.T) { + mux := Mux{} + type Payload struct { + Foo string `json:"foo"` + } + type Reply struct { + Bar string `json:"bar,omitempty"` + Baz string `json:"baz,omitempty"` + } + + mux.MustHandle("bar", func(payload *Payload) (*Reply, error) { + return &Reply{Bar: payload.Foo}, nil + }) + mux.MustHandle("baz", func(payload *Payload) (*Reply, error) { + return &Reply{Baz: payload.Foo}, nil + }) + ctx := context.Background() + assert := require.New(t) + { + reply, err := mux.Invoke(ctx, []byte(`[{"bar":{"foo":"bar"}},{"baz":{"foo":"baz"}}]`)) + assert.NoError(err) + assert.JSONEq(`[{"bar":"bar"},{"baz":"baz"}]`, string(reply)) + } +} diff --git a/pkg/x/lambdamux/demux.go b/pkg/x/lambdamux/demux.go new file mode 100644 index 0000000000..d830c94003 --- /dev/null +++ b/pkg/x/lambdamux/demux.go @@ -0,0 +1,114 @@ +package lambdamux + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + jsoniter "github.com/json-iterator/go" +) + +// Demuxer sets the routing strategy for a payload. +// +// It accepts the current JSON iterator and the original payload and returns the route payload and name. +type Demuxer interface { + Demux(iter *jsoniter.Iterator, payload []byte) ([]byte, string) +} + +// DemuxKeyValue uses the first key in a JSON object as route name and it's value as payload. +// +// For example a `DemuxKeyValue()` will route a payload `{"foo":{"bar":"baz"}}` +// to route `foo` with payload `{"bar":"baz"}`. +func DemuxKeyValue() Demuxer { + return &demuxKeyValue{} +} + +type demuxKeyValue struct{} + +func (d *demuxKeyValue) Demux(iter *jsoniter.Iterator, _ []byte) ([]byte, string) { + name := iter.ReadObject() + if name == "" { + return nil, "" + } + return iter.SkipAndReturnBytes(), name +} + +// DemuxPeekKey peeks into the value of a JSON object field to find the route name. +// +// For example a `DemuxPeekKey("method")` will route a payload `{"method":"foo", "bar":"baz"}}` +// to route `foo` with payload `{"method":"foo", "bar":"baz"}`. +func DemuxPeekKey(routeKey string) Demuxer { + return &demuxPeekKey{ + routeKey: routeKey, + } +} + +type demuxPeekKey struct { + routeKey string +} + +func (d *demuxPeekKey) Demux(iter *jsoniter.Iterator, payload []byte) ([]byte, string) { + for key := iter.ReadObject(); key != ""; key = iter.ReadObject() { + if key != d.routeKey { + iter.Skip() + continue + } + name := iter.ReadString() + if name == "" { + return nil, "" + } + return payload, name + } + return nil, "" +} + +// DemuxKeys uses the value of two separate keys of a JSON object as route name and payload. +// +// For example a `DemuxKeys("method","params")` will route a payload `{"method": "foo", "params":{"bar":"baz"}}` +// to route `foo` with payload `{"bar":"baz"}`. +// +func DemuxKeys(routeKey, payloadKey string) Demuxer { + return &demuxKeys{ + routeKey: routeKey, + payloadKey: payloadKey, + } +} + +type demuxKeys struct { + routeKey string + payloadKey string +} + +func (d *demuxKeys) Demux(iter *jsoniter.Iterator, payload []byte) (p []byte, name string) { + for key := iter.ReadObject(); key != ""; key = iter.ReadObject() { + switch key { + case d.routeKey: + name = iter.ReadString() + if p != nil { + return p, name + } + case d.payloadKey: + if name != "" { + return iter.SkipAndReturnBytes(), name + } + p = iter.SkipAndAppendBytes(make([]byte, 0, len(payload))) + default: + iter.Skip() + } + } + return nil, "" +} diff --git a/pkg/x/lambdamux/demux_test.go b/pkg/x/lambdamux/demux_test.go new file mode 100644 index 0000000000..b9e8aedaac --- /dev/null +++ b/pkg/x/lambdamux/demux_test.go @@ -0,0 +1,128 @@ +package lambdamux + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDemuxKeyValue(t *testing.T) { + mux := Mux{ + Demux: DemuxKeyValue(), + } + type Payload struct { + Bar string `json:"bar"` + } + type Reply struct { + Baz string `json:"baz"` + } + + mux.MustHandle("foo", func(payload *Payload) (*Reply, error) { + return &Reply{Baz: payload.Bar}, nil + }) + + ctx := context.Background() + assert := require.New(t) + { + reply, err := mux.Invoke(ctx, []byte(`{"foo":{"bar":"baz"}}`)) + assert.NoError(err) + assert.JSONEq(`{"baz":"baz"}`, string(reply)) + } + { + reply, err := mux.Invoke(ctx, []byte(`{"bar":{"bar":"baz"}}`)) + assert.True(errors.Is(err, ErrNotFound)) + assert.Nil(reply) + } +} + +func TestDemuxKeys(t *testing.T) { + mux := Mux{ + Demux: DemuxKeys("method", "params"), + } + type Payload struct { + Bar string `json:"bar"` + } + type Reply struct { + Baz string `json:"baz"` + } + + mux.MustHandle("foo", func(payload *Payload) (*Reply, error) { + return &Reply{Baz: payload.Bar}, nil + }) + + ctx := context.Background() + assert := require.New(t) + { + reply, err := mux.Invoke(ctx, []byte(`{"method": "foo", "params": {"bar":"baz"}}`)) + assert.NoError(err) + assert.JSONEq(`{"baz":"baz"}`, string(reply)) + } + { + reply, err := mux.Invoke(ctx, []byte(`{"method": "bar", "params": {"bar":"baz"}}`)) + assert.True(errors.Is(err, ErrNotFound)) + assert.Nil(reply) + } + { + reply, err := mux.Invoke(ctx, []byte(`{"method": "bar"}`)) + assert.True(errors.Is(err, ErrNotFound)) + assert.Nil(reply) + } + { + reply, err := mux.Invoke(ctx, []byte(`{"params": {"bar":"baz"}}`)) + assert.True(errors.Is(err, ErrNotFound)) + assert.Nil(reply) + } +} +func TestDemuxPeekKey(t *testing.T) { + mux := Mux{ + Demux: DemuxPeekKey("method"), + } + type Payload struct { + Bar string `json:"bar"` + } + type Reply struct { + Baz string `json:"baz"` + } + + mux.MustHandle("foo", func(payload *Payload) (*Reply, error) { + return &Reply{Baz: payload.Bar}, nil + }) + + ctx := context.Background() + assert := require.New(t) + { + reply, err := mux.Invoke(ctx, []byte(`{"method": "foo", "bar":"baz"}`)) + assert.NoError(err) + assert.JSONEq(`{"baz":"baz"}`, string(reply)) + } + { + reply, err := mux.Invoke(ctx, []byte(`{"method": "bar", "bar":"baz"}`)) + assert.True(errors.Is(err, ErrNotFound)) + assert.Nil(reply) + } + { + reply, err := mux.Invoke(ctx, []byte(`{"bar":"baz"}`)) + assert.True(errors.Is(err, ErrNotFound)) + assert.Nil(reply) + } +} diff --git a/pkg/x/lambdamux/lambadmux.go b/pkg/x/lambdamux/lambadmux.go new file mode 100644 index 0000000000..a13f43a3f6 --- /dev/null +++ b/pkg/x/lambdamux/lambadmux.go @@ -0,0 +1,72 @@ +package lambdamux + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + "errors" + "strings" + + jsoniter "github.com/json-iterator/go" +) + +const DefaultHandlerPrefix = "Invoke" + +// Handler is identical to lambda.Handler. +type Handler interface { + Invoke(ctx context.Context, payload []byte) ([]byte, error) +} + +// HandlerFunc is a function implementing Handler +type HandlerFunc func(ctx context.Context, payload []byte) ([]byte, error) + +var _ Handler = (HandlerFunc)(nil) + +// Invoke implements lambda.Handler +func (f HandlerFunc) Invoke(ctx context.Context, payload []byte) ([]byte, error) { + return f(ctx, payload) +} + +type RouteHandler interface { + Handler + Route() string +} + +// ErrNotFound is a well-known error that a route was not found. +// Using std errors.New here since we don't want a stack +var ErrNotFound = errors.New(`route not found`) + +// RouteError is the error implemented by errors returned from a route handler +type RouteError interface { + error + Route() string +} + +var defaultJSON = jsoniter.ConfigCompatibleWithStandardLibrary + +func resolveJSON(api jsoniter.API) jsoniter.API { + if api != nil { + return api + } + return defaultJSON +} + +func IgnoreCase(name string) string { + return strings.ToUpper(name) +} diff --git a/pkg/x/lambdamux/middleware.go b/pkg/x/lambdamux/middleware.go new file mode 100644 index 0000000000..36eef163bf --- /dev/null +++ b/pkg/x/lambdamux/middleware.go @@ -0,0 +1,79 @@ +package lambdamux + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + "errors" + "time" +) + +// Chain tries handlers in sequence while a NotFound error is returned. +func Chain(handlers ...Handler) Handler { + return chainHandler(handlers) +} + +type chainHandler []Handler + +func (c chainHandler) Invoke(ctx context.Context, payload []byte) ([]byte, error) { + for _, handler := range c { + reply, err := handler.Invoke(ctx, payload) + if err != nil { + if errors.Is(err, ErrNotFound) { + continue + } + return nil, err + } + return reply, nil + } + return nil, ErrNotFound +} + +func CacheProxy(maxAge time.Duration, handler Handler) Handler { + if maxAge <= 0 { + return handler + } + type cacheEntry struct { + Output []byte + UpdatedAt time.Time + } + cache := map[string]*cacheEntry{} + var lastInsertAt time.Time + return HandlerFunc(func(ctx context.Context, input []byte) ([]byte, error) { + entry, ok := cache[string(input)] + if ok && time.Since(entry.UpdatedAt) < maxAge { + return entry.Output, nil + } + output, err := handler.Invoke(ctx, input) + if err != nil { + return nil, err + } + now := time.Now() + // Reset the whole cache if last insert was too old to avoid memory leaks + if time.Since(lastInsertAt) > maxAge { + cache = map[string]*cacheEntry{} + lastInsertAt = now + } + cache[string(input)] = &cacheEntry{ + Output: output, + UpdatedAt: now, + } + return output, nil + }) +} diff --git a/pkg/x/lambdamux/mux.go b/pkg/x/lambdamux/mux.go new file mode 100644 index 0000000000..1146f03451 --- /dev/null +++ b/pkg/x/lambdamux/mux.go @@ -0,0 +1,234 @@ +package lambdamux + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + + jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" +) + +// Mux dispatches handling of a Lambda events +type Mux struct { + // If set, it will normalize route names. + RouteName func(name string) string + // Decorate can intercept new handlers and add decorations + Decorate func(key string, handler Handler) Handler + // Demux sets the routing method. + Demux Demuxer // defaults to DemuxKeyValue + // JSON + JSON jsoniter.API // defaults to jsoniter.ConfigCompatibleWithStandardLibrary + // Validate sets a custom validation function to be injected into every route handler + Validate func(payload interface{}) error + // IgnoreDuplicates will not return errors when duplicate route handlers are added to the mux + IgnoreDuplicates bool + + handlers map[string]RouteHandler +} + +// Handlers returns all handlers added to the mux. +func (m *Mux) Handlers() (handlers []RouteHandler) { + if len(m.handlers) == 0 { + return + } + handlers = make([]RouteHandler, 0, len(m.handlers)) + for _, handler := range m.handlers { + handlers = append(handlers, handler) + } + return +} + +// MustHandleMethods add all routes from a struct to the Mux or panics. +// It overrides previously defined routes without error if IgnoreDuplicates is set +func (m *Mux) MustHandleMethods(receivers ...interface{}) { + if err := m.HandleMethodsPrefix("", receivers...); err != nil { + panic(err) + } +} + +// HandleMethodsPrefix add all routes from a struct to the Mux. +// It fails if a method does not meet the signature requirements. +// It overrides previously defined routes without error if IgnoreDuplicates is set +func (m *Mux) HandleMethodsPrefix(prefix string, receivers ...interface{}) error { + for _, receiver := range receivers { + routes, err := routeMethods(prefix, receiver) + if err != nil { + return err + } + if err := m.handleRoutes(routes...); err != nil { + return err + } + } + return nil +} + +func (m *Mux) handleRoutes(routes ...*routeHandler) error { + for _, route := range routes { + name := route.name + if err := m.Handle(name, route); err != nil { + return err + } + } + return nil +} +func (m *Mux) MustHandle(name string, handler interface{}) { + if err := m.Handle(name, handler); err != nil { + panic(err) + } +} + +// Handle applies any decoration and adds a handler to the mux. +// It fails if the handler does not meet the signature requirements. +// It overrides previously defined routes without error if IgnoreDuplicates is set. +func (m *Mux) Handle(name string, handler interface{}) error { + key := name + if m.RouteName != nil { + key = m.RouteName(name) + } + if key == "" { + return errors.Errorf("invalid route name %q", name) + } + + var route RouteHandler + switch h := handler.(type) { + case *routeHandler: + r := *h + r.name = key + if m.JSON != nil { + r.jsonAPI = m.JSON + } + if m.Validate != nil { + r.validate = m.Validate + } + route = &r + case Handler: + route = &namedHandler{ + Handler: h, + route: key, + } + default: + r, err := buildRoute(key, handler) + if err != nil { + return err + } + if m.JSON != nil { + r.jsonAPI = m.JSON + } + if m.Validate != nil { + r.validate = m.Validate + } + route = r + } + + if decorate := m.Decorate; decorate != nil { + d := decorate(key, route) + // Allow Decorate to filter routes + if d == nil { + return nil + } + route = &namedHandler{ + Handler: d, + route: key, + } + } + + if !m.IgnoreDuplicates { + if _, duplicate := m.handlers[key]; duplicate { + return errors.Errorf("duplicate route handler for %q", name) + } + } + + if m.handlers == nil { + m.handlers = map[string]RouteHandler{} + } + m.handlers[key] = route + return nil +} + +func (m *Mux) Invoke(ctx context.Context, payload []byte) ([]byte, error) { + iter := resolveJSON(m.JSON).BorrowIterator(payload) + defer iter.Pool().ReturnIterator(iter) + switch next := iter.WhatIsNext(); next { + case jsoniter.ObjectValue: + p, name := m.demux(iter, payload) + if err := iter.Error; err != nil { + return nil, errors.Wrap(err, `invalid JSON payload`) + } + handler, err := m.Get(name) + if err != nil { + return nil, err + } + return handler.Invoke(ctx, p) + case jsoniter.ArrayValue: + b := borrowBatch() + defer b.Recycle() + if err := b.ReadJobs(m, iter); err != nil { + return nil, err + } + return m.runBatch(ctx, b) + default: + return nil, errors.Wrapf(ErrNotFound, `invalid JSON payload %q`, next) + } +} + +var defaultDemux = &demuxKeyValue{} + +func (m *Mux) demux(iter *jsoniter.Iterator, payload []byte) ([]byte, string) { + if d := m.Demux; d != nil { + return d.Demux(iter, payload) + } + return defaultDemux.Demux(iter, payload) +} + +func (m *Mux) Get(name string) (Handler, error) { + if name == "" { + return nil, errors.Wrap(ErrNotFound, `invalid payload`) + } + key := name + if m.RouteName != nil { + key = m.RouteName(key) + if key == "" { + return nil, errors.Wrapf(ErrNotFound, `invalid route key %q`, name) + } + } + if handler, ok := m.handlers[key]; ok { + return handler, nil + } + return nil, errors.Wrapf(ErrNotFound, "route %q not found", key) +} + +type namedHandler struct { + Handler + route string +} + +var _ RouteHandler = (*namedHandler)(nil) + +func (h *namedHandler) Route() string { + return h.route +} + +func (h *namedHandler) Invoke(ctx context.Context, payload []byte) ([]byte, error) { + reply, err := h.Handler.Invoke(ctx, payload) + if err != nil { + return nil, newRouteError(h.route, err) + } + return reply, nil +} diff --git a/pkg/x/lambdamux/mux_test.go b/pkg/x/lambdamux/mux_test.go new file mode 100644 index 0000000000..7e68facd90 --- /dev/null +++ b/pkg/x/lambdamux/mux_test.go @@ -0,0 +1,62 @@ +package lambdamux + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +type TestAPI struct{} + +type Foo struct { + Bar string `json:"bar"` +} + +func (*TestAPI) GetFoo() *Foo { + return &Foo{ + Bar: "baz", + } +} +func (*TestAPI) GetFooWithContext(_ context.Context) *Foo { + return &Foo{ + Bar: "baz", + } +} + +func TestMux(t *testing.T) { + mux := Mux{} + ctx := context.Background() + assert := require.New(t) + mux.MustHandleMethods(&TestAPI{}) + { + payload := []byte(`{"GetFoo":{}}`) + reply, err := mux.Invoke(ctx, payload) + assert.NoError(err) + assert.JSONEq(`{"bar":"baz"}`, string(reply)) + } + { + payload := []byte(`{"GetFooWithContext":{}}`) + reply, err := mux.Invoke(ctx, payload) + assert.NoError(err) + assert.JSONEq(`{"bar":"baz"}`, string(reply)) + } +} diff --git a/pkg/x/lambdamux/route.go b/pkg/x/lambdamux/route.go new file mode 100644 index 0000000000..fc37e7fac7 --- /dev/null +++ b/pkg/x/lambdamux/route.go @@ -0,0 +1,294 @@ +package lambdamux + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + "fmt" + "reflect" + "strings" + + jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" +) + +// Route is a named route method +type routeHandler struct { + name string + method reflect.Value + input reflect.Type + output reflect.Type + withContext bool + withError bool + validate func(interface{}) error + jsonAPI jsoniter.API +} + +// MustBuildRoute builds a route for a handler or panics +func mustBuildRoute(routeName string, handler interface{}) *routeHandler { + route, err := buildRoute(routeName, handler) + if err != nil { + panic(err) + } + return route +} + +// BuildRoute builds a route for a handler. +// If the handler does not meet the signature requirements it returns an error. +func buildRoute(routeName string, handler interface{}) (*routeHandler, error) { + val := reflect.ValueOf(handler) + route, err := buildRouteHandler(routeName, val) + if err != nil { + return nil, errors.WithMessagef(err, `invalid %q handler %s`, routeName, val.Type()) + } + return route, nil +} + +func routeMethods(prefix string, receiver interface{}) ([]*routeHandler, error) { + var routes []*routeHandler + val := reflect.ValueOf(receiver) + typ := val.Type() + switch typ.Kind() { + case reflect.Ptr: + case reflect.Interface: + case reflect.Struct: + return nil, errors.Errorf(`non-pointer receiver %s`, typ) + default: + return nil, errors.Errorf(`invalid receiver type %s`, typ) + } + if val.IsNil() { + return nil, errors.Errorf(`nil receiver %v`, val) + } + numMethod := typ.NumMethod() + for i := 0; i < numMethod; i++ { + method := typ.Method(i) + if method.PkgPath != "" { + // unexported method + continue + } + + routeName := method.Name + if prefix != "" { + if !strings.HasPrefix(routeName, prefix) { + continue + } + routeName = strings.TrimPrefix(routeName, prefix) + } + + route, err := buildRouteHandler(routeName, val.Method(i)) + if err != nil { + return nil, errors.WithMessagef(err, `invalid %q handler method`, method.Name) + } + routes = append(routes, route) + } + return routes, nil +} + +func buildRouteHandler(name string, method reflect.Value) (*routeHandler, error) { + typ := method.Type() + if typ.Kind() != reflect.Func { + return nil, errors.New(`invalid func value`) + } + route := routeHandler{ + name: name, + method: method, + jsonAPI: defaultJSON, + validate: nopValidate, + } + if err := route.setInput(typ); err != nil { + return nil, errors.WithMessagef(err, "invalid signature input %s", typ) + } + if err := route.setOutput(typ); err != nil { + return nil, errors.WithMessagef(err, "invalid signature output %s", typ) + } + return &route, nil +} + +func (r *routeHandler) setInput(typ reflect.Type) error { + switch typ.NumIn() { + case 2: + ctx, in := typ.In(0), typ.In(1) + if ctx != typContext { + return errors.New("first input is not context.Context") + } + r.withContext = true + if in.Kind() != reflect.Ptr { + return errors.New("second input is not a pointer") + } + r.input = in.Elem() + return nil + case 1: + in := typ.In(0) + if in == typContext { + r.withContext = true + return nil + } + if in.Kind() != reflect.Ptr { + return errors.New("input is not a pointer") + } + r.input = in.Elem() + return nil + case 0: + return nil + } + return errors.Errorf(`invalid signature input %s`, typ) +} + +var ( + typContext = reflect.TypeOf((*context.Context)(nil)).Elem() + typError = reflect.TypeOf((*error)(nil)).Elem() +) + +func (r *routeHandler) setOutput(typ reflect.Type) error { + switch typ.NumOut() { + case 0: + return errors.New(`no output`) + case 1: + out := typ.Out(0) + if out == typError { + r.withError = true + return nil + } + if out.Kind() != reflect.Ptr { + return errors.New(`output type non pointer`) + } + r.output = out.Elem() + return nil + case 2: + typOut, typErr := typ.Out(0), typ.Out(1) + if typErr != typError { + return errors.New(`second output is not error`) + } + r.withError = true + if typOut.Kind() != reflect.Ptr { + return errors.New(`first output is not a pointer`) + } + r.output = typOut.Elem() + return nil + default: + return errors.New(`too many outputs`) + } +} + +func (r *routeHandler) Route() string { + return r.name +} + +var emptyResult = []byte(`{}`) + +// Invoke implements Handler +func (r *routeHandler) Invoke(ctx context.Context, input []byte) ([]byte, error) { + params, err := r.callParams(ctx, input) + if err != nil { + return nil, r.wrapErr(err) + } + result, err := r.call(params) + if err != nil { + return nil, r.wrapErr(err) + } + if result == nil { + return emptyResult, nil + } + output, err := r.jsonAPI.Marshal(result.Interface()) + if err != nil { + return nil, r.wrapErr(errors.Wrap(err, "failed to marshal reply")) + } + return output, nil +} + +func (r *routeHandler) wrapErr(err error) error { + if err != nil { + return newRouteError(r.Route(), err) + } + return nil +} + +func (r *routeHandler) callParams(ctx context.Context, payload []byte) ([]reflect.Value, error) { + in := make([]reflect.Value, 0, 2) + if r.withContext { + in = append(in, reflect.ValueOf(ctx)) + } + if r.input != nil { + inputVal := reflect.New(r.input) + val := inputVal.Interface() + if err := r.jsonAPI.Unmarshal(payload, val); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal payload") + } + if err := r.validate(val); err != nil { + return nil, errors.WithMessage(err, "invalid payload") + } + in = append(in, inputVal) + } + return in, nil +} + +func (r *routeHandler) call(in []reflect.Value) (*reflect.Value, error) { + switch out := r.method.Call(in); len(out) { + case 2: + outVal, errVal := &out[0], &out[1] + if errVal.IsZero() || errVal.IsNil() { + return outVal, nil + } + return nil, errVal.Interface().(error) + case 1: + outVal := &out[0] + if r.withError { + if outVal.IsNil() { + return nil, nil + } + return nil, outVal.Interface().(error) + } + return outVal, nil + default: + return nil, errors.New(`invalid route signature`) + } +} + +func newRouteError(route string, err error) error { + if e, ok := err.(*routeError); ok { + err = e.err + } + return &routeError{ + routeName: route, + err: err, + } +} + +type routeError struct { + routeName string + err error +} + +var _ RouteError = (*routeError)(nil) + +func (e *routeError) Error() string { + return fmt.Sprintf("route %q error: %s", e.routeName, e.err) +} + +func (e *routeError) Unwrap() error { + return e.err +} + +func (e *routeError) Route() string { + return e.routeName +} + +func nopValidate(_ interface{}) error { + return nil +} diff --git a/pkg/x/lambdamux/route_test.go b/pkg/x/lambdamux/route_test.go new file mode 100644 index 0000000000..f410f5dead --- /dev/null +++ b/pkg/x/lambdamux/route_test.go @@ -0,0 +1,86 @@ +package lambdamux + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +type testAPI struct { +} + +func (*testAPI) InvokeFoo(_ context.Context) error { + return nil +} + +func TestStructRoutes(t *testing.T) { + assert := require.New(t) + routes, err := routeMethods(DefaultHandlerPrefix, &testAPI{}) + assert.NoError(err) + assert.Len(routes, 1) + assert.Nil(routes[0].input) + assert.Nil(routes[0].output) + + mux := Mux{ + IgnoreDuplicates: true, + } + assert.NoError(mux.HandleMethodsPrefix(DefaultHandlerPrefix, &testAPI{})) + assert.NoError(mux.HandleMethodsPrefix(DefaultHandlerPrefix, &testAPI{})) + output, err := mux.Invoke(context.Background(), json.RawMessage(`{"Foo":{}}`)) + assert.NoError(err) + assert.Equal("{}", string(output)) +} + +func TestSignatures(t *testing.T) { + assert := require.New(t) + type Input struct{} + type Output struct{} + mustBuildRoute("foo", func() error { return nil }) + mustBuildRoute("bar", func(context.Context) error { return nil }) + mustBuildRoute("baz", func(context.Context) (*Output, error) { return nil, nil }) + mustBuildRoute("foo", func(context.Context) *Output { return nil }) + mustBuildRoute("foo", func(*Input) error { return nil }) + mustBuildRoute("foo", func(*Input) (*Output, error) { return nil, nil }) + mustBuildRoute("foo", func(context.Context, *Input) error { return nil }) + mustBuildRoute("foo", func(context.Context, *Input) (*Output, error) { return nil, nil }) + mustBuildRoute("foo", func() (*Output, error) { return nil, nil }) + for _, method := range []interface{}{ + func(*Input, *Input) error { return nil }, + func(Input) error { return nil }, + func(*Input) {}, + func(Input) (*Output, error) { return nil, nil }, + func(context.Context, Input) error { return nil }, + func(context.Context, string) error { return nil }, + func(context.Context, *Input, *Input) error { return nil }, + func(context.Context, *Input) (Output, error) { return Output{}, nil }, + func(context.Context, *Input) Output { return Output{} }, + func(context.Context, *Input) (string, error) { return "", nil }, + func(context.Context, *Input) (*Output, *Output, error) { return nil, nil, nil }, + func(context.Context, *Input) (*Output, *Output) { return nil, nil }, + func() (Output, error) { return Output{}, nil }, + 42, + } { + _, err := buildRoute("foo", method) + assert.Error(err) + } +} diff --git a/tools/mage/build_namespace.go b/tools/mage/build_namespace.go index 5ad8eb5169..3479f8b70c 100644 --- a/tools/mage/build_namespace.go +++ b/tools/mage/build_namespace.go @@ -41,7 +41,16 @@ var build = Build{} // API Generate API source files from GraphQL + Swagger func (b Build) API() { - mg.Deps(b.generateSwaggerClients, b.generateWebTypescript) + mg.Deps(b.generateSwaggerClients, b.generateWebTypescript, b.goGenerate) +} + +func (b Build) goGenerate() error { + const generatePattern = "./..." + logger.Info("build:api: generating Go code with go:generate") + if err := sh.Run("go", "generate", generatePattern); err != nil { + return fmt.Errorf("go:generate failed: %s", err) + } + return nil } func (b Build) generateSwaggerClients() error { diff --git a/web/__generated__/schema.tsx b/web/__generated__/schema.tsx index 2f9ca77c37..1c70f1b04c 100644 --- a/web/__generated__/schema.tsx +++ b/web/__generated__/schema.tsx @@ -445,6 +445,11 @@ export enum ListAlertsSortFieldsEnum { CreatedAt = 'createdAt', } +export type ListAvailableLogTypesResponse = { + __typename?: 'ListAvailableLogTypesResponse'; + logTypes?: Maybe>>; +}; + export type ListComplianceItemsResponse = { __typename?: 'ListComplianceItemsResponse'; items?: Maybe>>; @@ -869,6 +874,7 @@ export type Query = { policy?: Maybe; policies?: Maybe; policiesForResource?: Maybe; + listAvailableLogTypes?: Maybe; listComplianceIntegrations: Array; listLogIntegrations: Array; organizationStats?: Maybe; @@ -1428,6 +1434,7 @@ export type ResolversTypes = { ListPoliciesResponse: ResolverTypeWrapper; PolicySummary: ResolverTypeWrapper; PoliciesForResourceInput: PoliciesForResourceInput; + ListAvailableLogTypesResponse: ResolverTypeWrapper; LogIntegration: ResolversTypes['S3LogIntegration'] | ResolversTypes['SqsLogSourceIntegration']; OrganizationStatsInput: OrganizationStatsInput; OrganizationStatsResponse: ResolverTypeWrapper; @@ -1565,6 +1572,7 @@ export type ResolversParentTypes = { ListPoliciesResponse: ListPoliciesResponse; PolicySummary: PolicySummary; PoliciesForResourceInput: PoliciesForResourceInput; + ListAvailableLogTypesResponse: ListAvailableLogTypesResponse; LogIntegration: | ResolversParentTypes['S3LogIntegration'] | ResolversParentTypes['SqsLogSourceIntegration']; @@ -1906,6 +1914,14 @@ export type ListAlertsResponseResolvers< __isTypeOf?: IsTypeOfResolverFn; }; +export type ListAvailableLogTypesResponseResolvers< + ContextType = any, + ParentType extends ResolversParentTypes['ListAvailableLogTypesResponse'] = ResolversParentTypes['ListAvailableLogTypesResponse'] +> = { + logTypes?: Resolver>>, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type ListComplianceItemsResponseResolvers< ContextType = any, ParentType extends ResolversParentTypes['ListComplianceItemsResponse'] = ResolversParentTypes['ListComplianceItemsResponse'] @@ -2422,6 +2438,11 @@ export type QueryResolvers< ContextType, RequireFields >; + listAvailableLogTypes?: Resolver< + Maybe, + ParentType, + ContextType + >; listComplianceIntegrations?: Resolver< Array, ParentType, @@ -2758,6 +2779,7 @@ export type Resolvers = { IntegrationTemplate?: IntegrationTemplateResolvers; JiraConfig?: JiraConfigResolvers; ListAlertsResponse?: ListAlertsResponseResolvers; + ListAvailableLogTypesResponse?: ListAvailableLogTypesResponseResolvers; ListComplianceItemsResponse?: ListComplianceItemsResponseResolvers; ListGlobalPythonModulesResponse?: ListGlobalPythonModulesResponseResolvers; ListPoliciesResponse?: ListPoliciesResponseResolvers; diff --git a/web/__tests__/__mocks__/builders.generated.ts b/web/__tests__/__mocks__/builders.generated.ts index 88a9cd0f2b..b70c4147fc 100644 --- a/web/__tests__/__mocks__/builders.generated.ts +++ b/web/__tests__/__mocks__/builders.generated.ts @@ -62,6 +62,7 @@ import { JiraConfigInput, ListAlertsInput, ListAlertsResponse, + ListAvailableLogTypesResponse, ListComplianceItemsResponse, ListGlobalPythonModuleInput, ListGlobalPythonModulesResponse, @@ -700,6 +701,15 @@ export const buildListAlertsResponse = ( }; }; +export const buildListAvailableLogTypesResponse = ( + overrides: Partial = {} +): ListAvailableLogTypesResponse => { + return { + __typename: 'ListAvailableLogTypesResponse', + logTypes: 'logTypes' in overrides ? overrides.logTypes : ['silver'], + }; +}; + export const buildListComplianceItemsResponse = ( overrides: Partial = {} ): ListComplianceItemsResponse => {