diff --git a/deployments/bootstrap_gateway.yml b/deployments/bootstrap_gateway.yml index f0963a5885..8177424346 100644 --- a/deployments/bootstrap_gateway.yml +++ b/deployments/bootstrap_gateway.yml @@ -81,6 +81,9 @@ Parameters: AllowedPattern: '^[a-zA-Z0-9_-]{15,}$' Mappings: + AthenaDataSources: + Dynamodb: + Name: ddb Functions: CustomResource: Memory: 1024 # use a larger size to run faster, since this is used rarely the small cost difference is not a concern @@ -142,7 +145,6 @@ Resources: - athena:GetQuery* - athena:StartQueryExecution - athena:StopQueryExecution - - athena:UpdateWorkGroup - guardduty:Create* - guardduty:DeletePublishingDestination - guardduty:Get* @@ -415,6 +417,39 @@ Resources: # here as for the compliance-api LatencyThresholdMs: 3000 + # https://docs.aws.amazon.com/athena/latest/ug/athena-prebuilt-data-connectors-dynamodb.html + AthenaDDBConnector: + Type: AWS::Serverless::Application + Properties: + Location: + # FIXME: this is currently a FIXED arn but the connector is not GA. Once GA it might be regional or different + ApplicationId: arn:aws:serverlessrepo:us-east-1:292517598671:applications/AthenaDynamoDBConnector + SemanticVersion: 2020.33.1 + Parameters: + AthenaCatalogName: !FindInMap [AthenaDataSources, Dynamodb, Name] # this is now called the "DataSource" + SpillBucket: !Ref AthenaResultsBucket + + AnthenaDDBDataSource: # this binds the lambda created in AthenaDDBConnector resource to Athena + DependsOn: AthenaDDBConnector + Type: AWS::Athena::DataCatalog + Properties: + Name: !FindInMap [AthenaDataSources, Dynamodb, Name] + Description: Dynamodb data source + Type: LAMBDA + Parameters: + function: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:ddb + + AthenaWorkGroup: + Type: AWS::Athena::WorkGroup + Properties: + # FIXME: switch to 'Panther' when DDB connector is GA https://docs.aws.amazon.com/athena/latest/ug/connect-to-a-data-source.html + Name: AmazonAthenaPreviewFunctionality # Panther + Description: The Panther workgroup + RecursiveDeleteOption: true + WorkGroupConfiguration: + ResultConfiguration: + OutputLocation: !Sub s3://${AthenaResultsBucket}/panther + Outputs: AppClientId: Description: Cognito user pool client ID @@ -441,3 +476,7 @@ Outputs: ResourcesApiEndpoint: Description: HTTPS endpoint for the resources api Value: !Sub ${ResourcesApi}.execute-api.${AWS::Region}.${AWS::URLSuffix} + + AthenaWorkGroup: + Description: The Athena workgroup for Panther queries + Value: !Ref AthenaWorkGroup diff --git a/deployments/cloud_security.yml b/deployments/cloud_security.yml index 4730c2576a..74693649ef 100644 --- a/deployments/cloud_security.yml +++ b/deployments/cloud_security.yml @@ -122,6 +122,16 @@ Conditions: TracingEnabled: !Not [!Equals ['', !Ref TracingMode]] Resources: + ###### Update Glue Table Schemas for Deployed Tables ##### + UpdateGlueTables: + Type: Custom::UpdateCloudSecurityTables + Properties: + # Update in case CustomResourceVersion has changed + CustomResourceVersion: !Ref CustomResourceVersion + ResourcesTableARN: !GetAtt ResourcesTable.Arn + ComplianceTableARN: !GetAtt ComplianceTable.Arn + ServiceToken: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:panther-cfn-custom-resources + ##### Alert Processor ##### AlertProcessorQueue: Type: AWS::SQS::Queue diff --git a/deployments/log_analysis.yml b/deployments/log_analysis.yml index 1b2182558f..f97fe05fa1 100644 --- a/deployments/log_analysis.yml +++ b/deployments/log_analysis.yml @@ -121,20 +121,11 @@ Conditions: TracingEnabled: !Not [!Equals ['', !Ref TracingMode]] Resources: - ##### Configure Athena ##### - AthenaConfigure: - Type: Custom::AthenaInit # this will associate the bucket as the default location for results - Properties: - AthenaResultsBucket: !Ref AthenaResultsBucket - CustomResourceVersion: !Ref CustomResourceVersion - ServiceToken: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:panther-cfn-custom-resources - ###### Update Glue Table Schemas for Deployed Tables ##### UpdateGlueTables: DependsOn: - - AthenaConfigure - UpdaterFunction - Type: Custom::UpdateGlueTables + Type: Custom::UpdateLogProcessorTables Properties: # Update in case TablesSignature or CustomResourceVersion has changed TablesSignature: !Ref TablesSignature diff --git a/internal/compliance/awsglue/awsglue.go b/internal/compliance/awsglue/awsglue.go new file mode 100644 index 0000000000..62739f87e7 --- /dev/null +++ b/internal/compliance/awsglue/awsglue.go @@ -0,0 +1,281 @@ +package awsglue + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/arn" + "github.com/aws/aws-sdk-go/service/glue" + "github.com/aws/aws-sdk-go/service/glue/glueiface" + "github.com/pkg/errors" + + "github.com/panther-labs/panther/pkg/awsutils" +) + +const ( + CloudSecurityDatabase = "panther_cloudsecurity" + CloudSecurityDatabaseDescription = "Hold tables related to Panther cloud security scanning" + + // https://github.com/awslabs/aws-athena-query-federation/tree/master/athena-dynamodb + + // FIXME: Update the description when the DDB connector is GA + ResourcesTableDDB = "panther-resources" + ResourcesTable = "resources" + ResourcesTableDescription = "(ddb.panther_cloudsecurity.panther-resources) The resources discovered by Panther scanning" + + ComplianceTableDDB = "panther-compliance" + ComplianceTable = "compliance" + ComplianceTableDescription = "(ddb.panther_cloudsecurity.panther-compliance) The policies and statuses from Panther scanning" +) + +var ( + // FIXME: Remove when the DDB connector is GA + // Available Regions – The Athena federated query feature is available in preview in the US East (N. Virginia), + // Asia Pacific (Mumbai), Europe (Ireland), and US West (Oregon) Regions. + anthenaDDBConnectorRegions = map[string]struct{}{ + "us-east-1": {}, + "ap-south-1": {}, + "eu-west-1": {}, + "us-west-2": {}, + } +) + +func CreateOrUpdateCloudSecurityDatabase(glueClient glueiface.GlueAPI) error { + dbInput := &glue.DatabaseInput{ + Description: aws.String(CloudSecurityDatabaseDescription), + LocationUri: aws.String("dynamo-db-flag"), + Name: aws.String(CloudSecurityDatabase), + } + + _, err := glueClient.CreateDatabase(&glue.CreateDatabaseInput{ + CatalogId: nil, + DatabaseInput: dbInput, + }) + if awsutils.IsAnyError(err, glue.ErrCodeAlreadyExistsException) { + return nil // nothing to do + } + return errors.Wrap(err, "could not create cloud security database") +} + +func CreateOrUpdateResourcesTable(glueClient glueiface.GlueAPI, locationARN string) error { + // FIXME: Remove when the DDB connector is GA + parsedARN, err := arn.Parse(locationARN) + if err != nil { + return err + } + if _, found := anthenaDDBConnectorRegions[parsedARN.Region]; !found { + return nil // not supported + } + + tableInput := &glue.TableInput{ + Name: aws.String(ResourcesTable), + Description: aws.String(ResourcesTableDescription), + Parameters: map[string]*string{ + // per https://github.com/awslabs/aws-athena-query-federation/tree/master/athena-dynamodb + "classification": aws.String("dynamodb"), + "sourceTable": aws.String(ResourcesTableDDB), + // for attrs with upper case + // nolint:lll + "columnMapping": aws.String(`expiresat=expiresAt,lastmodified=lastModified,integrationid=integrationId,integrationtype=integrationType`), + }, + StorageDescriptor: &glue.StorageDescriptor{ + Location: &locationARN, + + Columns: []*glue.Column{ + /* Commenting out for now: always 'aws' + { + Name: aws.String("integrationtype"), + Type: aws.String("string"), + Comment: aws.String("Indicates what type of integration this resource came from"), + }, + */ + { + Name: aws.String("deleted"), + Type: aws.String("boolean"), + Comment: aws.String("True if this is the snapshot of a deleted resource."), + }, + { + Name: aws.String("integrationid"), + Type: aws.String("string"), + Comment: aws.String("The unique ID indicating of the source integration."), + }, + { + Name: aws.String("attributes"), + Type: aws.String("string"), + Comment: aws.String("The JSON representation of the resource."), + }, + /* Commenting out: not useful + { + Name: aws.String("lowerid"), + Type: aws.String("string"), + Comment: aws.String("The resource ID converted to all lower case letters."), + }, + */ + { + Name: aws.String("lastmodified"), + Type: aws.String("string"), + Comment: aws.String("Timestamp of the most recent scan of this resource occurred."), + }, + { + Name: aws.String("id"), + Type: aws.String("string"), + Comment: aws.String("The panther wide unique identifier of the resource."), + }, + { + Name: aws.String("type"), + Type: aws.String("string"), + Comment: aws.String(" The type of resource (see https://docs.runpanther.io/cloud-security/resources)."), + }, + /* Commenting out: not useful + { + Name: aws.String("expiresat"), + Type: aws.String("bigint"), + Comment: aws.String("Unix timestamp representing when this resource will age out of the resources table."), + }, + */ + }, + }, + TableType: aws.String("EXTERNAL_TABLE"), + } + + createTableInput := &glue.CreateTableInput{ + DatabaseName: aws.String(CloudSecurityDatabase), + TableInput: tableInput, + } + + _, err = glueClient.CreateTable(createTableInput) + if err != nil { + if awsutils.IsAnyError(err, glue.ErrCodeAlreadyExistsException) { + // need to do an update + updateTableInput := &glue.UpdateTableInput{ + DatabaseName: aws.String(CloudSecurityDatabase), + TableInput: tableInput, + } + _, err := glueClient.UpdateTable(updateTableInput) + return errors.Wrapf(err, "failed to update table %s.%s", CloudSecurityDatabase, ResourcesTable) + } + return errors.Wrapf(err, "failed to create table %s.%s", CloudSecurityDatabase, ResourcesTable) + } + + return nil +} + +func CreateOrUpdateComplianceTable(glueClient glueiface.GlueAPI, locationARN string) error { + // FIXME: Remove when the DDB connector is GA + parsedARN, err := arn.Parse(locationARN) + if err != nil { + return err + } + if _, found := anthenaDDBConnectorRegions[parsedARN.Region]; !found { + return nil // not supported + } + + tableInput := &glue.TableInput{ + Name: aws.String(ComplianceTable), + Description: aws.String(ComplianceTableDescription), + Parameters: map[string]*string{ + // per https://github.com/awslabs/aws-athena-query-federation/tree/master/athena-dynamodb + "classification": aws.String("dynamodb"), + "sourceTable": aws.String(ComplianceTableDDB), + // for attrs with upper case + // nolint:lll + "columnMapping": aws.String(`policyseverity=policySeverity,errormessage=errorMessage,expiresat=expiresAt,lastupdated=lastUpdated,policyid=policyId,resourceid=resourceId,resourcetype=resourceType,integrationid=integrationId`), + }, + StorageDescriptor: &glue.StorageDescriptor{ + Location: &locationARN, + + Columns: []*glue.Column{ + { + Name: aws.String("lastupdated"), + Type: aws.String("string"), + Comment: aws.String("That last date the specified policy was evaluated against the specified resource."), + }, + { + Name: aws.String("resourceid"), + Type: aws.String("string"), + Comment: aws.String("The panther wide unique identifier of the resource being evaluated."), + }, + { + Name: aws.String("policyseverity"), + Type: aws.String("string"), + Comment: aws.String("The severity of the policy being evaluated."), + }, + { + Name: aws.String("policyid"), + Type: aws.String("string"), + Comment: aws.String("The unique identifier of the policy being evaluated."), + }, + { + Name: aws.String("integrationid"), + Type: aws.String("string"), + Comment: aws.String("The unique ID indicating of the source integration."), + }, + { + Name: aws.String("suppressed"), + Type: aws.String("boolean"), + Comment: aws.String("True if this compliance status is currently being omitted from compliance findings."), + }, + /* Commenting out: not useful + { + Name: aws.String("expiresat"), + Type: aws.String("bigint"), + Comment: aws.String("Unix timestamp representing when this resource will age out of the resources table."), + }, + */ + { + Name: aws.String("resourcetype"), + Type: aws.String("string"), + Comment: aws.String("The type of the specified resource."), + }, + { + Name: aws.String("status"), + Type: aws.String("string"), + Comment: aws.String("Whether the policy evaluation of this resource resulted in a PASS, FAIL, or ERROR state."), + }, + { + Name: aws.String("errormessage"), + Type: aws.String("string"), + Comment: aws.String("If an error occurred, the associated error message."), + }, + }, + }, + TableType: aws.String("EXTERNAL_TABLE"), + } + + createTableInput := &glue.CreateTableInput{ + DatabaseName: aws.String(CloudSecurityDatabase), + TableInput: tableInput, + } + + _, err = glueClient.CreateTable(createTableInput) + if err != nil { + if awsutils.IsAnyError(err, glue.ErrCodeAlreadyExistsException) { + // need to do an update + updateTableInput := &glue.UpdateTableInput{ + DatabaseName: aws.String(CloudSecurityDatabase), + TableInput: tableInput, + } + _, err := glueClient.UpdateTable(updateTableInput) + return errors.Wrapf(err, "failed to update table %s.%s", CloudSecurityDatabase, ResourcesTable) + } + return errors.Wrapf(err, "failed to create table %s.%s", CloudSecurityDatabase, ResourcesTable) + } + + return nil +} diff --git a/internal/core/custom_resources/resources/athena.go b/internal/core/custom_resources/resources/athena.go deleted file mode 100644 index 7084ecb452..0000000000 --- a/internal/core/custom_resources/resources/athena.go +++ /dev/null @@ -1,55 +0,0 @@ -package resources - -/** - * Panther is a Cloud-Native SIEM for the Modern Security Team. - * Copyright (C) 2020 Panther Labs Inc - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import ( - "context" - "fmt" - - "github.com/aws/aws-lambda-go/cfn" - - "github.com/panther-labs/panther/pkg/awsathena" -) - -type AthenaInitProperties struct { - AthenaResultsBucket string `validate:"required"` -} - -func customAthenaInit(_ context.Context, event cfn.Event) (string, map[string]interface{}, error) { - const resourceID = "custom:athena:init" - switch event.RequestType { - case cfn.RequestCreate, cfn.RequestUpdate: - var props AthenaInitProperties - if err := parseProperties(event.ResourceProperties, &props); err != nil { - return resourceID, nil, err - } - - // Workgroup "primary" is default. - const workgroup = "primary" - if err := awsathena.WorkgroupAssociateS3(awsSession, workgroup, props.AthenaResultsBucket); err != nil { - return resourceID, nil, fmt.Errorf("failed to associate %s Athena workgroup with %s bucket: %v", - workgroup, props.AthenaResultsBucket, err) - } - - return resourceID, nil, nil - - default: // ignore deletes - return event.PhysicalResourceID, nil, nil - } -} diff --git a/internal/core/custom_resources/resources/cloud_security_tables.go b/internal/core/custom_resources/resources/cloud_security_tables.go new file mode 100644 index 0000000000..67a1c90c9e --- /dev/null +++ b/internal/core/custom_resources/resources/cloud_security_tables.go @@ -0,0 +1,88 @@ +package resources + +/** + * Panther is a Cloud-Native SIEM for the Modern Security Team. + * Copyright (C) 2020 Panther Labs Inc + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import ( + "context" + "fmt" + + "github.com/aws/aws-lambda-go/cfn" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/glue" + "github.com/pkg/errors" + "go.uber.org/zap" + + cloudsecglue "github.com/panther-labs/panther/internal/compliance/awsglue" + "github.com/panther-labs/panther/internal/log_analysis/awsglue" +) + +type UpdateCloudSecurityTablesProperties struct { + ResourcesTableARN string + ComplianceTableARN string +} + +func customCloudSecurityTables(_ context.Context, event cfn.Event) (string, map[string]interface{}, error) { + switch event.RequestType { + case cfn.RequestCreate, cfn.RequestUpdate: + // It's important to always return this physicalResourceID + const physicalResourceID = "custom:glue:update-cloud-security-tables" + var props UpdateCloudSecurityTablesProperties + if err := parseProperties(event.ResourceProperties, &props); err != nil { + zap.L().Error("failed to parse resource properties", zap.Error(err)) + return physicalResourceID, nil, err + } + if err := updateCloudSecurityTables(&props); err != nil { + zap.L().Error("failed to update glue tables", zap.Error(err)) + return physicalResourceID, nil, err + } + return physicalResourceID, nil, nil + case cfn.RequestDelete: + zap.L().Info("deleting database", zap.String("database", cloudsecglue.CloudSecurityDatabase)) + if _, err := awsglue.DeleteDatabase(glueClient, cloudsecglue.CloudSecurityDatabase); err != nil { + var awsErr awserr.Error + if errors.As(err, &awsErr) && awsErr.Code() == glue.ErrCodeEntityNotFoundException { + zap.L().Info("already deleted", zap.String("database", cloudsecglue.CloudSecurityDatabase)) + } else { + return "", nil, errors.Wrapf(err, "failed deleting %s", cloudsecglue.CloudSecurityDatabase) + } + } + return event.PhysicalResourceID, nil, nil + default: + return "", nil, fmt.Errorf("unknown request type %s", event.RequestType) + } +} + +func updateCloudSecurityTables(props *UpdateCloudSecurityTablesProperties) error { + err := cloudsecglue.CreateOrUpdateCloudSecurityDatabase(glueClient) + if err != nil { + return err + } + + err = cloudsecglue.CreateOrUpdateResourcesTable(glueClient, props.ResourcesTableARN) + if err != nil { + return err + } + + err = cloudsecglue.CreateOrUpdateComplianceTable(glueClient, props.ComplianceTableARN) + if err != nil { + return err + } + + return nil +} diff --git a/internal/core/custom_resources/resources/glue.go b/internal/core/custom_resources/resources/log_processor_tables.go similarity index 91% rename from internal/core/custom_resources/resources/glue.go rename to internal/core/custom_resources/resources/log_processor_tables.go index b8de4cb220..26afbe1eed 100644 --- a/internal/core/custom_resources/resources/glue.go +++ b/internal/core/custom_resources/resources/log_processor_tables.go @@ -35,23 +35,23 @@ import ( "github.com/panther-labs/panther/pkg/awsutils" ) -type UpdateGlueTablesProperties struct { +type UpdateLogProcessorTablesProperties struct { // TablesSignature should change every time the tables change (for CF master.yml this can be the Panther version) TablesSignature string `validate:"required"` ProcessedDataBucket string `validate:"required"` } -func customUpdateGlueTables(ctx context.Context, event cfn.Event) (string, map[string]interface{}, error) { +func customUpdateLogProcessorTables(ctx context.Context, event cfn.Event) (string, map[string]interface{}, error) { switch event.RequestType { case cfn.RequestCreate, cfn.RequestUpdate: // It's important to always return this physicalResourceID - const physicalResourceID = "custom:glue:update-tables" - var props UpdateGlueTablesProperties + const physicalResourceID = "custom:glue:update-log-processor-tables" + var props UpdateLogProcessorTablesProperties if err := parseProperties(event.ResourceProperties, &props); err != nil { zap.L().Error("failed to parse resource properties", zap.Error(err)) return physicalResourceID, nil, err } - if err := updateGlueTables(ctx, &props); err != nil { + if err := updateLogProcessorTables(ctx, &props); err != nil { zap.L().Error("failed to update glue tables", zap.Error(err)) return physicalResourceID, nil, err } @@ -74,7 +74,7 @@ func customUpdateGlueTables(ctx context.Context, event cfn.Event) (string, map[s } } -func updateGlueTables(ctx context.Context, props *UpdateGlueTablesProperties) error { +func updateLogProcessorTables(ctx context.Context, props *UpdateLogProcessorTablesProperties) error { // ensure databases are all there for pantherDatabase, pantherDatabaseDescription := range awsglue.PantherDatabases { zap.L().Info("creating database", zap.String("database", pantherDatabase)) diff --git a/internal/core/custom_resources/resources/map.go b/internal/core/custom_resources/resources/map.go index 7bfb51bbf3..fd1bda8b83 100644 --- a/internal/core/custom_resources/resources/map.go +++ b/internal/core/custom_resources/resources/map.go @@ -56,14 +56,6 @@ var CustomResources = map[string]cfn.CustomResourceFunction{ // PhysicalId: custom:alarms:appsync:$API_ID "Custom::AppSyncAlarms": customAppSyncAlarms, - // Initialize Athena - // - // Parameters: - // AthenaResultsBucket: string (required) - // Outputs: None - // PhysicalId: custom:athena:init - "Custom::AthenaInit": customAthenaInit, - // CloudWatch alarms for Dynamo errors, throttles, and latency // // Parameters: @@ -103,13 +95,21 @@ var CustomResources = map[string]cfn.CustomResourceFunction{ // Deleting this resource has no effect on the user pool. "Custom::CognitoUserPoolMfa": customCognitoUserPoolMfa, - // Updates databases and table schemas + // Updates databases and table schemas from the log processor + // + // Parameters: + // DeploymentId: string (required) + // Outputs: None + // PhysicalId: custom:glue:update-log-processor-tables + "Custom::UpdateLogProcessorTables": customUpdateLogProcessorTables, + + // Updates databases and table schemas from the cloud security // // Parameters: // DeploymentId: string (required) // Outputs: None - // PhysicalId: custom:glue:update-tables - "Custom::UpdateGlueTables": customUpdateGlueTables, + // PhysicalId: custom:glue:update-cloud-security-tables + "Custom::UpdateCloudSecurityTables": customCloudSecurityTables, // Add a GuardDuty publishing destination //