From d09a0d3001ac1ae204962c2b22a3f5989ec6b20c Mon Sep 17 00:00:00 2001 From: jgrillo Date: Wed, 20 May 2020 12:28:30 -0400 Subject: [PATCH 1/9] trigger on prereleases as well --- .github/workflows/grapl-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/grapl-release.yml b/.github/workflows/grapl-release.yml index 51ab58638d..5c70da18dd 100644 --- a/.github/workflows/grapl-release.yml +++ b/.github/workflows/grapl-release.yml @@ -2,7 +2,7 @@ name: Grapl Release on: release: - types: [released] + types: [released, prereleased] jobs: release: From 47295ea4c22fea65dfba93ef3ca643e0cbb02cee Mon Sep 17 00:00:00 2001 From: jgrillo Date: Wed, 20 May 2020 12:38:04 -0400 Subject: [PATCH 2/9] actions/checkout@v2 should checkout the correct branch anyway? --- .github/workflows/grapl-release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/grapl-release.yml b/.github/workflows/grapl-release.yml index 5c70da18dd..b0abb3fd62 100644 --- a/.github/workflows/grapl-release.yml +++ b/.github/workflows/grapl-release.yml @@ -10,8 +10,6 @@ jobs: steps: - uses: actions/checkout@v2 - with: - ref: $GITHUB_REF - name: Determine release channel run: | From 6f320ae76aee5a4d97043fe611ac1e56fe4645aa Mon Sep 17 00:00:00 2001 From: jgrillo Date: Wed, 20 May 2020 13:07:24 -0400 Subject: [PATCH 3/9] bash --- .github/workflows/grapl-release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/grapl-release.yml b/.github/workflows/grapl-release.yml index b0abb3fd62..c615ae575f 100644 --- a/.github/workflows/grapl-release.yml +++ b/.github/workflows/grapl-release.yml @@ -13,13 +13,13 @@ jobs: - name: Determine release channel run: | - BRANCH = ${{ github.event.release.target_commitish }} \ - if [ $BRANCH = "master" ]; then \ - CHANNEL = "latest" \ - else \ - CHANNEL = "beta" \ - fi \ - echo '::set-env name=CHANNEL::$CHANNEL' + BRANCH=${{ github.event.release.target_commitish }} + if [[ "$BRANCH"="master" ]]; then + CHANNEL="latest" + else + CHANNEL="beta" + fi + echo "::set-env name=CHANNEL::$CHANNEL" - name: Build Grapl env: From e28c73a337b9c088b973761c617293088562c4d0 Mon Sep 17 00:00:00 2001 From: jgrillo Date: Wed, 20 May 2020 15:37:13 -0400 Subject: [PATCH 4/9] fix tagged builds --- .github/workflows/grapl-release.yml | 2 +- docker-compose.build.yml | 32 +- grapl-cdk/index.js | 894 ------------------ src/python/analyzer_executor/Dockerfile | 4 +- src/python/engagement-creator/Dockerfile | 2 +- src/python/engagement_edge/Dockerfile | 2 +- .../grapl-model-plugin-deployer/Dockerfile | 2 +- src/python/grapl-notebook/Dockerfile | 2 +- src/rust/graph-descriptions/Dockerfile | 2 +- 9 files changed, 23 insertions(+), 919 deletions(-) delete mode 100644 grapl-cdk/index.js diff --git a/.github/workflows/grapl-release.yml b/.github/workflows/grapl-release.yml index c615ae575f..63ba1e8271 100644 --- a/.github/workflows/grapl-release.yml +++ b/.github/workflows/grapl-release.yml @@ -25,7 +25,7 @@ jobs: env: VERSION: ${{ github.event.release.tag_name }} run: | - TAG=$VERSION docker-compose -f docker-compose.yml -f docker-compose.build.yml build --build-arg release_target=release + TAG="$VERSION-$CHANNEL" docker-compose -f docker-compose.yml -f docker-compose.build.yml build --build-arg release_target=release docker system prune -f - name: Log in to Docker registry diff --git a/docker-compose.build.yml b/docker-compose.build.yml index 2c83788ec4..195fc38262 100644 --- a/docker-compose.build.yml +++ b/docker-compose.build.yml @@ -6,7 +6,7 @@ services: # grapl-rust-src-build: - image: grapl/grapl-rust-src-build:${TAG:-latest} + image: grapl/grapl-rust-src-build:latest build: context: src/rust target: grapl-rust-src-build @@ -16,7 +16,7 @@ services: build: context: src/rust/sysmon-subgraph-generator target: grapl-sysmon-subgraph-generator - depends_on: + depends_on: - grapl-rust-src-build grapl-node-identifier: @@ -24,7 +24,7 @@ services: build: context: src/rust/node-identifier target: grapl-node-identifier - depends_on: + depends_on: - grapl-rust-src-build grapl-node-identifier-retry-handler: @@ -32,7 +32,7 @@ services: build: context: src/rust/node-identifier target: grapl-node-identifier-retry-handler - depends_on: + depends_on: - grapl-rust-src-build grapl-graph-merger: @@ -40,7 +40,7 @@ services: build: context: src/rust/graph-merger target: grapl-graph-merger - depends_on: + depends_on: - grapl-rust-src-build grapl-analyzer-dispatcher: @@ -48,7 +48,7 @@ services: build: context: src/rust/analyzer-dispatcher target: grapl-analyzer-dispatcher - depends_on: + depends_on: - grapl-rust-src-build # @@ -56,17 +56,17 @@ services: # grapl-python-build: - image: grapl/grapl-python-build:${TAG:-latest} + image: grapl/grapl-python-build:latest build: context: src/python/grapl-python-build target: grapl-python-build grapl-graph-descriptions-python-build: - image: grapl/grapl-graph-descriptions-python-build:${TAG:-latest} + image: grapl/grapl-graph-descriptions-python-build:latest build: context: src/rust/graph-descriptions target: grapl-graph-descriptions-python-build - depends_on: + depends_on: - grapl-python-build grapl-analyzer-executor: @@ -74,7 +74,7 @@ services: build: context: ./src/python/analyzer_executor target: grapl-analyzer-executor - depends_on: + depends_on: - grapl-graph-descriptions-python-build grapl-engagement-creator: @@ -82,7 +82,7 @@ services: build: context: ./src/python/engagement-creator target: grapl-engagement-creator - depends_on: + depends_on: - grapl-graph-descriptions-python-build grapl-engagement-edge: @@ -90,7 +90,7 @@ services: build: context: ./src/python/engagement_edge target: grapl-engagement-edge - depends_on: + depends_on: - grapl-graph-descriptions-python-build grapl-model-plugin-deployer: @@ -98,11 +98,9 @@ services: build: context: ./src/python/grapl-model-plugin-deployer/ dockerfile: ./Dockerfile - depends_on: + depends_on: - grapl-graph-descriptions-python-build - - grapl-notebook: image: grapl/grapl-notebook:${TAG:-latest} build: @@ -135,12 +133,12 @@ services: image: grapl/grapl-graph-provision:${TAG:-latest} build: context: ./src/python/engagement_edge/ - depends_on: + depends_on: - grapl-graph-descriptions-python-build grapl-dynamodb-provision: image: grapl/grapl-dynamodb-provision:${TAG:-latest} build: context: ./src/python/engagement_edge/ - depends_on: + depends_on: - grapl-graph-descriptions-python-build diff --git a/grapl-cdk/index.js b/grapl-cdk/index.js deleted file mode 100644 index 0a8d9dccb4..0000000000 --- a/grapl-cdk/index.js +++ /dev/null @@ -1,894 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -const aws_lambda_event_sources_1 = require("@aws-cdk/aws-lambda-event-sources"); -const aws_ec2_1 = require("@aws-cdk/aws-ec2"); -const aws_lambda_1 = require("@aws-cdk/aws-lambda"); -const core_1 = require("@aws-cdk/core"); -const aws_ecs_1 = require("@aws-cdk/aws-ecs"); -const AWS = require('aws-sdk'); -const uuidv4 = require('uuid/v4'); -const sagemaker = require("@aws-cdk/aws-sagemaker"); -const cloudmap = require("@aws-cdk/aws-servicediscovery"); -const s3Subs = require("@aws-cdk/aws-s3-notifications"); -const s3deploy = require("@aws-cdk/aws-s3-deployment"); -const snsSubs = require("@aws-cdk/aws-sns-subscriptions"); -const elasticache = require("@aws-cdk/aws-elasticache"); -const cdk = require("@aws-cdk/core"); -const s3 = require("@aws-cdk/aws-s3"); -const sns = require("@aws-cdk/aws-sns"); -const sqs = require("@aws-cdk/aws-sqs"); -const ec2 = require("@aws-cdk/aws-ec2"); -const servicediscovery = require("@aws-cdk/aws-servicediscovery"); -const lambda = require("@aws-cdk/aws-lambda"); -const iam = require("@aws-cdk/aws-iam"); -const dynamodb = require("@aws-cdk/aws-dynamodb"); -const ecs = require("@aws-cdk/aws-ecs"); -const apigateway = require("@aws-cdk/aws-apigateway"); -const env = require('node-env-file'); -const dir = require('node-dir'); -class RedisCluster extends cdk.Construct { - constructor(scope, id, vpc) { - super(scope, id); - // Define a group for telling Elasticache which subnets to put cache nodes in. - const subnetGroup = new elasticache.CfnSubnetGroup(this, `${id}-subnet-group`, { - description: `List of subnets used for redis cache ${id}`, - subnetIds: vpc.privateSubnets.map(function (subnet) { - return subnet.subnetId; - }), - cacheSubnetGroupName: id + 'SubnetGroupName' - }); - // The security group that defines network level access to the cluster - this.securityGroup = new ec2.SecurityGroup(this, `${id}-security-group`, { vpc: vpc }); - this.connections = new ec2.Connections({ - securityGroups: [this.securityGroup], - defaultPort: ec2.Port.tcp(6379) - }); - this.connections.allowFromAnyIpv4(aws_ec2_1.Port.tcp(6379)); - // The cluster resource itself. - this.cluster = new elasticache.CfnCacheCluster(this, `${id}-cluster`, { - cacheNodeType: 'cache.t2.small', - engine: 'redis', - numCacheNodes: 1, - autoMinorVersionUpgrade: true, - cacheSubnetGroupName: subnetGroup.cacheSubnetGroupName, - vpcSecurityGroupIds: [ - this.securityGroup.securityGroupId - ] - }); - } -} -class Queues { - constructor(stack, queue_name) { - this.dead_letter_queue = new sqs.Queue(stack, queue_name + '-dead-letter'); - this.retry_queue = new sqs.Queue(stack, queue_name + '-retry', { - deadLetterQueue: { queue: this.dead_letter_queue, maxReceiveCount: 10 }, - visibilityTimeout: core_1.Duration.seconds(360) - }); - this.queue = new sqs.Queue(stack, queue_name, { - deadLetterQueue: { queue: this.retry_queue, maxReceiveCount: 5 }, - visibilityTimeout: core_1.Duration.seconds(180) - }); - } -} -class UserAuthDb extends cdk.Stack { - constructor(parent, id) { - super(parent, id + '-stack'); - this.user_auth_table = new dynamodb.Table(this, 'user_auth_table', { - tableName: "user_auth_table", - partitionKey: { - name: 'username', - type: dynamodb.AttributeType.STRING - }, - serverSideEncryption: true, - billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, - }); - } - allowRead(service) { - this.user_auth_table.grantReadData(service.event_handler.role); - this.user_auth_table.grantReadData(service.event_retry_handler.role); - } - allowReadWrite(service) { - this.user_auth_table.grantReadData(service.event_handler.role); - this.user_auth_table.grantReadData(service.event_retry_handler.role); - } - allowReadFromRole(role) { - this.user_auth_table.grantReadData(role); - } - allowReadWriteFromRole(role) { - this.user_auth_table.grantReadWriteData(role); - } -} -class EngagementEdge extends cdk.Stack { - constructor(parent, name, hostname, jwt_secret, engagement_graph, user_auth_table, vpc) { - super(parent, name + '-stack'); - this.name = name + process.env.BUCKET_PREFIX; - this.integrationName = name + process.env.BUCKET_PREFIX + 'Integration'; - this.event_handler = new lambda.Function(this, name, { - runtime: aws_lambda_1.Runtime.PYTHON_3_7, - handler: `engagement_edge.app`, - code: lambda.Code.fromAsset(`./engagement_edge.zip`), - vpc: vpc, - environment: { - "EG_ALPHAS": engagement_graph.alphaNames.join(","), - "JWT_SECRET": jwt_secret, - "USER_AUTH_TABLE": user_auth_table.user_auth_table.tableName, - "BUCKET_PREFIX": process.env.BUCKET_PREFIX - }, - timeout: core_1.Duration.seconds(25), - memorySize: 256, - }); - user_auth_table.allowReadFromRole(this.event_handler.role); - this.integration = new apigateway.LambdaRestApi(this, this.integrationName, { - handler: this.event_handler, - }); - } -} -class Service { - constructor(stack, name, environment, vpc, retry_code_name, opt) { - let runtime = null; - if (opt && opt.runtime) { - runtime = opt.runtime; - } - else { - runtime = { name: "provided", supportsInlineCode: true }; - } - let handler = null; - if (runtime === aws_lambda_1.Runtime.PYTHON_3_7) { - handler = `${name}.lambda_handler`; - } - else { - handler = name; - } - const queues = new Queues(stack, name + '-queue'); - if (environment) { - environment.QUEUE_URL = queues.queue.queueUrl; - environment.RUST_BACKTRACE = "1"; - } - let event_handler = new lambda.Function(stack, name, { - runtime: runtime, - handler: handler, - code: lambda.Code.asset(`./${name}.zip`), - vpc: vpc, - environment: { - IS_RETRY: "False", - ...environment - }, - timeout: core_1.Duration.seconds(180), - memorySize: 256, - }); - if (!retry_code_name) { - retry_code_name = name; - } - if (environment) { - environment.QUEUE_URL = queues.retry_queue.queueUrl; - } - let event_retry_handler = new lambda.Function(stack, name + '-retry-handler', { - runtime: runtime, - handler: handler, - code: lambda.Code.asset(`./${retry_code_name}.zip`), - vpc: vpc, - environment: { - IS_RETRY: "True", - ...environment - }, - timeout: core_1.Duration.seconds(360), - memorySize: 512, - }); - event_handler.addEventSource(new aws_lambda_event_sources_1.SqsEventSource(queues.queue, { batchSize: 1 })); - event_retry_handler.addEventSource(new aws_lambda_event_sources_1.SqsEventSource(queues.retry_queue, { batchSize: 1 })); - queues.queue.grantConsumeMessages(event_handler); - queues.retry_queue.grantConsumeMessages(event_retry_handler); - this.queues = queues; - this.event_handler = event_handler; - this.event_retry_handler = event_retry_handler; - } - readsFrom(bucket, with_list) { - let policy = new iam.PolicyStatement(); - policy.addActions('s3:GetObject', 's3:ActionGetBucket'); - if (with_list === true) { - policy.addActions('s3:ListObjects'); - } - policy.addResources(bucket.bucketArn); - this.event_handler.addToRolePolicy(policy); - this.event_retry_handler.addToRolePolicy(policy); - // TODO: This is adding more permissions than necessary - bucket.grantRead(this.event_handler.role); - bucket.grantRead(this.event_retry_handler.role); - } - publishesToTopic(publishes_to) { - const topicPolicy = new iam.PolicyStatement(); - topicPolicy.addActions('sns:CreateTopic'); - topicPolicy.addResources(publishes_to.topicArn); - this.event_handler.addToRolePolicy(topicPolicy); - this.event_retry_handler.addToRolePolicy(topicPolicy); - publishes_to.grantPublish(this.event_handler.role); - publishes_to.grantPublish(this.event_retry_handler.role); - } - publishesToBucket(publishes_to) { - publishes_to.grantWrite(this.event_handler.role); - publishes_to.grantWrite(this.event_retry_handler.role); - } -} -class SessionIdentityCache extends cdk.Stack { - constructor(parent, vpc) { - super(parent, 'session-identity-cache-stack'); - // const zone = new route53.PrivateHostedZone(this, 'HostedZone', { - // zoneName: 'sessionid.cache', - // vpc_props - // }); - } -} -class EventEmitter { - constructor(stack, eventName) { - this.bucket = - new s3.Bucket(stack, eventName + '-bucket', { - bucketName: process.env.BUCKET_PREFIX + eventName + "-bucket" - }); - // SNS Topics - this.topic = - new sns.Topic(stack, `${eventName}-topic`, { - topicName: `${eventName}-topic` - }); - this.bucket - .addEventNotification(s3.EventType.OBJECT_CREATED, new s3Subs.SnsDestination(this.topic)); - } -} -class EventEmitters extends cdk.Stack { - constructor(parent, id) { - super(parent, id + '-stack'); - let raw_logs_bucket = new s3.Bucket(this, id + '-raw-log-bucket', { - bucketName: process.env.BUCKET_PREFIX + "-raw-log-bucket" - }); - let sysmon_logs_bucket = new s3.Bucket(this, id + '-sysmon-log-bucket', { - bucketName: process.env.BUCKET_PREFIX + "-sysmon-log-bucket" - }); - let identity_mappings_bucket = new s3.Bucket(this, id + '-identity-mappings-bucket', { - bucketName: process.env.BUCKET_PREFIX + "-identity-mappings-bucket" - }); - let unid_subgraphs_generated_bucket = new s3.Bucket(this, id + '-unid-subgraphs-generated-bucket', { - bucketName: process.env.BUCKET_PREFIX + "-unid-subgraphs-generated-bucket" - }); - let subgraphs_generated_bucket = new s3.Bucket(this, id + '-subgraphs-generated-bucket', { - bucketName: process.env.BUCKET_PREFIX + "-subgraphs-generated-bucket" - }); - let analyzers_bucket = new s3.Bucket(this, id + '-analyzers-bucket', { - bucketName: process.env.BUCKET_PREFIX + "-analyzers-bucket" - }); - let model_plugins_bucket = new s3.Bucket(this, id + '-model-plugins-bucket', { - bucketName: process.env.BUCKET_PREFIX + "-model-plugins-bucket" - }); - let subgraphs_merged_bucket = new s3.Bucket(this, id + '-subgraphs-merged-bucket', { - bucketName: process.env.BUCKET_PREFIX + "-subgraphs-merged-bucket" - }); - let dispatched_analyzer_bucket = new s3.Bucket(this, id + '-dispatched-analyzer-bucket', { - bucketName: process.env.BUCKET_PREFIX + "-dispatched-analyzer-bucket" - }); - let analyzer_matched_subgraphs_bucket = new s3.Bucket(this, id + '-analyzer-matched-subgraphs-bucket', { - bucketName: process.env.BUCKET_PREFIX + "-analyzer-matched-subgraphs-bucket" - }); - // SNS Topics - let incident_topic = new sns.Topic(this, id + '-incident-topic', { - topicName: 'incident-topic' - }); - let raw_logs_topic = new sns.Topic(this, id + '-raw-log-topic', { - topicName: 'raw-log-topic' - }); - let sysmon_logs_topic = new sns.Topic(this, id + '-sysmon-log-topic', { - topicName: 'sysmon-log-topic' - }); - let identity_mappings_topic = new sns.Topic(this, id + '-identity-mappings-topic', { - topicName: 'identity-mappings-topic' - }); - let unid_subgraphs_generated_topic = new sns.Topic(this, id + '-unid-subgraphs-generated-topic', { - topicName: 'unid-subgraphs-generated-topic' - }); - let subgraphs_generated_topic = new sns.Topic(this, id + '-subgraphs-generated-topic', { - topicName: 'subgraphs-generated-topic' - }); - let subgraph_merged_topic = new sns.Topic(this, id + '-subgraphs-merged-topic', { - topicName: 'subgraphs-merged-topic' - }); - let dispatched_analyzer_topic = new sns.Topic(this, id + '-dispatched-analyzer-topic', { - topicName: 'dispatched-analyzer-topic' - }); - let analyzer_matched_subgraphs_topic = new sns.Topic(this, id + '-analyzer-matched-subgraphs-topic', { - topicName: 'analyzer-matched-subgraphs-topic' - }); - let engagements_created_topic = new sns.Topic(this, id + '-engagements-created-topic', { - topicName: 'engagements-created-topic' - }); - // S3 -> SNS Events - raw_logs_bucket - .addEventNotification(s3.EventType.OBJECT_CREATED, new s3Subs.SnsDestination(raw_logs_topic)); - sysmon_logs_bucket - .addEventNotification(s3.EventType.OBJECT_CREATED, new s3Subs.SnsDestination(sysmon_logs_topic)); - identity_mappings_bucket - .addEventNotification(s3.EventType.OBJECT_CREATED, new s3Subs.SnsDestination(identity_mappings_topic)); - unid_subgraphs_generated_bucket - .addEventNotification(s3.EventType.OBJECT_CREATED, new s3Subs.SnsDestination(unid_subgraphs_generated_topic)); - subgraphs_generated_bucket - .addEventNotification(s3.EventType.OBJECT_CREATED, new s3Subs.SnsDestination(subgraphs_generated_topic)); - subgraphs_merged_bucket - .addEventNotification(s3.EventType.OBJECT_CREATED, new s3Subs.SnsDestination(subgraph_merged_topic)); - dispatched_analyzer_bucket - .addEventNotification(s3.EventType.OBJECT_CREATED, new s3Subs.SnsDestination(dispatched_analyzer_topic)); - analyzer_matched_subgraphs_bucket - .addEventNotification(s3.EventType.OBJECT_CREATED, new s3Subs.SnsDestination(analyzer_matched_subgraphs_topic)); - this.raw_logs_bucket = raw_logs_bucket; - this.sysmon_logs_bucket = sysmon_logs_bucket; - this.identity_mappings_bucket = identity_mappings_bucket; - this.unid_subgraphs_generated_bucket = unid_subgraphs_generated_bucket; - this.subgraphs_generated_bucket = subgraphs_generated_bucket; - this.analyzers_bucket = analyzers_bucket; - this.model_plugins_bucket = model_plugins_bucket; - this.subgraphs_merged_bucket = subgraphs_merged_bucket; - this.dispatched_analyzer_bucket = dispatched_analyzer_bucket; - this.analyzer_matched_subgraphs_bucket = analyzer_matched_subgraphs_bucket; - this.incident_topic = incident_topic; - this.raw_logs_topic = raw_logs_topic; - this.sysmon_logs_topic = sysmon_logs_topic; - this.identity_mappings_topic = identity_mappings_topic; - this.unid_subgraphs_generated_topic = unid_subgraphs_generated_topic; - this.subgraphs_generated_topic = subgraphs_generated_topic; - this.subgraph_merged_topic = subgraph_merged_topic; - this.dispatched_analyzer_topic = dispatched_analyzer_topic; - this.analyzer_matched_subgraphs_topic = analyzer_matched_subgraphs_topic; - this.engagements_created_topic = engagements_created_topic; - } -} -class SysmonSubgraphGenerator extends cdk.Stack { - constructor(parent, id, reads_from, subscribes_to, writes_to, vpc) { - super(parent, id + '-stack'); - const sysmon_event_cache = new RedisCluster(this, id + 'sysmoneventcache', vpc); - sysmon_event_cache.connections.allowFromAnyIpv4(aws_ec2_1.Port.allTcp()); - const environment = { - "BUCKET_PREFIX": process.env.BUCKET_PREFIX, - "EVENT_CACHE_ADDR": sysmon_event_cache.cluster.attrRedisEndpointAddress, - "EVENT_CACHE_PORT": sysmon_event_cache.cluster.attrRedisEndpointPort, - }; - const service = new Service(this, 'sysmon-subgraph-generator', environment, vpc); - service.event_handler.connections.allowToAnyIpv4(aws_ec2_1.Port.allTraffic()); - service.event_retry_handler.connections.allowToAnyIpv4(aws_ec2_1.Port.allTraffic()); - service.readsFrom(reads_from); - addSubscription(this, subscribes_to, new snsSubs.SqsSubscription(service.queues.queue), true); - service.publishesToBucket(writes_to); - } -} -class GenericSubgraphGenerator extends cdk.Stack { - constructor(parent, id, reads_from, subscribes_to, writes_to, vpc) { - super(parent, id + '-stack'); - const generic_event_cache = new RedisCluster(this, id + 'genericeventcache', vpc); - generic_event_cache.connections.allowFromAnyIpv4(aws_ec2_1.Port.allTcp()); - const environment = { - "BUCKET_PREFIX": process.env.BUCKET_PREFIX, - "GENERIC_EVENT_CACHE_ADDR": generic_event_cache.cluster.attrRedisEndpointAddress, - "GENERIC_EVENT_CACHE_PORT": generic_event_cache.cluster.attrRedisEndpointPort, - }; - const service = new Service(this, 'generic-subgraph-generator', environment, vpc); - service.readsFrom(reads_from); - service.event_handler.connections.allowToAnyIpv4(aws_ec2_1.Port.tcp(parseInt(generic_event_cache.cluster.attrRedisEndpointPort))); - service.event_retry_handler.connections.allowToAnyIpv4(aws_ec2_1.Port.tcp(parseInt(generic_event_cache.cluster.attrRedisEndpointPort))); - addSubscription(this, subscribes_to, new snsSubs.SqsSubscription(service.queues.queue), true); - service.publishesToBucket(writes_to); - } -} -function addSubscription(scope, topic, subscription, raw) { - const config = subscription.bind(topic); - new sns.Subscription(scope, 'Subscription', { - topic: topic, - endpoint: config.endpoint, - filterPolicy: config.filterPolicy, - protocol: config.protocol, - rawMessageDelivery: raw || config.rawMessageDelivery - }); -} -class NodeIdentifier extends cdk.Stack { - constructor(parent, id, reads_from, subscribes_to, writes_to, history_db, vpc) { - super(parent, id + '-stack'); - const retry_identity_cache = new RedisCluster(this, id + 'retrycache', vpc); - retry_identity_cache.connections.allowFromAnyIpv4(aws_ec2_1.Port.allTcp()); - const environment = { - "BUCKET_PREFIX": process.env.BUCKET_PREFIX, - "RETRY_IDENTITY_CACHE_ADDR": retry_identity_cache.cluster.attrRedisEndpointAddress, - "RETRY_IDENTITY_CACHE_PORT": retry_identity_cache.cluster.attrRedisEndpointPort, - }; - const service = new Service(this, 'node-identifier', environment, vpc, 'node-identifier-retry-handler'); - service.readsFrom(reads_from); - history_db.allowReadWrite(service); - service.publishesToBucket(writes_to); - addSubscription(this, subscribes_to, new snsSubs.SqsSubscription(service.queues.queue), true); - service.event_handler.connections.allowToAnyIpv4(aws_ec2_1.Port.tcp(parseInt(retry_identity_cache.cluster.attrRedisEndpointPort))); - service.event_retry_handler.connections.allowToAnyIpv4(aws_ec2_1.Port.tcp(parseInt(retry_identity_cache.cluster.attrRedisEndpointPort))); - service.event_handler.connections.allowToAnyIpv4(ec2.Port.tcp(443), 'Allow outbound to S3'); - service.event_retry_handler.connections.allowToAnyIpv4(ec2.Port.tcp(443), 'Allow outbound to S3'); - } -} -class GraphMerger extends cdk.Stack { - constructor(parent, id, reads_from, subscribes_to, writes_to, master_graph, vpc) { - super(parent, id + '-stack'); - const graph_merge_cache = new RedisCluster(this, id + 'mergedcache', vpc); - graph_merge_cache.connections.allowFromAnyIpv4(aws_ec2_1.Port.allTcp()); - graph_merge_cache.connections.allowFromAnyIpv4(aws_ec2_1.Port.allTcp()); - const environment = { - "SUBGRAPH_MERGED_BUCKET": writes_to.bucketName, - "BUCKET_PREFIX": process.env.BUCKET_PREFIX, - "MG_ALPHAS": master_graph.alphaNames.join(","), - "MERGED_CACHE_ADDR": graph_merge_cache.cluster.attrRedisEndpointAddress, - "MERGED_CACHE_PORT": graph_merge_cache.cluster.attrRedisEndpointPort, - }; - const service = new Service(this, 'graph-merger', environment, vpc); - service.readsFrom(reads_from); - service.publishesToBucket(writes_to); - addSubscription(this, subscribes_to, new snsSubs.SqsSubscription(service.queues.queue), true); - // - // service.event_handler.connections - // .allowToAnyIpv4(new ec2.Port({ - // - // }), 'Allow outbound to S3'); - // service.event_retry_handler.connections - // .allowToAnyIpv4(ec2.Port.allTcp(), 'Allow outbound to S3'); - } -} -class AnalyzerDispatch extends cdk.Stack { - constructor(parent, id, subscribes_to, // The SNS Topic that we must subscribe to our queue - writes_to, reads_from, analyzer_bucket, vpc) { - super(parent, id + '-stack'); - const dispatch_event_cache = new RedisCluster(this, id + 'dispatcheventcache', vpc); - dispatch_event_cache.connections.allowFromAnyIpv4(aws_ec2_1.Port.allTcp()); - const environment = { - "BUCKET_PREFIX": process.env.BUCKET_PREFIX, - "EVENT_CACHE_ADDR": dispatch_event_cache.cluster.attrRedisEndpointAddress, - "EVENT_CACHE_PORT": dispatch_event_cache.cluster.attrRedisEndpointPort, - "DISPATCHED_ANALYZER_BUCKET": writes_to.bucketName, - "SUBGRAPH_MERGED_BUCKET": reads_from.bucketName, - }; - const service = new Service(this, 'analyzer-dispatcher', environment, vpc); - service.publishesToBucket(writes_to); - // We need the List capability to find each of the analyzers - service.readsFrom(analyzer_bucket, true); - service.readsFrom(reads_from); - addSubscription(this, subscribes_to, new snsSubs.SqsSubscription(service.queues.queue), true); - service.event_handler.connections.allowToAnyIpv4(ec2.Port.allTcp(), 'Allow outbound to S3'); - service.event_retry_handler.connections.allowToAnyIpv4(ec2.Port.allTcp(), 'Allow outbound to S3'); - } -} -class AnalyzerExecutor extends cdk.Stack { - constructor(parent, id, subscribes_to, reads_analyzers_from, reads_events_from, writes_events_to, model_plugins_bucket, master_graph, vpc) { - super(parent, id + '-stack'); - this.count_cache = new RedisCluster(this, id + 'countcache', vpc); - this.hit_cache = new RedisCluster(this, id + 'hitcache', vpc); - this.message_cache = new RedisCluster(this, id + 'msgcache', vpc); - const environment = { - "ANALYZER_MATCH_BUCKET": writes_events_to.bucketName, - "BUCKET_PREFIX": process.env.BUCKET_PREFIX, - "MG_ALPHAS": master_graph.alphaNames.join(","), - "COUNTCACHE_ADDR": this.count_cache.cluster.attrRedisEndpointAddress, - "COUNTCACHE_PORT": this.count_cache.cluster.attrRedisEndpointPort, - "MESSAGECACHE_ADDR": this.message_cache.cluster.attrRedisEndpointAddress, - "MESSAGECACHE_PORT": this.message_cache.cluster.attrRedisEndpointPort, - "HITCACHE_ADDR": this.hit_cache.cluster.attrRedisEndpointAddress, - "HITCACHE_PORT": this.hit_cache.cluster.attrRedisEndpointPort, - "GRPC_ENABLE_FORK_SUPPORT": "1", - }; - const service = new Service(this, 'analyzer-executor', environment, vpc, null, { - runtime: aws_lambda_1.Runtime.PYTHON_3_7 - }); - this.count_cache.connections.allowFromAnyIpv4(aws_ec2_1.Port.tcp(6379)); - this.hit_cache.connections.allowFromAnyIpv4(aws_ec2_1.Port.tcp(6379)); - this.message_cache.connections.allowFromAnyIpv4(aws_ec2_1.Port.tcp(6379)); - service.publishesToBucket(writes_events_to); - // We need the List capability to find each of the analyzers - service.readsFrom(reads_analyzers_from, true); - service.readsFrom(model_plugins_bucket, true); - service.readsFrom(reads_events_from); - // Need to be able to GetObject in order to HEAD, can be replaced with - // a cache later, but safe so long as there is no LIST - let policy = new iam.PolicyStatement(); - policy.addActions('s3:GetObject'); - policy.addResources(writes_events_to.bucketArn); - service.event_handler.addToRolePolicy(policy); - service.event_retry_handler.addToRolePolicy(policy); - addSubscription(this, subscribes_to, new snsSubs.SqsSubscription(service.queues.queue), true); - service.event_handler.connections.allowToAnyIpv4(ec2.Port.allTraffic(), 'Allow outbound to S3'); - service.event_retry_handler.connections.allowToAnyIpv4(ec2.Port.allTraffic(), 'Allow outbound to S3'); - } -} -class EngagementCreator extends cdk.Stack { - constructor(parent, id, reads_from, subscribes_to, publishes_to, master_graph, engagement_graph, vpc) { - super(parent, id + '-stack'); - const environment = { - "MG_ALPHAS": master_graph.alphaNames.join(","), - "EG_ALPHAS": engagement_graph.alphaNames.join(","), - }; - const service = new Service(this, 'engagement-creator', environment, vpc, null, { - runtime: aws_lambda_1.Runtime.PYTHON_3_7 - }); - service.readsFrom(reads_from); - service.publishesToTopic(publishes_to); - addSubscription(this, subscribes_to, new snsSubs.SqsSubscription(service.queues.queue), true); - service.event_handler.connections.allowToAnyIpv4(ec2.Port.allTcp(), 'Allow outbound to S3'); - service.event_retry_handler.connections.allowToAnyIpv4(ec2.Port.allTcp(), 'Allow outbound to S3'); - } -} -class Networks extends cdk.Stack { - constructor(parent, id) { - super(parent, id + '-stack'); - this.grapl_vpc = new ec2.Vpc(this, 'GraplVPC', { - natGateways: 1, - enableDnsHostnames: true, - enableDnsSupport: true, - }); - } -} -class Zero { - constructor(parent, stack, graph, id, cluster, peer, idx) { - const zeroTask = new ecs.Ec2TaskDefinition(stack, id, { - networkMode: aws_ecs_1.NetworkMode.AWS_VPC, - }); - let command = ["dgraph", "zero", `--my=${id}.${graph}.grapl:5080`, - "--replicas=3", - `--idx=${idx}`, - "--alsologtostderr"]; - if (peer) { - command.push(`--peer=${peer}.${graph}.grapl:5080`); - } - const logDriver = new ecs.AwsLogDriver({ - streamPrefix: `ecs${graph + id}`, - }); - zeroTask.addContainer(id + 'Container', { - // --my is our own hostname (graph + id) - // --peer is the other dgraph zero hostname - image: ecs.ContainerImage.fromRegistry("dgraph/dgraph"), - command, - logging: logDriver, - memoryReservationMiB: 1024, - }); - const zeroService = new ecs.Ec2Service(stack, id + 'Service', { - cluster, - taskDefinition: zeroTask, - cloudMapOptions: { - name: id, - dnsRecordType: servicediscovery.DnsRecordType.A, - dnsTtl: core_1.Duration.seconds(300), - } - }); - this.name = `${id}.${graph}.grapl`; - zeroService.connections.allowFromAnyIpv4(ec2.Port.allTcp()); - } -} -class Alpha { - constructor(parent, stack, graph, id, cluster, zero) { - const alphaTask = new ecs.Ec2TaskDefinition(stack, id, { - networkMode: aws_ecs_1.NetworkMode.AWS_VPC, - }); - const logDriver = new ecs.AwsLogDriver({ - streamPrefix: `ecs${graph + id}`, - }); - alphaTask.addContainer(id + graph + 'Container', { - image: ecs.ContainerImage.fromRegistry("dgraph/dgraph"), - command: [ - "dgraph", "alpha", `--my=${id}.${graph}.grapl:7080`, - "--lru_mb=1024", `--zero=${zero}.${graph}.grapl:5080`, - "--alsologtostderr" - ], - logging: logDriver, - memoryReservationMiB: 2048, - }); - const alphaService = new ecs.Ec2Service(stack, id + 'Service', { - cluster, - taskDefinition: alphaTask, - cloudMapOptions: { - name: id, - dnsRecordType: servicediscovery.DnsRecordType.A, - dnsTtl: core_1.Duration.seconds(300), - } - }); - this.name = `${id}.${graph}.grapl`; - alphaService.connections.allowFromAnyIpv4(ec2.Port.allTcp()); - } -} -class DGraphEcs extends cdk.Stack { - constructor(parent, id, vpc, zeroCount, alphaCount) { - super(parent, id + '-stack'); - const cluster = new ecs.Cluster(this, id + '-EcsCluster', { - vpc: vpc - }); - cluster.connections.allowInternally(aws_ec2_1.Port.allTcp()); - this.cluster = cluster; - const namespace = cluster.addDefaultCloudMapNamespace({ - name: id + '.grapl', - type: cloudmap.NamespaceType.DNS_PRIVATE, - vpc - }); - cluster.addCapacity(id + "ZeroGroupCapacity", { - instanceType: new ec2.InstanceType("t3a.small"), - minCapacity: zeroCount, - desiredCapacity: zeroCount, - maxCapacity: zeroCount, - }); - const zero0 = new Zero(parent, this, id, 'zero0', cluster, null, 1); - for (let i = 1; i < zeroCount; i++) { - new Zero(parent, this, id, `zero${i}`, cluster, 'zero0', 1 + i); - } - this.alphaNames = []; - cluster.addCapacity(id + "AlphaGroupCapacity", { - instanceType: new ec2.InstanceType("t3a.2xlarge"), - minCapacity: alphaCount, - desiredCapacity: alphaCount, - maxCapacity: alphaCount, - }); - for (let i = 0; i < alphaCount; i++) { - const alpha = new Alpha(parent, this, id, `alpha${i}`, // increment for each alpha - cluster, "zero0"); - this.alphaNames.push(alpha.name); - } - } -} -class EngagementNotebook extends cdk.Stack { - constructor(parent, id, user_auth_db, vpc) { - super(parent, id + '-notebook-stack'); - this.securityGroup = new ec2.SecurityGroup(this, `${id}-notebook-security-group`, { vpc: vpc }); - this.connections = new ec2.Connections({ - securityGroups: [this.securityGroup], - defaultPort: ec2.Port.allTcp() - }); - const role = new iam.Role(this, id + 'notebook-role', { - assumedBy: new iam.ServicePrincipal('sagemaker.amazonaws.com') - }); - user_auth_db.allowReadWriteFromRole(role); - const _notebook = new sagemaker.CfnNotebookInstance(this, id + '-sagemaker-endpoint', { - instanceType: 'ml.t2.medium', - securityGroupIds: [this.securityGroup.securityGroupId], - subnetId: vpc.privateSubnets[0].subnetId, - directInternetAccess: 'Enabled', - roleArn: role.roleArn - }); - } -} -const fs = require('fs'), path = require('path'); -const replaceInFile = (toModify, toReplace, replaceWith, outputFile) => { - return fs.readFile(toModify, (err, data) => { - if (err) { - return console.log(err); - } - const replaced = data - .split(toReplace) - .join(replaceWith) - .split("const isLocal = true;") - .join("const isLocal = false;"); - if (outputFile) { - fs.writeFile(outputFile, replaced, 'utf8', (err) => { - if (err) - return console.log(err); - }); - } - else { - fs.writeFile(toModify, replaced, 'utf8', (err) => { - if (err) - return console.log(err); - }); - } - }, 'utf8'); -}; -const getEdgeGatewayId = (integrationName, cb) => { - let apigateway = new AWS.APIGateway(); - apigateway.getRestApis({}, function (err, data) { - if (err) { - console.log('Error getting edge gateway ID', err); - } - for (const item of data.items) { - if (item.name === integrationName) { - console.log(`restApi ID ${item.id}`); - cb(item.id); - return; - } - } - console.warn(false, 'Could not find any integrations. Ensure you have deployed engagement edge.'); - }); -}; -class EngagementUx extends cdk.Stack { - constructor(parent, id, edge) { - super(parent, id + '-stack'); - const bucketName = process.env.BUCKET_PREFIX + id + '-bucket'; - const edgeBucket = new s3.Bucket(this, bucketName, { - bucketName, - publicReadAccess: true, - websiteIndexDocument: 'index.html', - }); - if (!fs.existsSync(path.join(__dirname, 'edge_ux_package/'))) { - fs.mkdirSync(path.join(__dirname, 'edge_ux_package/')); - } - getEdgeGatewayId(edge.name + 'Integration', (gatewayId) => { - const edgeUrl = `https://${gatewayId}.execute-api.${AWS.config.region}.amazonaws.com/prod/`; - const toReplace = "__engagement_ux_standin__hostname__"; - const replacement = `${edgeUrl}`; - console.log(__dirname); - dir.readFiles(path.join(__dirname, 'edge_ux/'), function (err, content, filename, next) { - if (err) - throw err; - const targetDir = path.dirname(filename).replace("edge_ux", "edge_ux_package"); - if (!fs.existsSync(targetDir)) { - fs.mkdirSync(targetDir, { recursive: true }); - } - const newPath = filename.replace("edge_ux", "edge_ux_package"); - replaceInFile(filename, toReplace, replacement, newPath); - next(); - }, function (err, files) { - if (err) - throw err; - }); - }); - new s3deploy.BucketDeployment(this, id + 'Ux', { - sources: [s3deploy.Source.asset('./edge_ux_package')], - destinationBucket: edgeBucket, - }); - } -} -class HistoryDb extends cdk.Stack { - constructor(parent, id) { - super(parent, id + '-stack'); - this.proc_history = new dynamodb.Table(this, 'process_history_table', { - tableName: "process_history_table", - partitionKey: { - name: 'pseudo_key', - type: dynamodb.AttributeType.STRING - }, - sortKey: { - name: 'create_time', - type: dynamodb.AttributeType.NUMBER - }, - billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, - }); - this.file_history = new dynamodb.Table(this, 'file_history_table', { - tableName: "file_history_table", - partitionKey: { - name: 'pseudo_key', - type: dynamodb.AttributeType.STRING - }, - sortKey: { - name: 'create_time', - type: dynamodb.AttributeType.NUMBER - }, - billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, - }); - this.outbound_connection_history = new dynamodb.Table(this, 'outbound_connection_history_table', { - tableName: "outbound_connection_history_table", - partitionKey: { - name: 'pseudo_key', - type: dynamodb.AttributeType.STRING - }, - sortKey: { - name: 'create_time', - type: dynamodb.AttributeType.NUMBER - }, - billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, - }); - this.inbound_connection_history = new dynamodb.Table(this, 'inbound_connection_history_table', { - tableName: "inbound_connection_history_table", - partitionKey: { - name: 'pseudo_key', - type: dynamodb.AttributeType.STRING - }, - sortKey: { - name: 'create_time', - type: dynamodb.AttributeType.NUMBER - }, - billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, - }); - this.network_connection_history = new dynamodb.Table(this, 'network_connection_history', { - tableName: "network_connection_history_table", - partitionKey: { - name: 'pseudo_key', - type: dynamodb.AttributeType.STRING - }, - sortKey: { - name: 'create_time', - type: dynamodb.AttributeType.NUMBER - }, - billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, - }); - this.ip_connection_history = new dynamodb.Table(this, 'ip_connection_history_table', { - tableName: "ip_connection_history_table", - partitionKey: { - name: 'pseudo_key', - type: dynamodb.AttributeType.STRING - }, - sortKey: { - name: 'create_time', - type: dynamodb.AttributeType.NUMBER - }, - billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, - }); - this.asset_history = new dynamodb.Table(this, 'asset_id_mappings', { - tableName: "asset_id_mappings", - partitionKey: { - name: 'pseudo_key', - type: dynamodb.AttributeType.STRING - }, - sortKey: { - name: 'c_timestamp', - type: dynamodb.AttributeType.NUMBER - }, - billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, - }); - this.dynamic_session_table = new dynamodb.Table(this, 'dynamic_session_table', { - tableName: "dynamic_session_table", - partitionKey: { - name: 'pseudo_key', - type: dynamodb.AttributeType.STRING - }, - sortKey: { - name: 'create_time', - type: dynamodb.AttributeType.NUMBER - }, - billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, - }); - this.static_mapping_table = new dynamodb.Table(this, 'static_mapping_table', { - tableName: "static_mapping_table", - partitionKey: { - name: 'pseudo_key', - type: dynamodb.AttributeType.STRING - }, - billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, - }); - this.node_id_retry_table = new dynamodb.Table(this, 'node_id_retry_table', { - tableName: "node_id_retry_table", - partitionKey: { - name: 'pseudo_key', - type: dynamodb.AttributeType.STRING - }, - billingMode: dynamodb.BillingMode.PAY_PER_REQUEST, - timeToLiveAttribute: "ttl_ts" - }); - } - allowReadWrite(service) { - this.proc_history.grantReadWriteData(service.event_handler.role); - this.file_history.grantReadWriteData(service.event_handler.role); - this.outbound_connection_history.grantReadWriteData(service.event_handler.role); - this.inbound_connection_history.grantReadWriteData(service.event_handler.role); - this.network_connection_history.grantReadWriteData(service.event_handler.role); - this.ip_connection_history.grantReadWriteData(service.event_handler.role); - this.asset_history.grantReadWriteData(service.event_handler.role); - this.node_id_retry_table.grantReadWriteData(service.event_handler.role); - this.static_mapping_table.grantReadWriteData(service.event_handler.role); - this.dynamic_session_table.grantReadWriteData(service.event_handler.role); - this.proc_history.grantReadWriteData(service.event_retry_handler.role); - this.file_history.grantReadWriteData(service.event_retry_handler.role); - this.outbound_connection_history.grantReadWriteData(service.event_retry_handler.role); - this.inbound_connection_history.grantReadWriteData(service.event_retry_handler.role); - this.network_connection_history.grantReadWriteData(service.event_retry_handler.role); - this.ip_connection_history.grantReadWriteData(service.event_retry_handler.role); - this.asset_history.grantReadWriteData(service.event_retry_handler.role); - this.node_id_retry_table.grantReadWriteData(service.event_retry_handler.role); - this.static_mapping_table.grantReadWriteData(service.event_retry_handler.role); - this.dynamic_session_table.grantReadWriteData(service.event_retry_handler.role); - } -} -class Grapl extends cdk.App { - constructor() { - super(); - env(__dirname + '/.env'); - const mgZeroCount = Number(process.env.MG_ZEROS_COUNT) || 1; - const mgAlphaCount = Number(process.env.MG_ALPHAS_COUNT) || 1; - const egZeroCount = Number(process.env.EG_ZEROS_COUNT) || 1; - const egAlphaCount = Number(process.env.EG_ALPHAS_COUNT) || 1; - const jwtSecret = process.env.JWT_SECRET || uuidv4(); - let event_emitters = new EventEmitters(this, 'grapl-event-emitters'); - const network = new Networks(this, 'graplvpcs'); - const history_db = new HistoryDb(this, 'graplhistorydb'); - const master_graph = new DGraphEcs(this, 'mastergraphcluster', network.grapl_vpc, mgZeroCount, mgAlphaCount); - const engagement_graph = new DGraphEcs(this, 'engagementgraphcluster', network.grapl_vpc, egZeroCount, egAlphaCount); - new GenericSubgraphGenerator(this, 'grapl-generic-subgraph-generator', event_emitters.raw_logs_bucket, event_emitters.raw_logs_topic, event_emitters.unid_subgraphs_generated_bucket, network.grapl_vpc); - new SysmonSubgraphGenerator(this, 'grapl-sysmon-subgraph-generator', event_emitters.sysmon_logs_bucket, event_emitters.sysmon_logs_topic, event_emitters.unid_subgraphs_generated_bucket, network.grapl_vpc); - new NodeIdentifier(this, 'grapl-node-identifier', event_emitters.unid_subgraphs_generated_bucket, event_emitters.unid_subgraphs_generated_topic, event_emitters.subgraphs_generated_bucket, history_db, network.grapl_vpc); - new GraphMerger(this, 'grapl-graph-merger', event_emitters.subgraphs_generated_bucket, event_emitters.subgraphs_generated_topic, event_emitters.subgraphs_merged_bucket, master_graph, network.grapl_vpc); - new AnalyzerDispatch(this, 'grapl-analyzer-dispatcher', event_emitters.subgraph_merged_topic, event_emitters.dispatched_analyzer_bucket, event_emitters.subgraphs_merged_bucket, event_emitters.analyzers_bucket, network.grapl_vpc); - new AnalyzerExecutor(this, 'grapl-analyzer-executor', event_emitters.dispatched_analyzer_topic, event_emitters.analyzers_bucket, event_emitters.dispatched_analyzer_bucket, event_emitters.analyzer_matched_subgraphs_bucket, master_graph, network.grapl_vpc); - new EngagementCreator(this, 'grapl-engagement-creator', event_emitters.analyzer_matched_subgraphs_bucket, event_emitters.analyzer_matched_subgraphs_topic, event_emitters.engagements_created_topic, master_graph, engagement_graph, network.grapl_vpc); - const user_auth_table = new UserAuthDb(this, 'grapl-user-auth-table'); - const engagement_edge = new EngagementEdge(this, 'engagementedge', 'engagementedge', jwtSecret, engagement_graph, user_auth_table, network.grapl_vpc); - new EngagementNotebook(this, 'engagements', user_auth_table, network.grapl_vpc); - new EngagementUx(this, 'engagement-ux', engagement_edge); - } -} -new Grapl().synth(); diff --git a/src/python/analyzer_executor/Dockerfile b/src/python/analyzer_executor/Dockerfile index b85a328f6b..8fdc30a70e 100644 --- a/src/python/analyzer_executor/Dockerfile +++ b/src/python/analyzer_executor/Dockerfile @@ -1,4 +1,4 @@ -FROM grapl/grapl-graph-descriptions-python-build AS analyzer-executor-build +FROM grapl/grapl-graph-descriptions-python-build:latest AS analyzer-executor-build USER grapl WORKDIR /home/grapl COPY --chown=grapl . analyzer_executor @@ -16,4 +16,4 @@ ENV USER grapl WORKDIR /home/grapl COPY --from=analyzer-executor-build /home/grapl/lambda.zip lambda.zip COPY --from=analyzer-executor-build /home/grapl/venv venv -COPY --from=analyzer-executor-build /home/grapl/analyzer_executor analyzer_executor \ No newline at end of file +COPY --from=analyzer-executor-build /home/grapl/analyzer_executor analyzer_executor diff --git a/src/python/engagement-creator/Dockerfile b/src/python/engagement-creator/Dockerfile index 2ad7b9fd96..c4a82458ce 100644 --- a/src/python/engagement-creator/Dockerfile +++ b/src/python/engagement-creator/Dockerfile @@ -1,4 +1,4 @@ -FROM grapl/grapl-graph-descriptions-python-build AS engagement-creator-build +FROM grapl/grapl-graph-descriptions-python-build:latest AS engagement-creator-build USER grapl WORKDIR /home/grapl COPY --chown=grapl . engagement-creator diff --git a/src/python/engagement_edge/Dockerfile b/src/python/engagement_edge/Dockerfile index a4c3fc5d2f..f08b6a366a 100644 --- a/src/python/engagement_edge/Dockerfile +++ b/src/python/engagement_edge/Dockerfile @@ -1,4 +1,4 @@ -FROM grapl/grapl-graph-descriptions-python-build AS engagement-edge-build +FROM grapl/grapl-graph-descriptions-python-build:latest AS engagement-edge-build USER grapl WORKDIR /home/grapl COPY --chown=grapl . engagement_edge diff --git a/src/python/grapl-model-plugin-deployer/Dockerfile b/src/python/grapl-model-plugin-deployer/Dockerfile index 36d3e5e4ec..2aa4223845 100644 --- a/src/python/grapl-model-plugin-deployer/Dockerfile +++ b/src/python/grapl-model-plugin-deployer/Dockerfile @@ -1,4 +1,4 @@ -FROM grapl/grapl-graph-descriptions-python-build AS grapl-model-plugin-deployer-build +FROM grapl/grapl-graph-descriptions-python-build:latest AS grapl-model-plugin-deployer-build USER grapl WORKDIR /home/grapl COPY --chown=grapl . grapl-model-plugin-deployer diff --git a/src/python/grapl-notebook/Dockerfile b/src/python/grapl-notebook/Dockerfile index 020e4396a9..e2b245385a 100644 --- a/src/python/grapl-notebook/Dockerfile +++ b/src/python/grapl-notebook/Dockerfile @@ -4,6 +4,6 @@ RUN pip install --user --upgrade grapl_analyzerlib COPY ./jupyter_notebook_config.py /home/jovyan/.jupyter/jupyter_notebook_config.py COPY ./GraplProvision.ipynb /home/jovyan/ COPY ./Demo_Engagement.ipynb /home/jovyan/ -COPY ./model_plugins/ /home/jovyan/model_plugins +RUN mkdir -p /home/jovyan/model_plugins CMD jupyter notebook diff --git a/src/rust/graph-descriptions/Dockerfile b/src/rust/graph-descriptions/Dockerfile index 74d0e53a73..a9bec12377 100644 --- a/src/rust/graph-descriptions/Dockerfile +++ b/src/rust/graph-descriptions/Dockerfile @@ -1,4 +1,4 @@ -FROM grapl/grapl-python-build AS grapl-graph-descriptions-python-build +FROM grapl/grapl-python-build:latest AS grapl-graph-descriptions-python-build USER grapl WORKDIR /home/grapl COPY --chown=grapl . graph-descriptions From 30dd8c6610824e8e8beefe305841d670e0f8b817 Mon Sep 17 00:00:00 2001 From: jgrillo Date: Wed, 20 May 2020 16:44:43 -0400 Subject: [PATCH 5/9] add graphql-endpoint to build scripts --- .github/workflows/grapl-release.yml | 36 +++++++++++++++++++++-------- docker-compose.build.yml | 2 +- src/js/graphql_endpoint/Dockerfile | 8 +++---- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/.github/workflows/grapl-release.yml b/.github/workflows/grapl-release.yml index 63ba1e8271..f6e15dbd33 100644 --- a/.github/workflows/grapl-release.yml +++ b/.github/workflows/grapl-release.yml @@ -45,6 +45,7 @@ jobs: docker push grapl/grapl-engagement-edge:$VERSION-$CHANNEL docker push grapl/grapl-model-plugin-deployer:$VERSION-$CHANNEL docker push grapl/grapl-engagement-view:$VERSION-$CHANNEL + docker push grapl/grapl-graphql-endpoint:$VERSION-$CHANNEL docker push grapl/grapl-graph-provision:$VERSION-$CHANNEL docker push grapl/grapl-dynamodb-provision:$VERSION-$CHANNEL @@ -52,58 +53,62 @@ jobs: env: VERSION: ${{ github.event.release.tag_name }} run: | - docker create -ti --name cp-sysmon-subgraph-generator grapl/grapl-sysmon-subgraph-generator:$VERSION bash + docker create -ti --name cp-sysmon-subgraph-generator grapl/grapl-sysmon-subgraph-generator:$VERSION-$CHANNEL bash docker cp-sysmon-subgraph-generator:/target/release/sysmon-subgraph-generator . docker rm -f cp-sysmon-subgraph-generator zip -9 ./sysmon-subgraph-generator ./sysmon-subgraph-generator-$VERSION-$CHANNEL.zip rm ./sysmon-subgraph-generator - docker create -ti --name cp-generic-subgraph-generator grapl/grapl-generic-subgraph-generator:$VERSION bash + docker create -ti --name cp-generic-subgraph-generator grapl/grapl-generic-subgraph-generator:$VERSION-$CHANNEL bash docker cp cp-generic-subgraph-generator:/target/release/generic-subgraph-generator . docker rm -f cp-generic-subgraph-generator zip -9 ./generic-subgraph-generator ./generic-subgraph-generator-$VERSION-$CHANNEL.zip rm ./generic-subgraph-generator - docker create -ti --name cp-node-identifier grapl/grapl-node-identifier:$VERSION bash + docker create -ti --name cp-node-identifier grapl/grapl-node-identifier:$VERSION-$CHANNEL bash docker cp cp-node-identifier:/target/release/node-identifier . docker rm -f cp-node-identifier zip -9 ./node-identifier ./node-identifier-$VERSION-$CHANNEL.zip rm ./node-identifier - docker create -ti --name cp-node-identifier-retry-handler grapl/grapl-node-identifier-retry-handler:$VERSION bash + docker create -ti --name cp-node-identifier-retry-handler grapl/grapl-node-identifier-retry-handler:$VERSION-$CHANNEL bash docker cp cp-node-identifier-retry-handler:/target/release/node-identifier-retry-handler . docker rm -f cp-node-identifier-retry-handler zip -9 ./node-identifier-retry-handler ./node-identifier-retry-handler-$VERSION-$CHANNEL.zip rm ./node-identifier-retry-handler - docker create -ti --name cp-graph-merger grapl/grapl-graph-merger:$VERSION bash + docker create -ti --name cp-graph-merger grapl/grapl-graph-merger:$VERSION-$CHANNEL bash docker cp cp-graph-merger:/target/release/graph-merger . docker rm -f cp-graph-merger zip -9 ./graph-merger ./graph-merger-$VERSION-$CHANNEL.zip rm ./graph-merger - docker create -ti --name cp-analyzer-dispatcher grapl/grapl-analyzer-dispatcher:$VERSION bash + docker create -ti --name cp-analyzer-dispatcher grapl/grapl-analyzer-dispatcher:$VERSION-$CHANNEL bash docker cp cp-analyzer-dispatcher:/target/release/analyzer-dispatcher . docker rm -f cp-analyzer-dispatcher zip -9 ./analyzer-dispatcher ./analyzer-dispatcher-$VERSION-$CHANNEL.zip rm ./analyzer-dispatcher - docker create -ti --name cp-analyzer-executor grapl/grapl-analyzer-executor:$VERSION bash + docker create -ti --name cp-analyzer-executor grapl/grapl-analyzer-executor:$VERSION-$CHANNEL bash docker cp cp-analyzer-executor:/lambda.zip analyzer-executor-$VERSION-$CHANNEL.zip docker rm -f cp-analyzer-executor - docker create -ti --name cp-engagement-creator grapl/grapl-engagement-creator:$VERSION bash + docker create -ti --name cp-engagement-creator grapl/grapl-engagement-creator:$VERSION-$CHANNEL bash docker cp cp-engagement-creator:/lambda.zip engagement-creator-$VERSION-$CHANNEL.zip docker rm -f cp-engagement-creator - docker create -ti --name cp-engagement-edge grapl/grapl-engagement-edge:$VERSION bash + docker create -ti --name cp-engagement-edge grapl/grapl-engagement-edge:$VERSION-$CHANNEL bash docker cp cp-engagement-edge:/lambda.zip engagement-edge-$VERSION-$CHANNEL.zip docker rm -f cp-engagement-edge - docker create -ti --name cp-model-plugin-deployer grapl/grapl-model-plugin-deployer:$VERSION bash + docker create -ti --name cp-model-plugin-deployer grapl/grapl-model-plugin-deployer:$VERSION-$CHANNEL bash docker cp cp-model-plugin-deployer:/lambda.zip model-plugin-deployer-$VERSION-$CHANNEL.zip docker rm -f cp-model-plugin-deployer + docker create -ti --name cp-graphql-endpoint grapl/grapl-graphql-endpoint:$VERSION-$CHANNEL bash + docker cp cp-graphql-endpoint:/lambda.zip graphql-endpoint-$VERSION-$CHANNEL.zip + docker rm -f cp-graphql-endpoint + - name: Upload sysmon-subgraph-generator to Github uses: actions/upload-release-asset@v1 env: @@ -213,3 +218,14 @@ jobs: asset_path: ./model-plugin-deployer-$VERSION-$CHANNEL.zip asset_name: model-plugin-deployer-$VERSION-$CHANNEL.zip asset_content_type: application/zip + + - name: Upload graphql-endpoint to Github + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ github.event.release.tag_name }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./graphql-endpoint-$VERSION-$CHANNEL.zip + asset_name: graphql-endpoint-$VERSION-$CHANNEL.zip + asset_content_type: application/zip diff --git a/docker-compose.build.yml b/docker-compose.build.yml index 195fc38262..e0fdcc2591 100644 --- a/docker-compose.build.yml +++ b/docker-compose.build.yml @@ -123,7 +123,7 @@ services: image: grapl/grapl-graphql-endpoint:${TAG:-latest} build: context: ./src/js/graphql_endpoint - dockerfile: ./Dockerfile + target: grapl-graphql-endpoint # # Utility services diff --git a/src/js/graphql_endpoint/Dockerfile b/src/js/graphql_endpoint/Dockerfile index 60223d0c0d..2756f08e23 100644 --- a/src/js/graphql_endpoint/Dockerfile +++ b/src/js/graphql_endpoint/Dockerfile @@ -1,4 +1,4 @@ -FROM node:alpine3.10 as build +FROM node:alpine3.10 AS grapl-graphql-endpoint-build WORKDIR /lambda RUN apk upgrade && apk add zip @@ -18,7 +18,5 @@ COPY modules/* ./modules/ COPY server.js . RUN zip --quiet -9r /lambda.zip . -#FROM scratch - -#COPY --from=0 /lambda.zip / - +FROM node:alpine3.10 AS grapl-graphql-endpoint +COPY --from=grapl-graphql-endpoint-build /lambda.zip / From 62ab7e8d87f77f94695795045e098c2efd7b3d71 Mon Sep 17 00:00:00 2001 From: jgrillo Date: Wed, 20 May 2020 17:01:16 -0400 Subject: [PATCH 6/9] fix graphql endpoint Dockerfile --- src/js/graphql_endpoint/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/js/graphql_endpoint/Dockerfile b/src/js/graphql_endpoint/Dockerfile index 2756f08e23..d5b79f7735 100644 --- a/src/js/graphql_endpoint/Dockerfile +++ b/src/js/graphql_endpoint/Dockerfile @@ -19,4 +19,6 @@ COPY server.js . RUN zip --quiet -9r /lambda.zip . FROM node:alpine3.10 AS grapl-graphql-endpoint +RUN apk add --no-cache zip COPY --from=grapl-graphql-endpoint-build /lambda.zip / +RUN unzip -q lambda.zip From e48498ba5b44ec0c30ebfdd60ae844ce6e526cff Mon Sep 17 00:00:00 2001 From: jgrillo Date: Wed, 20 May 2020 19:05:17 -0400 Subject: [PATCH 7/9] docker cp --- .github/workflows/grapl-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/grapl-release.yml b/.github/workflows/grapl-release.yml index f6e15dbd33..0e28a1f995 100644 --- a/.github/workflows/grapl-release.yml +++ b/.github/workflows/grapl-release.yml @@ -54,7 +54,7 @@ jobs: VERSION: ${{ github.event.release.tag_name }} run: | docker create -ti --name cp-sysmon-subgraph-generator grapl/grapl-sysmon-subgraph-generator:$VERSION-$CHANNEL bash - docker cp-sysmon-subgraph-generator:/target/release/sysmon-subgraph-generator . + docker cp cp-sysmon-subgraph-generator:/target/release/sysmon-subgraph-generator . docker rm -f cp-sysmon-subgraph-generator zip -9 ./sysmon-subgraph-generator ./sysmon-subgraph-generator-$VERSION-$CHANNEL.zip rm ./sysmon-subgraph-generator From 6aab4b454ed3a707c25d5cd0b697e8b7a9d2e9d1 Mon Sep 17 00:00:00 2001 From: jgrillo Date: Wed, 20 May 2020 22:54:07 -0400 Subject: [PATCH 8/9] zip arguments reversed >_< --- .github/workflows/grapl-release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/grapl-release.yml b/.github/workflows/grapl-release.yml index a2e1ecb8b6..e94760fe75 100644 --- a/.github/workflows/grapl-release.yml +++ b/.github/workflows/grapl-release.yml @@ -56,37 +56,37 @@ jobs: docker create -ti --name cp-sysmon-subgraph-generator grapl/grapl-sysmon-subgraph-generator:$VERSION-$CHANNEL bash docker cp cp-sysmon-subgraph-generator:/sysmon-subgraph-generator . docker rm -f cp-sysmon-subgraph-generator - zip -9 ./sysmon-subgraph-generator ./sysmon-subgraph-generator-$VERSION-$CHANNEL.zip + zip -9 sysmon-subgraph-generator-$VERSION-$CHANNEL.zip ./sysmon-subgraph-generator rm ./sysmon-subgraph-generator docker create -ti --name cp-generic-subgraph-generator grapl/grapl-generic-subgraph-generator:$VERSION-$CHANNEL bash docker cp cp-generic-subgraph-generator:/generic-subgraph-generator . docker rm -f cp-generic-subgraph-generator - zip -9 ./generic-subgraph-generator ./generic-subgraph-generator-$VERSION-$CHANNEL.zip + zip -9 generic-subgraph-generator-$VERSION-$CHANNEL.zip ./generic-subgraph-generator rm ./generic-subgraph-generator docker create -ti --name cp-node-identifier grapl/grapl-node-identifier:$VERSION-$CHANNEL bash docker cp cp-node-identifier:/node-identifier . docker rm -f cp-node-identifier - zip -9 ./node-identifier ./node-identifier-$VERSION-$CHANNEL.zip + zip -9 node-identifier-$VERSION-$CHANNEL.zip ./node-identifier rm ./node-identifier docker create -ti --name cp-node-identifier-retry-handler grapl/grapl-node-identifier-retry-handler:$VERSION-$CHANNEL bash docker cp cp-node-identifier-retry-handler:/node-identifier-retry-handler . docker rm -f cp-node-identifier-retry-handler - zip -9 ./node-identifier-retry-handler ./node-identifier-retry-handler-$VERSION-$CHANNEL.zip + zip -9 node-identifier-retry-handler-$VERSION-$CHANNEL.zip ./node-identifier-retry-handler rm ./node-identifier-retry-handler docker create -ti --name cp-graph-merger grapl/grapl-graph-merger:$VERSION-$CHANNEL bash docker cp cp-graph-merger:/graph-merger . docker rm -f cp-graph-merger - zip -9 ./graph-merger ./graph-merger-$VERSION-$CHANNEL.zip + zip -9 graph-merger-$VERSION-$CHANNEL.zip ./graph-merger rm ./graph-merger docker create -ti --name cp-analyzer-dispatcher grapl/grapl-analyzer-dispatcher:$VERSION-$CHANNEL bash docker cp cp-analyzer-dispatcher:/analyzer-dispatcher . docker rm -f cp-analyzer-dispatcher - zip -9 ./analyzer-dispatcher ./analyzer-dispatcher-$VERSION-$CHANNEL.zip + zip -9 analyzer-dispatcher-$VERSION-$CHANNEL.zip ./analyzer-dispatcher rm ./analyzer-dispatcher docker create -ti --name cp-analyzer-executor grapl/grapl-analyzer-executor:$VERSION-$CHANNEL bash From 5b3a2ec217fe954283d39ee5ce876299e060550a Mon Sep 17 00:00:00 2001 From: jgrillo Date: Thu, 21 May 2020 07:59:21 -0400 Subject: [PATCH 9/9] python lambda.zips are located at /home/grapl --- .github/workflows/grapl-release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/grapl-release.yml b/.github/workflows/grapl-release.yml index 41396d2331..e29fc7c202 100644 --- a/.github/workflows/grapl-release.yml +++ b/.github/workflows/grapl-release.yml @@ -91,19 +91,19 @@ jobs: rm ./analyzer-dispatcher docker create -ti --name cp-analyzer-executor grapl/grapl-analyzer-executor:$VERSION-$CHANNEL bash - docker cp cp-analyzer-executor:/lambda.zip analyzer-executor-$VERSION-$CHANNEL.zip + docker cp cp-analyzer-executor:/home/grapl/lambda.zip analyzer-executor-$VERSION-$CHANNEL.zip docker rm -f cp-analyzer-executor docker create -ti --name cp-engagement-creator grapl/grapl-engagement-creator:$VERSION-$CHANNEL bash - docker cp cp-engagement-creator:/lambda.zip engagement-creator-$VERSION-$CHANNEL.zip + docker cp cp-engagement-creator:/home/grapl/lambda.zip engagement-creator-$VERSION-$CHANNEL.zip docker rm -f cp-engagement-creator docker create -ti --name cp-engagement-edge grapl/grapl-engagement-edge:$VERSION-$CHANNEL bash - docker cp cp-engagement-edge:/lambda.zip engagement-edge-$VERSION-$CHANNEL.zip + docker cp cp-engagement-edge:/home/grapl/lambda.zip engagement-edge-$VERSION-$CHANNEL.zip docker rm -f cp-engagement-edge docker create -ti --name cp-model-plugin-deployer grapl/grapl-model-plugin-deployer:$VERSION-$CHANNEL bash - docker cp cp-model-plugin-deployer:/lambda.zip model-plugin-deployer-$VERSION-$CHANNEL.zip + docker cp cp-model-plugin-deployer:/home/grapl/lambda.zip model-plugin-deployer-$VERSION-$CHANNEL.zip docker rm -f cp-model-plugin-deployer docker create -ti --name cp-graphql-endpoint grapl/grapl-graphql-endpoint:$VERSION-$CHANNEL bash