Initial AWS Agent wrapper module

.circleci/config.yml

@@ -0,0 +1,78 @@
+version: 2.1
+
+commands:
+  cmd-trufflehog-scan:
+    parameters:
+      disable_entropy:
+        default: true
+        description: Should we disable truffleHog's entropy detection?
+        type: boolean
+      max_history:
+        default: "1"
+        description: How far back to scan in git revisions
+        type: string
+      regexp_rules:
+        default: ""
+        description: Override default regexp rules with this file.
+        type: string
+      allowlist_file:
+        default: ".circleci/trufflehog_config/allowlist.json"
+        description: Add items to this file to allow you to override specific findings.
+        type: string
+      repo_path:
+        default: .
+        description: Scan alternate local or remote repo
+        type: string
+      current_branch:
+        default: ${CIRCLE_BRANCH}
+        description: Git branch to use
+        type: string
+    steps:
+      - run:
+          command: >
+            trufflehog --regex --json \
+                       --branch << parameters.current_branch >> \
+                       <<# parameters.allowlist_file >> --allow << parameters.allowlist_file >> <</ parameters.allowlist_file >> \
+                       <<# parameters.max_history >> --max_depth=<< parameters.max_history >> <</ parameters.max_history>> \
+                       <<# parameters.disable_entropy >> --entropy=False <</ parameters.disable_entropy >> \
+                       <<# parameters.regexp_rules >> --rules=<< parameters.regexp_rules >> <</ parameters.regexp_rules >> \
+                       << parameters.repo_path >> \
+                       | jq '{"reason":.reason,"path": .path}'
+          name: Scan using truffleHog
+
+jobs:
+  run-trufflehog-scan:
+    docker:
+      - image: cimg/python:3.11
+    parameters:
+      current_branch:
+        default: ${CIRCLE_BRANCH}
+        description: Git branch to use
+        type: string
+    steps:
+      - checkout
+      - run:
+          name: Install truffleHog
+          command: pip install truffleHog
+      - cmd-trufflehog-scan:
+          current_branch: << parameters.current_branch >>
+
+  run-sanity-check:
+    docker:
+      - image: docker.mirror.hashicorp.services/hashicorp/terraform:light
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: apk add --update make
+      - run:
+          name: Format and validate
+          command: make sanity-check
+
+workflows:
+  version: 2
+
+  validate:
+    jobs:
+      - run-trufflehog-scan
+      - run-sanity-check

.circleci/trufflehog_config/allowlist.json

@@ -0,0 +1 @@
+{}

.gitignore

@@ -32,3 +32,6 @@ override.tf.json
 # Ignore CLI configuration files
 .terraformrc
 terraform.rc
+
+# IDE
+.idea/

CODEOWNERS

@@ -0,0 +1,2 @@
+# Default owners
+* @monte-carlo-data/apollo

LICENSE

@@ -0,0 +1,20 @@
+Copyright 2023 Monte Carlo Data, Inc.
+
+The Software contained herein (the “Software”) is the intellectual property of Monte Carlo Data, Inc. (“Licensor”),
+and Licensor retains all intellectual property rights in the Software, including any and all derivatives, changes and
+improvements thereto. Only customers who have entered into a commercial agreement with Licensor for use or
+purchase of the Software (“Licensee”) are licensed or otherwise authorized to use the Software, and any Licensee
+agrees that it obtains no copyright or other intellectual property rights to the Software, except for the license
+expressly granted below or in accordance with the terms of their commercial agreement with Licensor (the
+“Agreement”). Subject to the terms and conditions of the Agreement, Licensor grants Licensee a non-exclusive,
+non-transferable, non-sublicensable, revocable, limited right and license to use the Software, in each case solely
+internally within Licensee’s organization for non-commercial purposes and only in connection with the service
+provided by Licensor pursuant to the Agreement, and in object code form only. Without Licensor’s express prior
+written consent, Licensee may not, directly or indirectly, (i) distribute the Software, any portion thereof, or any
+modifications, enhancements, or derivative works of any of the foregoing (collectively, the “Derivatives”) to any
+third party, (ii) license, market, sell, offer for sale or otherwise attempt to commercialize any Software, Derivatives,
+or portions thereof, (iii) use the Software, Derivatives, or any portion thereof for the benefit of any third party, (iv)
+use the Software, Derivatives, or any portion thereof in any manner or with respect to any commercial activity
+which competes, or is reasonably likely to compete, with any business that Licensor conducts, proposes to conduct
+or demonstrably anticipates conducting, at any time; or (v) seek any patent or other intellectual property rights or
+protections over or in connection with any Software of Derivatives.

Makefile

@@ -0,0 +1,10 @@
+.PHONY: default sanity-check
+
+default:
+	@echo "Read the readme"
+
+sanity-check:
+	# Validate TF configuration files and formatting. Used in CI pipeline.
+	terraform init -backend=false
+	terraform fmt -recursive -check -diff
+	terraform validate

README.md

@@ -1 +1,89 @@
-# terraform-aws-mcd-agent
+# Monte Carlo AWS Agent Module (Beta)
+
+This module deploys Monte Carlo's [containerized agent](https://hub.docker.com/r/montecarlodata/agent)* (Beta) on AWS
+Lambda, along with storage, roles etc.
+
+See [here](https://docs.getmontecarlo.com/docs/platform-architecture) for architecture details and alternative
+deployment options.
+
+## Prerequisites
+
+- [Terraform](https://developer.hashicorp.com/terraform/downloads) (>= 1.3)
+- [AWS CLI](https://aws.amazon.com/cli/).
+  [Authentication reference](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration)
+
+## Usage
+
+Basic usage of this module:
+
+```
+module "apollo" {
+  source = "monte-carlo-data/mcd-agent/aws"
+}
+
+output "function_arn" {
+  value       = module.apollo.mcd_agent_function_arn
+  description = "Agent Function ARN. To be used in registering."
+}
+
+output "invoker_role_arn" {
+  value       = module.apollo.mcd_agent_invoker_role_arn
+  description = "Assumable role ARN. To be used in registering."
+}
+
+output "invoker_role_external_id" {
+  value       = module.apollo.mcd_agent_invoker_role_external_id
+  description = "Assumable role External ID. To be used in registering."
+}
+```
+
+After which you must register your agent with Monte Carlo. See
+[here](https://docs.getmontecarlo.com/docs/create-and-register-an-aws-agent) for more details, options, and
+documentation.
+
+## Inputs
+
+| **Name**          | **Description**                                                                                                                                                                                                                                                                                                                                                                                                                             | **Type**     | **Default**                                           |
+|-------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------|-------------------------------------------------------|
+| image             | URI of the Agent container image (ECR Repo). Note that the region is automatically derived from the region variable.                                                                                                                                                                                                                                                                                                                        | string       | 752656882040.dkr.ecr.*.amazonaws.com/mcd-agent:latest |
+| cloud_account_id  | Select the Monte Carlo account your collection service is hosted in. This can be found in the '[settings/integrations/collectors](https://getmontecarlo.com/settings/integrations/collectors)' tab on the UI or via the '[montecarlo collectors list](https://clidocs.getmontecarlo.com/#montecarlo-collectors-list)' command on the CLI                                                                                                    | string       | 190812797848                                          |
+| private_subnets   | Optionally connect the agent to a VPC by specifying at least two private subnet IDs in that VPC.                                                                                                                                                                                                                                                                                                                                            | list(string) | []                                                    |
+| region            | The AWS region to deploy the agent into.                                                                                                                                                                                                                                                                                                                                                                                                    | string       | us-east-1                                             |
+| remote_upgradable | Allow the agent image and configuration to be remotely upgraded by Monte Carlo. Note that this sets a lifecycle to ignore any changes in Terraform to the image used after the initial deployment. If not set to 'true' you will be responsible for upgrading the image (e.g. specifying a new tag) for any bug fixes and improvements. Changing this value after initial deployment might replace your agent and require (re)registration. | bool         | true                                                  |
+
+## Outputs
+
+| **Name**                           | **Description**                                        |
+|------------------------------------|--------------------------------------------------------|
+| mcd_agent_function_arn             | Agent Function ARN. To be used in registering.         |
+| mcd_agent_invoker_role_arn         | Assumable role ARN. To be used in registering.         |
+| mcd_agent_invoker_role_external_id | Assumable role External ID. To be used in registering. |
+| mcd_agent_security_group_name      | Security group ID.                                     |
+| mcd_agent_storage_bucket_arn       | Storage bucket ARN.                                    |
+
+## Releases and Development
+
+The README and sample agent in the `examples/agent` directory is a good starting point to familiarize
+yourself with using the agent.
+
+Note that all Terraform files must conform to the standards of `terraform fmt` and
+the [standard module structure](https://developer.hashicorp.com/terraform/language/modules/develop).
+CircleCI will sanity check formatting and for valid tf config files.
+It is also recommended you use Terraform Cloud as a backend.
+Otherwise, as normal, please follow Monte Carlo's code guidelines during development and review.
+
+When ready to release simply add a new version tag, e.g. v0.0.42, and push that tag to GitHub.
+See additional
+details [here](https://developer.hashicorp.com/terraform/registry/modules/publish#releasing-new-versions).
+
+## License
+
+See [LICENSE](LICENSE) for more information.
+
+## Security
+
+See [SECURITY](SECURITY.md) for more
+information.
+---
+*Note that due to an AWS limitation the agent image is also uploaded and then sourced from AWS ECR when executed on
+Lambda.

SECURITY.md

@@ -0,0 +1,17 @@
+# Reporting Security Vulnerabilities
+
+Monte Carlo values independent security research and believes responsible vulnerability disclosure ensures user security and privacy.
+
+Please do **NOT** raise a GitHub Issue to report a security vulnerability. If you believe you have found a security vulnerability, please submit a report to [email protected]. Proof of concepts are appreciated. We provide additional information on [how to report security vulnerabilities to Monte Carlo](https://trust.montecarlodata.com/resources/ZTMzNjM0YmMtZDQ3Yi00NWJmLTk0NmQtNWEwNzJkMWJmMDZj).
+
+We ask that you do not use other channels or contact project contributors directly.
+
+Non-vulnerability related security issues such as new great new ideas for security features are welcome on GitHub Issues. 
+
+## Security Updates, Alerts and Bulletins
+
+Security updates will be released on a regular cadence and are aligned with the severity of the issue. More details can be found on our [Trust Center portal](https://trust.montecarlodata.com). Additional information, including our vulnerability management policy, is available on our [Trust Center portal](https://trust.montecarlodata.com).
+
+## Security-Related Information
+
+We may provide security related information such as a threat model, considerations for secure use, or any known security issues in our documentation. Please note that documentation and sample code are intended to demonstrate a concept and may not be sufficiently hardened for production use.

examples/agent/Makefile

@@ -0,0 +1,38 @@
+.PHONY: default apply test exec clean check-profile
+
+default:
+	@echo "Read the readme"
+
+apply: check-profile
+	@# Apply this example.
+	terraform init && AWS_PROFILE=$(profile) terraform apply -auto-approve
+
+test: check-profile
+	@# Test agent reachability via the health endpoint.
+	@# This command expects that you have have applied this example. Use exec to do both steps.
+	@temp_role=$$(aws sts assume-role --profile $(profile) \
+									  --role-arn "$$(terraform output --raw invoker_role_arn)" \
+									  --external-id "$$(terraform output --raw invoker_role_external_id)" \
+									  --role-session-name 'mc-agent-test'); \
+	AWS_ACCESS_KEY_ID=$$(jq -r .Credentials.AccessKeyId <<<"$$temp_role") \
+	AWS_SECRET_ACCESS_KEY=$$(jq -r .Credentials.SecretAccessKey <<<"$$temp_role") \
+	AWS_SESSION_TOKEN=$$(jq -r .Credentials.SessionToken <<<"$$temp_role") \
+	AWS_REGION=$$(terraform output --raw function_arn | cut -d':' -f4) \
+	  aws lambda invoke \
+	  --function-name $$(terraform output --raw function_arn) \
+	  --cli-binary-format raw-in-base64-out \
+	  --payload '{"path": "/api/v1/test/health", "httpMethod": "GET", "queryStringParameters": {"trace_id": "123456789", "full": true}}' \
+	  /dev/stdout | jq '.body | select( . != null ) | fromjson'
+
+exec: apply sleep-30 test  # Sleep to wait on initial IAM propagation. Subsequent usage should require no delay.
+
+clean: check-profile
+	@# Delete this example.
+	@# WARNING -- This command will delete resources and local state files. This is unrecoverable.
+	AWS_PROFILE=$(profile) terraform destroy -auto-approve && rm -rf .terraform* terraform.tfstate*
+
+sleep-%:
+	sleep $(@:sleep-%=%)
+
+check-profile:
+	@[ "${profile}" ] || ( echo "profile is not set"; exit 1 )

examples/agent/README.md

@@ -0,0 +1,45 @@
+# Agent Sample
+
+This example deploys a pre-release Agent that is connected to a VPC. The VPC is created
+with the help of [this](https://github.com/terraform-aws-modules/terraform-aws-vpc) module.
+
+Note that the pre-release agent is in active development and not intended for production usage.
+
+## Prerequisites
+
+See the Prerequisites subsection in the module.
+
+## Usage
+
+To provision this example and access (test) the agent locally:
+
+```
+make exec profile=<AWS_PROFILE>
+```
+
+Note that this convenience command auto-approves the plan, and executes a test request. To review
+interactively please use `terraform plan + apply` instead.
+
+See [here](https://github.com/monte-carlo-data/apollo-agent) for agent usage and docs. You should be able to use any
+endpoint as documented.
+
+And don't forget to delete any resources when you're done (e.g. `terraform destroy`).
+
+## Addendum
+
+During development, you might want configure Terraform Cloud as the backend. To do so you can add the following snippet:
+
+```
+terraform {
+  cloud {
+    organization = "<org>"
+
+    workspaces {
+      name = "<workspace>"
+    }
+  }
+}
+```
+
+This also requires you to execute `terraform login` before initializing. You will also either have to update the
+working directory to include the agent module or set the execution mode to "Local".

examples/agent/main.tf

@@ -0,0 +1,48 @@
+data "aws_region" "current" {}
+
+module "vpc" {
+  source = "terraform-aws-modules/vpc/aws"
+
+  name               = "apollo-vpc"
+  cidr               = "10.0.0.0/16"
+  azs                = formatlist("${data.aws_region.current.name}%s", ["a", "b"])
+  private_subnets    = ["10.0.0.0/24", "10.0.1.0/24"]
+  public_subnets     = ["10.0.2.0/24", "10.0.3.0/24"]
+  enable_nat_gateway = true
+  single_nat_gateway = true
+}
+
+resource "aws_vpc_endpoint" "s3" {
+  vpc_id          = module.vpc.vpc_id
+  service_name    = "com.amazonaws.${data.aws_region.current.name}.s3"
+  route_table_ids = concat(module.vpc.private_route_table_ids, module.vpc.public_route_table_ids)
+}
+
+module "apollo" {
+  source = "../../"
+
+  cloud_account_id = "682816785079"
+  image            = "404798114945.dkr.ecr.*.amazonaws.com/mcd-pre-release-agent:latest"
+  region           = data.aws_region.current.name
+  private_subnets  = module.vpc.private_subnets
+}
+
+output "function_arn" {
+  value       = module.apollo.mcd_agent_function_arn
+  description = "Agent Function ARN. To be used in registering."
+}
+
+output "invoker_role_arn" {
+  value       = module.apollo.mcd_agent_invoker_role_arn
+  description = "Assumable role ARN. To be used in registering."
+}
+
+output "invoker_role_external_id" {
+  value       = module.apollo.mcd_agent_invoker_role_external_id
+  description = "Assumable role External ID. To be used in registering."
+}
+
+output "public_ip" {
+  value       = module.vpc.nat_public_ips
+  description = "IP address from which agent resources access the Internet (e.g. for IP whitelisting)."
+}