diff --git a/Prerequisites-notebook.ipynb b/Prerequisites-notebook.ipynb index 2bdb5e3..f9f5440 100644 --- a/Prerequisites-notebook.ipynb +++ b/Prerequisites-notebook.ipynb @@ -5,11 +5,18 @@ "id": "79a5fe45-1150-49e5-80de-858fe8d58785", "metadata": {}, "source": [ - "# This notebook focuses on the prerequisites section of Trapheus. Executing this notebook will create the following resources\n", - "* Verify an email address to be used to send email alerts\n", - "* Create a S3 bucket where the system is going to store the cloud formation templates\n", - "* Create a VPC with 2 Private Subnets in 2 availability zones. Make sure the region passed is the same as in AWS credentials\n", - "* Create an RDS instance with the given name with MySQL DB, with credentials (root/pass12345)\n", + "# This notebook focuses on the Installation of Trapheus along with its prerequisites section. Executing this notebook will create the resources if it doesn't exist.\n", + "\n", + "Below are the parameters needed for the Trapheus installation\n", + "\n", + "* s3-bucket : [Optional] The name of the CloudFormation template S3 bucket from the Pre-Requisites.\n", + "* vpcID : [Required] The id of the VPC from the Pre-Requisites. The lambdas from the Trapheus state machine will be created in this VPC.\n", + "* Subnets : [Required] A comma separated list of private subnet ids (region specific) from the Pre-Requisites VPC.\n", + "* SenderEmail : [Required] The SES sending email configured in the Pre-Requisites\n", + "* RecipientEmail : [Required] Comma separated list of recipient email addresses configured in Pre-Requisites.\n", + "* UseVPCAndSubnets : [Optional] Whether to use the vpc and subnets to create a security group and link the security group and vpc to the lambdas. When UseVPCAndSubnets left out (default) or set to 'true', lambdas are connected to a VPC in your account, and by default the function can't access the RDS (or other services) if VPC doesn't provide access (either by routing outbound traffic to a NAT gateway in a public subnet, or having a VPC endpoint, both of which incur cost or require more setup). If set to 'false', the lambdas will run in a default Lambda owned VPC that has access to RDS (and other AWS services).\n", + "* SlackWebhookUrls : [Optional] Comma separated list of Slack webhooks for failure alerts.\n", + "\n", "\n", "> Note: Make sure to have AWS credentials configured. Refer to [Setting up AWS credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)" ] @@ -21,10 +28,22 @@ "metadata": {}, "outputs": [], "source": [ - "email_id=input(\"Enter your email id to verify in SES: \")\n", - "region=input(\"Enter the region to create the resources: \")\n", - "s3_bucket_name=input(\"Enter the name of the S3 bucket to be created(Proposed Name: trapheus-cfn-s3-[account-id]-[region]): \")\n", - "db_instance_name=input(\"Enter the name for the RDS instance to be created: \")" + "region=input(\"Enter the region [for instance, us-west-2]: \")\n", + "if region == \"\":\n", + " raise SystemExit(\"Region is empty. Provide the region to create the resources\")\n", + "s3_bucket_name=input(\"Enter the name of the S3 bucket. New bucket will be created if it doesn't exist(Proposed Name: trapheus-cfn-s3-[account-id]-[region]): \")\n", + "isVpc = input('Do you want to use vpc[y/n]: ')\n", + "if isVpc == 'y' or isVpc == 'Y':\n", + " vpc_id=input(\"Enter the VPC ID. Leaving it empty will create a new VPC along with subnets: \")\n", + " if len(vpc_id) > 0:\n", + " subnets = input('Enter comma separated list of PRIVATE subnets: ')\n", + "sender_email_id=input(\"Enter sender email to send email FROM in case of failure: \")\n", + "receiver_email_id=input(\"Enter recipient email to send email TO in case of failure: \")\n", + "create_rds=input(\"Do you want to create an RDS instance[y/n]\")\n", + "if create_rds == 'y' or create_rds == 'Y' :\n", + " db_instance_name=input(\"Enter a RDS instance name\")\n", + "slack_webhook_urls = input('Enter slack webhooks to publish failure notifications to: ')\n", + "stack_name = input('Enter a cloud formation stack name: ')" ] }, { @@ -46,23 +65,40 @@ "source": [ "# Verify email\n", "ses = boto3.client('ses')\n", - "response = ses.verify_email_identity(\n", - " EmailAddress = email_id\n", - ")" + "if len(sender_email_id) > 0:\n", + " response = ses.verify_email_identity(\n", + " EmailAddress = sender_email_id\n", + " )\n", + "else:\n", + " print(\"Sender email id not provided. Skipping this step\")\n", + "\n", + "if len(receiver_email_id) > 0:\n", + " response = ses.verify_email_identity(\n", + " EmailAddress = receiver_email_id\n", + " )\n", + "else:\n", + " print(\"Receiver email id not provided. Skipping this step\")" ] }, { "cell_type": "code", "execution_count": null, - "id": "6b87538d-9771-42a5-9217-2a25a9709998", + "id": "dd5d494e-c0d6-408a-877b-40a6ada11bad", "metadata": {}, "outputs": [], "source": [ - "#Create S3 bucket\n", + "# Validate and Create S3 bucket\n", "s3 = boto3.resource('s3')\n", - "s3.create_bucket(Bucket=s3_bucket_name,CreateBucketConfiguration={\n", - " 'LocationConstraint': region\n", - " })" + "bucket = s3.Bucket(s3_bucket_name)\n", + "\n", + "if bucket.creation_date:\n", + " print(\"The bucket already exists. Proceeding to next step.\")\n", + "else:\n", + " print(\"The bucket does not exist. Creating new one with the given name\")\n", + " s3Response = s3.create_bucket(Bucket=s3_bucket_name,CreateBucketConfiguration={\n", + " 'LocationConstraint': region})\n", + " print(\"created bucket {}\".format(s3_bucket_name))\n", + " " ] }, { @@ -72,57 +108,48 @@ "metadata": {}, "outputs": [], "source": [ - "# Get availability zones\n", - "response = ec2Client.describe_availability_zones(\n", - " Filters=[\n", - " {\n", - " 'Name': 'region-name',\n", - " 'Values': [region]\n", - " },\n", - " {\n", - " 'Name': 'state',\n", - " 'Values': ['available']\n", - " },\n", - " ]\n", - ")\n", - "if len(response['AvailabilityZones']) < 2:\n", - " raise SystemExit(\"Stopping the execution. We need at least 2 availability zones to create private subnets. Make sure the region is set correctly in aws credentials\")\n", - "zone1 = response['AvailabilityZones'][0]['ZoneName']\n", - "zone2 = response['AvailabilityZones'][1]['ZoneName']\n", - "print(zone1, zone2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "37a1e945-bddf-42e7-aa5c-b4689617bf22", - "metadata": {}, - "outputs": [], - "source": [ - "#Create VPC\n", + "# If VPC id is not provided, then create VPC along with private subnets\n", "ec2 = boto3.resource('ec2')\n", "ec2Client = boto3.client('ec2')\n", - "vpc = ec2.create_vpc(CidrBlock='10.0.0.0/16')\n", - "vpc.wait_until_available()\n", - "vpc_id=vpc.id\n", - "print(vpc.id)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f8bd5dc5-e8e6-4304-97ee-98f3f40b6e51", - "metadata": {}, - "outputs": [], - "source": [ - "#Create Private Subnets\n", - "subnet1 = ec2.create_subnet(CidrBlock='10.0.0.0/19', VpcId=vpc_id, AvailabilityZone=zone1)\n", - "subnet1_id=subnet1.id\n", - "print(subnet1.id)\n", + "if vpc_id == \"\":\n", + " print(\"Creating VPC...\")\n", + " # Get availability zones\n", + " response = ec2Client.describe_availability_zones(\n", + " Filters=[\n", + " {\n", + " 'Name': 'region-name',\n", + " 'Values': [region]\n", + " },\n", + " {\n", + " 'Name': 'state',\n", + " 'Values': ['available']\n", + " },\n", + " ]\n", + " )\n", + " if len(response['AvailabilityZones']) < 2:\n", + " raise SystemExit(\"Stopping the execution. We need at least 2 availability zones to create private subnets. Make sure the region is set correctly in aws credentials\")\n", + " zone1 = response['AvailabilityZones'][0]['ZoneName']\n", + " zone2 = response['AvailabilityZones'][1]['ZoneName']\n", + " # print(zone1, zone2)\n", + " \n", + " #Create VPC\n", + "\n", + " vpc = ec2.create_vpc(CidrBlock='10.0.0.0/16')\n", + " vpc.wait_until_available()\n", + " vpc_id=vpc.id\n", + " #print(vpc.id)\n", + " \n", + " #Create Private Subnets\n", + " subnet1 = ec2.create_subnet(CidrBlock='10.0.0.0/19', VpcId=vpc_id, AvailabilityZone=zone1)\n", + " subnet1_id=subnet1.id\n", + " #print(subnet1.id)\n", + " \n", + " subnet2 = ec2.create_subnet(CidrBlock='10.0.32.0/19', VpcId=vpc_id,AvailabilityZone=zone2)\n", + " #print(subnet2.id)\n", + " subnet2_id=subnet2.id\n", + " subnets = subnet1_id + \",\" + subnet2_id\n", "\n", - "subnet2 = ec2.create_subnet(CidrBlock='10.0.32.0/19', VpcId=vpc_id,AvailabilityZone=zone2)\n", - "print(subnet2.id)\n", - "subnet2_id=subnet2.id\n" + " print(\"Created VPC {} with private subnets {},{}\".format(vpc_id, subnet1_id, subnet2_id))" ] }, { @@ -132,93 +159,107 @@ "metadata": {}, "outputs": [], "source": [ - "#Create RDS DB Subnet\n", - "conn = boto3.client('rds')\n", + "# create a new RDS instance\n", + "if (create_rds == 'y' or create_rds == 'Y') and len(db_instance_name) > 0:\n", + " print(\"Creating RDS Instance...\")\n", + " #Create RDS DB Subnet\n", + " conn = boto3.client('rds')\n", "\n", - "response = conn.create_db_subnet_group(\n", - " DBSubnetGroupName='Trapheus-rdssubnetgrp',\n", - " DBSubnetGroupDescription='rdssubnetgrp',\n", - " SubnetIds=[subnet1_id,subnet2_id],\n", - ")\n" + " db_subnet_name = db_instance_name+\"_dbsubnet\"\n", + " print(db_subnet_name)\n", + " response = conn.create_db_subnet_group(\n", + " DBSubnetGroupName=db_subnet_name,\n", + " DBSubnetGroupDescription='rdssubnetgrp',\n", + " SubnetIds=[subnet1_id,subnet2_id],\n", + " )\n", + " \n", + " # Get DB security group id\n", + " response = ec2Client.describe_security_groups(\n", + " Filters=[\n", + " {\n", + " 'Name': 'vpc-id',\n", + " 'Values': [vpc_id,]\n", + " },\n", + " {\n", + " 'Name': 'group-name',\n", + " 'Values': [\"default\"]\n", + " },\n", + " ],\n", + " )\n", + " db_security_group_id=response['SecurityGroups'][0]['GroupId']\n", + " print(db_security_group_id)\n", + " \n", + " # Create RDS\n", + " response = conn.create_db_instance(\n", + " AllocatedStorage=10,\n", + " DBInstanceIdentifier=db_instance_name,\n", + " DBInstanceClass=\"db.t2.small\",\n", + " Engine=\"mysql\",\n", + " MasterUsername=\"root\",\n", + " MasterUserPassword=\"pass12345\",\n", + " Port=3306,\n", + " VpcSecurityGroupIds=[db_security_group_id],\n", + " DBSubnetGroupName = db_subnet_name,\n", + " StorageEncrypted=True\n", + " )\n", + " print(response)\n", + " db_name = response['DBInstance']['DBInstanceIdentifier']\n", + " print(\"RDS instance created: DB name={}, with credentials (root/pass12345)\".format(db_name))" ] }, { "cell_type": "code", "execution_count": null, - "id": "7fa2e8d8-f4b4-4f95-a3cf-5271743556b6", + "id": "c646117a-350e-4850-9344-4fcc862e475a", "metadata": {}, "outputs": [], "source": [ - "# Get DB security group id\n", - "response = ec2Client.describe_security_groups(\n", - " Filters=[\n", - " {\n", - " 'Name': 'vpc-id',\n", - " 'Values': [vpc_id,]\n", - " },\n", - " {\n", - " 'Name': 'group-name',\n", - " 'Values': [\"default\"]\n", - " },\n", - " ],\n", - ")\n", - "db_security_group_id=response['SecurityGroups'][0]['GroupId']\n", - "print(db_security_group_id)\n" + "print(\"S3 bucket: {} \".format(s3_bucket_name))\n", + "print(\"Region: {}\".format(region))\n", + "print(\"Stack name: {}\".format(stack_name))\n", + "print(\"Sender email: {}\".format(sender_email_id))\n", + "print(\"Recipent email: {}\".format(receiver_email_id))\n", + "print(\"SlackWebhooks: {}\".format(slack_webhook_urls))\n", + "print(\"isVPC: {}\".format(isVpc))\n", + "print(\"VPC: {}\".format(vpc_id))\n", + "print(\"Subnets: {}\".format(subnets))\n" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "c8b8d57f-dae2-4f53-b29c-a193a7b09b95", + "cell_type": "markdown", + "id": "c5bbc722-6433-466a-8fd7-b03b65b627c9", "metadata": {}, - "outputs": [], "source": [ - "db_instance_name" + "### Installing Trapheus with the above parameters" ] }, { "cell_type": "code", "execution_count": null, - "id": "63b095cf-43ba-44c0-9d57-b5257be7ed3c", + "id": "b491f084-5b05-46ba-a660-127523b92cb9", "metadata": {}, "outputs": [], "source": [ - "# Create RDS\n", - "response = conn.create_db_instance(\n", - " AllocatedStorage=10,\n", - " DBInstanceIdentifier=db_instance_name,\n", - " DBInstanceClass=\"db.t2.small\",\n", - " Engine=\"mysql\",\n", - " MasterUsername=\"root\",\n", - " MasterUserPassword=\"pass12345\",\n", - " Port=3306,\n", - " VpcSecurityGroupIds=[db_security_group_id],\n", - " DBSubnetGroupName = \"Trapheus-rdssubnetgrp\",\n", - " StorageEncrypted=True\n", - " )" + "pip install -r requirements.txt" ] }, { "cell_type": "code", "execution_count": null, - "id": "c646117a-350e-4850-9344-4fcc862e475a", + "id": "2d970e67-fbca-4a91-91ac-a5aaa4c03e83", "metadata": {}, "outputs": [], "source": [ - "print(\"verified email: {}\".format(email_id))\n", - "print(\"S3 bucket: {} \".format(s3_bucket_name))\n", - "print(\"Region: {}\".format(region))\n", - "print(\"Availability zones: {}, {}\".format(zone1,zone2))\n", - "print(\"VPC id: {}\".format(vpc.id))\n", - "print(\"Subnet ids: {},{}\".format(subnet1.id, subnet2.id))\n" - ] - }, - { - "cell_type": "markdown", - "id": "cdc509d5-d0c6-40dc-94a7-3dc1d8c8eb71", - "metadata": {}, - "source": [ - "# Use the above information while running Trapheus install notebook" + "%run install.py \\\n", + " --s3bucket $s3_bucket_name \\\n", + " --region $region \\\n", + " --stackname $stack_name \\\n", + " --senderemail $sender_email_id \\\n", + " --recipientemail $receiver_email_id \\\n", + " --slackwebhooks $slack_webhook_urls \\\n", + " --isvpc $isVpc \\\n", + " --vpcid $vpc_id \\\n", + " --subnets $subnets" ] } ], diff --git a/Trapheus-install-notebook.ipynb b/Trapheus-install-notebook.ipynb deleted file mode 100644 index ffff444..0000000 --- a/Trapheus-install-notebook.ipynb +++ /dev/null @@ -1,60 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "a2f31174-16d1-4a7b-a1ca-ad36c3e49053", - "metadata": {}, - "source": [ - "# We need the below parameters to install Trapheus. If you are new, you can use Prerequisites workbook and create the resources\n", - "* s3-bucket : [Optional] The name of the CloudFormation template S3 bucket from the Pre-Requisites.\n", - "* vpcID : [Required] The id of the VPC from the Pre-Requisites. The lambdas from the Trapheus state machine will be created in this VPC.\n", - "* Subnets : [Required] A comma separated list of private subnet ids (region specific) from the Pre-Requisites VPC.\n", - "* SenderEmail : [Required] The SES sending email configured in the Pre-Requisites\n", - "* RecipientEmail : [Required] Comma separated list of recipient email addresses configured in Pre-Requisites.\n", - "* UseVPCAndSubnets : [Optional] Whether to use the vpc and subnets to create a security group and link the security group and vpc to the lambdas. When UseVPCAndSubnets left out (default) or set to 'true', lambdas are connected to a VPC in your account, and by default the function can't access the RDS (or other services) if VPC doesn't provide access (either by routing outbound traffic to a NAT gateway in a public subnet, or having a VPC endpoint, both of which incur cost or require more setup). If set to 'false', the lambdas will run in a default Lambda owned VPC that has access to RDS (and other AWS services).\n", - "* SlackWebhookUrls : [Optional] Comma separated list of Slack webhooks for failure alerts." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1bff0d83-8d17-4179-91ca-a3ffe3de94fb", - "metadata": {}, - "outputs": [], - "source": [ - "pip install -r requirements.txt" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51116639-ca0b-4df1-b4a7-c7fd2e7c2faf", - "metadata": {}, - "outputs": [], - "source": [ - "%run install.py" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/install.py b/install.py index 7f161c3..6baac19 100644 --- a/install.py +++ b/install.py @@ -1,6 +1,13 @@ #!/usr/bin/env python3 import subprocess +import argparse + +args = ["--s3bucket", "--region", "--stackname", "--senderemail", "--recipientemail", "--slackwebhooks", "--isvpc", "--vpcid", "--subnets"] +parser = argparse.ArgumentParser() +for arg in args: + parser.add_argument(arg, default=None, required=False, nargs='?', const='') +cli_args = parser.parse_args() def execute_subprocess(command_list): print(f'Executing command list: {command_list}') @@ -10,15 +17,20 @@ def execute_subprocess(command_list): print(f'Error: {e}') raise Exception(e) +def get_input(var_name, input_text): + if arg_val := getattr(cli_args, var_name, None): + return arg_val + else: + return input(input_text) def main(): package_command_list = ['sam', 'package', '--template-file', 'template.yaml', '--output-template-file', 'deploy.yaml'] deploy_command_list = ['sam', 'deploy', '--template-file', 'deploy.yaml', '--capabilities', 'CAPABILITY_NAMED_IAM'] - s3_bucket = input('Enter the s3 bucket name created as part of pre-requisite: ') + s3_bucket = get_input('s3bucket','Enter the s3 bucket name created as part of pre-requisite: ') if s3_bucket: package_command_list.append('--s3-bucket') package_command_list.append(s3_bucket) - region = input('Enter the region [for instance, us-west-2]: ') + region = get_input('region','Enter the region [for instance, us-west-2]: ') if region: package_command_list.append('--region') package_command_list.append(region) @@ -30,27 +42,27 @@ def main(): execute_subprocess(package_command_list) - stack_name = input('Enter a stack name: ') + stack_name = get_input('stackname','Enter a stack name: ') if stack_name: deploy_command_list.append('--stack-name') deploy_command_list.append(stack_name) deploy_command_list.append('--parameter-overrides') - sender_email = input('Enter sender email to send email FROM in case of failure: ') + sender_email = get_input('senderemail','Enter sender email to send email FROM in case of failure: ') if sender_email: deploy_command_list.append('SenderEmail=' + sender_email) - recipient_email = input('Enter recipient email to send email TO in case of failure: ') + recipient_email = get_input('recipientemail','Enter recipient email to send email TO in case of failure: ') if recipient_email: deploy_command_list.append('RecipientEmail=' + recipient_email) - slack_webhook_urls = input('Enter slack webhooks to publish failure notifications to: ') + slack_webhook_urls = get_input('slackwebhooks','Enter slack webhooks to publish failure notifications to: ') if slack_webhook_urls: deploy_command_list.append('--SlackWebhookUrls=' + slack_webhook_urls) - vpc = input('Are ypu using vpc[y/n]: ') + vpc = get_input('isvpc','Are ypu using vpc[y/n]: ') if vpc == 'y' or vpc == 'Y': deploy_command_list.append('UseVPCAndSubnets=true') - vpc_id = input('Enter vpc ID: ') + vpc_id = get_input('vpcid','Enter vpc ID: ') if vpc_id: deploy_command_list.append('vpcId=' + vpc_id) - subnets = input('Enter comma seperated list of PRIVATE subnets: ') + subnets = get_input('subnets','Enter comma seperated list of PRIVATE subnets: ') if subnets: deploy_command_list.append('Subnets=' +subnets) print()