Updating the notebook as per review comments

Signed-off-by: kokikathir <[email protected]>
intuit · Oct 29, 2023 · eee3e89 · eee3e89
1 parent deb8f8c
commit eee3e89
Show file tree

Hide file tree

Showing 3 changed files with 184 additions and 191 deletions.
diff --git a/Prerequisites-notebook.ipynb b/Prerequisites-notebook.ipynb
@@ -5,11 +5,18 @@
    "id": "79a5fe45-1150-49e5-80de-858fe8d58785",
    "metadata": {},
    "source": [
-    "# This notebook focuses on the prerequisites section of Trapheus. Executing this notebook will create the following resources\n",
-    "* Verify an email address to be used to send email alerts\n",
-    "* Create a S3 bucket where the system is going to store the cloud formation templates\n",
-    "* Create a VPC with 2 Private Subnets in 2 availability zones. Make sure the region passed is the same as in AWS credentials\n",
-    "* Create an RDS instance with the given name with MySQL DB, with credentials (root/pass12345)\n",
+    "# This notebook focuses on the Installation of Trapheus along with its prerequisites section. Executing this notebook will create the resources if it doesn't exist.\n",
+    "\n",
+    "Below are the parameters needed for the Trapheus installation\n",
+    "\n",
+    "* s3-bucket : [Optional] The name of the CloudFormation template S3 bucket from the Pre-Requisites.\n",
+    "* vpcID : [Required] The id of the VPC from the Pre-Requisites. The lambdas from the Trapheus state machine will be created in this VPC.\n",
+    "* Subnets : [Required] A comma separated list of private subnet ids (region specific) from the Pre-Requisites VPC.\n",
+    "* SenderEmail : [Required] The SES sending email configured in the Pre-Requisites\n",
+    "* RecipientEmail : [Required] Comma separated list of recipient email addresses configured in Pre-Requisites.\n",
+    "* UseVPCAndSubnets : [Optional] Whether to use the vpc and subnets to create a security group and link the security group and vpc to the lambdas. When UseVPCAndSubnets left out (default) or set to 'true', lambdas are connected to a VPC in your account, and by default the function can't access the RDS (or other services) if VPC doesn't provide access (either by routing outbound traffic to a NAT gateway in a public subnet, or having a VPC endpoint, both of which incur cost or require more setup). If set to 'false', the lambdas will run in a default Lambda owned VPC that has access to RDS (and other AWS services).\n",
+    "* SlackWebhookUrls : [Optional] Comma separated list of Slack webhooks for failure alerts.\n",
+    "\n",
     "\n",
     "> Note:  Make sure to have AWS credentials configured. Refer to [Setting up AWS credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)"
    ]
@@ -21,10 +28,22 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "email_id=input(\"Enter your email id to verify in SES: \")\n",
-    "region=input(\"Enter the region to create the resources: \")\n",
-    "s3_bucket_name=input(\"Enter the name of the S3 bucket to be created(Proposed Name: trapheus-cfn-s3-[account-id]-[region]): \")\n",
-    "db_instance_name=input(\"Enter the name for the RDS instance to be created: \")"
+    "region=input(\"Enter the region [for instance, us-west-2]: \")\n",
+    "if region == \"\":\n",
+    "    raise SystemExit(\"Region is empty. Provide the region to create the resources\")\n",
+    "s3_bucket_name=input(\"Enter the name of the S3 bucket. New bucket will be created if it doesn't exist(Proposed Name: trapheus-cfn-s3-[account-id]-[region]): \")\n",
+    "isVpc = input('Do you want to use vpc[y/n]: ')\n",
+    "if isVpc == 'y' or isVpc == 'Y':\n",
+    "    vpc_id=input(\"Enter the VPC ID. Leaving it empty will create a new VPC along with subnets: \")\n",
+    "    if len(vpc_id) > 0:\n",
+    "        subnets = input('Enter comma separated list of PRIVATE subnets: ')\n",
+    "sender_email_id=input(\"Enter sender email to send email FROM in case of failure: \")\n",
+    "receiver_email_id=input(\"Enter recipient email to send email TO in case of failure:  \")\n",
+    "create_rds=input(\"Do you want to create an RDS instance[y/n]\")\n",
+    "if create_rds == 'y' or create_rds == 'Y' :\n",
+    "    db_instance_name=input(\"Enter a RDS instance name\")\n",
+    "slack_webhook_urls = input('Enter slack webhooks to publish failure notifications to: ')\n",
+    "stack_name = input('Enter a cloud formation stack name: ')"
    ]
   },
   {
@@ -46,23 +65,40 @@
    "source": [
     "# Verify email\n",
     "ses = boto3.client('ses')\n",
-    "response = ses.verify_email_identity(\n",
-    "  EmailAddress = email_id\n",
-    ")"
+    "if len(sender_email_id) > 0:\n",
+    "    response = ses.verify_email_identity(\n",
+    "      EmailAddress = sender_email_id\n",
+    "    )\n",
+    "else:\n",
+    "    print(\"Sender email id not provided. Skipping this step\")\n",
+    "\n",
+    "if len(receiver_email_id) > 0:\n",
+    "    response = ses.verify_email_identity(\n",
+    "      EmailAddress = receiver_email_id\n",
+    "    )\n",
+    "else:\n",
+    "    print(\"Receiver email id not provided. Skipping this step\")"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "6b87538d-9771-42a5-9217-2a25a9709998",
+   "id": "dd5d494e-c0d6-408a-877b-40a6ada11bad",
    "metadata": {},
    "outputs": [],
    "source": [
-    "#Create S3 bucket\n",
+    "# Validate and Create S3 bucket\n",
     "s3 = boto3.resource('s3')\n",
-    "s3.create_bucket(Bucket=s3_bucket_name,CreateBucketConfiguration={\n",
-    "        'LocationConstraint': region\n",
-    "    })"
+    "bucket = s3.Bucket(s3_bucket_name)\n",
+    "\n",
+    "if bucket.creation_date:\n",
+    "   print(\"The bucket already exists. Proceeding to next step.\")\n",
+    "else:\n",
+    "   print(\"The bucket does not exist. Creating new one with the given name\")\n",
+    "   s3Response = s3.create_bucket(Bucket=s3_bucket_name,CreateBucketConfiguration={\n",
+    "    'LocationConstraint': region})\n",
+    "   print(\"created bucket {}\".format(s3_bucket_name))\n",
+    "   "
    ]
   },
   {
@@ -72,57 +108,48 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Get availability zones\n",
-    "response = ec2Client.describe_availability_zones(\n",
-    "    Filters=[\n",
-    "        {\n",
-    "            'Name': 'region-name',\n",
-    "            'Values': [region]\n",
-    "        },\n",
-    "        {\n",
-    "            'Name': 'state',\n",
-    "            'Values': ['available']\n",
-    "        },\n",
-    "    ]\n",
-    ")\n",
-    "if len(response['AvailabilityZones']) < 2:\n",
-    "    raise SystemExit(\"Stopping the execution. We need at least 2 availability zones to create private subnets. Make sure the region is set correctly in aws credentials\")\n",
-    "zone1 = response['AvailabilityZones'][0]['ZoneName']\n",
-    "zone2 = response['AvailabilityZones'][1]['ZoneName']\n",
-    "print(zone1, zone2)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "37a1e945-bddf-42e7-aa5c-b4689617bf22",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "#Create VPC\n",
+    "# If VPC id is not provided, then create VPC along with private subnets\n",
     "ec2 = boto3.resource('ec2')\n",
     "ec2Client = boto3.client('ec2')\n",
-    "vpc = ec2.create_vpc(CidrBlock='10.0.0.0/16')\n",
-    "vpc.wait_until_available()\n",
-    "vpc_id=vpc.id\n",
-    "print(vpc.id)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f8bd5dc5-e8e6-4304-97ee-98f3f40b6e51",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "#Create Private Subnets\n",
-    "subnet1 = ec2.create_subnet(CidrBlock='10.0.0.0/19', VpcId=vpc_id, AvailabilityZone=zone1)\n",
-    "subnet1_id=subnet1.id\n",
-    "print(subnet1.id)\n",
+    "if vpc_id == \"\":\n",
+    "    print(\"Creating VPC...\")\n",
+    "    # Get availability zones\n",
+    "    response = ec2Client.describe_availability_zones(\n",
+    "        Filters=[\n",
+    "            {\n",
+    "                'Name': 'region-name',\n",
+    "                'Values': [region]\n",
+    "            },\n",
+    "            {\n",
+    "                'Name': 'state',\n",
+    "                'Values': ['available']\n",
+    "            },\n",
+    "        ]\n",
+    "    )\n",
+    "    if len(response['AvailabilityZones']) < 2:\n",
+    "        raise SystemExit(\"Stopping the execution. We need at least 2 availability zones to create private subnets. Make sure the region is set correctly in aws credentials\")\n",
+    "    zone1 = response['AvailabilityZones'][0]['ZoneName']\n",
+    "    zone2 = response['AvailabilityZones'][1]['ZoneName']\n",
+    "    # print(zone1, zone2)\n",
+    "    \n",
+    "    #Create VPC\n",
+    "\n",
+    "    vpc = ec2.create_vpc(CidrBlock='10.0.0.0/16')\n",
+    "    vpc.wait_until_available()\n",
+    "    vpc_id=vpc.id\n",
+    "    #print(vpc.id)\n",
+    "    \n",
+    "    #Create Private Subnets\n",
+    "    subnet1 = ec2.create_subnet(CidrBlock='10.0.0.0/19', VpcId=vpc_id, AvailabilityZone=zone1)\n",
+    "    subnet1_id=subnet1.id\n",
+    "    #print(subnet1.id)\n",
+    "    \n",
+    "    subnet2 = ec2.create_subnet(CidrBlock='10.0.32.0/19', VpcId=vpc_id,AvailabilityZone=zone2)\n",
+    "    #print(subnet2.id)\n",
+    "    subnet2_id=subnet2.id\n",
+    "    subnets = subnet1_id + \",\" + subnet2_id\n",
     "\n",
-    "subnet2 = ec2.create_subnet(CidrBlock='10.0.32.0/19', VpcId=vpc_id,AvailabilityZone=zone2)\n",
-    "print(subnet2.id)\n",
-    "subnet2_id=subnet2.id\n"
+    "    print(\"Created VPC {} with private subnets {},{}\".format(vpc_id, subnet1_id, subnet2_id))"
    ]
   },
   {
@@ -132,93 +159,107 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "#Create RDS DB Subnet\n",
-    "conn = boto3.client('rds')\n",
+    "# create a new RDS instance\n",
+    "if (create_rds == 'y' or create_rds == 'Y') and len(db_instance_name) > 0:\n",
+    "    print(\"Creating RDS Instance...\")\n",
+    "    #Create RDS DB Subnet\n",
+    "    conn = boto3.client('rds')\n",
     "\n",
-    "response = conn.create_db_subnet_group(\n",
-    "    DBSubnetGroupName='Trapheus-rdssubnetgrp',\n",
-    "    DBSubnetGroupDescription='rdssubnetgrp',\n",
-    "    SubnetIds=[subnet1_id,subnet2_id],\n",
-    ")\n"
+    "    db_subnet_name = db_instance_name+\"_dbsubnet\"\n",
+    "    print(db_subnet_name)\n",
+    "    response = conn.create_db_subnet_group(\n",
+    "        DBSubnetGroupName=db_subnet_name,\n",
+    "        DBSubnetGroupDescription='rdssubnetgrp',\n",
+    "        SubnetIds=[subnet1_id,subnet2_id],\n",
+    "    )\n",
+    "    \n",
+    "    # Get DB security group id\n",
+    "    response = ec2Client.describe_security_groups(\n",
+    "        Filters=[\n",
+    "            {\n",
+    "                'Name': 'vpc-id',\n",
+    "                'Values': [vpc_id,]\n",
+    "            },\n",
+    "        {\n",
+    "                'Name': 'group-name',\n",
+    "                'Values': [\"default\"]\n",
+    "            },\n",
+    "        ],\n",
+    "    )\n",
+    "    db_security_group_id=response['SecurityGroups'][0]['GroupId']\n",
+    "    print(db_security_group_id)\n",
+    "    \n",
+    "    # Create RDS\n",
+    "    response = conn.create_db_instance(\n",
+    "            AllocatedStorage=10,\n",
+    "            DBInstanceIdentifier=db_instance_name,\n",
+    "            DBInstanceClass=\"db.t2.small\",\n",
+    "            Engine=\"mysql\",\n",
+    "            MasterUsername=\"root\",\n",
+    "            MasterUserPassword=\"pass12345\",\n",
+    "            Port=3306,\n",
+    "            VpcSecurityGroupIds=[db_security_group_id],\n",
+    "            DBSubnetGroupName = db_subnet_name,\n",
+    "            StorageEncrypted=True\n",
+    "        )\n",
+    "    print(response)\n",
+    "    db_name = response['DBInstance']['DBInstanceIdentifier']\n",
+    "    print(\"RDS instance created: DB name={}, with credentials (root/pass12345)\".format(db_name))"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "7fa2e8d8-f4b4-4f95-a3cf-5271743556b6",
+   "id": "c646117a-350e-4850-9344-4fcc862e475a",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Get DB security group id\n",
-    "response = ec2Client.describe_security_groups(\n",
-    "    Filters=[\n",
-    "        {\n",
-    "            'Name': 'vpc-id',\n",
-    "            'Values': [vpc_id,]\n",
-    "        },\n",
-    "    {\n",
-    "            'Name': 'group-name',\n",
-    "            'Values': [\"default\"]\n",
-    "        },\n",
-    "    ],\n",
-    ")\n",
-    "db_security_group_id=response['SecurityGroups'][0]['GroupId']\n",
-    "print(db_security_group_id)\n"
+    "print(\"S3 bucket: {} \".format(s3_bucket_name))\n",
+    "print(\"Region: {}\".format(region))\n",
+    "print(\"Stack name: {}\".format(stack_name))\n",
+    "print(\"Sender email: {}\".format(sender_email_id))\n",
+    "print(\"Recipent email: {}\".format(receiver_email_id))\n",
+    "print(\"SlackWebhooks: {}\".format(slack_webhook_urls))\n",
+    "print(\"isVPC: {}\".format(isVpc))\n",
+    "print(\"VPC: {}\".format(vpc_id))\n",
+    "print(\"Subnets: {}\".format(subnets))\n"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c8b8d57f-dae2-4f53-b29c-a193a7b09b95",
+   "cell_type": "markdown",
+   "id": "c5bbc722-6433-466a-8fd7-b03b65b627c9",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "db_instance_name"
+    "### Installing Trapheus with the above parameters"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "63b095cf-43ba-44c0-9d57-b5257be7ed3c",
+   "id": "b491f084-5b05-46ba-a660-127523b92cb9",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Create RDS\n",
-    "response = conn.create_db_instance(\n",
-    "        AllocatedStorage=10,\n",
-    "        DBInstanceIdentifier=db_instance_name,\n",
-    "        DBInstanceClass=\"db.t2.small\",\n",
-    "        Engine=\"mysql\",\n",
-    "        MasterUsername=\"root\",\n",
-    "        MasterUserPassword=\"pass12345\",\n",
-    "        Port=3306,\n",
-    "        VpcSecurityGroupIds=[db_security_group_id],\n",
-    "        DBSubnetGroupName = \"Trapheus-rdssubnetgrp\",\n",
-    "        StorageEncrypted=True\n",
-    "    )"
+    "pip install -r requirements.txt"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "c646117a-350e-4850-9344-4fcc862e475a",
+   "id": "2d970e67-fbca-4a91-91ac-a5aaa4c03e83",
    "metadata": {},
    "outputs": [],
    "source": [
-    "print(\"verified email: {}\".format(email_id))\n",
-    "print(\"S3 bucket: {} \".format(s3_bucket_name))\n",
-    "print(\"Region: {}\".format(region))\n",
-    "print(\"Availability zones: {}, {}\".format(zone1,zone2))\n",
-    "print(\"VPC id: {}\".format(vpc.id))\n",
-    "print(\"Subnet ids: {},{}\".format(subnet1.id, subnet2.id))\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "cdc509d5-d0c6-40dc-94a7-3dc1d8c8eb71",
-   "metadata": {},
-   "source": [
-    "# Use the above information while running Trapheus install notebook"
+    "%run install.py \\\n",
+    "    --s3bucket $s3_bucket_name \\\n",
+    "    --region $region \\\n",
+    "    --stackname $stack_name \\\n",
+    "    --senderemail $sender_email_id \\\n",
+    "    --recipientemail $receiver_email_id \\\n",
+    "    --slackwebhooks $slack_webhook_urls \\\n",
+    "    --isvpc $isVpc \\\n",
+    "    --vpcid $vpc_id \\\n",
+    "    --subnets $subnets"
    ]
   }
  ],