-
Notifications
You must be signed in to change notification settings - Fork 55
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #181 from kokikathir/master
Creating jupyter notebook workshop for prerequisities and installation
- Loading branch information
Showing
2 changed files
with
308 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,287 @@ | ||
| { | ||
| "cells": [ | ||
| { | ||
| "cell_type": "markdown", | ||
| "id": "79a5fe45-1150-49e5-80de-858fe8d58785", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "# This notebook focuses on the Installation of Trapheus along with its prerequisites section. Executing this notebook will create the resources if it doesn't exist.\n", | ||
| "\n", | ||
| "Below are the parameters needed for the Trapheus installation\n", | ||
| "\n", | ||
| "* s3-bucket : [Optional] The name of the CloudFormation template S3 bucket from the Pre-Requisites.\n", | ||
| "* vpcID : [Required] The id of the VPC from the Pre-Requisites. The lambdas from the Trapheus state machine will be created in this VPC.\n", | ||
| "* Subnets : [Required] A comma separated list of private subnet ids (region specific) from the Pre-Requisites VPC.\n", | ||
| "* SenderEmail : [Required] The SES sending email configured in the Pre-Requisites\n", | ||
| "* RecipientEmail : [Required] Comma separated list of recipient email addresses configured in Pre-Requisites.\n", | ||
| "* UseVPCAndSubnets : [Optional] Whether to use the vpc and subnets to create a security group and link the security group and vpc to the lambdas. When UseVPCAndSubnets left out (default) or set to 'true', lambdas are connected to a VPC in your account, and by default the function can't access the RDS (or other services) if VPC doesn't provide access (either by routing outbound traffic to a NAT gateway in a public subnet, or having a VPC endpoint, both of which incur cost or require more setup). If set to 'false', the lambdas will run in a default Lambda owned VPC that has access to RDS (and other AWS services).\n", | ||
| "* SlackWebhookUrls : [Optional] Comma separated list of Slack webhooks for failure alerts.\n", | ||
| "\n", | ||
| "\n", | ||
| "> Note: Make sure to have AWS credentials configured. Refer to [Setting up AWS credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "ab1d9b7a-0f99-4935-a03f-5f6807a7d113", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "region=input(\"Enter the region [for instance, us-west-2]: \")\n", | ||
| "if region == \"\":\n", | ||
| " raise SystemExit(\"Region is empty. Provide the region to create the resources\")\n", | ||
| "s3_bucket_name=input(\"Enter the name of the S3 bucket. New bucket will be created if it doesn't exist(Proposed Name: trapheus-cfn-s3-[account-id]-[region]): \")\n", | ||
| "isVpc = input('Do you want to use vpc[y/n]: ')\n", | ||
| "if isVpc == 'y' or isVpc == 'Y':\n", | ||
| " vpc_id=input(\"Enter the VPC ID. Leaving it empty will create a new VPC along with subnets: \")\n", | ||
| " if len(vpc_id) > 0:\n", | ||
| " subnets = input('Enter comma separated list of PRIVATE subnets: ')\n", | ||
| "sender_email_id=input(\"Enter sender email to send email FROM in case of failure: \")\n", | ||
| "receiver_email_id=input(\"Enter recipient email to send email TO in case of failure: \")\n", | ||
| "create_rds=input(\"Do you want to create an RDS instance[y/n]\")\n", | ||
| "if create_rds == 'y' or create_rds == 'Y' :\n", | ||
| " db_instance_name=input(\"Enter a RDS instance name\")\n", | ||
| "slack_webhook_urls = input('Enter slack webhooks to publish failure notifications to: ')\n", | ||
| "stack_name = input('Enter a cloud formation stack name: ')" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "9f78c60c-9e29-4194-ad1a-415aac889e18", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "import boto3" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "c1dd5f60-a9fd-4126-9cdc-b6bb4739c9f6", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "# Verify email\n", | ||
| "ses = boto3.client('ses')\n", | ||
| "if len(sender_email_id) > 0:\n", | ||
| " response = ses.verify_email_identity(\n", | ||
| " EmailAddress = sender_email_id\n", | ||
| " )\n", | ||
| "else:\n", | ||
| " print(\"Sender email id not provided. Skipping this step\")\n", | ||
| "\n", | ||
| "if len(receiver_email_id) > 0:\n", | ||
| " response = ses.verify_email_identity(\n", | ||
| " EmailAddress = receiver_email_id\n", | ||
| " )\n", | ||
| "else:\n", | ||
| " print(\"Receiver email id not provided. Skipping this step\")" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "dd5d494e-c0d6-408a-877b-40a6ada11bad", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "# Validate and Create S3 bucket\n", | ||
| "s3 = boto3.resource('s3')\n", | ||
| "bucket = s3.Bucket(s3_bucket_name)\n", | ||
| "\n", | ||
| "if bucket.creation_date:\n", | ||
| " print(\"The bucket already exists. Proceeding to next step.\")\n", | ||
| "else:\n", | ||
| " print(\"The bucket does not exist. Creating new one with the given name\")\n", | ||
| " s3Response = s3.create_bucket(Bucket=s3_bucket_name,CreateBucketConfiguration={\n", | ||
| " 'LocationConstraint': region})\n", | ||
| " print(\"created bucket {}\".format(s3_bucket_name))\n", | ||
| " " | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "affe7b5c-5c67-43f3-aa5f-37c291307175", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "# If VPC id is not provided, then create VPC along with private subnets\n", | ||
| "ec2 = boto3.resource('ec2')\n", | ||
| "ec2Client = boto3.client('ec2')\n", | ||
| "if vpc_id == \"\":\n", | ||
| " print(\"Creating VPC...\")\n", | ||
| " # Get availability zones\n", | ||
| " response = ec2Client.describe_availability_zones(\n", | ||
| " Filters=[\n", | ||
| " {\n", | ||
| " 'Name': 'region-name',\n", | ||
| " 'Values': [region]\n", | ||
| " },\n", | ||
| " {\n", | ||
| " 'Name': 'state',\n", | ||
| " 'Values': ['available']\n", | ||
| " },\n", | ||
| " ]\n", | ||
| " )\n", | ||
| " if len(response['AvailabilityZones']) < 2:\n", | ||
| " raise SystemExit(\"Stopping the execution. We need at least 2 availability zones to create private subnets. Make sure the region is set correctly in aws credentials\")\n", | ||
| " zone1 = response['AvailabilityZones'][0]['ZoneName']\n", | ||
| " zone2 = response['AvailabilityZones'][1]['ZoneName']\n", | ||
| " # print(zone1, zone2)\n", | ||
| " \n", | ||
| " #Create VPC\n", | ||
| "\n", | ||
| " vpc = ec2.create_vpc(CidrBlock='10.0.0.0/16')\n", | ||
| " vpc.wait_until_available()\n", | ||
| " vpc_id=vpc.id\n", | ||
| " #print(vpc.id)\n", | ||
| " \n", | ||
| " #Create Private Subnets\n", | ||
| " subnet1 = ec2.create_subnet(CidrBlock='10.0.0.0/19', VpcId=vpc_id, AvailabilityZone=zone1)\n", | ||
| " subnet1_id=subnet1.id\n", | ||
| " #print(subnet1.id)\n", | ||
| " \n", | ||
| " subnet2 = ec2.create_subnet(CidrBlock='10.0.32.0/19', VpcId=vpc_id,AvailabilityZone=zone2)\n", | ||
| " #print(subnet2.id)\n", | ||
| " subnet2_id=subnet2.id\n", | ||
| " subnets = subnet1_id + \",\" + subnet2_id\n", | ||
| "\n", | ||
| " print(\"Created VPC {} with private subnets {},{}\".format(vpc_id, subnet1_id, subnet2_id))" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "1b170e4b-18f2-43a0-99b4-286633512f48", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "# create a new RDS instance\n", | ||
| "if (create_rds == 'y' or create_rds == 'Y') and len(db_instance_name) > 0:\n", | ||
| " print(\"Creating RDS Instance...\")\n", | ||
| " #Create RDS DB Subnet\n", | ||
| " conn = boto3.client('rds')\n", | ||
| "\n", | ||
| " db_subnet_name = db_instance_name+\"_dbsubnet\"\n", | ||
| " print(db_subnet_name)\n", | ||
| " response = conn.create_db_subnet_group(\n", | ||
| " DBSubnetGroupName=db_subnet_name,\n", | ||
| " DBSubnetGroupDescription='rdssubnetgrp',\n", | ||
| " SubnetIds=[subnet1_id,subnet2_id],\n", | ||
| " )\n", | ||
| " \n", | ||
| " # Get DB security group id\n", | ||
| " response = ec2Client.describe_security_groups(\n", | ||
| " Filters=[\n", | ||
| " {\n", | ||
| " 'Name': 'vpc-id',\n", | ||
| " 'Values': [vpc_id,]\n", | ||
| " },\n", | ||
| " {\n", | ||
| " 'Name': 'group-name',\n", | ||
| " 'Values': [\"default\"]\n", | ||
| " },\n", | ||
| " ],\n", | ||
| " )\n", | ||
| " db_security_group_id=response['SecurityGroups'][0]['GroupId']\n", | ||
| " print(db_security_group_id)\n", | ||
| " \n", | ||
| " # Create RDS\n", | ||
| " response = conn.create_db_instance(\n", | ||
| " AllocatedStorage=10,\n", | ||
| " DBInstanceIdentifier=db_instance_name,\n", | ||
| " DBInstanceClass=\"db.t2.small\",\n", | ||
| " Engine=\"mysql\",\n", | ||
| " MasterUsername=\"root\",\n", | ||
| " MasterUserPassword=\"pass12345\",\n", | ||
| " Port=3306,\n", | ||
| " VpcSecurityGroupIds=[db_security_group_id],\n", | ||
| " DBSubnetGroupName = db_subnet_name,\n", | ||
| " StorageEncrypted=True\n", | ||
| " )\n", | ||
| " print(response)\n", | ||
| " db_name = response['DBInstance']['DBInstanceIdentifier']\n", | ||
| " print(\"RDS instance created: DB name={}, with credentials (root/pass12345)\".format(db_name))" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "c646117a-350e-4850-9344-4fcc862e475a", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "print(\"S3 bucket: {} \".format(s3_bucket_name))\n", | ||
| "print(\"Region: {}\".format(region))\n", | ||
| "print(\"Stack name: {}\".format(stack_name))\n", | ||
| "print(\"Sender email: {}\".format(sender_email_id))\n", | ||
| "print(\"Recipent email: {}\".format(receiver_email_id))\n", | ||
| "print(\"SlackWebhooks: {}\".format(slack_webhook_urls))\n", | ||
| "print(\"isVPC: {}\".format(isVpc))\n", | ||
| "print(\"VPC: {}\".format(vpc_id))\n", | ||
| "print(\"Subnets: {}\".format(subnets))\n" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "id": "c5bbc722-6433-466a-8fd7-b03b65b627c9", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "### Installing Trapheus with the above parameters" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "b491f084-5b05-46ba-a660-127523b92cb9", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "pip install -r requirements.txt" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "2d970e67-fbca-4a91-91ac-a5aaa4c03e83", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "%run install.py \\\n", | ||
| " --s3bucket $s3_bucket_name \\\n", | ||
| " --region $region \\\n", | ||
| " --stackname $stack_name \\\n", | ||
| " --senderemail $sender_email_id \\\n", | ||
| " --recipientemail $receiver_email_id \\\n", | ||
| " --slackwebhooks $slack_webhook_urls \\\n", | ||
| " --isvpc $isVpc \\\n", | ||
| " --vpcid $vpc_id \\\n", | ||
| " --subnets $subnets" | ||
| ] | ||
| } | ||
| ], | ||
| "metadata": { | ||
| "kernelspec": { | ||
| "display_name": "Python 3 (ipykernel)", | ||
| "language": "python", | ||
| "name": "python3" | ||
| }, | ||
| "language_info": { | ||
| "codemirror_mode": { | ||
| "name": "ipython", | ||
| "version": 3 | ||
| }, | ||
| "file_extension": ".py", | ||
| "mimetype": "text/x-python", | ||
| "name": "python", | ||
| "nbconvert_exporter": "python", | ||
| "pygments_lexer": "ipython3", | ||
| "version": "3.11.4" | ||
| } | ||
| }, | ||
| "nbformat": 4, | ||
| "nbformat_minor": 5 | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters