From fc4d0412912e4a9af32e90db09aea1b092a50709 Mon Sep 17 00:00:00 2001 From: Buckingham Date: Fri, 6 Dec 2024 09:31:03 +0000 Subject: [PATCH] Update_061224_1 --- terraform/environments/ppud/alb_external.tf | 12 +-- terraform/environments/ppud/alb_internal.tf | 6 +- .../environments/ppud/endpointservice.tf | 6 +- .../lambda_scripts/ppud_email_report_prod.py | 51 ++++++++----- .../lambda_scripts/send_cpu_graph_prod.py | 75 ++++++++++++------- 5 files changed, 84 insertions(+), 66 deletions(-) diff --git a/terraform/environments/ppud/alb_external.tf b/terraform/environments/ppud/alb_external.tf index a672ed7cdb4..70ed8fb6b79 100644 --- a/terraform/environments/ppud/alb_external.tf +++ b/terraform/environments/ppud/alb_external.tf @@ -4,17 +4,13 @@ resource "aws_lb" "PPUD-ALB" { # checkov:skip=CKV2_AWS_28: "ALB is already protected by WAF" # checkov:skip=CKV_AWS_152: "ALB target groups only have 2 targets so cross zone load balancing is not required" + # checkov:skip=CKV_AWS_91: "ELB Logging not required" count = local.is-development == true ? 1 : 0 name = "PPUD-ALB" internal = false load_balancer_type = "application" security_groups = [aws_security_group.PPUD-ALB.id] subnets = [data.aws_subnet.public_subnets_b.id, data.aws_subnet.public_subnets_c.id] - access_logs { - bucket = aws_s3_bucket.moj-log-files-dev[0].id - prefix = "alb-logs" - enabled = true - } enable_deletion_protection = true drop_invalid_header_fields = true @@ -88,16 +84,12 @@ resource "aws_lb_target_group_attachment" "PPUD-PORTAL-1" { resource "aws_lb" "WAM-ALB" { # checkov:skip=CKV2_AWS_28: "ALB is already protected by WAF" # checkov:skip=CKV_AWS_152: "ALB target groups only have 2 targets so cross zone load balancing is not required" + # checkov:skip=CKV_AWS_91: "ELB Logging not required" name = local.application_data.accounts[local.environment].WAM_ALB internal = false load_balancer_type = "application" security_groups = [aws_security_group.WAM-ALB.id] subnets = [data.aws_subnet.public_subnets_a.id, data.aws_subnet.public_subnets_b.id] - # access_logs { - # bucket = aws_s3_bucket.moj-log-files-dev[0].id - # prefix = "alb-logs" - # enabled = true - # } enable_deletion_protection = true drop_invalid_header_fields = true diff --git a/terraform/environments/ppud/alb_internal.tf b/terraform/environments/ppud/alb_internal.tf index 782d7296952..9f4add3406b 100644 --- a/terraform/environments/ppud/alb_internal.tf +++ b/terraform/environments/ppud/alb_internal.tf @@ -4,6 +4,7 @@ resource "aws_lb" "PPUD-internal-ALB" { # checkov:skip=CKV_AWS_152: "ALB target groups only have 2 targets so cross zone load balancing is not required" + # checkov:skip=CKV_AWS_91: "ELB Logging not required" count = local.is-development == false ? 1 : 0 name = local.application_data.accounts[local.environment].PPUD_Internal_ALB internal = true @@ -11,11 +12,6 @@ resource "aws_lb" "PPUD-internal-ALB" { load_balancer_type = "application" security_groups = [aws_security_group.PPUD-ALB.id] subnets = [data.aws_subnet.private_subnets_b.id, data.aws_subnet.private_subnets_c.id] - # access_logs { - # bucket = aws_s3_bucket.moj-log-files-uat[0].id - # prefix = "alb-logs" - # enabled = true - # } enable_deletion_protection = true drop_invalid_header_fields = true diff --git a/terraform/environments/ppud/endpointservice.tf b/terraform/environments/ppud/endpointservice.tf index ed408b30d57..c871d55e7c4 100644 --- a/terraform/environments/ppud/endpointservice.tf +++ b/terraform/environments/ppud/endpointservice.tf @@ -17,6 +17,7 @@ resource "aws_vpc_endpoint_service_allowed_principal" "HomeOffice" { resource "aws_lb" "ppud_internal_nlb" { # checkov:skip=CKV2_AWS_28: "ALB is already protected by WAF" # checkov:skip=CKV_AWS_152: "ALB target groups only have 2 targets so cross zone load balancing is not required" + # checkov:skip=CKV_AWS_91: "ELB Logging not required" count = local.is-production == true ? 1 : 0 name = "ppud-internal-nlb" internal = true @@ -24,11 +25,6 @@ resource "aws_lb" "ppud_internal_nlb" { subnets = [data.aws_subnet.private_subnets_b.id, data.aws_subnet.private_subnets_c.id] security_groups = [aws_security_group.PPUD-ALB.id] enable_deletion_protection = true - #access_logs { - # bucket = aws_s3_bucket.moj-log-files-prod[0].id - # prefix = "alb-logs" - # enabled = true - #} tags = { Name = "${var.networking[0].business-unit}-${local.environment}" diff --git a/terraform/environments/ppud/lambda_scripts/ppud_email_report_prod.py b/terraform/environments/ppud/lambda_scripts/ppud_email_report_prod.py index 381b8a2fc89..6d037f5eef1 100644 --- a/terraform/environments/ppud/lambda_scripts/ppud_email_report_prod.py +++ b/terraform/environments/ppud/lambda_scripts/ppud_email_report_prod.py @@ -5,23 +5,30 @@ import re import io import base64 +import smtplib from datetime import datetime, timedelta from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText # Initialize boto3 clients s3 = boto3.client('s3') -ses = boto3.client('ses') +# ses = boto3.client('ses') # Configuration CURRENT_DATE = datetime.now().strftime('%a %d %b %Y') -bucket_name = 'moj-lambda-layers-dev' +bucket_name = 'moj-lambda-layers-prod' file_names = ['monday.log', 'tuesday.log', 'wednesday.log', 'thursday.log', 'friday.log', 'saturday.log', 'sunday.log'] -SENDER = 'noreply@internaltest.ppud.justice.gov.uk' +SENDER = 'donotreply@cjsm.secure-email.ppud.justice.gov.uk' RECIPIENTS = ['nick.buckingham@colt.net'] SUBJECT = f'AWS Weekly PPUD Email Report - {CURRENT_DATE}' AWS_REGION = 'eu-west-2' +# SMTP Configuration +SMTP_SERVER = "10.27.9.39" +SMTP_PORT = 25 +MAIL_FROM = "donotreply@cjsm.secure-email.ppud.justice.gov.uk" +EMAIL_TO = ["nick.buckingham@colt.net"] + def retrieve_file_from_s3(bucket, key): response = s3.get_object(Bucket=bucket, Key=key) content = response['Body'].read().decode('utf-8') @@ -42,13 +49,8 @@ def create_graph(data): plt.title('PPUD Emails Sent') plt.tight_layout() - # buf = io.BytesIO() - # plt.savefig(buf, format='png') - # buf.seek(0) - # return buf.getvalue() - # Save the graph to a temporary buffer - temp_file = "/tmp/ppud_emails_send.png" + temp_file = "/tmp/ppud_emails_sent.png" plt.savefig(temp_file) plt.close() @@ -60,12 +62,11 @@ def create_graph(data): os.remove(temp_file) return encoded_string -# Function to send an email via SES def send_email_with_graph(graph_base64): """ Send an email with the graph embedded in the email body using AWS SES. """ - ses_client = boto3.client("ses", region_name=AWS_REGION) + # ses_client = boto3.client("ses", region_name=REGION) # Email body with the embedded image email_body = f""" @@ -88,17 +89,27 @@ def send_email_with_graph(graph_base64): # Attach the HTML body msg.attach(MIMEText(email_body, "html")) - # Send the email + # Send the email with AWS SES + # try: + # response = ses_client.send_raw_email( + # Source=SENDER, + # Destinations=RECIPIENTS, + # RawMessage={"Data": msg.as_string()}, + # ) + # print("Email sent! Message ID:", response["MessageId"]) + # except Exception as e: + # print("Error sending email:", e) + # raise + + # Send the email with an EC2 Instance Mail Relay try: - response = ses_client.send_raw_email( - Source=SENDER, - Destinations=RECIPIENTS, - RawMessage={"Data": msg.as_string()}, - ) - print("Email sent! Message ID:", response["MessageId"]) + with smtplib.SMTP(SMTP_SERVER, SMTP_PORT) as server: + # server.starttls() + # server.login(SENDER, EMAIL_PASSWORD) + server.sendmail(SENDER, RECIPIENTS, msg.as_string()) + print("Email sent successfully.") except Exception as e: - print("Error sending email:", e) - raise + print(f"Error sending email: {e}") def lambda_handler(event, context): pattern = r'to=<' diff --git a/terraform/environments/ppud/lambda_scripts/send_cpu_graph_prod.py b/terraform/environments/ppud/lambda_scripts/send_cpu_graph_prod.py index 0927365bc61..efb903f2b7c 100644 --- a/terraform/environments/ppud/lambda_scripts/send_cpu_graph_prod.py +++ b/terraform/environments/ppud/lambda_scripts/send_cpu_graph_prod.py @@ -1,3 +1,5 @@ +# Python script to + import boto3 import os os.environ['MPLCONFIGDIR'] = "/tmp/graph" @@ -5,24 +7,35 @@ from datetime import datetime, timedelta import io import base64 +import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText # Initialize boto3 clients cloudwatch = boto3.client('cloudwatch') +# ses_client = boto3.client('ses', region_name=REGION) # Configuration -current_date = datetime.now().strftime('%a %d %b %Y') +CURRENT_DATE = datetime.now().strftime('%a %d %b %Y') INSTANCE_ID = "i-029d2b17679dab982" -start_time = datetime(2024, 12, 4, 8, 0, 0) -end_time = datetime(2024, 12, 4, 17, 0, 0) +SERVER = "022" +#START_TIME = datetime(2024, 12, 4, 8, 0, 0) +#END_TIME = datetime(2024, 12, 4, 14, 0, 0) +END_TIME = datetime.utcnow() +START_TIME = END_TIME - timedelta(hours=9) SENDER = "donotreply@cjsm.secure-email.ppud.justice.gov.uk" RECIPIENTS = ["nick.buckingham@colt.net"] -SUBJECT = f'EC2 CPU Utilization Report - {current_date}' +SUBJECT = f'AWS EC2 CPU Utilization Report - {SERVER} - {CURRENT_DATE}' REGION = "eu-west-2" IMAGE_ID = "ami-02f8251c8cdf2464f" INSTANCE_TYPE = "m5.xlarge" +# SMTP Configuration +SMTP_SERVER = "10.27.9.39" +SMTP_PORT = 25 +MAIL_FROM = "donotreply@cjsm.secure-email.ppud.justice.gov.uk" +EMAIL_TO = ["nick.buckingham@colt.net"] + def get_metric_data(namespace, metric_name, dimensions): response = cloudwatch.get_metric_data( MetricDataQueries=[ @@ -40,8 +53,8 @@ def get_metric_data(namespace, metric_name, dimensions): 'ReturnData': True }, ], - StartTime=start_time, - EndTime=end_time + StartTime=START_TIME, + EndTime=END_TIME ) return response['MetricDataResults'][0] @@ -51,11 +64,11 @@ def create_graph(cpu_data, converttopdf_data, pdfcrawler2app_data, winword_data, plt.plot(converttopdf_data['Timestamps'], converttopdf_data['Values'], label='Convert to PDF CPU Utilization', marker="o", linestyle="--", color="royalblue") plt.plot(pdfcrawler2app_data['Timestamps'], pdfcrawler2app_data['Values'], label='PDF Crawler CPU Utilization', marker="o", linestyle="--", color="cyan") plt.plot(winword_data['Timestamps'], winword_data['Values'], label='Microsoft Word CPU Utilization', marker="o", linestyle="--", color="orange") - plt.plot(wmiprvse_data['Timestamps'], wmiprvse_data['Values'], label='WMIPrvSE CPU Utilization', marker="o", linestyle="--", color="red") - plt.plot(createthumbnails_data['Timestamps'], createthumbnails_data['Values'], label='Create Thumbnails CPU Utilization', marker="o", linestyle="--", color="springgreen") + plt.plot(wmiprvse_data['Timestamps'], wmiprvse_data['Values'], label='WmiPrvSE CPU Utilization', marker="o", linestyle="--", color="red") + plt.plot(createthumbnails_data['Timestamps'], createthumbnails_data['Values'], label='Create Thumbnails CPU Utilization', marker="o", linestyle="--", color="darkviolet") plt.xlabel('Time') plt.ylabel('CPU Utilization (%)') - plt.title(f'EC2 CPU Utilization - {INSTANCE_ID} - {current_date}') + plt.title(f'EC2 CPU Utilization - {SERVER} - {CURRENT_DATE}') plt.legend() plt.grid(True) plt.tight_layout() @@ -84,7 +97,7 @@ def email_image_to_users(graph_base64):

Hi Team,

-

Please find below the CPU utilization metrics for EC2 instance {INSTANCE_ID} for today from 08:00 to 17:00.

+

Please find below the CPU utilization metrics for EC2 instance {SERVER} for today from 08:00 to 17:00.

CPU Utilization Graph

This is an automated email.

@@ -100,25 +113,35 @@ def email_image_to_users(graph_base64): # Attach the HTML body msg.attach(MIMEText(email_body, "html")) - # Send the email + # Send the email with AWS SES + # try: + # response = ses_client.send_raw_email( + # Source=SENDER, + # Destinations=RECIPIENTS, + # RawMessage={"Data": msg.as_string()}, + # ) + # print("Email sent! Message ID:", response["MessageId"]) + # except Exception as e: + # print("Error sending email:", e) + # raise + + # Send the email with an EC2 Instance Mail Relay try: - response = ses_client.send_raw_email( - Source=SENDER, - Destinations=RECIPIENTS, - RawMessage={"Data": msg.as_string()}, - ) - print("Email sent! Message ID:", response["MessageId"]) + with smtplib.SMTP(SMTP_SERVER, SMTP_PORT) as server: + # server.starttls() + # server.login(SENDER, EMAIL_PASSWORD) + server.sendmail(SENDER, RECIPIENTS, msg.as_string()) + print("Email sent successfully.") except Exception as e: - print("Error sending email:", e) - raise - + print(f"Error sending email: {e}") + def lambda_handler(event, context): cpu_data = get_metric_data('AWS/EC2', 'CPUUtilization', [{'Name': 'InstanceId', 'Value': INSTANCE_ID}]) - converttopdf_data = get_metric_data('CWAgent', 'procstat cpu_usage', [{'Name': 'InstanceId', 'Value': INSTANCE_ID}, {'Name': 'process_name', 'Value': 'converttopdf.exe'}, {'Name': 'exe', 'Value': 'converttopdf'}, {'Name': 'ImageId', 'Value': IMAGE_ID}, {'Name': 'InstanceType', 'Value': INSTANCE_TYPE}]) - pdfcrawler2app_data = get_metric_data('CWAgent', 'procstat cpu_usage', [{'Name': 'InstanceId', 'Value': INSTANCE_ID}, {'Name': 'process_name', 'Value': 'pdfcrawler2app.exe'}, {'Name': 'exe', 'Value': 'pdfcrawler2app'}, {'Name': 'ImageId', 'Value': IMAGE_ID}, {'Name': 'InstanceType', 'Value': INSTANCE_TYPE}]) - winword_data = get_metric_data('CWAgent', 'procstat cpu_usage', [{'Name': 'InstanceId', 'Value': INSTANCE_ID}, {'Name': 'process_name', 'Value': 'winword.exe'}, {'Name': 'exe', 'Value': 'winword'}, {'Name': 'ImageId', 'Value': IMAGE_ID}, {'Name': 'InstanceType', 'Value': INSTANCE_TYPE}]) - wmiprvse_data = get_metric_data('CWAgent', 'procstat cpu_usage', [{'Name': 'InstanceId', 'Value': INSTANCE_ID}, {'Name': 'process_name', 'Value': 'wmiprcse.exe'}, {'Name': 'exe', 'Value': 'wmiprcse'}, {'Name': 'ImageId', 'Value': IMAGE_ID}, {'Name': 'InstanceType', 'Value': INSTANCE_TYPE}]) - createthumbnails_data = get_metric_data('CWAgent', 'procstat cpu_usage', [{'Name': 'InstanceId', 'Value': INSTANCE_ID}, {'Name': 'process_name', 'Value': 'createthumbnails.exe'}, {'Name': 'exe', 'Value': 'createthumbnails'}, {'Name': 'ImageId', 'Value': IMAGE_ID}, {'Name': 'InstanceType', 'Value': INSTANCE_TYPE}]) + converttopdf_data = get_metric_data('CWAgent', 'procstat cpu_usage', [{'Name': 'InstanceId', 'Value': INSTANCE_ID}, {'Name': 'process_name', 'Value': 'ConvertToPDF.exe'}, {'Name': 'exe', 'Value': 'ConvertToPDF'}, {'Name': 'ImageId', 'Value': IMAGE_ID}, {'Name': 'InstanceType', 'Value': INSTANCE_TYPE}]) + pdfcrawler2app_data = get_metric_data('CWAgent', 'procstat cpu_usage', [{'Name': 'InstanceId', 'Value': INSTANCE_ID}, {'Name': 'process_name', 'Value': 'PDFCrawler2App.exe'}, {'Name': 'exe', 'Value': 'PDFCrawler2App'}, {'Name': 'ImageId', 'Value': IMAGE_ID}, {'Name': 'InstanceType', 'Value': INSTANCE_TYPE}]) + winword_data = get_metric_data('CWAgent', 'procstat cpu_usage', [{'Name': 'InstanceId', 'Value': INSTANCE_ID}, {'Name': 'process_name', 'Value': 'WINWORD.exe'}, {'Name': 'exe', 'Value': 'WINWORD'}, {'Name': 'ImageId', 'Value': IMAGE_ID}, {'Name': 'InstanceType', 'Value': INSTANCE_TYPE}]) + wmiprvse_data = get_metric_data('CWAgent', 'procstat cpu_usage', [{'Name': 'InstanceId', 'Value': INSTANCE_ID}, {'Name': 'process_name', 'Value': 'WmiPrvSE.exe'}, {'Name': 'exe', 'Value': 'WmiPrvSE'}, {'Name': 'ImageId', 'Value': IMAGE_ID}, {'Name': 'InstanceType', 'Value': INSTANCE_TYPE}]) + createthumbnails_data = get_metric_data('CWAgent', 'procstat cpu_usage', [{'Name': 'InstanceId', 'Value': INSTANCE_ID}, {'Name': 'process_name', 'Value': 'CreateThumbnails.exe'}, {'Name': 'exe', 'Value': 'CreateThumbnails'}, {'Name': 'ImageId', 'Value': IMAGE_ID}, {'Name': 'InstanceType', 'Value': INSTANCE_TYPE}]) # Create a graph and encode it as base64 print("Creating graph...") @@ -131,5 +154,5 @@ def lambda_handler(event, context): return { 'statusCode': 200, - 'body': 'Graph uploaded to S3 successfully!' + 'body': 'Graph successfully emailed!' }