From dd96c42cf62a3858ad4e49ad1f906f1f28c9f208 Mon Sep 17 00:00:00 2001 From: Mitch Dawson Date: Wed, 11 Oct 2023 21:17:03 +0100 Subject: [PATCH] :white_check_mark: Initial Changes --- .../data-platform/tests/run_smoke_tests.py | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/terraform/environments/data-platform/tests/run_smoke_tests.py b/terraform/environments/data-platform/tests/run_smoke_tests.py index fef13c42c96..bfbd9fb37ce 100644 --- a/terraform/environments/data-platform/tests/run_smoke_tests.py +++ b/terraform/environments/data-platform/tests/run_smoke_tests.py @@ -1,13 +1,18 @@ import base64 import hashlib -import requests import json -import boto3 +import requests +import sys import time +import boto3 + file_path = "test_data.csv" -database = "test_product5" -table = "testing" +data_product_name = "example_prison_data_product" +table_name = "testing" +base_url = "https://hsolkci589.execute-api.eu-west-2.amazonaws.com/development/" +presigned_url = f"data-product/{data_product_name}/table/{table_name}/upload" +url = base_url + presigned_url glue = boto3.client("glue") @@ -24,12 +29,24 @@ def md5_hash_file_contents(file) -> str: file_md5_hash = md5_hash_file_contents(file_path) +body = { + "filename": "testdata.csv", + "contentMD5": file_md5_hash, +} + # Get presigned url -response = requests.get( - url="https://hsolkci589.execute-api.eu-west-2.amazonaws.com/development/upload_data", - params={"database": database, "table": table, "contentMD5": file_md5_hash}, +response = requests.post( + url=base_url + presigned_url, + json=body, headers={"authorizationToken": "placeholder"}, ) + +if response.status_code != 200: + print(f"get presigned url error status code: {response.status_code}") + print(f"get presigned url error response: {response.text}") + print("Exiting...") + sys.exit(1) + response_json = json.loads(response.text) post_policy_form_data = response_json["URL"]["fields"] multipart_form_data = { @@ -43,19 +60,19 @@ def md5_hash_file_contents(file) -> str: upload_response = requests.post(response_json["URL"]["url"], files=multipart_form_data) print(upload_response.status_code, upload_response.text) -print(f"Waiting for {database}.{table} to create in athena") +print(f"Waiting for {data_product_name}.{table_name} to create in athena") time.sleep(10) # Check for created table try: - glue.get_table(DatabaseName=database, Name=table) - print(f"{database}.{table} found in glue") + glue.get_table(data_product_nameName=data_product_name, Name=table_name) + print(f"{data_product_name}.{table_name} found in glue") except Exception as e: raise e # Clean up created table try: - glue.delete_table(DatabaseName=database, Name=table) - print(f"{database}.{table} deleted from glue") + glue.delete_table(data_product_nameName=data_product_name, Name=table_name) + print(f"{data_product_name}.{table_name} deleted from glue") except Exception as e: raise e