From 1a4b9274e84bf3c2fa83f7df4ad94215de291b34 Mon Sep 17 00:00:00 2001 From: Wesley Pettit Date: Mon, 12 Jun 2023 15:31:26 -0700 Subject: [PATCH 1/4] load-test: remove task def counter which is not used Signed-off-by: Wesley Pettit --- load_tests/load_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/load_tests/load_test.py b/load_tests/load_test.py index 047b60515..3b13936fa 100644 --- a/load_tests/load_test.py +++ b/load_tests/load_test.py @@ -80,10 +80,6 @@ def calculate_total_input_number(throughput): # 1. Configure task definition for each load test based on existing templates # 2. Register generated task definition def generate_task_definition(session, throughput, input_logger, s3_fluent_config_arn): - if not hasattr(generate_task_definition, "counter"): - generate_task_definition.counter = 0 # it doesn't exist yet, so initialize it - generate_task_definition.counter += 1 - # Generate configuration information for STD and TCP tests std_config = resource_resolver.get_input_configuration(PLATFORM, resource_resolver.STD_INPUT_PREFIX, throughput) custom_config = resource_resolver.get_input_configuration(PLATFORM, resource_resolver.CUSTOM_INPUT_PREFIX, throughput) From b463edcfb347bde9536357991db67870d8bb3d46 Mon Sep 17 00:00:00 2001 From: Wesley Pettit Date: Mon, 12 Jun 2023 15:41:31 -0700 Subject: [PATCH 2/4] load-test: make task def name unique across each test case Signed-off-by: Wesley Pettit --- load_tests/load_test.py | 2 +- load_tests/task_definitions/cloudwatch.json | 2 +- load_tests/task_definitions/firehose.json | 2 +- load_tests/task_definitions/kinesis.json | 2 +- load_tests/task_definitions/s3.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/load_tests/load_test.py b/load_tests/load_test.py index 3b13936fa..a273c6f4a 100644 --- a/load_tests/load_test.py +++ b/load_tests/load_test.py @@ -210,7 +210,7 @@ def run_ecs_tests(): response = client.run_task( cluster=ecs_cluster_name, launchType='EC2', - taskDefinition=f'{PREFIX}{OUTPUT_PLUGIN}-{throughput}' + taskDefinition=f'{PREFIX}{OUTPUT_PLUGIN}-{throughput}-{input_logger['name']}' ) names[f'{OUTPUT_PLUGIN}_{throughput}_task_arn'] = response['tasks'][0]['taskArn'] diff --git a/load_tests/task_definitions/cloudwatch.json b/load_tests/task_definitions/cloudwatch.json index 8108447e3..0a9648b4a 100644 --- a/load_tests/task_definitions/cloudwatch.json +++ b/load_tests/task_definitions/cloudwatch.json @@ -1,5 +1,5 @@ { - "family": "load-test-fluent-bit-cloudwatch-$THROUGHPUT", + "family": "load-test-fluent-bit-cloudwatch-$THROUGHPUT-$INPUT_NAME", "taskRoleArn": "$TASK_ROLE_ARN", "executionRoleArn": "$TASK_EXECUTION_ROLE_ARN", "networkMode": "bridge", diff --git a/load_tests/task_definitions/firehose.json b/load_tests/task_definitions/firehose.json index 3801e8ce1..33b1fb436 100644 --- a/load_tests/task_definitions/firehose.json +++ b/load_tests/task_definitions/firehose.json @@ -1,5 +1,5 @@ { - "family": "load-test-fluent-bit-firehose-$THROUGHPUT", + "family": "load-test-fluent-bit-firehose-$THROUGHPUT-$INPUT_NAME", "taskRoleArn": "$TASK_ROLE_ARN", "executionRoleArn": "$TASK_EXECUTION_ROLE_ARN", "networkMode": "bridge", diff --git a/load_tests/task_definitions/kinesis.json b/load_tests/task_definitions/kinesis.json index 0678e953b..5ce6f8e0f 100644 --- a/load_tests/task_definitions/kinesis.json +++ b/load_tests/task_definitions/kinesis.json @@ -1,5 +1,5 @@ { - "family": "load-test-fluent-bit-kinesis-$THROUGHPUT", + "family": "load-test-fluent-bit-kinesis-$THROUGHPUT-$INPUT_NAME", "taskRoleArn": "$TASK_ROLE_ARN", "executionRoleArn": "$TASK_EXECUTION_ROLE_ARN", "networkMode": "bridge", diff --git a/load_tests/task_definitions/s3.json b/load_tests/task_definitions/s3.json index 5cbe31a6c..a25b487c7 100644 --- a/load_tests/task_definitions/s3.json +++ b/load_tests/task_definitions/s3.json @@ -1,5 +1,5 @@ { - "family": "load-test-fluent-bit-s3-$THROUGHPUT", + "family": "load-test-fluent-bit-s3-$THROUGHPUT-$INPUT_NAME", "taskRoleArn": "$TASK_ROLE_ARN", "executionRoleArn": "$TASK_EXECUTION_ROLE_ARN", "networkMode": "bridge", From e190e3031111f86a2638ee4bb41340203673c534 Mon Sep 17 00:00:00 2001 From: Wesley Pettit Date: Mon, 12 Jun 2023 15:42:09 -0700 Subject: [PATCH 3/4] load-test: make parse_json_template always replace all vars Signed-off-by: Wesley Pettit --- load_tests/load_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/load_tests/load_test.py b/load_tests/load_test.py index a273c6f4a..46c3a660f 100644 --- a/load_tests/load_test.py +++ b/load_tests/load_test.py @@ -394,7 +394,7 @@ def parse_json_template(template, dict): for key in dict: if(key[0] == '$'): data = data.replace(key, dict[key]) - elif(key == OUTPUT_PLUGIN): + else: for sub_key in dict[key]: data = data.replace(sub_key, dict[key][sub_key]) return data From 3adfc45d1d5d7809a3821ffccca0e871d8b874da Mon Sep 17 00:00:00 2001 From: Wesley Pettit Date: Mon, 12 Jun 2023 15:46:31 -0700 Subject: [PATCH 4/4] load-test: refresh session for each waiter to prevent token expiration Signed-off-by: Wesley Pettit --- load_tests/load_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/load_tests/load_test.py b/load_tests/load_test.py index 46c3a660f..c16b84c0e 100644 --- a/load_tests/load_test.py +++ b/load_tests/load_test.py @@ -220,12 +220,12 @@ def run_ecs_tests(): # wait for tasks and validate for input_logger in INPUT_LOGGERS: # Wait until task stops and start validation - session = get_sts_boto_session() processes = [] - - client = session.client('ecs') - waiter = client.get_waiter('tasks_stopped') + for throughput in THROUGHPUT_LIST: + session = get_sts_boto_session() + client = session.client('ecs') + waiter = client.get_waiter('tasks_stopped') task_arn = names[f'{OUTPUT_PLUGIN}_{throughput}_task_arn'] waiter.wait( cluster=ecs_cluster_name,