From 80ee274a8cc8d64984bc267e04eea54519c6f0d7 Mon Sep 17 00:00:00 2001 From: Stewart Wallace Date: Tue, 19 Oct 2021 17:22:33 +0100 Subject: [PATCH] Consistent retries, fixing tests --- Makefile | 6 +++ src/account_processing.yml | 48 +++++++++---------- .../account_processing/create_account.py | 4 +- .../process_account_files.py | 9 ++-- .../register_account_for_support.py | 9 ++-- .../account_processing/requirements.txt | 4 +- .../tests/test_account_alias.py | 5 +- .../tests/test_account_creation.py | 3 ++ tox.ini | 1 + 9 files changed, 48 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index fdf4d728e..a666fef77 100644 --- a/Makefile +++ b/Makefile @@ -13,3 +13,9 @@ test: lint: # Linter performs static analysis to catch latent bugs find src/ -iname "*.py" -not -path ".aws-sam/*" | xargs pylint --rcfile .pylintrc + +build: + sam build -u + +deploy: + sam deploy --capabilities CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND diff --git a/src/account_processing.yml b/src/account_processing.yml index de31c6cca..db8029cd3 100644 --- a/src/account_processing.yml +++ b/src/account_processing.yml @@ -474,10 +474,10 @@ Resources: "Type": "Task", "Resource": "${AccountAliasConfigFunction.Arn}", "Retry": [{ - "ErrorEquals": ["RetryError"], - "IntervalSeconds": 10, - "BackoffRate": 1.0, - "MaxAttempts": 20 + "ErrorEquals": ["States.TaskFailed"], + "IntervalSeconds": 3, + "BackoffRate": 1.5, + "MaxAttempts": 30 }], "Next": "ConfigureAccountTags" }, @@ -485,10 +485,10 @@ Resources: "Type": "Task", "Resource": "${CreateAccountFunction.Arn}", "Retry": [{ - "ErrorEquals": ["RetryError"], - "IntervalSeconds": 10, - "BackoffRate": 1.0, - "MaxAttempts": 20 + "ErrorEquals": ["States.TaskFailed"], + "IntervalSeconds": 3, + "BackoffRate": 1.5, + "MaxAttempts": 30 }], "Next": "WaitFor10Seconds" }, @@ -501,10 +501,10 @@ Resources: "Type": "Task", "Resource": "${RegisterAccountForSupportFunction.Arn}", "Retry": [{ - "ErrorEquals": ["RetryError"], - "IntervalSeconds": 10, - "BackoffRate": 1.0, - "MaxAttempts": 20 + "ErrorEquals": ["States.TaskFailed"], + "IntervalSeconds": 3, + "BackoffRate": 1.5, + "MaxAttempts": 30 }], "Next": "ConfigureAccountAlias" }, @@ -512,10 +512,10 @@ Resources: "Type": "Task", "Resource": "${AccountTagConfigFunction.Arn}", "Retry": [{ - "ErrorEquals": ["RetryError"], - "IntervalSeconds": 10, - "BackoffRate": 1.0, - "MaxAttempts": 20 + "ErrorEquals": ["States.TaskFailed"], + "IntervalSeconds": 3, + "BackoffRate": 1.5, + "MaxAttempts": 30 }], "Next": "ConfigureAccountOU" }, @@ -523,10 +523,10 @@ Resources: "Type": "Task", "Resource": "${AccountOUConfigFunction.Arn}", "Retry": [{ - "ErrorEquals": ["RetryError"], - "IntervalSeconds": 10, - "BackoffRate": 1.0, - "MaxAttempts": 20 + "ErrorEquals": ["States.TaskFailed"], + "IntervalSeconds": 3, + "BackoffRate": 1.5, + "MaxAttempts": 30 }], "Next": "DeleteDefaultVPCChoice" }, @@ -545,10 +545,10 @@ Resources: "Type": "Task", "Resource": "${GetAccountRegionsFunction.Arn}", "Retry": [{ - "ErrorEquals": ["RetryError"], - "IntervalSeconds": 10, - "BackoffRate": 1.0, - "MaxAttempts": 20 + "ErrorEquals": ["States.TaskFailed"], + "IntervalSeconds": 3, + "BackoffRate": 1.5, + "MaxAttempts": 30 }], "Next": "DeleteDefaultVPCMap" }, diff --git a/src/lambda_codebase/account_processing/create_account.py b/src/lambda_codebase/account_processing/create_account.py index 1bc021e3c..e307f9280 100644 --- a/src/lambda_codebase/account_processing/create_account.py +++ b/src/lambda_codebase/account_processing/create_account.py @@ -7,10 +7,10 @@ import os from aws_xray_sdk.core import patch_all -patch_all() -ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME") import boto3 +patch_all() +ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME") def create_account(account, adf_role_name, org_client): allow_billing = "ALLOW" if account.get("allow_billing", False) else "DENY" diff --git a/src/lambda_codebase/account_processing/process_account_files.py b/src/lambda_codebase/account_processing/process_account_files.py index 94544616a..ab4558ef6 100644 --- a/src/lambda_codebase/account_processing/process_account_files.py +++ b/src/lambda_codebase/account_processing/process_account_files.py @@ -10,16 +10,13 @@ import json import os from typing import Tuple -from aws_xray_sdk.core.patcher import patch import yaml -from aws_xray_sdk.core import xray_recorder, patch_all - -patch_all() - +import boto3 +from aws_xray_sdk.core import patch_all from organizations import Organizations -import boto3 +patch_all() ACCOUNT_MANAGEMENT_STATEMACHINE = os.getenv("ACCOUNT_MANAGEMENT_STATEMACHINE_ARN") diff --git a/src/lambda_codebase/account_processing/register_account_for_support.py b/src/lambda_codebase/account_processing/register_account_for_support.py index ffcdf2ce5..263007b3a 100644 --- a/src/lambda_codebase/account_processing/register_account_for_support.py +++ b/src/lambda_codebase/account_processing/register_account_for_support.py @@ -8,17 +8,14 @@ """ from enum import Enum -from logger import configure_logger -from aws_xray_sdk.core import patch_all -patch_all() - import boto3 from botocore.exceptions import ClientError, BotoCoreError from botocore.config import Config - +from logger import configure_logger +from aws_xray_sdk.core import patch_all LOGGER = configure_logger(__name__) - +patch_all() class SupportLevel(Enum): BASIC = "basic" diff --git a/src/lambda_codebase/account_processing/requirements.txt b/src/lambda_codebase/account_processing/requirements.txt index 97f04e731..0230d520d 100644 --- a/src/lambda_codebase/account_processing/requirements.txt +++ b/src/lambda_codebase/account_processing/requirements.txt @@ -1,3 +1,3 @@ -pyyaml +pyyaml>=5.4.1 wrapt==1.12 -aws-xray-sdk \ No newline at end of file +aws-xray-sdk==2.8.0 \ No newline at end of file diff --git a/src/lambda_codebase/account_processing/tests/test_account_alias.py b/src/lambda_codebase/account_processing/tests/test_account_alias.py index e8ec67737..347683e0d 100644 --- a/src/lambda_codebase/account_processing/tests/test_account_alias.py +++ b/src/lambda_codebase/account_processing/tests/test_account_alias.py @@ -4,11 +4,14 @@ import boto3 from botocore.stub import Stubber - +from aws_xray_sdk import global_sdk_config from ..configure_account_alias import create_account_alias +global_sdk_config.set_sdk_enabled(False) + # pylint: disable=W0106 def test_account_alias(): + test_account = {"Id": 1234567890, "alias": "MyCoolAlias"} iam_client = boto3.client("iam") stubber = Stubber(iam_client) diff --git a/src/lambda_codebase/account_processing/tests/test_account_creation.py b/src/lambda_codebase/account_processing/tests/test_account_creation.py index 3261d7bb4..0871c62da 100644 --- a/src/lambda_codebase/account_processing/tests/test_account_creation.py +++ b/src/lambda_codebase/account_processing/tests/test_account_creation.py @@ -5,8 +5,11 @@ import unittest import boto3 from botocore.stub import Stubber +from aws_xray_sdk import global_sdk_config + from ..create_account import create_account +global_sdk_config.set_sdk_enabled(False) # pylint: disable=W0106 class SuccessTestCase(unittest.TestCase): diff --git a/tox.ini b/tox.ini index a40296ac6..7996b8d16 100644 --- a/tox.ini +++ b/tox.ini @@ -31,6 +31,7 @@ deps = -r{toxinidir}/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt -r{toxinidir}/src/lambda_codebase/initial_commit/requirements.txt -r{toxinidir}/src/lambda_codebase/account/requirements.txt + -r{toxinidir}/src/lambda_codebase/account_processing/requirements.txt commands = make test make lint