Merge pull request #20 from awslabs/uploadAssetWorkflow

feat: Added uploadAssetWorkflow lambda function

README.md

@@ -1,6 +1,7 @@
 # Visual Asset Management System(VAMS) 
 
-![Logo](./web/logo_white.png)
+![Logo](./web/logo_dark.png#gh-light-mode-only)
+![Logo](./web/logo_white.png#gh-dark-mode-only)
 
 ![Build](https://github.com/awslabs/visual-asset-management-system/actions/workflows/ci.yml/badge.svg) 
 ## Introduction

backend/.flake8

@@ -1,5 +1,5 @@
 [flake8]
-max-line-length = 88
+max-line-length = 120
 extend-ignore = E203
 exclude =
     ./backend/common/*

backend/backend/functions/assets/upload_asset_workflow/lambda_handler.py

@@ -0,0 +1,32 @@
+import os
+from typing import Any, Dict
+from aws_lambda_powertools.utilities.typing import LambdaContext
+from aws_lambda_powertools.utilities.parser import parse, ValidationError
+from backend.functions.assets.upload_asset_workflow.request_handler import UploadAssetWorkflowRequestHandler
+from backend.logging.logger import safeLogger
+
+from backend.models.assets import UploadAssetWorkflowRequestModel
+from backend.models.common import APIGatewayProxyResponseV2, internal_error, success, validation_error
+import boto3
+
+logger = safeLogger(service_name="UploadAssetWorkflow")
+handler = UploadAssetWorkflowRequestHandler(
+    sfn_client=boto3.client('stepfunctions'),
+    state_machine_arn=os.environ["UPLOAD_WORKFLOW_ARN"]
+)
+
+
+def lambda_handler(event: Dict[Any, Any], context: LambdaContext) -> APIGatewayProxyResponseV2:
+    try:
+        request = parse(event['body'], model=UploadAssetWorkflowRequestModel)
+        logger.info(request)
+        response = handler.process_request(request=request)
+        return success(body=response.dict())
+    except ValidationError as v:
+        logger.exception("ValidationError")
+        return validation_error(body={
+            'message': str(v)
+        })
+    except Exception as e:
+        logger.exception("Exception")
+        return internal_error(body={'message': str(e)})

backend/backend/functions/assets/upload_asset_workflow/request_handler.py

@@ -0,0 +1,27 @@
+import json
+from backend.logging.logger import safeLogger
+from backend.models.assets import (
+    GetUploadAssetWorkflowStepFunctionInput,
+    UploadAssetWorkflowRequestModel,
+    UploadAssetWorkflowResponseModel,
+)
+from mypy_boto3_stepfunctions import Client
+
+logger = safeLogger(child=True)
+
+
+class UploadAssetWorkflowRequestHandler:
+
+    def __init__(self, sfn_client: Client, state_machine_arn: str) -> None:
+        self.sfn_client = sfn_client
+        self.stat_machine_arn = state_machine_arn
+
+    def process_request(self, request: UploadAssetWorkflowRequestModel) -> UploadAssetWorkflowResponseModel:
+        stepfunction_request = GetUploadAssetWorkflowStepFunctionInput(request)
+        sfn_response = self.sfn_client.start_execution(
+            stateMachineArn=self.stat_machine_arn,
+            input=json.dumps(stepfunction_request.dict())
+        )
+        logger.info(f"Started uploadAssetWorkflow: Execution Id: {sfn_response['executionArn']}")
+        response: UploadAssetWorkflowResponseModel = UploadAssetWorkflowResponseModel(message='Success')
+        return response

backend/backend/logging/logger.py

@@ -0,0 +1,34 @@
+from aws_lambda_powertools import Logger
+from aws_lambda_powertools.logging.formatter import LambdaPowertoolsFormatter
+
+location_format = "[%(funcName)s] %(module)s"
+date_format = "%m/%d/%Y %I:%M:%S %p"
+
+
+def mask_sensitive_data(event):
+    # remove sensitive data from request object before logging
+    keys_to_redact = ["authorization"]
+    result = {}
+    for k, v in event.items():
+        if isinstance(v, dict):
+            result[k] = mask_sensitive_data(v)
+        elif k in keys_to_redact:
+            result[k] = "<redacted>"
+        else:
+            result[k] = v
+    return result
+
+
+class CustomFormatter(LambdaPowertoolsFormatter):
+    def serialize(self, log: dict) -> str:
+        """Serialize final structured log dict to JSON str"""
+        log = mask_sensitive_data(event=log)  # rename message key to event
+        return self.json_serializer(log)  # use configured json serializer
+
+
+def safeLogger(**kwargs):
+    return Logger(
+        logger_formatter=CustomFormatter(),
+        location=location_format,
+        datefmt=date_format,
+        **kwargs)

backend/backend/models/assets.py

@@ -0,0 +1,52 @@
+from pydantic import BaseModel, Json
+from aws_lambda_powertools.utilities.parser.models import (
+    APIGatewayProxyEventV2Model
+)
+
+
+class AssetPreviewLocationModel(BaseModel):
+    Bucket: str
+    Key: str
+
+
+class UploadAssetModel(BaseModel):
+    databaseId: str
+    assetId: str
+    bucket: str
+    key: str
+    assetType: str
+    description: str
+    isDistributable: bool
+    Comment: str
+    previewLocation: AssetPreviewLocationModel
+    specifiedPipelines: list[str]
+
+
+class UploadAssetWorkflowRequestModel(BaseModel):
+    uploadAssetBody: UploadAssetModel
+
+
+class UploadAssetWorkflowResponseModel(BaseModel):
+    message: str
+
+
+class UploadAssetWorkflowRequest(APIGatewayProxyEventV2Model):
+    body: Json[UploadAssetWorkflowRequestModel]  # type: ignore[assignment]
+
+
+class UploadAssetStepFunctionRequest(BaseModel):
+    body: UploadAssetModel
+
+
+class UploadAssetWorkflowStepFunctionInput(BaseModel):
+    uploadAssetBody: UploadAssetStepFunctionRequest
+
+
+def GetUploadAssetWorkflowStepFunctionInput(
+        uploadAssetWorkflowRequestModel: UploadAssetWorkflowRequestModel
+) -> UploadAssetWorkflowStepFunctionInput:
+    return UploadAssetWorkflowStepFunctionInput(
+        uploadAssetBody=UploadAssetStepFunctionRequest(
+            body=uploadAssetWorkflowRequestModel.uploadAssetBody
+        )
+    )

backend/backend/models/common.py

@@ -0,0 +1,43 @@
+import json
+from typing import Any, Dict, TypedDict
+
+
+class APIGatewayProxyResponseV2(TypedDict):
+    isBase64Encoded: bool
+    statusCode: int
+    headers: Dict[str, str]
+    body: str
+
+
+def commonHeaders() -> Dict[str, str]:
+    return {
+        'Content-Type': 'application/json',
+        'Access-Control-Allow-Origin': '*'
+    }
+
+
+def success(status_code: int = 200, body: Any = {'message': 'Success'}) -> APIGatewayProxyResponseV2:
+    return APIGatewayProxyResponseV2(
+        isBase64Encoded=False,
+        statusCode=status_code,
+        headers=commonHeaders(),
+        body=json.dumps(body)
+    )
+
+
+def validation_error(status_code: int = 422, body: dict = {'message': 'Validation Error'}) -> APIGatewayProxyResponseV2:
+    return APIGatewayProxyResponseV2(
+        isBase64Encoded=False,
+        statusCode=status_code,
+        headers=commonHeaders(),
+        body=json.dumps(body)
+    )
+
+
+def internal_error(status_code: int = 500, body: Any = {'message': 'Validation Error'}) -> APIGatewayProxyResponseV2:
+    return APIGatewayProxyResponseV2(
+        isBase64Encoded=False,
+        statusCode=status_code,
+        headers=commonHeaders(),
+        body=json.dumps(body)
+    )

backend/pyproject.toml

@@ -10,13 +10,17 @@ requests = "^2.28.1"
 stepfunctions = "^2.3.0"
 protobuf = "3.20.*"
 aws-lambda-powertools = "^2.7.1"
+pydantic = "^1.10.4"
+boto3-stubs = {extras = ["stepfunctions"], version = "^1.26.69"}
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.2"
 boto3 = "^1.24.45"
 flake8 = "^6.0.0"
 coverage = "^7.1.0"
 mypy = "^1.0.0"
+pytest-mock = "^3.10.0"
+moto = "^4.1.2"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

backend/requirements.txt

@@ -1,31 +1,40 @@
-attrs==22.1.0; python_version >= "3.5"
-boto3==1.24.45; python_version >= "3.7"
-botocore==1.27.45; python_version >= "3.7"
-certifi==2022.6.15; python_version >= "3.7" and python_version < "4"
-charset-normalizer==2.1.0; python_version >= "3.7" and python_version < "4" and python_full_version >= "3.6.0"
-dill==0.3.5.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.7.0"
-google-pasta==0.2.0
-idna==3.3; python_version >= "3.7" and python_version < "4"
-importlib-metadata==4.12.0; python_version >= "3.7"
+attrs==22.1.0; python_version >= "3.6"
+aws-lambda-powertools==2.7.1; python_full_version >= "3.7.4" and python_full_version < "4.0.0"
+boto3-stubs==1.26.69; python_version >= "3.7"
+boto3==1.25.5; python_version >= "3.7"
+botocore-stubs==1.29.69; python_version >= "3.7" and python_version < "4.0"
+botocore==1.28.5; python_version >= "3.7"
+certifi==2022.12.7; python_version >= "3.7" and python_version < "4"
+charset-normalizer==2.1.1; python_version >= "3.7" and python_version < "4" and python_full_version >= "3.6.0"
+contextlib2==21.6.0; python_version >= "3.6"
+dill==0.3.6; python_version >= "3.7"
+google-pasta==0.2.0; python_version >= "3.6"
+idna==3.4; python_version >= "3.7" and python_version < "4"
+importlib-metadata==4.13.0; python_version >= "3.7"
 jmespath==1.0.1; python_version >= "3.7"
-multiprocess==0.70.13; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.7.0"
-numpy==1.23.1
-packaging==21.3; python_version >= "3.6"
-pandas==1.4.3; python_version >= "3.8"
-pathos==0.2.9; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.7.0"
-pox==0.3.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.7.0"
-ppft==1.7.6.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.7.0"
-protobuf3-to-dict==0.1.5
-protobuf==3.20.1; python_version >= "3.7"
-pyparsing==3.0.9; python_full_version >= "3.6.8" and python_version >= "3.6"
+multiprocess==0.70.14; python_version >= "3.7"
+mypy-boto3-stepfunctions==1.26.21; python_version >= "3.7"
+numpy==1.23.4
+packaging==23.0; python_version >= "3.7"
+pandas==1.5.1; python_version >= "3.8"
+pathos==0.3.0; python_version >= "3.7"
+pox==0.3.2; python_version >= "3.7"
+ppft==1.7.6.6; python_version >= "3.7"
+protobuf3-to-dict==0.1.5; python_version >= "3.6"
+protobuf==3.20.3; python_version >= "3.7"
+pydantic==1.10.4; python_version >= "3.7"
 python-dateutil==2.8.2; python_version >= "3.8" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.8"
-pytz==2022.1; python_version >= "3.8"
+pytz==2022.5; python_version >= "3.8"
 pyyaml==6.0; python_version >= "3.6"
 requests==2.28.1; python_version >= "3.7" and python_version < "4"
 s3transfer==0.6.0; python_version >= "3.7"
-sagemaker==2.75.1
-six==1.16.0; python_version >= "3.8" and python_full_version < "3.0.0" or python_full_version >= "3.7.0" and python_version >= "3.8"
-smdebug-rulesconfig==1.0.1; python_version >= "2.7"
+sagemaker==2.116.0; python_version >= "3.6"
+schema==0.7.5; python_version >= "3.6"
+six==1.16.0; python_version >= "3.8" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.8"
+smdebug-rulesconfig==1.0.1; python_version >= "3.6"
 stepfunctions==2.3.0
-urllib3==1.26.11; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7"
-zipp==3.8.1; python_version >= "3.7"
+types-awscrt==0.16.10; python_version >= "3.7" and python_version < "4.0"
+types-s3transfer==0.6.0.post5; python_version >= "3.7" and python_version < "4.0"
+typing-extensions==4.4.0; python_full_version >= "3.7.4" and python_full_version < "4.0.0" and python_version >= "3.7"
+urllib3==1.26.12; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7"
+zipp==3.10.0; python_version >= "3.7"

backend/tests/functions/assets/upload_asset_workflow/test_lambda_handler.py

@@ -0,0 +1,79 @@
+import json
+from unittest.mock import patch
+from backend.functions.assets.upload_asset_workflow.request_handler import UploadAssetWorkflowRequestHandler
+from backend.models.assets import (
+    AssetPreviewLocationModel, UploadAssetModel, UploadAssetWorkflowRequestModel, UploadAssetWorkflowResponseModel
+)
+from moto import mock_stepfunctions
+import pytest
+
+
+@pytest.fixture()
+def sample_request():
+    event = {'body': {}}
+    requst = json.dumps(UploadAssetWorkflowRequestModel(
+            uploadAssetBody=UploadAssetModel(
+                databaseId='1',
+                assetId='test',
+                bucket='test_bucket',
+                key='test_file',
+                assetType='step',
+                description='Testing',
+                isDistributable=False,
+                specifiedPipelines=[],
+                Comment='Testing',
+                previewLocation=AssetPreviewLocationModel(
+                    Bucket='test_bucket',
+                    Key='test_preview_key'
+                )
+            )
+        ).dict()
+    )
+    event['body'] = requst
+    return event
+
+
+def mock_process_request(self, request):
+    return UploadAssetWorkflowResponseModel(message='Success')
+
+
+def mock_process_request_returns_error(self, request):
+    raise Exception('StepFunction')
+
+
+@patch.object(UploadAssetWorkflowRequestHandler, "process_request", mock_process_request)
+@mock_stepfunctions
+def test_request_handler_success(sample_request, monkeypatch):
+    monkeypatch.setenv("UPLOAD_WORKFLOW_ARN", "TestArn")
+    monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
+    from backend.functions.assets.upload_asset_workflow.lambda_handler import lambda_handler
+    response = lambda_handler(sample_request, None)
+    assert response['statusCode'] == 200
+
+
+@patch.object(UploadAssetWorkflowRequestHandler, "process_request", mock_process_request_returns_error)
+@mock_stepfunctions
+def test_request_handler_500(sample_request, monkeypatch):
+    monkeypatch.setenv("UPLOAD_WORKFLOW_ARN", "TestArn")
+    monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
+    from backend.functions.assets.upload_asset_workflow.lambda_handler import lambda_handler
+    response = lambda_handler(sample_request, None)
+    assert response['statusCode'] == 500
+
+
+@mock_stepfunctions
+def test_request_handler_validation_error(monkeypatch):
+    monkeypatch.setenv("UPLOAD_WORKFLOW_ARN", "TestArn")
+    monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
+    from backend.functions.assets.upload_asset_workflow.lambda_handler import lambda_handler
+    response = lambda_handler({'body': {}}, None)
+    assert response['statusCode'] == 422
+
+
+@mock_stepfunctions
+def test_request_handler_exception(monkeypatch, sample_request):
+    monkeypatch.setenv("UPLOAD_WORKFLOW_ARN", "TestArn")
+    monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
+    from backend.functions.assets.upload_asset_workflow.lambda_handler import lambda_handler
+    response = lambda_handler(sample_request, None)
+    assert response['statusCode'] == 500