Merge remote-tracking branch 'origin/main'

CHANGELOG.md

@@ -15,6 +15,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### **Changed**
 - fixed model deploy cross-account permissions
 - added bucket and model package group names as stack outputs in the `sagemaker-templates` module
+- refactor inputs for `mlflow-fargate` and `mlflow-image` 
+- refactor inputs for `sagemaker-studio` 
+- rename seedfarmer project name to `aiops`
+- chore: adding some missing auto_delete attributes
+- chore: Add `auto_delete` to `mlflow-fargate` elb access logs bucket
 
 ## v1.1.0
 

data/mwaa/requirements/constraint-requirements.txt

@@ -34,7 +34,7 @@ Authlib==1.3.0
 Babel==2.14.0
 ConfigUpdater==3.2
 Deprecated==1.2.14
-Flask-AppBuilder==4.3.10
+Flask-AppBuilder==4.3.11
 Flask-Babel==2.0.0
 Flask-Bcrypt==1.0.1
 Flask-Caching==2.1.0
@@ -48,7 +48,7 @@ Flask==2.2.5
 GitPython==3.1.41
 JPype1==1.5.0
 JayDeBeApi==1.2.3
-Jinja2==3.1.3
+Jinja2==3.1.4
 Js2Py==0.74
 Mako==1.3.0
 Markdown==3.5.2
@@ -65,7 +65,7 @@ SQLAlchemy==1.4.51
 SecretStorage==3.3.3
 Sphinx==5.3.0
 WTForms==3.1.2
-Werkzeug==2.2.3
+Werkzeug==3.0.3
 adal==1.2.7
 adlfs==2023.12.0
 aiobotocore==2.11.2
@@ -659,7 +659,7 @@ tomli_w==1.0.0
 tomlkit==0.12.3
 tornado==6.4
 towncrier==23.11.0
-tqdm==4.66.1
+tqdm==4.66.3
 traitlets==5.14.1
 trino==0.327.0
 trove-classifiers==2024.1.8

data/mwaa/requirements/requirements.txt

@@ -14,9 +14,9 @@ pathos==0.3.2
 pox==0.3.4 
 ppft==1.7.6.8
 s3fs==2023.12.2 
-sagemaker==2.214.3 
+sagemaker==2.218.0 
 schema==0.7.5 
 scipy==1.13.0
 smdebug-rulesconfig==1.0.1 
 tblib==3.0.0 
-tqdm==4.66.1
+tqdm==4.66.3

manifests/mlflow-experiments-tracking/storage-modules.yaml

@@ -1,5 +1,5 @@
 name: ecr-mlflow
-path: git::https://github.com/awslabs/idf-modules.git//modules/storage/ecr?ref=release/1.3.0&depth=1
+path: git::https://github.com/awslabs/idf-modules.git//modules/storage/ecr?ref=release/1.7.0&depth=1
 targetAccount: primary
 parameters:
   - name: image-tag-mutability
@@ -12,7 +12,7 @@ parameters:
   - name: encryption-type
     value: SSE
   - name: retention-type
-    value: RETAIN
+    value: DESTROY
 ---
 name: mlflow-mysql
 path: git::https://github.com/awslabs/idf-modules.git//modules/database/rds?ref=release/1.4.0&depth=1

manifests/mlops-sagemaker-multiacc/storage-modules.yaml

@@ -12,4 +12,4 @@ parameters:
   - name: encryption-type
     value: SSE
   - name: retention-type
-    value: RETAIN
+    value: DESTROY

manifests/mlops-sagemaker/storage-modules.yaml

@@ -12,4 +12,4 @@ parameters:
   - name: encryption-type
     value: SSE
   - name: retention-type
-    value: RETAIN
+    value: DESTROY

manifests/storage-modules.yaml

@@ -19,7 +19,7 @@ parameters:
   - name: encryption-type
     value: SSE
   - name: retention-type
-    value: RETAIN
+    value: DESTROY
 ---
 name: mlflow-mysql
 path: git::https://github.com/awslabs/idf-modules.git//modules/database/rds?ref=release/1.7.0&depth=1

modules/mlflow/mlflow-fargate/app.py

@@ -1,108 +1,33 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
-
-import json
-import os
-from typing import Optional, cast
-
 import aws_cdk
 import cdk_nag
 
-from stack import MlflowFargateStack, RDSSettings
-
-
-def _param(name: str) -> str:
-    return f"SEEDFARMER_PARAMETER_{name}"
-
-
-project_name = os.getenv("SEEDFARMER_PROJECT_NAME", "")
-deployment_name = os.getenv("SEEDFARMER_DEPLOYMENT_NAME", "")
-module_name = os.getenv("SEEDFARMER_MODULE_NAME", "")
-app_prefix = f"{project_name}-{deployment_name}-{module_name}"
-
-DEFAULT_ECS_CLUSTER_NAME = None
-DEFAULT_SERVICE_NAME = None
-DEFAULT_TASK_CPU_UNITS = 4 * 1024
-DEFAULT_TASK_MEMORY_LIMIT_MB = 8 * 1024
-DEFAULT_AUTOSCALE_MAX_CAPACITY = 2
-DEFAULT_LB_ACCESS_LOGS_BUCKET_NAME = None
-DEFAULT_LB_ACCESS_LOGS_BUCKET_PREFIX = None
-DEFAULT_EFS_REMOVAL_POLICY = "RETAIN"
-
-environment = aws_cdk.Environment(
-    account=os.environ["CDK_DEFAULT_ACCOUNT"],
-    region=os.environ["CDK_DEFAULT_REGION"],
-)
-
-vpc_id = os.getenv(_param("VPC_ID"))
-subnet_ids = json.loads(os.getenv(_param("SUBNET_IDS"), "[]"))
-ecs_cluster_name = os.getenv(_param("ECS_CLUSTER_NAME"), DEFAULT_ECS_CLUSTER_NAME)
-service_name = os.getenv(_param("SERVICE_NAME"), DEFAULT_SERVICE_NAME)
-ecr_repo_name = os.getenv(_param("ECR_REPOSITORY_NAME"))
-task_cpu_units = os.getenv(_param("TASK_CPU_UNITS"), DEFAULT_TASK_CPU_UNITS)
-task_memory_limit_mb = os.getenv(_param("TASK_MEMORY_LIMIT_MB"), DEFAULT_TASK_MEMORY_LIMIT_MB)
-autoscale_max_capacity = os.getenv(_param("AUTOSCALE_MAX_CAPACITY"), DEFAULT_AUTOSCALE_MAX_CAPACITY)
-artifacts_bucket_name = os.getenv(_param("ARTIFACTS_BUCKET_NAME"))
-lb_access_logs_bucket_name = os.getenv(_param("LB_ACCESS_LOGS_BUCKET_NAME"), DEFAULT_LB_ACCESS_LOGS_BUCKET_NAME)
-lb_access_logs_bucket_prefix = os.getenv(_param("LB_ACCESS_LOGS_BUCKET_PREFIX"), DEFAULT_LB_ACCESS_LOGS_BUCKET_PREFIX)
-efs_removal_policy = os.getenv(_param("EFS_REMOVAL_POLICY"), DEFAULT_EFS_REMOVAL_POLICY)
-
-rds_hostname = os.getenv(_param("RDS_HOSTNAME"))
-rds_port = os.getenv(_param("RDS_PORT"))
-rds_security_group_id = os.getenv(_param("RDS_SECURITY_GROUP_ID"))
-rds_credentials_secret_arn = os.getenv(_param("RDS_CREDENTIALS_SECRET_ARN"))
-
-
-if not vpc_id:
-    raise ValueError("Missing input parameter vpc-id")
-
-if not ecr_repo_name:
-    raise ValueError("Missing input parameter ecr-repository-name")
-
-if not artifacts_bucket_name:
-    raise ValueError("Missing input parameter artifacts-bucket-name")
-
-
-rds_settings: Optional[RDSSettings]
-if all([rds_hostname, rds_port, rds_security_group_id, rds_credentials_secret_arn]):
-    rds_settings = RDSSettings(
-        hostname=cast(str, rds_hostname),
-        port=int(cast(str, rds_port)),
-        security_group_id=cast(str, rds_security_group_id),
-        credentials_secret_arn=cast(str, rds_credentials_secret_arn),
-    )
-elif not any([rds_hostname, rds_port, rds_security_group_id, rds_credentials_secret_arn]):
-    rds_settings = None
-else:
-    raise ValueError(
-        "Invalid combination of input parameters rds-hostname, rds-port, rds-security-group-id, "
-        "and rds-credentials-secret-arn. "
-        "They must either all be specified or none of them."
-    )
-
+from settings import ApplicationSettings
+from stack import MlflowFargateStack
 
+app_settings = ApplicationSettings()
 app = aws_cdk.App()
 
 stack = MlflowFargateStack(
     scope=app,
-    id=app_prefix,
-    app_prefix=app_prefix,
-    vpc_id=vpc_id,
-    subnet_ids=subnet_ids,
-    ecs_cluster_name=ecs_cluster_name,
-    service_name=service_name,
-    ecr_repo_name=ecr_repo_name,
-    task_cpu_units=int(task_cpu_units),
-    task_memory_limit_mb=int(task_memory_limit_mb),
-    autoscale_max_capacity=int(autoscale_max_capacity),
-    artifacts_bucket_name=artifacts_bucket_name,
-    lb_access_logs_bucket_name=lb_access_logs_bucket_name,
-    lb_access_logs_bucket_prefix=lb_access_logs_bucket_prefix,
-    efs_removal_policy=efs_removal_policy,
-    rds_settings=rds_settings,
+    id=app_settings.settings.app_prefix,
+    vpc_id=app_settings.parameters.vpc_id,
+    subnet_ids=app_settings.parameters.subnet_ids,
+    ecs_cluster_name=app_settings.parameters.ecs_cluster_name,
+    service_name=app_settings.parameters.service_name,
+    ecr_repo_name=app_settings.parameters.ecr_repository_name,
+    task_cpu_units=app_settings.parameters.task_cpu_units,
+    task_memory_limit_mb=app_settings.parameters.task_memory_limit_mb,
+    autoscale_max_capacity=app_settings.parameters.autoscale_max_capacity,
+    artifacts_bucket_name=app_settings.parameters.artifacts_bucket_name,
+    lb_access_logs_bucket_name=app_settings.parameters.lb_access_logs_bucket_name,
+    lb_access_logs_bucket_prefix=app_settings.parameters.lb_access_logs_bucket_prefix,
+    efs_removal_policy=app_settings.parameters.efs_removal_policy,
+    rds_settings=app_settings.parameters.rds_settings,
     env=aws_cdk.Environment(
-        account=os.environ["CDK_DEFAULT_ACCOUNT"],
-        region=os.environ["CDK_DEFAULT_REGION"],
+        account=app_settings.default.account,
+        region=app_settings.default.region,
     ),
 )
 
@@ -123,4 +48,12 @@ def _param(name: str) -> str:
 
 aws_cdk.Aspects.of(app).add(cdk_nag.AwsSolutionsChecks(log_ignores=True))
 
+if app_settings.parameters.tags:
+    for tag_key, tag_value in app_settings.parameters.tags.items():
+        aws_cdk.Tags.of(app).add(tag_key, tag_value)
+
+aws_cdk.Tags.of(app).add("SeedFarmerDeploymentName", app_settings.settings.deployment_name)
+aws_cdk.Tags.of(app).add("SeedFarmerModuleName", app_settings.settings.module_name)
+aws_cdk.Tags.of(app).add("SeedFarmerProjectName", app_settings.settings.project_name)
+
 app.synth()

modules/mlflow/mlflow-fargate/requirements.in

@@ -2,3 +2,5 @@ aws-cdk-lib~=2.129
 cdk-nag~=2.28.42
 boto3~=1.34
 cdk_ecr_deployment~=3.0
+pydantic~=2.7.1
+pydantic-settings~=2.2.1

modules/mlflow/mlflow-fargate/requirements.txt

@@ -1,9 +1,11 @@
 #
-# This file is autogenerated by pip-compile with Python 3.9
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile --output-file=requirements.txt requirements.in
 #
+annotated-types==0.6.0
+    # via pydantic
 attrs==23.2.0
     # via
     #   cattrs
@@ -36,8 +38,6 @@ constructs==10.3.0
     #   aws-cdk-lib
     #   cdk-ecr-deployment
     #   cdk-nag
-exceptiongroup==1.2.0
-    # via cattrs
 importlib-resources==6.1.1
     # via jsii
 jmespath==1.0.1
@@ -63,10 +63,20 @@ publication==0.0.3
     #   cdk-nag
     #   constructs
     #   jsii
+pydantic==2.7.1
+    # via
+    #   -r requirements.in
+    #   pydantic-settings
+pydantic-core==2.18.2
+    # via pydantic
+pydantic-settings==2.2.1
+    # via -r requirements.in
 python-dateutil==2.8.2
     # via
     #   botocore
     #   jsii
+python-dotenv==1.0.1
+    # via pydantic-settings
 s3transfer==0.10.0
     # via boto3
 six==1.16.0
@@ -83,9 +93,8 @@ typeguard==2.13.3
     #   jsii
 typing-extensions==4.9.0
     # via
-    #   cattrs
     #   jsii
+    #   pydantic
+    #   pydantic-core
 urllib3==1.26.18
     # via botocore
-zipp==3.17.0
-    # via importlib-resources