SageMaker Groundtruth module added and tested

manifests/mlops-sagemaker/deployment.yaml

@@ -12,6 +12,8 @@ groups:
     path: manifests/mlops-sagemaker/sagemaker-templates-modules.yaml
   - name: sagemaker-kernels
     path: manifests/mlops-sagemaker/kernels-modules.yaml
+  - name: ground-truth
+    path: manifests/mlops-sagemaker/sagemaker-groundtruth-modules.yaml
 targetAccountMappings:
   - alias: primary
     accountId:

manifests/mlops-sagemaker/sagemaker-groundtruth-modules.yaml

@@ -0,0 +1,2 @@
+name: ground-truth
+path: modules/sagemaker/sagemaker-groundtruth

modules/sagemaker/sagemaker-groundtruth/README.md

@@ -0,0 +1,35 @@
+## Introduction
+sagemaker-groundtruth
+
+
+## Description
+
+A short description of the module.
+
+## Architecture
+
+
+
+## Deployment
+
+
+## Inputs/Outputs
+
+
+### Input Parameters
+
+
+#### Required
+
+
+#### Optional
+
+
+#### Input Example
+
+
+### Module Metadata Outputs
+
+
+
+#### Output Example

modules/sagemaker/sagemaker-groundtruth/app.py

@@ -0,0 +1,115 @@
+import aws_cdk as cdk
+import yaml
+from aws_cdk import (
+    Stack,
+)
+from constructs import Construct
+from cdk_nag import NagSuppressions
+from cdk_nag import AwsSolutionsChecks
+from lib.stacks.init import LabelingInitStack as InitStack
+from lib.stacks.labeling_pipeline import LabelingPipelineStack
+from lib.stacks.statemachine_pipeline import ExecuteStateMachinePipeline
+
+app = cdk.App()
+
+
+class AppConfig(cdk.Stack):
+    def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
+        super().__init__(scope, construct_id, **kwargs)
+
+        with open("config.yaml") as file:
+            config_yaml = yaml.safe_load(file)
+        with open("repo_config.yaml") as repo_file:
+            repo_config_yaml = yaml.safe_load(repo_file)
+
+        self.repo_type = repo_config_yaml["repoType"]
+        self.repo_name = repo_config_yaml["repoName"]
+        self.branch_name = repo_config_yaml["branchName"]
+        # self.github_connection_arn = repo_config_yaml['githubConnectionArn']
+        # self.github_repo_owner = repo_config_yaml['githubRepoOwner']
+        self.pipeline_assets_prefix = config_yaml["pipelineAssetsPrefix"]
+        self.labeling_job_private_workteam_arn = config_yaml[
+            "labelingJobPrivateWorkteamArn"
+        ]
+        self.use_private_workteam_for_labeling = config_yaml[
+            "usePrivateWorkteamForLabeling"
+        ]
+        self.use_private_workteam_for_verification = config_yaml[
+            "usePrivateWorkteamForVerification"
+        ]
+        self.verification_job_private_workteam_arn = config_yaml[
+            "verificationJobPrivateWorkteamArn"
+        ]
+        self.max_labels_per_labeling_job = config_yaml["maxLabelsPerLabelingJob"]
+        self.labeling_pipeline_schedule = config_yaml["labelingPipelineSchedule"]
+        self.feature_group_name = config_yaml["featureGroupName"]
+        # self.assets_bucket= str(config_yaml['assets_bucket'])
+        self.model_package_group_name = config_yaml["modelPackageGroupName"]
+        self.model_package_group_description = config_yaml[
+            "modelPackageGroupDescription"
+        ]
+        # self.feature_group_name=str(cdk.Fn.import_value("aiopsfeatureGroup")),
+        # self.assets_bucket=str(cdk.Fn.import_value("aiopsDataBucket")),
+
+
+# class LabelingPipelineStack(Stack):
+#    def __init__(self, scope: Construct, construct_id: str, app_config: AppConfig, **kwargs) -> None:
+#        super().__init__(scope, construct_id, **kwargs)
+
+# Define your labeling pipeline resources here
+
+
+def add_security_checks(app: cdk.App, stacks: list[Stack]):
+    for stack in stacks:
+        NagSuppressions.add_stack_suppressions(
+            stack,
+            [
+                {
+                    "id": "AwsSolutions-IAM4",
+                    "reason": "Suppress disallowed use of managed policies for increased simplicity as this is a sample. Scope down in production!",
+                },
+                {
+                    "id": "AwsSolutions-IAM5",
+                    "reason": "Suppress disallowed use of wildcards in IAM policies for increased simplicity as this is a sample. Scope down in production!",
+                },
+                {
+                    "id": "AwsSolutions-L1",
+                    "reason": "Using fixed python version for lambda functions as sample needs to be stable",
+                },
+                {
+                    "id": "AwsSolutions-CB3",
+                    "reason": "Suppress warning for use of privileged mode for codebuild, as this is required for docker image build",
+                },
+                {
+                    "id": "AwsSolutions-CB4",
+                    "reason": "Suppress required use of KMS for CodeBuild as it incurs additional cost. Consider using KMS for Codebuild in production",
+                },
+            ],
+        )
+    AwsSolutionsChecks(verbose=True)
+
+
+def main():
+    app_config = AppConfig(app, "aiops-config")
+    init_stack = InitStack(app, "aiops-init-stack", app_config)
+    labeling_pipeline_stack = LabelingPipelineStack(
+        app, "aiops-labeling-infra-stack", app_config
+    )
+    # Add the dependency
+    labeling_pipeline_stack.add_dependency(init_stack)
+
+    # Statemachine stack
+    statemachine_pipeline_stack = ExecuteStateMachinePipeline(
+        app, "aiops-statemachine-pipeline", app_config
+    )
+    statemachine_pipeline_stack.add_dependency(init_stack)
+    statemachine_pipeline_stack.add_dependency(labeling_pipeline_stack)
+
+    add_security_checks(
+        app, [init_stack, labeling_pipeline_stack, statemachine_pipeline_stack]
+    )
+    app.synth()
+
+
+if __name__ == "__main__":
+    main()

modules/sagemaker/sagemaker-groundtruth/config.yaml

@@ -0,0 +1,25 @@
+# the codecommit repo where the pipeline should pull its source from
+repoName: "aiops-to-greengrass-cs"
+# the branch to use
+branchName: "main"
+# S3 prefix where pipeline assets will be stored
+pipelineAssetsPrefix: "pipeline/labeling"
+# whether to use a private worteam for Labeling
+usePrivateWorkteamForLabeling: false
+# whether to use a private worteam for verifaction
+usePrivateWorkteamForVerification: false
+# maximum number of labels per labeling job
+maxLabelsPerLabelingJob: 200
+# the arn of the private workteam for labeling (only used if usePrivateWorkteamForLabeling is true)
+labelingJobPrivateWorkteamArn: "arn:aws:sagemaker:eu-west-1:0000000000000:workteam/private-crowd/GT1"
+# the arn of the private workteam for labeling (only used if usePrivateWorkteamForLabeling is true)
+verificationJobPrivateWorkteamArn: "arn:aws:sagemaker:eu-west-1:0000000000000:workteam/private-crowd/GT1"
+# labeling pipeline schedule, triggering once a month on the 1st to keep cost to a minimum , fell free to change this
+labelingPipelineSchedule: "cron(0 12 1 * ? *)"
+# featureGroupName in SageMaker Feature Store, where features should be saved
+featureGroupName: "tag-quality-inspection"
+assets_bucket: "tag-quality-inspection-bkt-2024"
+# modelPackageGroupName in model Registry
+modelPackageGroupName: "TagQualityInspectionPackageGroup"
+# modelPackageGroupName in model Registry
+modelPackageGroupDescription: "Contains models for quality inspection of metal tags"

modules/sagemaker/sagemaker-groundtruth/deployspec.yaml

@@ -0,0 +1,38 @@
+publishGenericEnvVariables: true
+deploy:
+  phases:
+    install:
+      commands:
+      - npm install -g [email protected]
+      - pip install -r requirements.txt
+    pre_build:
+      commands:
+      - echo "Prebuild stage"
+    build:
+      commands:
+       - cdk deploy --all --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json
+       - cat cdk-exports.json
+       # Export metadata
+       - seedfarmer metadata convert -f cdk-exports.json || true
+       - export SEEDFARMER_MODULE_METADATA=$(cat SEEDFARMER_MODULE_METADATA)
+       - echo "bash deploy.sh"
+    post_build:
+      commands:
+      - echo "Deploy successful"
+destroy:
+  phases:
+    install:
+      commands:
+      - npm install -g [email protected]
+      - pip install -r requirements.txt
+    pre_build:
+      commands:
+      - cdk destroy --force --app "python app.py"
+      - echo "Prebuild stage"
+    build:
+      commands:
+      - echo "DESTROY!"
+    post_build:
+      commands:
+      - echo "Destroy successful"
+build_type: BUILD_GENERAL1_SMALL

modules/sagemaker/sagemaker-groundtruth/lib/assets/groundtruth/labeling_job/class_labels.json

@@ -0,0 +1 @@
+{"labels": [{"label": "scratch"}]}

modules/sagemaker/sagemaker-groundtruth/lib/assets/groundtruth/verification_job/data.json

@@ -0,0 +1 @@
+{"document-version":"2021-05-13","labels":[{"label":"Label correct"},{"label":"Incorrect label - missed object"},{"label":"Incorrect label - bounding box not accurate enough"}]}