Merge pull request #15 from aws-solutions-library-samples/v3.2

V3.2 lambdavpcstack

.gitignore

@@ -6,3 +6,6 @@ search-web-knn/node_modules
 *.pyc
 .idea/
 .~*.py
+*.ipynb_checkpoints
+myenv
+venv

deployment/README.md

@@ -29,6 +29,14 @@ cd ./deployment
 npm install -g aws-cdk
 ```
 
+启动Docker
+```
+sudo yum install -y npm docker
+sudo yum install docker -y
+sudo service docker start
+sudo chmod 666 /var/run/docker.sock
+```
+
 
 3. Bootstrap the CDK to provision all the infrastructure needed for the CDK to make changes to your AWS account
 

deployment/lib/ss_notebook.py

@@ -57,6 +57,7 @@ def __init__(self, scope: Construct, construct_id: str,search_engine_key: str, *
         self.notebook_job_role.add_managed_policy(_iam.ManagedPolicy.from_aws_managed_policy_name('AmazonSageMakerFullAccess'))
         self.notebook_job_role.add_managed_policy(_iam.ManagedPolicy.from_aws_managed_policy_name('SecretsManagerReadWrite'))
         self.notebook_job_role.add_managed_policy(_iam.ManagedPolicy.from_aws_managed_policy_name('AmazonDynamoDBFullAccess'))
+        self.notebook_job_role.add_managed_policy(_iam.ManagedPolicy.from_aws_managed_policy_name('AmazonBedrockFullAccess'))
 
         # notebookDeployment is more flexible , while using function is more faster with fixed value
         if self.node.try_get_context("notebook_deployment"):

deployment/setup.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+sudo yum install -y pip npm 
+sudo npm install -g aws-cdk
+
+# Get the region of the EC2 instance
+TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+AZ=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/placement/availability-zone)
+REGION=$(echo $AZ | sed 's/[a-zA-Z]$//')
+
+#pip install based on region
+if [[ $REGION == *"cn"* ]]; then
+    pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
+else
+    pip install -r requirements.txt
+fi
+

lambda/endpoint_list/lambda_function.py

@@ -0,0 +1,26 @@
+import boto3
+import json
+
+def lambda_handler(event, context):
+    # 创建SageMaker客户端
+    sagemaker = boto3.client('sagemaker')
+
+    # 调用list_endpoints方法获取端点列表
+    response = sagemaker.list_endpoints()
+
+    # 从响应中提取处于"InService"状态的所有端点的名称
+    endpoint_names = [
+        endpoint['EndpointName'] for endpoint in response['Endpoints']
+        if endpoint['EndpointStatus'] == 'InService'
+    ]
+
+    # 返回处于"InService"状态的端点名称列表
+    return {
+        'statusCode': 200,
+        'headers': {
+            'Access-Control-Allow-Headers': 'Content-Type',
+            'Access-Control-Allow-Origin': '*',
+            'Access-Control-Allow-Methods': 'OPTIONS,GET'
+        },
+        'body': json.dumps(endpoint_names)
+    }

lambda/knowledge_base_handler/crud/create.py

@@ -0,0 +1,60 @@
+import os
+import json
+import boto3
+from uuid import uuid4
+import time
+
+TABLE_NAME = os.getenv('TABLE_NAME', '')
+PRIMARY_KEY = os.getenv('PRIMARY_KEY', '')
+
+dynamodb = boto3.resource('dynamodb')
+table = dynamodb.Table(TABLE_NAME)
+
+RESERVED_RESPONSE = "Error: You're using AWS reserved keywords as attributes"
+DYNAMODB_EXECUTION_ERROR = "Error: Execution update, caused a Dynamodb error, please take a look at your CloudWatch Logs."
+
+def update_item(key, update_expression, expression_values, expression_keys):
+    # Create a DynamoDB resource
+    dynamodb = boto3.resource('dynamodb')
+
+    # Get the table
+    table = dynamodb.Table(TABLE_NAME)
+
+    # Update the item
+    response = table.update_item(
+        Key=key,
+        UpdateExpression=update_expression,
+        ExpressionAttributeValues=expression_values,
+        ExpressionAttributeNames=expression_keys
+    )
+
+    return response
+
+def handler(event, context):
+    if 'body' not in event:
+        return {'statusCode': 400, 'body': 'invalid request, you are missing the parameter body'}
+
+    item = event['body'] if isinstance(event['body'], dict) else json.loads(event['body'])
+    item["jobInfo"]["createdAt"] = int(time.time())
+    item  = item["jobInfo"]
+    #add a create time in to item["jobInfo"] use current timestamp
+    print(f"item:{item}")
+    if not item.get(id):
+        item[PRIMARY_KEY] = str(uuid4())
+    print(f"item:{item}")    
+
+    try:
+        table.put_item(Item=item)
+        #return {'statusCode': 201, 'body': f'job id:{item[PRIMARY_KEY]}'}
+        return {
+            'statusCode': 200,
+            'headers': {
+                'Access-Control-Allow-Headers': 'Content-Type',
+                'Access-Control-Allow-Origin': '*',
+                'Access-Control-Allow-Methods': 'OPTIONS,POST,GET'
+            },
+            'body': json.dumps(item)
+            }
+    except Exception as e:
+        error_response = RESERVED_RESPONSE if 'ValidationException' in str(e) and 'reserved keyword' in str(e) else DYNAMODB_EXECUTION_ERROR
+        return {'statusCode': 500, 'body': error_response}

lambda/knowledge_base_handler/crud/delete-one.py

@@ -0,0 +1,20 @@
+import os
+import json
+import boto3
+
+TABLE_NAME = os.getenv('TABLE_NAME', '')
+PRIMARY_KEY = os.getenv('PRIMARY_KEY', '')
+
+dynamodb = boto3.resource('dynamodb')
+table = dynamodb.Table(TABLE_NAME)
+
+def handler(event, context):
+    requested_item_id = event['pathParameters']['id'] if 'pathParameters' in event and 'id' in event['pathParameters'] else None
+    if not requested_item_id:
+        return {'statusCode': 400, 'body': 'Error: You are missing the path parameter id'}
+
+    try:
+        table.delete_item(Key={PRIMARY_KEY: requested_item_id})
+        return {'statusCode': 200, 'body': ''}
+    except Exception as e:
+        return {'statusCode': 500, 'body': json.dumps(str(e))}

lambda/knowledge_base_handler/crud/get-all.py

@@ -0,0 +1,38 @@
+import os
+import json
+import boto3
+from decimal import Decimal
+
+class DecimalEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Decimal):
+            return f'{obj}'
+        return super(DecimalEncoder, self).default(obj)
+
+TABLE_NAME = os.getenv('TABLE_NAME', '')
+
+dynamodb = boto3.resource('dynamodb')
+table = dynamodb.Table(TABLE_NAME)
+
+def handler(event, context):
+    try:
+        response = table.scan()
+        return {
+            'statusCode': 200,
+            'headers': {
+                'Access-Control-Allow-Headers': 'Content-Type',
+                'Access-Control-Allow-Origin': '*',
+                'Access-Control-Allow-Methods': 'OPTIONS,POST,GET'
+            },
+            'body': json.dumps(response['Items'], cls=DecimalEncoder)
+            }
+    except Exception as e:
+        return {
+            'statusCode': 500,
+            'headers': {
+                'Access-Control-Allow-Headers': 'Content-Type',
+                'Access-Control-Allow-Origin': '*',
+                'Access-Control-Allow-Methods': 'OPTIONS,POST,GET'
+            },
+            'body': json.dumps(str(e))
+            }

lambda/knowledge_base_handler/crud/get-one.py

@@ -0,0 +1,23 @@
+import os
+import json
+import boto3
+
+TABLE_NAME = os.getenv('TABLE_NAME', '')
+PRIMARY_KEY = os.getenv('PRIMARY_KEY', '')
+
+dynamodb = boto3.resource('dynamodb')
+table = dynamodb.Table(TABLE_NAME)
+
+def handler(event, context):
+    requested_item_id = event['pathParameters']['id'] if 'pathParameters' in event and 'id' in event['pathParameters'] else None
+    if not requested_item_id:
+        return {'statusCode': 400, 'body': 'Error: You are missing the path parameter id'}
+
+    try:
+        response = table.get_item(Key={PRIMARY_KEY: requested_item_id})
+        if 'Item' in response:
+            return {'statusCode': 200, 'body': json.dumps(response['Item'])}
+        else:
+            return {'statusCode': 404}
+    except Exception as e:
+        return {'statusCode': 500, 'body': json.dumps(str(e))}

lambda/knowledge_base_handler/crud/requirements.txt

@@ -0,0 +1 @@
+

lambda/knowledge_base_handler/crud/update-one.py

@@ -0,0 +1,45 @@
+import os
+import json
+import boto3
+
+TABLE_NAME = os.getenv('TABLE_NAME', '')
+PRIMARY_KEY = os.getenv('PRIMARY_KEY', '')
+
+dynamodb = boto3.resource('dynamodb')
+table = dynamodb.Table(TABLE_NAME)
+
+def handler(event, context):
+    if 'body' not in event:
+        return {'statusCode': 400, 'body': 'invalid request, you are missing the parameter body'}
+
+    print(event)
+
+    edited_item_id = event['pathParameters']['id'] if 'pathParameters' in event and 'id' in event['pathParameters'] else None
+    if not edited_item_id:
+        return {'statusCode': 400, 'body': 'invalid request, you are missing the path parameter id'}
+
+    edited_item = event['body'] if isinstance(event['body'], dict) else json.loads(event['body'])
+    if not edited_item or len(edited_item) < 1:
+        return {'statusCode': 400, 'body': 'invalid request, no arguments provided'}
+
+    UpdateExpression='SET ' + ', '.join(f'#{k}=:{k}' for k in edited_item.keys())
+    ExpressionAttributeValues={f':{k}': v for k, v in edited_item.items()}
+    ExpressionAttributeNames={f'#{k}': k for k in edited_item.keys()}
+
+    print(UpdateExpression)
+    print("*************************")
+    print(ExpressionAttributeValues)
+    print("*************************")
+    print(ExpressionAttributeNames)
+
+    try:
+        table.update_item(
+            Key={PRIMARY_KEY: edited_item_id},
+            UpdateExpression='SET ' + ', '.join(f'#{k}=:{k}' for k in edited_item.keys()),
+            ExpressionAttributeValues={f':{k}': v for k, v in edited_item.items()},
+            ExpressionAttributeNames={f'#{k}': k for k in edited_item.keys()},
+            ReturnValues='UPDATED_NEW'
+        )
+        return {'statusCode': 200, 'body': ''}
+    except Exception as e:
+        return {'statusCode': 500, 'body': json.dumps(str(e))}

lambda/knowledge_base_handler/lambda_function.py → lambda/knowledge_base_handler/indices/lambda_function.py

@@ -4,7 +4,7 @@
 from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth, helpers
 
 # get opensearch env 
-host =  os.environ.get('host')
+host =  os.environ.get('HOST')
 index =  os.environ.get('index')
 
 # retrieve secret manager value by key using boto3                                             

lambda/knowledge_base_handler/indices/requirements.txt

@@ -0,0 +1 @@
+

lambda/knowledge_base_handler/job/Dockerfile

@@ -0,0 +1,13 @@
+FROM public.ecr.aws/lambda/python:3.9
+
+# Copy requirements.txt
+COPY requirements.txt ${LAMBDA_TASK_ROOT}
+
+# Install the specified packages
+RUN pip install -r requirements.txt -t ./
+
+# Copy function code
+COPY . ${LAMBDA_TASK_ROOT}
+
+# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
+CMD [ "lambda_function_sqs.lambda_handler" ]