add online +batch inference notebooks and cli examples for clip embed…

…dings (#2696)

* start batch notebook

* prototyping batch inference

* add files for online inference notebook and cli examples -> these files aren't accurate they are just for model card links

* fix text input

* add batch endpoint and online endpoint notebooks

* add cli examples excluding prepare_data.py script

* update requests and batches in prepare_data

* run and revise batch inference notebook

* run and revise batch inference cli example

* run black reformatter

* respond to PR comments

* correct directory name extraction

* black reformatting

* add timestamp to batch endpoint name

* fix path for sample iamge

cli/foundation-models/system/inference/image-text-embeddings/deploy-batch.yaml

@@ -0,0 +1,9 @@
+$schema: https://azuremlschemas.azureedge.net/latest/batchDeployment.schema.json
+name: demo
+description: "Batch endpoint for image and text embeddings task"
+type: model
+resources:
+    instance_count: 1
+settings:
+    mini_batch_size: 1
+

cli/foundation-models/system/inference/image-text-embeddings/deploy-online.yaml

@@ -0,0 +1,12 @@
+$schema: https://azuremlschemas.azureedge.net/latest/managedOnlineDeployment.schema.json
+name: demo
+instance_type: Standard_DS3_v2
+instance_count: 1
+liveness_probe:
+  initial_delay: 180
+  period: 180
+  failure_threshold: 49
+  timeout: 299
+request_settings:
+  request_timeout_ms: 90000
+

cli/foundation-models/system/inference/image-text-embeddings/image-text-embeddings-batch-endpoint.sh

@@ -0,0 +1,165 @@
+set -x
+# The commands in this file map to steps in this notebook: https://aka.ms/azureml-infer-batch-sdk-image-classification
+# The sample scoring file available in the same folder as the above notebook.
+
+# script inputs
+registry_name="azureml"
+subscription_id="<SUBSCRIPTION_ID>"
+resource_group_name="<RESOURCE_GROUP>"
+workspace_name="<WORKSPACE_NAME>"
+
+# This is the model from system registry that needs to be deployed
+model_name="OpenAI-CLIP-Image-Text-Embeddings-vit-base-patch32"
+model_label="latest"
+
+deployment_compute="cpu-cluster"
+# todo: fetch deployment_sku from the min_inference_sku tag of the model
+deployment_sku="Standard_DS3_v2"
+
+
+version=$(date +%s)
+endpoint_name="clip-embeddings-batch-$version"
+deployment_name="demo-$version"
+
+# Prepare data for deployment
+data_path="data_batch"
+python ./prepare_data.py --mode "batch" --data_path $data_path
+# sample request data folders of csv files with image and text columns
+image_csv_folder="./data_batch/fridgeObjects/image_batch"
+text_csv_folder="./data_batch/fridgeObjects/text_batch"
+image_text_csv_folder="./data_batch/fridgeObjects/image_text_batch"
+
+# 1. Setup pre-requisites
+if [ "$subscription_id" = "<SUBSCRIPTION_ID>" ] || \
+   ["$resource_group_name" = "<RESOURCE_GROUP>" ] || \
+   [ "$workspace_name" = "<WORKSPACE_NAME>" ]; then 
+    echo "Please update the script with the subscription_id, resource_group_name and workspace_name"
+    exit 1
+fi
+
+az account set -s $subscription_id
+workspace_info="--resource-group $resource_group_name --workspace-name $workspace_name"
+
+# 2. Check if the model exists in the registry
+# Need to confirm model show command works for registries outside the tenant (aka system registry)
+if ! az ml model show --name $model_name --label $model_label --registry-name $registry_name 
+then
+    echo "Model $model_name:$model_label does not exist in registry $registry_name"
+    exit 1
+fi
+
+# Get the latest model version
+model_version=$(az ml model show --name $model_name --label $model_label --registry-name $registry_name --query version --output tsv)
+
+# 3. Check if compute $deployment_compute exists, else create it
+if az ml compute show --name $deployment_compute $workspace_info
+then
+    echo "Compute cluster $deployment_compute already exists"
+else
+    echo "Creating compute cluster $deployment_compute"
+    az ml compute create --name $deployment_compute --type amlcompute --min-instances 0 --max-instances 2 --size $deployment_sku $workspace_info || {
+        echo "Failed to create compute cluster $deployment_compute"
+        exit 1
+    }
+fi
+
+# 4. Deploy the model to an endpoint
+# Create batch endpoint
+az ml batch-endpoint create --name $endpoint_name $workspace_info  || {
+    echo "endpoint create failed"; exit 1;
+}
+
+# Deploy model from registry to endpoint in workspace
+az ml batch-deployment create --file ./deploy-batch.yaml $workspace_info --set \
+  endpoint_name=$endpoint_name model=azureml://registries/$registry_name/models/$model_name/versions/$model_version \
+  compute=$deployment_compute \
+  name=$deployment_name || {
+    echo "deployment create failed"; exit 1;
+}
+
+# 5.1 Try a scoring request with csv file for image embeddings
+
+# Check if scoring data file exists
+if [ -d $image_csv_folder ]; then
+    echo "Invoking endpoint $endpoint_name with following input:\n\n"
+    echo "\n\n"
+else
+    echo "Scoring file $image_csv_folder does not exist"
+    exit 1
+fi
+
+# Invoke the endpoint
+# Note: If job failed with Out of Memory Error then 
+# please try splitting your input into smaller csv files or
+# decrease the mini_batch_size for the deployment (see deploy-batch.yaml).
+csv_inference_job=$(az ml batch-endpoint invoke --name $endpoint_name \
+ --deployment-name $deployment_name --input $image_csv_folder --input-type \
+  uri_folder $workspace_info --query name --output tsv) || {
+    echo "endpoint invoke failed"; exit 1;
+}
+
+# wait for the job to complete
+az ml job stream --name $csv_inference_job $workspace_info || {
+    echo "job stream failed"; exit 1;
+}
+
+# 5.2 Try a scoring request with csv file for text embeddings
+
+# Check if scoring data file exists
+if [ -d $text_csv_folder ]; then
+    echo "Invoking endpoint $endpoint_name with following input:\n\n"
+    echo "\n\n"
+else
+    echo "Scoring file $text_csv_folder does not exist"
+    exit 1
+fi
+
+# Invoke the endpoint
+# Note: If job failed with Out of Memory Error then 
+# please try splitting your input into smaller csv files or
+# decrease the mini_batch_size for the deployment (see deploy-batch.yaml).
+csv_inference_job=$(az ml batch-endpoint invoke --name $endpoint_name \
+ --deployment-name $deployment_name --input $text_csv_folder --input-type \
+  uri_folder $workspace_info --query name --output tsv) || {
+    echo "endpoint invoke failed"; exit 1;
+}
+
+# wait for the job to complete
+az ml job stream --name $csv_inference_job $workspace_info || {
+    echo "job stream failed"; exit 1;
+}
+
+# 5.3 Try a scoring request with csv file for image and text embeddings
+
+# Check if scoring data file exists
+if [ -d $image_text_csv_folder ]; then
+    echo "Invoking endpoint $endpoint_name with following input:\n\n"
+    echo "\n\n"
+else
+    echo "Scoring file $image_text_csv_folder does not exist"
+    exit 1
+fi
+
+# Invoke the endpoint
+# Note: If job failed with Out of Memory Error then 
+# please try splitting your input into smaller csv files or
+# decrease the mini_batch_size for the deployment (see deploy-batch.yaml).
+csv_inference_job=$(az ml batch-endpoint invoke --name $endpoint_name \
+ --deployment-name $deployment_name --input $image_text_csv_folder --input-type \
+  uri_folder $workspace_info --query name --output tsv) || {
+    echo "endpoint invoke failed"; exit 1;
+}
+
+# wait for the job to complete
+az ml job stream --name $csv_inference_job $workspace_info || {
+    echo "job stream failed"; exit 1;
+}
+
+# 6. Delete the endpoint
+# Batch endpoints use compute resources only when jobs are submitted. You can keep the 
+# batch endpoint for your reference without worrying about compute bills, or choose to delete the endpoint. 
+# If you created your compute cluster to have zero minimum instances and scale down soon after being idle, 
+# you won't be charged for an unused compute.
+az ml batch-endpoint delete --name $endpoint_name $workspace_info --yes || {
+    echo "endpoint delete failed"; exit 1;
+}

cli/foundation-models/system/inference/image-text-embeddings/image-text-embeddings-online-endpoint.sh

@@ -0,0 +1,110 @@
+set -x
+# The commands in this file map to steps in this notebook: <TODO>
+# The sample scoring file available in the same folder as the above notebook
+
+# script inputs
+registry_name="azureml"
+subscription_id="<SUBSCRIPTION_ID>"
+resource_group_name="<RESOURCE_GROUP>"
+workspace_name="<WORKSPACE_NAME>"
+
+# This is the model from system registry that needs to be deployed
+model_name="OpenAI-CLIP-Image-Text-Embeddings-vit-base-patch32"
+model_label="latest"
+
+version=$(date +%s)
+endpoint_name="clip-embeddings-$version"
+
+# Todo: fetch deployment_sku from the min_inference_sku tag of the model
+deployment_sku="Standard_DS3_v2"
+
+# Prepare data for deployment
+data_path="./data_online"
+python ./prepare_data.py --data_path $data_path --mode "online"
+# sample_request_data
+image_request_data="$data_path/fridgeObjects/image_request_data.json"
+text_request_data="$data_path/fridgeObjects/text_request_data.json"
+image_text_request_data="$data_path/fridgeObjects/image_text_request_data.json"
+# 1. Setup pre-requisites
+if [ "$subscription_id" = "<SUBSCRIPTION_ID>" ] || \
+   ["$resource_group_name" = "<RESOURCE_GROUP>" ] || \
+   [ "$workspace_name" = "<WORKSPACE_NAME>" ]; then 
+    echo "Please update the script with the subscription_id, resource_group_name and workspace_name"
+    exit 1
+fi
+
+az account set -s $subscription_id
+workspace_info="--resource-group $resource_group_name --workspace-name $workspace_name"
+
+# 2. Check if the model exists in the registry
+# Need to confirm model show command works for registries outside the tenant (aka system registry)
+if ! az ml model show --name $model_name --label $model_label --registry-name $registry_name 
+then
+    echo "Model $model_name:$model_label does not exist in registry $registry_name"
+    exit 1
+fi
+
+# Get the latest model version
+model_version=$(az ml model show --name $model_name --label $model_label --registry-name $registry_name --query version --output tsv)
+
+# 3. Deploy the model to an endpoint
+# Create online endpoint 
+az ml online-endpoint create --name $endpoint_name $workspace_info  || {
+    echo "endpoint create failed"; exit 1;
+}
+
+# Deploy model from registry to endpoint in workspace
+az ml online-deployment create --file deploy-online.yaml $workspace_info --all-traffic --set \
+  endpoint_name=$endpoint_name model=azureml://registries/$registry_name/models/$model_name/versions/$model_version \
+  instance_type=$deployment_sku || {
+    echo "deployment create failed"; exit 1;
+}
+
+# 4.1 Try a sample scoring request for image embeddings
+
+# Check if scoring data file exists
+if [ -f $image_request_data ]; then
+    echo "Invoking endpoint $endpoint_name with $image_request_data\n\n"
+else
+    echo "Scoring file $image_request_data does not exist"
+    exit 1
+fi
+
+az ml online-endpoint invoke --name $endpoint_name --request-file $image_request_data $workspace_info || {
+    echo "endpoint invoke failed"; exit 1;
+}
+# 4.2 Try a sample scoring request for text embeddings
+
+# Check if scoring data file exists
+if [ -f $text_request_data ]; then
+    echo "Invoking endpoint $endpoint_name with $text_request_data\n\n"
+else
+    echo "Scoring file $text_request_data does not exist"
+    exit 1
+fi
+
+az ml online-endpoint invoke --name $endpoint_name --request-file $text_request_data $workspace_info || {
+    echo "endpoint invoke failed"; exit 1;
+}
+# 4.1 Try a sample scoring request for image and text embeddings
+
+# Check if scoring data file exists
+if [ -f $image_text_request_data ]; then
+    echo "Invoking endpoint $endpoint_name with $image_text_request_data\n\n"
+else
+    echo "Scoring file $image_text_request_data does not exist"
+    exit 1
+fi
+
+az ml online-endpoint invoke --name $endpoint_name --request-file $image_text_request_data $workspace_info || {
+    echo "endpoint invoke failed"; exit 1;
+}
+
+# 6. Delete the endpoint and sample_request_data.json
+az ml online-endpoint delete --name $endpoint_name $workspace_info --yes || {
+    echo "endpoint delete failed"; exit 1;
+}
+
+rm $image_request_data
+rm $text_request_data
+rm $image_text_request_data