MMD OD/IS sdk, cli examples; Update HuggingFace Classification examples

cli/foundation-models/system/finetune/image-classification/multiclass-classification/hftransformers-fridgeobjects-multiclass-classification-pipeline.yaml

@@ -57,7 +57,6 @@ jobs:
       training_data: ${{parent.inputs.training_data}}
       validation_data: ${{parent.inputs.validation_data}}
 
-      auto_hyperparameter_selection: False
       image_width: -1
       image_height: -1
       metric_for_best_model: accuracy

cli/foundation-models/system/finetune/image-classification/multilabel-classification/hftransformers-fridgeobjects-multilabel-classification-pipeline.yaml

@@ -58,7 +58,6 @@ jobs:
       training_data: ${{parent.inputs.training_data}}
       validation_data: ${{parent.inputs.validation_data}}
 
-      auto_hyperparameter_selection: False
       image_width: -1
       image_height: -1
       metric_for_best_model: iou

cli/foundation-models/system/finetune/image-instance-segmentation/deepspeed_configs/zero1.json

@@ -0,0 +1,42 @@
+{
+  "fp16": {
+    "enabled": true,
+    "loss_scale": 0,
+    "loss_scale_window": 1000,
+    "hysteresis": 2,
+    "min_loss_scale": 1
+  },
+  "zero_optimization": {
+    "stage": 1,
+    "allgather_partitions": true,
+    "allgather_bucket_size": 200000000,
+    "overlap_comm": true,
+    "reduce_scatter": true,
+    "reduce_bucket_size": 200000000,
+    "contiguous_gradients": false,
+    "cpu_offload": false
+  },
+  "zero_allow_untested_optimizer": true,
+  "optimizer": {
+    "type": "AdamW",
+    "params": {
+      "lr": "auto",
+      "betas": "auto",
+      "eps": "auto",
+      "weight_decay": "auto"
+    }
+  },
+  "scheduler": {
+    "type": "WarmupLR",
+    "params": {
+      "warmup_min_lr": "auto",
+      "warmup_max_lr": "auto",
+      "warmup_num_steps": "auto"
+    }
+  },
+  "steps_per_print": 2000,
+  "train_batch_size": "auto",
+  "train_micro_batch_size_per_gpu": "auto",
+  "gradient_accumulation_steps": "auto",
+  "wall_clock_breakdown": false
+}

cli/foundation-models/system/finetune/image-instance-segmentation/deploy.yaml

@@ -0,0 +1,4 @@
+$schema: https://azuremlschemas.azureedge.net/latest/managedOnlineDeployment.schema.json
+name: demo
+instance_type: Standard_DS3_v2
+instance_count: 1

cli/foundation-models/system/finetune/image-instance-segmentation/jsonl_converter.py

@@ -0,0 +1,227 @@
+import argparse
+import os
+import json
+import numpy as np
+import PIL.Image as Image
+import xml.etree.ElementTree as ET
+
+from simplification.cutil import simplify_coords
+from skimage import measure
+
+
+def convert_mask_to_polygon(
+    mask,
+    max_polygon_points=100,
+    score_threshold=0.5,
+    max_refinement_iterations=25,
+    edge_safety_padding=1,
+):
+    """Convert a numpy mask to a polygon outline in normalized coordinates.
+
+    :param mask: Pixel mask, where each pixel has an object (float) score in [0, 1], in size ([1, height, width])
+    :type: mask: <class 'numpy.array'>
+    :param max_polygon_points: Maximum number of (x, y) coordinate pairs in polygon
+    :type: max_polygon_points: Int
+    :param score_threshold: Score cutoff for considering a pixel as in object.
+    :type: score_threshold: Float
+    :param max_refinement_iterations: Maximum number of times to refine the polygon
+    trying to reduce the number of pixels to meet max polygon points.
+    :type: max_refinement_iterations: Int
+    :param edge_safety_padding: Number of pixels to pad the mask with
+    :type edge_safety_padding: Int
+    :return: normalized polygon coordinates
+    :rtype: list of list
+    """
+    # Convert to numpy bitmask
+    mask = mask[0]
+    mask_array = np.array((mask > score_threshold), dtype=np.uint8)
+    image_shape = mask_array.shape
+
+    # Pad the mask to avoid errors at the edge of the mask
+    embedded_mask = np.zeros(
+        (
+            image_shape[0] + 2 * edge_safety_padding,
+            image_shape[1] + 2 * edge_safety_padding,
+        ),
+        dtype=np.uint8,
+    )
+    embedded_mask[
+        edge_safety_padding : image_shape[0] + edge_safety_padding,
+        edge_safety_padding : image_shape[1] + edge_safety_padding,
+    ] = mask_array
+
+    # Find Image Contours
+    contours = measure.find_contours(embedded_mask, 0.5)
+    simplified_contours = []
+
+    for contour in contours:
+
+        # Iteratively reduce polygon points, if necessary
+        if max_polygon_points is not None:
+            simplify_factor = 0
+            while (
+                len(contour) > max_polygon_points
+                and simplify_factor < max_refinement_iterations
+            ):
+                contour = simplify_coords(contour, simplify_factor)
+                simplify_factor += 1
+
+        # Convert to [x, y, x, y, ....] coordinates and correct for padding
+        unwrapped_contour = [0] * (2 * len(contour))
+        unwrapped_contour[::2] = np.ceil(contour[:, 1]) - edge_safety_padding
+        unwrapped_contour[1::2] = np.ceil(contour[:, 0]) - edge_safety_padding
+
+        simplified_contours.append(unwrapped_contour)
+
+    return _normalize_contour(simplified_contours, image_shape)
+
+
+def _normalize_contour(contours, image_shape):
+
+    height, width = image_shape[0], image_shape[1]
+
+    for contour in contours:
+        contour[::2] = [x * 1.0 / width for x in contour[::2]]
+        contour[1::2] = [y * 1.0 / height for y in contour[1::2]]
+
+    return contours
+
+
+def binarise_mask(mask_fname):
+
+    mask = Image.open(mask_fname)
+    mask = np.array(mask)
+    # instances are encoded as different colors
+    obj_ids = np.unique(mask)
+    # first id is the background, so remove it
+    obj_ids = obj_ids[1:]
+
+    # split the color-encoded mask into a set of binary masks
+    binary_masks = mask == obj_ids[:, None, None]
+    return binary_masks
+
+
+def parsing_mask(mask_fname):
+
+    # For this particular dataset, initially each mask was merged (based on binary mask of each object)
+    # in the order of the bounding boxes described in the corresponding PASCAL VOC annotation file.
+    # Therefore, we have to extract each binary mask which is in the order of objects in the annotation file.
+    # https://github.com/microsoft/computervision-recipes/blob/master/utils_cv/detection/dataset.py
+    binary_masks = binarise_mask(mask_fname)
+    polygons = []
+    for bi_mask in binary_masks:
+
+        if len(bi_mask.shape) == 2:
+            bi_mask = bi_mask[np.newaxis, :]
+        polygon = convert_mask_to_polygon(bi_mask)
+        polygons.append(polygon)
+
+    return polygons
+
+
+def convert_mask_in_VOC_to_jsonl(dataset_dir: str, remote_path: str) -> None:
+    """
+    :param dataset_dir: Path to dataset directory, where user has images and anotations
+    :type: dataset_dir: String
+    :param remote_path: Remote path for dataset
+    :type: remote_path: String
+    """
+
+    dataset_parent_dir = os.path.dirname(dataset_dir)
+    print(dataset_dir, dataset_parent_dir)
+
+    # We'll copy each JSONL file within its related MLTable folder
+    training_mltable_path = os.path.join(dataset_parent_dir, "training-mltable-folder")
+    validation_mltable_path = os.path.join(
+        dataset_parent_dir, "validation-mltable-folder"
+    )
+
+    # First, let's create the folders if they don't exist
+    os.makedirs(training_mltable_path, exist_ok=True)
+    os.makedirs(validation_mltable_path, exist_ok=True)
+
+    train_validation_ratio = 5
+
+    # Path to the training and validation files
+    train_annotations_file = os.path.join(
+        training_mltable_path, "train_annotations.jsonl"
+    )
+    validation_annotations_file = os.path.join(
+        validation_mltable_path, "validation_annotations.jsonl"
+    )
+
+    # Path to the annotations
+    annotations_folder = os.path.join(dataset_dir, "annotations")
+    mask_folder = os.path.join(dataset_dir, "segmentation-masks")
+
+    # sample json line dictionary
+    json_line_sample = {
+        "image_url": remote_path,
+        "image_details": {"format": None, "width": None, "height": None},
+        "label": [],
+    }
+
+    # Read each annotation and convert it to jsonl line
+    with open(train_annotations_file, "w") as train_f:
+        with open(validation_annotations_file, "w") as validation_f:
+            for i, filename in enumerate(os.listdir(annotations_folder)):
+                if not filename.endswith(".xml"):
+                    print(f"Skipping unknown file: {filename}")
+                    continue
+
+                annotation_file_path = os.path.join(annotations_folder, filename)
+                print(f"Parsing {annotation_file_path}")
+
+                root = ET.parse(annotation_file_path).getroot()
+                width = int(root.find("size/width").text)
+                height = int(root.find("size/height").text)
+
+                # convert mask into polygon
+                mask_fname = os.path.join(mask_folder, filename[:-4] + ".png")
+                polygons = parsing_mask(mask_fname)
+
+                labels = []
+                for index, object in enumerate(root.findall("object")):
+                    name = object.find("name").text
+                    isCrowd = int(object.find("difficult").text)
+                    labels.append(
+                        {
+                            "label": name,
+                            "bbox": "null",
+                            "isCrowd": isCrowd,
+                            "polygon": polygons[index],
+                        }
+                    )
+
+                # build the jsonl file
+                image_filename = root.find("filename").text
+                _, file_extension = os.path.splitext(image_filename)
+                json_line = dict(json_line_sample)
+                json_line["image_url"] = (
+                    json_line["image_url"] + "images/" + image_filename
+                )
+                json_line["image_details"]["format"] = file_extension[1:]
+                json_line["image_details"]["width"] = width
+                json_line["image_details"]["height"] = height
+                json_line["label"] = labels
+
+                if i % train_validation_ratio == 0:
+                    # validation annotation
+                    validation_f.write(json.dumps(json_line) + "\n")
+                else:
+                    # train annotation
+                    train_f.write(json.dumps(json_line) + "\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(allow_abbrev=False)
+    parser.add_argument(
+        "--data_path",
+        type=str,
+        help="The directory contains images, annotations, and masks.",
+    )
+
+    args, remaining_args = parser.parse_known_args()
+    data_path = args.data_path
+
+    convert_mask_in_VOC_to_jsonl(data_path, None)

cli/foundation-models/system/finetune/image-instance-segmentation/mmdetection-fridgeobjects-instance-segmentation-pipeline.yml

@@ -0,0 +1,102 @@
+$schema: https://azuremlschemas.azureedge.net/latest/pipelineJob.schema.json
+type: pipeline
+
+experiment_name: AzureML-Train-Finetune-Vision-Instance-Segmentation-Samples
+
+inputs:
+  # dataset files
+  training_data:
+    type: mltable
+
+  validation_data:
+    type: mltable
+
+  # deepspeed config file
+  ds_finetune:
+    path: deepspeed_configs/zero1.json
+    type: uri_file
+  # compute
+  compute_model_import: sample-model-import-cluster
+  compute_finetune: sample-finetune-cluster-gpu-nc6
+  # model_name: microsoft/beit-base-patch16-224
+  # model - specify the foundation model available in the azureml system registry
+  # pytorch_model:
+  #     path: azureml://registries/azureml-preview/models/mask_rcnn_r50_fpn_1x_coco/versions/1
+  #     type: custom_model
+  # TODO: update the path from azureml-preview once registered
+  mlflow_model: 
+    path: azureml://registries/azureml-preview/models/mask_rcnn_r50_fpn_1x_coco/versions/1
+    type: mlflow_model
+
+outputs:
+  # map the output of the fine tuning job to the output of pipeline job so that we can easily register the fine tuned model
+  # registering the model is required to deploy the model to an online or batch endpoint
+  trained_model:
+    type: mlflow_model
+
+settings:
+  force_rerun: true
+  default_compute: azureml:sample-finetune-cluster-gpu-nc6
+
+jobs:
+  mmdetection_model_finetune_job:
+    type: pipeline
+    # TODO change the asset id
+    component: azureml://registries/azureml-preview/components/mmdetection_image_objectdetection_instancesegmentation_pipeline/labels/latest
+    inputs:
+
+      # Compute
+      compute_model_import: ${{parent.inputs.compute_model_import}}
+      compute_finetune: ${{parent.inputs.compute_finetune}}
+      instance_count: 1
+      process_count_per_instance: 1
+
+      # model selection
+      task_name: image-instance-segmentation
+
+      # model_name: ${{parent.inputs.model_name}}
+      # pytorch_model: ${{parent.inputs.pytorch_model}}
+      mlflow_model: ${{parent.inputs.mlflow_model}}
+      model_family: MmDetectionImage
+
+      # data
+      training_data: ${{parent.inputs.training_data}}
+      validation_data: ${{parent.inputs.validation_data}}
+
+      # finetuning parameters
+      apply_augmentations: True
+      image_min_size: -1
+      image_max_size: -1
+      metric_for_best_model: mean_average_precision
+      number_of_workers: 8
+      apply_deepspeed: False
+      deepspeed_config: ${{parent.inputs.ds_finetune}}
+      apply_ort: False
+      number_of_epochs: 15
+      max_steps: -1
+      training_batch_size: 4
+      validation_batch_size: 4
+      auto_find_batch_size: False
+      learning_rate: 5e-5
+      learning_rate_scheduler: warmup_linear
+      warmup_steps: 0
+      optimizer: adamw_hf
+      weight_decay: 0.0
+      extra_optim_args: ""
+      gradient_accumulation_step: 1
+      precision: 32
+      random_seed: 42
+      evaluation_strategy: epoch
+      evaluation_steps: 500
+      logging_strategy: epoch
+      logging_steps: 500
+      save_strategy: epoch
+      save_steps: 500
+      save_total_limit: -1
+      early_stopping: False
+      early_stopping_patience: 1
+      max_grad_norm: 1.0
+      resume_from_checkpoint: False
+
+    outputs:
+      mlflow_model_folder: ${{parent.outputs.trained_model}}