Skip to content

Commit

Permalink
Model related docs refine (Azure#31552)
Browse files Browse the repository at this point in the history
* model operation doc refine

* model entities doc refine

* resolve conflict

* fix review comments

* fix line length limit issue

* minor fix on _load_functions.py

* revert back change
  • Loading branch information
CoderKevinZhang authored Aug 22, 2023
1 parent 94f42bb commit 9367722
Show file tree
Hide file tree
Showing 5 changed files with 184 additions and 24 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,15 @@ class ModelConfiguration:
:type mode: str
:param mount_path: The mount path of the model.
:type mount_path: str
.. admonition:: Example:
.. literalinclude:: ../../../../../../../samples/ml_samples_misc.py
:start-after: [START model_configuration_entity_create]
:end-before: [END model_configuration_entity_create]
:language: python
:dedent: 8
:caption: Creating a Model Configuration object.
"""

def __init__(self, *, mode: str = None, mount_path: str = None):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,15 @@ class ModelPackageInput:
:type mode: Optional[str]
:param mount_path: The mount path for the input.
:type mount_path: Optional[str]
.. admonition:: Example:
.. literalinclude:: ../../../../../../../samples/ml_samples_misc.py
:start-after: [START model_package_input_entity_create]
:end-before: [END model_package_input_entity_create]
:language: python
:dedent: 8
:caption: Create a Model Package Input object.
"""

def __init__(
Expand Down Expand Up @@ -191,7 +200,16 @@ class ModelPackage(Resource, PackageRequest):
:param model_configuration: The model configuration.
:type model_configuration: Optional[~azure.ai.ml.entities.ModelConfiguration]
:param tags: The tags of the model package.
:type tags: Optiona[dict[str, str]]
:type tags: Optional[dict[str, str]]
.. admonition:: Example:
.. literalinclude:: ../../../../../../../samples/ml_samples_misc.py
:start-after: [START model_package_entity_create]
:end-before: [END model_package_entity_create]
:language: python
:dedent: 8
:caption: Create a Model Package object.
"""

def __init__(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@

from typing import Dict, Optional

from azure.ai.ml._schema._deployment.batch.model_batch_deployment_settings import ModelBatchDeploymentSettingsSchema
from azure.ai.ml._utils._experimental import experimental
from azure.ai.ml.constants._common import BASE_PATH_CONTEXT_KEY
from azure.ai.ml.constants._deployment import BatchDeploymentOutputAction
from azure.ai.ml.entities._deployment.deployment_settings import BatchRetrySettings
from azure.ai.ml._utils._experimental import experimental
from azure.ai.ml._schema._deployment.batch.model_batch_deployment_settings import ModelBatchDeploymentSettingsSchema


@experimental
Expand Down Expand Up @@ -39,6 +39,15 @@ class ModelBatchDeploymentSettings:
:type error_threshold: int
:param logging_level: Logging level for batch inference operation, defaults to "info"
:type logging_level: str
.. admonition:: Example:
.. literalinclude:: ../../../../../samples/ml_samples_misc.py
:start-after: [START model_batch_deployment_settings_entity_create]
:end-before: [END model_batch_deployment_settings_entity_create]
:language: python
:dedent: 8
:caption: Creating a Model Batch Deployment Settings object.
"""

def __init__(
Expand Down
126 changes: 107 additions & 19 deletions sdk/ml/azure-ai-ml/azure/ai/ml/operations/_model_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,12 @@

# pylint: disable=protected-access

from contextlib import contextmanager
from os import PathLike, path
from typing import Dict, Iterable, Optional, Union
from contextlib import contextmanager

from marshmallow.exceptions import ValidationError as SchemaValidationError

from azure.ai.ml._utils._experimental import experimental
from azure.ai.ml._artifacts._artifact_utilities import (
_check_and_upload_path,
_get_default_datastore_info,
Expand All @@ -21,40 +20,39 @@
CHANGED_ASSET_PATH_MSG,
CHANGED_ASSET_PATH_MSG_NO_PERSONAL_DATA,
)
from azure.ai.ml._utils._arm_id_utils import is_ARM_id_for_resource
from azure.ai.ml._utils._registry_utils import get_registry_client
from azure.ai.ml._exception_helper import log_and_raise_error
from azure.ai.ml._restclient.v2021_10_01_dataplanepreview import (
AzureMachineLearningWorkspaces as ServiceClient102021Dataplane,
)
from azure.ai.ml._restclient.v2023_04_01_preview.models import ListViewType, ModelVersion
from azure.ai.ml._restclient.v2023_04_01_preview import AzureMachineLearningWorkspaces as ServiceClient042023Preview
from azure.ai.ml._restclient.v2023_04_01_preview.models import ListViewType, ModelVersion
from azure.ai.ml._scope_dependent_operations import (
OperationConfig,
OperationsContainer,
OperationScope,
_ScopeDependentOperations,
OperationsContainer,
)
from azure.ai.ml.entities._assets._artifacts.code import Code

from azure.ai.ml.constants._common import ARM_ID_PREFIX
from azure.ai.ml._telemetry import ActivityType, monitor_with_activity
from azure.ai.ml._utils._arm_id_utils import is_ARM_id_for_resource
from azure.ai.ml._utils._asset_utils import (
_archive_or_restore,
_get_latest,
_resolve_label_to_asset,
_get_next_version_from_container,
_resolve_label_to_asset,
)
from azure.ai.ml._utils._experimental import experimental
from azure.ai.ml._utils._logger_utils import OpsLogger
from azure.ai.ml._utils._registry_utils import (
get_asset_body_for_registry_storage,
get_registry_client,
get_sas_uri_for_registry_asset,
get_storage_details_for_registry_assets,
)
from azure.ai.ml._utils._storage_utils import get_ds_name_and_path_prefix, get_storage_client
from azure.ai.ml._utils.utils import resolve_short_datastore_url, validate_ml_flow_folder
from azure.ai.ml.constants._common import ASSET_ID_FORMAT, AzureMLResourceType
from azure.ai.ml.entities._assets import Model, ModelPackage, Environment
from azure.ai.ml.constants._common import ARM_ID_PREFIX, ASSET_ID_FORMAT, AzureMLResourceType
from azure.ai.ml.entities._assets import Environment, Model, ModelPackage
from azure.ai.ml.entities._assets._artifacts.code import Code
from azure.ai.ml.entities._assets.workspace_asset_reference import WorkspaceAssetReference
from azure.ai.ml.entities._credentials import AccountKeyConfiguration
from azure.ai.ml.exceptions import (
Expand All @@ -64,8 +62,9 @@
ValidationErrorType,
ValidationException,
)
from azure.core.exceptions import ResourceNotFoundError
from azure.ai.ml.operations._datastore_operations import DatastoreOperations
from azure.core.exceptions import ResourceNotFoundError

from ._operation_orchestrator import OperationOrchestrator

ops_logger = OpsLogger(__name__)
Expand All @@ -77,6 +76,21 @@ class ModelOperations(_ScopeDependentOperations):
You should not instantiate this class directly. Instead, you should create an MLClient instance that instantiates it
for you and attaches it as an attribute.
:param operation_scope: Scope variables for the operations classes of an MLClient object.
:type operation_scope: ~azure.ai.ml._scope_dependent_operations.OperationScope
:param operation_config: Common configuration for operations classes of an MLClient object.
:type operation_config: ~azure.ai.ml._scope_dependent_operations.OperationConfig
:param service_client: Service client to allow end users to operate on Azure Machine Learning Workspace
resources (ServiceClient042023Preview or ServiceClient102021Dataplane).
:type service_client: typing.Union[
~azure.ai.ml._restclient.v2023_04_01_preview._azure_machine_learning_workspaces.AzureMachineLearningWorkspaces,
~azure.ai.ml._restclient.v2021_10_01_dataplanepreview._azure_machine_learning_workspaces.
AzureMachineLearningWorkspaces]
:param datastore_operations: Represents a client for performing operations on Datastores.
:type datastore_operations: ~azure.ai.ml.operations._datastore_operations.DatastoreOperations
:param all_operations: All operations classes of an MLClient object.
:type all_operations: ~azure.ai.ml._scope_dependent_operations.OperationsContainer
"""

# pylint: disable=unused-argument
Expand Down Expand Up @@ -116,6 +130,15 @@ def create_or_update(
:raises ~azure.ai.ml.exceptions.EmptyDirectoryError: Raised if local path provided points to an empty directory.
:return: Model asset object.
:rtype: ~azure.ai.ml.entities.Model
.. admonition:: Example:
.. literalinclude:: ../../../../samples/ml_samples_misc.py
:start-after: [START model_operations_create_or_update]
:end-before: [END model_operations_create_or_update]
:language: python
:dedent: 8
:caption: Create model example.
"""
try:
name = model.name
Expand Down Expand Up @@ -274,6 +297,15 @@ def get(self, name: str, version: Optional[str] = None, label: Optional[str] = N
Details will be provided in the error message.
:return: Model asset object.
:rtype: ~azure.ai.ml.entities.Model
.. admonition:: Example:
.. literalinclude:: ../../../../samples/ml_samples_misc.py
:start-after: [START model_operations_get]
:end-before: [END model_operations_get]
:language: python
:dedent: 8
:caption: Get model example.
"""
if version and label:
msg = "Cannot specify both version and label."
Expand Down Expand Up @@ -306,11 +338,23 @@ def get(self, name: str, version: Optional[str] = None, label: Optional[str] = N
def download(self, name: str, version: str, download_path: Union[PathLike, str] = ".") -> None:
"""Download files related to a model.
:param str name: Name of the model.
:param str version: Version of the model.
:param Union[PathLike, str] download_path: Local path as download destination,
defaults to current working directory of the current user. Contents will be overwritten.
:raise: ResourceNotFoundError if can't find a model matching provided name.
:param name: Name of the model.
:type name: str
:param version: Version of the model.
:type version: str
:param download_path: Local path as download destination, defaults to current working directory of the current
user. Contents will be overwritten.
:type download_path: Union[PathLike, str]
:raises ResourceNotFoundError: if can't find a model matching provided name.
.. admonition:: Example:
.. literalinclude:: ../../../../samples/ml_samples_misc.py
:start-after: [START model_operations_download]
:end-before: [END model_operations_download]
:language: python
:dedent: 8
:caption: Download files to model example.
"""

model_uri = self.get(name=name, version=version).path
Expand Down Expand Up @@ -379,6 +423,15 @@ def archive(
:type version: str
:param label: Label of the model asset. (mutually exclusive with version)
:type label: str
.. admonition:: Example:
.. literalinclude:: ../../../../samples/ml_samples_misc.py
:start-after: [START model_operations_archive]
:end-before: [END model_operations_archive]
:language: python
:dedent: 8
:caption: Archive a model example.
"""
_archive_or_restore(
asset_operations=self,
Expand All @@ -402,6 +455,15 @@ def restore(
:type version: str
:param label: Label of the model asset. (mutually exclusive with version)
:type label: str
.. admonition:: Example:
.. literalinclude:: ../../../../samples/ml_samples_misc.py
:start-after: [START model_operations_restore]
:end-before: [END model_operations_restore]
:language: python
:dedent: 8
:caption: Restore a model example.
"""
_archive_or_restore(
asset_operations=self,
Expand Down Expand Up @@ -431,7 +493,16 @@ def list(
:attr:`ListViewType.ACTIVE_ONLY`.
:type list_view_type: ListViewType
:return: An iterator like instance of Model objects
:rtype: ~azure.core.paging.ItemPaged[Model]
:rtype: ~azure.core.paging.ItemPaged[~azure.ai.ml.entities.Model]
.. admonition:: Example:
.. literalinclude:: ../../../../samples/ml_samples_misc.py
:start-after: [START model_operations_list]
:end-before: [END model_operations_list]
:language: python
:dedent: 8
:caption: List all models example.
"""
if name:
return (
Expand Down Expand Up @@ -485,6 +556,15 @@ def share(self, name, version, *, share_with_name, share_with_version, registry_
:paramtype registry_name: str
:return: Model asset object.
:rtype: ~azure.ai.ml.entities.Model
.. admonition:: Example:
.. literalinclude:: ../../../../samples/ml_samples_misc.py
:start-after: [START model_operations_share]
:end-before: [END model_operations_share]
:language: python
:dedent: 8
:caption: Share a model example.
"""

# Get workspace info to get workspace GUID
Expand Down Expand Up @@ -574,6 +654,14 @@ def package(self, name: str, version: str, package_request: ModelPackage, **kwar
:return: Environment object
:rtype: ~azure.ai.ml.entities.Environment
.. admonition:: Example:
.. literalinclude:: ../../../../samples/ml_samples_misc.py
:start-after: [START model_operations_package]
:end-before: [END model_operations_package]
:language: python
:dedent: 8
:caption: Package a model example.
"""

if not kwargs.get("skip_to_rest", False):
Expand Down
40 changes: 38 additions & 2 deletions sdk/ml/azure-ai-ml/samples/ml_samples_misc.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,8 @@

class MiscConfigurationOptions(object):
def ml_misc_config(self):
from azure.identity import DefaultAzureCredential

from azure.ai.ml import MLClient
from azure.identity import DefaultAzureCredential

subscription_id = os.environ["AZURE_SUBSCRIPTION_ID"]
resource_group = os.environ["RESOURCE_GROUP_NAME"]
Expand Down Expand Up @@ -120,6 +119,43 @@ def ml_misc_config(self):
)
# [END model_entity_create]

# [START model_batch_deployment_settings_entity_create]
from azure.ai.ml.entities._deployment.model_batch_deployment_settings import ModelBatchDeploymentSettings

modelBatchDeploymentSetting = ModelBatchDeploymentSettings(
mini_batch_size=256,
instance_count=5,
max_concurrency_per_instance=2,
output_file_name="output-file-name",
environment_variables={"env1": "value1", "env2": "value2"},
error_threshold=2,
logging_level=1,
)
# [END model_batch_deployment_settings_entity_create]

# [START model_configuration_entity_create]
from azure.ai.ml.entities._assets._artifacts._package.model_configuration import ModelConfiguration

modelConfiguration = ModelConfiguration(mode="model-mode", mount_path="model-mount-path")
# [END model_configuration_entity_create]

# [START model_package_input_entity_create]
from azure.ai.ml.entities._assets._artifacts._package.model_package import ModelPackageInput

modelPackageInput = ModelPackageInput(type="input-type", mode="input-mode", mount_path="input-mount-path")
# [END model_package_input_entity_create]

# [START model_package_entity_create]
from azure.ai.ml.entities._assets._artifacts._package.model_package import ModelPackage

modelPackage = ModelPackage(
target_environment_name="env-name",
target_environment_version="1.0",
environment_variables={"env1": "value1", "env2": "value2"},
tags={"tag1": "value1", "tag2": "value2"},
)
# [END model_package_entity_create]

# [START create_inputs_outputs]
from azure.ai.ml import Input, Output
from azure.ai.ml.entities import CommandJob, CommandJobLimits
Expand Down

0 comments on commit 9367722

Please sign in to comment.