From 0671018b6201702d437c36dac380b3bcdf5b3e27 Mon Sep 17 00:00:00 2001 From: Xingzhi Zhang <37076709+elliotzh@users.noreply.github.com> Date: Tue, 8 Aug 2023 10:40:54 +0800 Subject: [PATCH] fix: accept PathLike as code of CommandComponent (#31469) --- sdk/ml/azure-ai-ml/CHANGELOG.md | 2 +- .../ai/ml/_internal/entities/component.py | 2 +- .../_component/_additional_includes.py | 4 +- .../azure/ai/ml/entities/_component/code.py | 17 +- .../entities/_component/command_component.py | 11 - .../component/e2etests/test_component.py | 15 ++ ...mmand_component_with_pathlike_as_code.json | 245 ++++++++++++++++++ 7 files changed, 277 insertions(+), 19 deletions(-) create mode 100644 sdk/ml/azure-ai-ml/tests/recordings/component/e2etests/test_component.pyTestComponenttest_command_component_with_pathlike_as_code.json diff --git a/sdk/ml/azure-ai-ml/CHANGELOG.md b/sdk/ml/azure-ai-ml/CHANGELOG.md index 41ba6046d394..fc24269a02f1 100644 --- a/sdk/ml/azure-ai-ml/CHANGELOG.md +++ b/sdk/ml/azure-ai-ml/CHANGELOG.md @@ -8,8 +8,8 @@ ### Bugs Fixed - Local job runs will no longer fail if Docker registry has no username/password - - Fixed an issue that code asset doesn't work with relative symbol links. +- Fixed [Issue 31319](https://github.com/Azure/azure-sdk-for-python/issues/31319): can't accept `PathLike` for `CommandComponent.code`. ### Breaking Changes diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/_internal/entities/component.py b/sdk/ml/azure-ai-ml/azure/ai/ml/_internal/entities/component.py index c5f03421d571..12c74484b2e6 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/_internal/entities/component.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/_internal/entities/component.py @@ -202,7 +202,7 @@ def _get_base_path_for_code(self) -> Path: return Path(self._source_path).parent def _get_origin_code_value(self) -> Union[str, PathLike, None]: - return self.code or Path(".").as_posix() + return self.code or "." # endregion diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/_additional_includes.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/_additional_includes.py index 5e2b1c2d6776..8bea3d975a2c 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/_additional_includes.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/_additional_includes.py @@ -40,7 +40,7 @@ class AdditionalIncludes: def __init__( self, *, - origin_code_value: Union[None, str], + origin_code_value: Optional[str], base_path: Path, configs: List[Union[str, dict]] = None, ) -> None: @@ -457,7 +457,7 @@ def _generate_additional_includes_obj(self): return AdditionalIncludes( base_path=self._get_base_path_for_code(), configs=self._get_all_additional_includes_configs(), - origin_code_value=self._get_origin_code_value(), + origin_code_value=self._get_origin_code_in_str(), ) @contextmanager diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/code.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/code.py index 67192e186f30..1987b99fef70 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/code.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/code.py @@ -176,6 +176,15 @@ def _get_origin_code_value(self) -> Union[str, os.PathLike, None]: """ return getattr(self, self._get_code_field_name(), None) + def _get_origin_code_in_str(self) -> Optional[str]: + """Get origin code value in str to simplify following logic.""" + origin_code_value = self._get_origin_code_value() + if origin_code_value is None: + return None + if isinstance(origin_code_value, Path): + return origin_code_value.as_posix() + return str(origin_code_value) + def _append_diagnostics_and_check_if_origin_code_reliable_for_local_path_validation( self, base_validation_result: MutableValidationResult = None ) -> bool: @@ -199,7 +208,7 @@ def _append_diagnostics_and_check_if_origin_code_reliable_for_local_path_validat # If private features are enable and component has code value of type str we need to check # that it is a valid git path case. Otherwise, we should throw a ValidationError # saying that the code value is not valid - code_type = _get_code_type(self._get_origin_code_value()) + code_type = _get_code_type(self._get_origin_code_in_str()) if code_type == CodeType.GIT and not is_private_preview_enabled(): if base_validation_result is not None: base_validation_result.append_error( @@ -215,7 +224,7 @@ def _build_code(self) -> Optional[Code]: If built code is the same as its origin value, do nothing and yield None. Otherwise, yield a Code object pointing to the code. """ - origin_code_value = self._get_origin_code_value() + origin_code_value = self._get_origin_code_in_str() code_type = _get_code_type(origin_code_value) if code_type == CodeType.GIT: @@ -231,7 +240,7 @@ def _build_code(self) -> Optional[Code]: @contextmanager def _try_build_local_code(self): """Extract the logic of _build_code for local code for further override.""" - origin_code_value = self._get_origin_code_value() + origin_code_value = self._get_origin_code_in_str() if origin_code_value is None: yield None else: @@ -243,6 +252,6 @@ def _try_build_local_code(self): def _with_local_code(self): # TODO: remove this method after we have a better way to do this judge in cache_utils - origin_code_value = self._get_origin_code_value() + origin_code_value = self._get_origin_code_in_str() code_type = _get_code_type(origin_code_value) return code_type == CodeType.LOCAL diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/command_component.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/command_component.py index b1433ac376b2..9dadaa874d8a 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/command_component.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/command_component.py @@ -159,17 +159,6 @@ def __init__( self.instance_count = instance_count self.additional_includes = additional_includes or [] - # region AdditionalIncludesMixin - def _get_origin_code_value(self) -> Union[str, os.PathLike, None]: - if self.code is not None and isinstance(self.code, str): - # directly return code given it will be validated in self._validate_additional_includes - return self.code - - # self.code won't be a Code object, or it will fail schema validation according to CodeFields - return None - - # endregion - @property def instance_count(self) -> int: """The number of instances or nodes to be used by the compute target. diff --git a/sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py b/sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py index 8d63bf3647be..c65e2b826cae 100644 --- a/sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py +++ b/sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py @@ -404,6 +404,21 @@ def test_command_component_with_code(self, client: MLClient, randstr: Callable[[ assert component_resource.code assert is_ARM_id_for_resource(component_resource.code) + def test_command_component_with_pathlike_as_code(self, client: MLClient, randstr: Callable[[str], str]) -> None: + component_name = randstr("component_name") + + component = load_component(source="./tests/test_configs/components/basic_component_code_local_path.yml") + from pathlib import Path + + component.name = component_name + component.code = Path(component.code) + + component_resource = client.components.create_or_update(component) + assert component_resource.name == component_name + # make sure code is created + assert component_resource.code + assert is_ARM_id_for_resource(component_resource.code) + def test_component_list(self, client: MLClient, randstr: Callable[[str], str]) -> None: component_name = randstr("component name") diff --git a/sdk/ml/azure-ai-ml/tests/recordings/component/e2etests/test_component.pyTestComponenttest_command_component_with_pathlike_as_code.json b/sdk/ml/azure-ai-ml/tests/recordings/component/e2etests/test_component.pyTestComponenttest_command_component_with_pathlike_as_code.json new file mode 100644 index 000000000000..6224e7edf5e0 --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/recordings/component/e2etests/test_component.pyTestComponenttest_command_component_with_pathlike_as_code.json @@ -0,0 +1,245 @@ +{ + "Entries": [ + { + "RequestUri": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/codes/000000000000000000000/versions?api-version=2023-04-01\u0026hash=3b94ac6cb29712a56cb97d9765c2dacc29a08b4fcde68a55a600b8abadbcf91e\u0026hashVersion=202208", + "RequestMethod": "GET", + "RequestHeaders": { + "Accept": "application/json", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "User-Agent": "azure-ai-ml/1.9.0 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.9.10 (Windows-10-10.0.22621-SP0)" + }, + "RequestBody": null, + "StatusCode": 200, + "ResponseHeaders": { + "Cache-Control": "no-cache", + "Content-Encoding": "gzip", + "Content-Type": "application/json; charset=utf-8", + "Date": "Fri, 04 Aug 2023 05:25:50 GMT", + "Expires": "-1", + "Pragma": "no-cache", + "Request-Context": "appId=cid-v1:2d2e8e63-272e-4b3c-8598-4ee570a0e70d", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains", + "Transfer-Encoding": "chunked", + "Vary": [ + "Accept-Encoding", + "Accept-Encoding" + ], + "x-aml-cluster": "vienna-eastus-01", + "X-Content-Type-Options": "nosniff", + "x-ms-correlation-request-id": "21c6b84b-fb05-4fb1-a344-93512507ac64", + "x-ms-ratelimit-remaining-subscription-reads": "11995", + "x-ms-response-type": "standard", + "x-ms-routing-request-id": "JAPANEAST:20230804T052550Z:21c6b84b-fb05-4fb1-a344-93512507ac64", + "x-request-time": "0.044" + }, + "ResponseBody": { + "value": [ + { + "id": "/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/codes/37cdd127-bdae-4873-9afa-0b485d43c989/versions/1", + "name": "1", + "type": "Microsoft.MachineLearningServices/workspaces/codes/versions", + "properties": { + "description": null, + "tags": {}, + "properties": { + "hash_sha256": "3b94ac6cb29712a56cb97d9765c2dacc29a08b4fcde68a55a600b8abadbcf91e", + "hash_version": "202208" + }, + "isArchived": false, + "isAnonymous": false, + "codeUri": "https://sdknotebstorage763b8b075.blob.core.windows.net:443/86530095-7-0bba99e7-0445-5a98-b6df-ba42c8a99b2b/helloworld_components_with_env", + "provisioningState": "Succeeded" + }, + "systemData": { + "createdAt": "2023-08-02T03:56:51.1145408\u002B00:00", + "createdBy": "Xingzhi Zhang", + "createdByType": "User", + "lastModifiedAt": "2023-08-02T03:56:51.1145408\u002B00:00", + "lastModifiedBy": "Xingzhi Zhang", + "lastModifiedByType": "User" + } + } + ] + } + }, + { + "RequestUri": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/codes/37cdd127-bdae-4873-9afa-0b485d43c989/versions/1?api-version=2023-04-01", + "RequestMethod": "GET", + "RequestHeaders": { + "Accept": "application/json", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "User-Agent": "azure-ai-ml/1.9.0 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.9.10 (Windows-10-10.0.22621-SP0)" + }, + "RequestBody": null, + "StatusCode": 200, + "ResponseHeaders": { + "Cache-Control": "no-cache", + "Content-Encoding": "gzip", + "Content-Type": "application/json; charset=utf-8", + "Date": "Fri, 04 Aug 2023 05:25:51 GMT", + "Expires": "-1", + "Pragma": "no-cache", + "Request-Context": "appId=cid-v1:2d2e8e63-272e-4b3c-8598-4ee570a0e70d", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains", + "Transfer-Encoding": "chunked", + "Vary": [ + "Accept-Encoding", + "Accept-Encoding" + ], + "x-aml-cluster": "vienna-eastus-01", + "X-Content-Type-Options": "nosniff", + "x-ms-correlation-request-id": "35c1e6ae-306f-4831-bb3d-941196b43d6b", + "x-ms-ratelimit-remaining-subscription-reads": "11994", + "x-ms-response-type": "standard", + "x-ms-routing-request-id": "JAPANEAST:20230804T052551Z:35c1e6ae-306f-4831-bb3d-941196b43d6b", + "x-request-time": "0.050" + }, + "ResponseBody": { + "id": "/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/codes/37cdd127-bdae-4873-9afa-0b485d43c989/versions/1", + "name": "1", + "type": "Microsoft.MachineLearningServices/workspaces/codes/versions", + "properties": { + "description": null, + "tags": {}, + "properties": { + "hash_sha256": "0000000000000", + "hash_version": "0000000000000" + }, + "isArchived": false, + "isAnonymous": false, + "codeUri": "https://sdknotebstorage763b8b075.blob.core.windows.net:443/86530095-7-0bba99e7-0445-5a98-b6df-ba42c8a99b2b/helloworld_components_with_env", + "provisioningState": "Succeeded" + }, + "systemData": { + "createdAt": "2023-08-02T03:56:51.1145408\u002B00:00", + "createdBy": "Xingzhi Zhang", + "createdByType": "User", + "lastModifiedAt": "2023-08-02T03:56:51.1145408\u002B00:00", + "lastModifiedBy": "Xingzhi Zhang", + "lastModifiedByType": "User" + } + } + }, + { + "RequestUri": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/components/test_269918924253/versions/1?api-version=2022-10-01", + "RequestMethod": "PUT", + "RequestHeaders": { + "Accept": "application/json", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Content-Length": "981", + "Content-Type": "application/json", + "User-Agent": "azure-ai-ml/1.9.0 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.9.10 (Windows-10-10.0.22621-SP0)" + }, + "RequestBody": { + "properties": { + "description": "This is the basic command component", + "properties": { + "client_component_hash": "104acc03-ba5e-888f-08c2-c7843b3be91b" + }, + "tags": { + "tag": "tagvalue", + "owner": "sdkteam" + }, + "isAnonymous": false, + "isArchived": false, + "componentSpec": { + "command": "echo Hello World", + "code": "azureml:/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/codes/37cdd127-bdae-4873-9afa-0b485d43c989/versions/1", + "environment": "azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:33", + "name": "test_269918924253", + "description": "This is the basic command component", + "tags": { + "tag": "tagvalue", + "owner": "sdkteam" + }, + "version": "1", + "$schema": "https://azuremlschemas.azureedge.net/development/commandComponent.schema.json", + "display_name": "CommandComponentBasic", + "is_deterministic": true, + "outputs": { + "component_out_path": { + "type": "uri_folder" + } + }, + "type": "command", + "_source": "YAML.COMPONENT" + } + } + }, + "StatusCode": 201, + "ResponseHeaders": { + "Cache-Control": "no-cache", + "Content-Length": "1768", + "Content-Type": "application/json; charset=utf-8", + "Date": "Fri, 04 Aug 2023 05:25:53 GMT", + "Expires": "-1", + "Location": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/components/test_269918924253/versions/1?api-version=2022-10-01", + "Pragma": "no-cache", + "Request-Context": "appId=cid-v1:2d2e8e63-272e-4b3c-8598-4ee570a0e70d", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains", + "x-aml-cluster": "vienna-eastus-01", + "X-Content-Type-Options": "nosniff", + "x-ms-correlation-request-id": "1964ca7f-6c48-4835-8195-49dc77449fde", + "x-ms-ratelimit-remaining-subscription-writes": "1197", + "x-ms-response-type": "standard", + "x-ms-routing-request-id": "JAPANEAST:20230804T052553Z:1964ca7f-6c48-4835-8195-49dc77449fde", + "x-request-time": "0.685" + }, + "ResponseBody": { + "id": "/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/components/test_269918924253/versions/1", + "name": "1", + "type": "Microsoft.MachineLearningServices/workspaces/components/versions", + "properties": { + "description": null, + "tags": { + "tag": "tagvalue", + "owner": "sdkteam" + }, + "properties": { + "client_component_hash": "104acc03-ba5e-888f-08c2-c7843b3be91b" + }, + "isArchived": false, + "isAnonymous": false, + "componentSpec": { + "$schema": "https://azuremlschemas.azureedge.net/development/commandComponent.schema.json", + "name": "test_269918924253", + "version": "1", + "display_name": "CommandComponentBasic", + "is_deterministic": "True", + "type": "command", + "description": "This is the basic command component", + "tags": { + "tag": "tagvalue", + "owner": "sdkteam" + }, + "outputs": { + "component_out_path": { + "type": "uri_folder" + } + }, + "code": "azureml:/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/codes/37cdd127-bdae-4873-9afa-0b485d43c989/versions/1", + "environment": "azureml://registries/azureml/environments/AzureML-sklearn-1.0-ubuntu20.04-py38-cpu/versions/33", + "resources": { + "instance_count": "1" + }, + "command": "echo Hello World" + } + }, + "systemData": { + "createdAt": "2023-08-04T05:25:53.54076\u002B00:00", + "createdBy": "Xingzhi Zhang", + "createdByType": "User", + "lastModifiedAt": "2023-08-04T05:25:53.6146631\u002B00:00", + "lastModifiedBy": "Xingzhi Zhang", + "lastModifiedByType": "User" + } + } + } + ], + "Variables": { + "component_name": "test_269918924253" + } +}