From 150594cf9be0b2782f1ffbcabeccd20ec68bae3f Mon Sep 17 00:00:00 2001 From: Korin <0mza987@gmail.com> Date: Wed, 1 Feb 2023 15:11:37 +0800 Subject: [PATCH] [mldesigner] Centralize private imports for mldesigner (#27220) * Add mldesigner __init__ * sort modules order * add load_environment * remove public functions/entities from __init__.py * update new imports * update * update * update * update * have 3 category * reduce imports * update * update test * update * update * Revert "update" This reverts commit 6a20b1617adc1cdc82294fa0f0c2a774b10ab0f0. * update --- .../azure/ai/ml/dsl/_mldesigner/__init__.py | 42 ++++++++++++ .../azure/ai/ml/dsl/_mldesigner/_constants.py | 31 +++++++++ .../dsl/unittests/test_mldesigner_imports.py | 66 +++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_mldesigner/__init__.py create mode 100644 sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_mldesigner/_constants.py create mode 100644 sdk/ml/azure-ai-ml/tests/dsl/unittests/test_mldesigner_imports.py diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_mldesigner/__init__.py b/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_mldesigner/__init__.py new file mode 100644 index 000000000000..f3ca8ca911d0 --- /dev/null +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_mldesigner/__init__.py @@ -0,0 +1,42 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +""" +This file stores functions and objects that will be used in mldesigner package. +DO NOT change the module names in "all" list. If the interface has changed in source code, wrap it here and keep +original function/module names the same as before, otherwise mldesigner will be broken by this change. +""" + +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore + +from azure.ai.ml.entities._component.component_factory import component_factory +from azure.ai.ml.entities._job.pipeline._load_component import _generate_component_function +from azure.ai.ml.entities._inputs_outputs import _get_param_with_standard_annotation +from azure.ai.ml._internal.entities._additional_includes import _AdditionalIncludes # pylint: disable=unused-import +from azure.ai.ml._utils._asset_utils import get_ignore_file +from azure.ai.ml._utils.utils import try_enable_internal_components +from azure.ai.ml._internal.entities import InternalComponent # pylint: disable=unused-import +from azure.ai.ml.dsl._condition import condition +from azure.ai.ml.dsl._do_while import do_while +from azure.ai.ml.dsl._group_decorator import group + +from ._constants import V1_COMPONENT_TO_NODE + +component_factory_load_from_dict = component_factory.load_from_dict + + +__all__ = [ + # to be put in main package + "condition", + "do_while", + "group", + + # must keep + "get_ignore_file", + "_get_param_with_standard_annotation", + "_generate_component_function", + "component_factory_load_from_dict", + "V1_COMPONENT_TO_NODE", + "try_enable_internal_components", +] diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_mldesigner/_constants.py b/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_mldesigner/_constants.py new file mode 100644 index 000000000000..66e806cb338f --- /dev/null +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/dsl/_mldesigner/_constants.py @@ -0,0 +1,31 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +""" +This file stores constants that will be used in mldesigner package. +""" +from azure.ai.ml._internal._schema.component import NodeType as V1NodeType +from azure.ai.ml._internal.entities import ( + Ae365exepool, + Command as InternalCommand, + Parallel as InternalParallel, + DataTransfer, + Distributed, + HDInsight, + Hemera, + Scope, + Starlite, +) + +V1_COMPONENT_TO_NODE = { + V1NodeType.SCOPE: Scope, + V1NodeType.COMMAND: InternalCommand, + V1NodeType.PARALLEL: InternalParallel, + V1NodeType.DATA_TRANSFER: DataTransfer, + V1NodeType.DISTRIBUTED: Distributed, + V1NodeType.HDI: HDInsight, + V1NodeType.STARLITE: Starlite, + V1NodeType.HEMERA: Hemera, + V1NodeType.AE365EXEPOOL: Ae365exepool, +} diff --git a/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_mldesigner_imports.py b/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_mldesigner_imports.py new file mode 100644 index 000000000000..7d6ac6c12b39 --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/dsl/unittests/test_mldesigner_imports.py @@ -0,0 +1,66 @@ +import pytest +from azure.ai.ml import Input +from azure.ai.ml.entities import( + Component, + CommandComponent, + PipelineComponent, + ValidationResult, +) +from azure.ai.ml.dsl._mldesigner import( + _AdditionalIncludes, + InternalComponent, +) +from azure.ai.ml.entities._builders.base_node import BaseNode +from azure.ai.ml.entities._inputs_outputs import GroupInput +from azure.ai.ml.entities._job.pipeline._io import PipelineInput, NodeOutput, NodeInput + + +@pytest.mark.unittest +@pytest.mark.pipeline_test +class TestMldesignerImports: + """ + The assertions are NOT SUPPOSED TO BE CHANGED once they are added. + + The attributes are needed for a certain version of mldesigner package, modifying or deleting any of them will cause + compatibility issues. If there are new dependencies for mldesigner package, add new assertions in this file. + """ + def test_necessay_attributes(self): + assert hasattr(Component, "_customized_validate") + assert hasattr(Component, "_source_path") + assert hasattr(CommandComponent, "_to_dict") + assert hasattr(CommandComponent, "_source_path") + assert hasattr(PipelineComponent, "_to_dict") + assert hasattr(PipelineComponent, "_source_path") + assert hasattr(PipelineComponent, "jobs") + assert hasattr(InternalComponent, "_to_dict") + assert hasattr(InternalComponent, "_source_path") + assert hasattr(InternalComponent, "_additional_includes") + assert hasattr(_AdditionalIncludes, "with_includes") + assert hasattr(_AdditionalIncludes, "_code_path") + assert hasattr(_AdditionalIncludes, "_includes") + assert hasattr(ValidationResult, "passed") + assert hasattr(ValidationResult, "error_messages") + + def test_necessary_attributes_for_input(self): + input_obj = Input() + assert hasattr(input_obj, "type") + assert hasattr(input_obj, "_is_enum") + assert hasattr(input_obj, "default") + assert hasattr(input_obj, "min") + assert hasattr(input_obj, "max") + assert hasattr(input_obj, "optional") + assert hasattr(input_obj, "_is_literal") + assert hasattr(input_obj, "_get_python_builtin_type_str") + assert hasattr(input_obj, "_get_param_with_standard_annotation") + + node_input_obj = NodeInput(name="sdk", meta=input_obj) + assert hasattr(node_input_obj, "_meta") + assert hasattr(node_input_obj, "_data") + + def test_class_names(self): + """These class are undirectly used in mldesigner by their class names""" + assert BaseNode.__name__ == "BaseNode" + assert GroupInput.__name__ == "GroupInput" + assert PipelineInput.__name__ == "PipelineInput" + assert NodeInput.__name__ == "NodeInput" + assert NodeOutput.__name__ == "NodeOutput" \ No newline at end of file