From e53711c5ee5931964b3a82f808a37a04e54c1fdb Mon Sep 17 00:00:00 2001 From: Yaqi Ji Date: Tue, 30 Nov 2021 11:25:31 -0800 Subject: [PATCH] fix(sdk): fix resource spec test (#6979) * fix(sdk): fix load_test * concat placeholder * release notes * fix(sdk): fix resource spec test * release notes * test for ontology --- .../kfp/v2/compiler/pipeline_spec_builder.py | 4 +- .../compiler_cli_tests/compiler_cli_tests.py | 10 +- .../test_data/pipeline_with_ontology.json | 276 ++++++++--------- .../pipeline_with_resource_spec.json | 288 +++++++++--------- .../test_data/pipeline_with_resource_spec.py | 4 +- 5 files changed, 283 insertions(+), 299 deletions(-) diff --git a/sdk/python/kfp/v2/compiler/pipeline_spec_builder.py b/sdk/python/kfp/v2/compiler/pipeline_spec_builder.py index 608145ed23b3..7cfdcd791373 100644 --- a/sdk/python/kfp/v2/compiler/pipeline_spec_builder.py +++ b/sdk/python/kfp/v2/compiler/pipeline_spec_builder.py @@ -389,9 +389,9 @@ def build_container_spec_for_task( )) if task.container_spec.resources is not None: - container_spec.reources.cpu_limit = ( + container_spec.resources.cpu_limit = ( task.container_spec.resources.cpu_limit) - container_spec.reources.memory_limit = ( + container_spec.resources.memory_limit = ( task.container_spec.resources.memory_limit) if task.container_spec.resources.accelerator_count is not None: container_spec.resources.accelerator.CopyFrom( diff --git a/sdk/python/kfp/v2/compiler_cli_tests/compiler_cli_tests.py b/sdk/python/kfp/v2/compiler_cli_tests/compiler_cli_tests.py index 267c0373f84f..b947149baa96 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/compiler_cli_tests.py +++ b/sdk/python/kfp/v2/compiler_cli_tests/compiler_cli_tests.py @@ -103,9 +103,8 @@ def test_two_step_pipeline(self): def test_pipeline_with_importer(self): self._test_compile_py_to_json('pipeline_with_importer') - # TODO: re-enable the test, debug load_component_from_file error - # def test_pipeline_with_ontology(self): - # self._test_compile_py_to_json('pipeline_with_ontology') + def test_pipeline_with_ontology(self): + self._test_compile_py_to_json('pipeline_with_ontology') def test_pipeline_with_if_placeholder(self): self._test_compile_py_to_json('pipeline_with_if_placeholder') @@ -113,9 +112,8 @@ def test_pipeline_with_if_placeholder(self): def test_pipeline_with_concat_placeholder(self): self._test_compile_py_to_json('pipeline_with_concat_placeholder') - # TODO: re-enable the test, debug add_node_selector_constraint error - # def test_pipeline_with_resource_spec(self): - # self._test_compile_py_to_json('pipeline_with_resource_spec') + def test_pipeline_with_resource_spec(self): + self._test_compile_py_to_json('pipeline_with_resource_spec') # TODO: re-enable the test, debug load_component_from_text error # def test_pipeline_with_various_io_types(self): diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json index 97cf271e27d8..c97f55c7b606 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json @@ -1,169 +1,163 @@ { - "pipelineSpec": { - "components": { - "comp-ingestion": { - "executorLabel": "exec-ingestion", - "inputDefinitions": { - "parameters": { - "input_location": { - "parameterType": "STRING" - } - } - }, - "outputDefinitions": { - "artifacts": { - "examples": { - "artifactType": { - "schemaTitle": "system.Dataset", - "schemaVersion": "0.0.1" - } - } + "components": { + "comp-ingestion": { + "executorLabel": "exec-ingestion", + "inputDefinitions": { + "parameters": { + "input_location": { + "parameterType": "STRING" } } }, - "comp-trainer": { - "executorLabel": "exec-trainer", - "inputDefinitions": { - "artifacts": { - "examples": { - "artifactType": { - "schemaTitle": "system.Dataset", - "schemaVersion": "0.0.1" - } - } - }, - "parameters": { - "n_epochs": { - "parameterType": "NUMBER_INTEGER" - }, - "optimizer": { - "parameterType": "STRING" - } - } - }, - "outputDefinitions": { - "artifacts": { - "model": { - "artifactType": { - "schemaTitle": "system.Model", - "schemaVersion": "0.0.1" - } + "outputDefinitions": { + "artifacts": { + "examples": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" } } } } }, - "deploymentSpec": { - "executors": { - "exec-ingestion": { - "container": { - "args": [ - "--input-location", - "{{$.inputs.parameters['input_location']}}", - "--output-examples", - "{{$.outputs.artifacts['examples'].uri}}" - ], - "image": "gcr.io/my-project/my-ingestor" + "comp-trainer": { + "executorLabel": "exec-trainer", + "inputDefinitions": { + "artifacts": { + "examples": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } } }, - "exec-trainer": { - "container": { - "args": [ - "--input-examples", - "{{$.inputs.artifacts['examples'].uri}}", - "--optimizer", - "{{$.inputs.parameters['optimizer']}}", - "--n_epochs", - "{{$.inputs.parameters['n_epochs']}}", - "--output-model", - "{{$.outputs.artifacts['model'].uri}}" - ], - "image": "gcr.io/my-project/my-fancy-trainer" + "parameters": { + "n_epochs": { + "parameterType": "NUMBER_INTEGER" + }, + "optimizer": { + "parameterType": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "model": { + "artifactType": { + "schemaTitle": "system.Model", + "schemaVersion": "0.0.1" + } } } } - }, - "pipelineInfo": { - "name": "two-step-pipeline-with-ontology" - }, - "root": { - "dag": { - "tasks": { - "ingestion": { - "cachingOptions": { - "enableCache": true - }, - "componentRef": { - "name": "comp-ingestion" - }, - "inputs": { - "parameters": { - "input_location": { - "componentInputParameter": "input_location" - } + } + }, + "defaultPipelineRoot": "dummy_root", + "deploymentSpec": { + "executors": { + "exec-ingestion": { + "container": { + "args": [ + "--input-location", + "{{$.inputs.parameters['input_location']}}", + "--output-examples", + "{{$.outputs.artifacts['examples'].uri}}" + ], + "image": "gcr.io/my-project/my-ingestor" + } + }, + "exec-trainer": { + "container": { + "args": [ + "--input-examples", + "{{$.inputs.artifacts['examples'].uri}}", + "--optimizer", + "{{$.inputs.parameters['optimizer']}}", + "--n_epochs", + "{{$.inputs.parameters['n_epochs']}}", + "--output-model", + "{{$.outputs.artifacts['model'].uri}}" + ], + "image": "gcr.io/my-project/my-fancy-trainer" + } + } + } + }, + "pipelineInfo": { + "name": "two-step-pipeline-with-ontology" + }, + "root": { + "dag": { + "tasks": { + "ingestion": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-ingestion" + }, + "inputs": { + "parameters": { + "input_location": { + "componentInputParameter": "input_location" } - }, - "taskInfo": { - "name": "ingestion" } }, - "trainer": { - "cachingOptions": { - "enableCache": true - }, - "componentRef": { - "name": "comp-trainer" - }, - "dependentTasks": [ - "ingestion" - ], - "inputs": { - "artifacts": { - "examples": { - "taskOutputArtifact": { - "outputArtifactKey": "examples", - "producerTask": "ingestion" - } - } - }, - "parameters": { - "n_epochs": { - "componentInputParameter": "n_epochs" - }, - "optimizer": { - "componentInputParameter": "optimizer" + "taskInfo": { + "name": "ingestion" + } + }, + "trainer": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-trainer" + }, + "dependentTasks": [ + "ingestion" + ], + "inputs": { + "artifacts": { + "examples": { + "taskOutputArtifact": { + "outputArtifactKey": "examples", + "producerTask": "ingestion" } } }, - "taskInfo": { - "name": "trainer" + "parameters": { + "n_epochs": { + "componentInputParameter": "n_epochs" + }, + "optimizer": { + "componentInputParameter": "optimizer" + } } - } - } - }, - "inputDefinitions": { - "parameters": { - "input_location": { - "parameterType": "STRING" - }, - "n_epochs": { - "parameterType": "NUMBER_INTEGER" }, - "optimizer": { - "parameterType": "STRING" + "taskInfo": { + "name": "trainer" } } } }, - "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.6" - }, - "runtimeConfig": { - "gcsOutputDirectory": "dummy_root", - "parameterValues": { - "input_location": "gs://test-bucket/pipeline_root", - "n_epochs": 200.0, - "optimizer": "sgd" + "inputDefinitions": { + "parameters": { + "input_location": { + "defaultValue": "gs://test-bucket/pipeline_root", + "parameterType": "STRING" + }, + "n_epochs": { + "defaultValue": 200.0, + "parameterType": "NUMBER_INTEGER" + }, + "optimizer": { + "defaultValue": "sgd", + "parameterType": "STRING" + } + } } - } + }, + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json index c9e9476baefb..a6aacf502ee1 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json @@ -1,177 +1,171 @@ { - "pipelineSpec": { - "components": { - "comp-ingestion": { - "executorLabel": "exec-ingestion", - "inputDefinitions": { - "parameters": { - "input_location": { - "parameterType": "STRING" - } - } - }, - "outputDefinitions": { - "artifacts": { - "examples": { - "artifactType": { - "schemaTitle": "system.Dataset", - "schemaVersion": "0.0.1" - } - } + "components": { + "comp-ingestion": { + "executorLabel": "exec-ingestion", + "inputDefinitions": { + "parameters": { + "input_location": { + "parameterType": "STRING" } } }, - "comp-trainer": { - "executorLabel": "exec-trainer", - "inputDefinitions": { - "artifacts": { - "examples": { - "artifactType": { - "schemaTitle": "system.Dataset", - "schemaVersion": "0.0.1" - } - } - }, - "parameters": { - "n_epochs": { - "parameterType": "NUMBER_INTEGER" - }, - "optimizer": { - "parameterType": "STRING" - } - } - }, - "outputDefinitions": { - "artifacts": { - "model": { - "artifactType": { - "schemaTitle": "system.Model", - "schemaVersion": "0.0.1" - } + "outputDefinitions": { + "artifacts": { + "examples": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" } } } } }, - "deploymentSpec": { - "executors": { - "exec-ingestion": { - "container": { - "args": [ - "--input-location", - "{{$.inputs.parameters['input_location']}}", - "--output-examples", - "{{$.outputs.artifacts['examples'].uri}}" - ], - "image": "gcr.io/my-project/my-ingestor" + "comp-trainer": { + "executorLabel": "exec-trainer", + "inputDefinitions": { + "artifacts": { + "examples": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } } }, - "exec-trainer": { - "container": { - "args": [ - "--input-examples", - "{{$.inputs.artifacts['examples'].uri}}", - "--optimizer", - "{{$.inputs.parameters['optimizer']}}", - "--n_epochs", - "{{$.inputs.parameters['n_epochs']}}", - "--output-model", - "{{$.outputs.artifacts['model'].uri}}" - ], - "image": "gcr.io/my-project/my-fancy-trainer", - "resources": { - "accelerator": { - "count": "1", - "type": "TPU_V3" - }, - "cpuLimit": 4.0, - "memoryLimit": 15.032385536 + "parameters": { + "n_epochs": { + "parameterType": "NUMBER_INTEGER" + }, + "optimizer": { + "parameterType": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "model": { + "artifactType": { + "schemaTitle": "system.Model", + "schemaVersion": "0.0.1" } } } } - }, - "pipelineInfo": { - "name": "two-step-pipeline-with-resource-spec" - }, - "root": { - "dag": { - "tasks": { - "ingestion": { - "cachingOptions": { - "enableCache": true - }, - "componentRef": { - "name": "comp-ingestion" + } + }, + "defaultPipelineRoot": "dummy_root", + "deploymentSpec": { + "executors": { + "exec-ingestion": { + "container": { + "args": [ + "--input-location", + "{{$.inputs.parameters['input_location']}}", + "--output-examples", + "{{$.outputs.artifacts['examples'].uri}}" + ], + "image": "gcr.io/my-project/my-ingestor" + } + }, + "exec-trainer": { + "container": { + "args": [ + "--input-examples", + "{{$.inputs.artifacts['examples'].uri}}", + "--optimizer", + "{{$.inputs.parameters['optimizer']}}", + "--n_epochs", + "{{$.inputs.parameters['n_epochs']}}", + "--output-model", + "{{$.outputs.artifacts['model'].uri}}" + ], + "image": "gcr.io/my-project/my-fancy-trainer", + "resources": { + "accelerator": { + "count": "1", + "type": "tpu-v3" }, - "inputs": { - "parameters": { - "input_location": { - "componentInputParameter": "input_location" - } + "cpuLimit": 4.0, + "memoryLimit": 15.032385536 + } + } + } + } + }, + "pipelineInfo": { + "name": "two-step-pipeline-with-resource-spec" + }, + "root": { + "dag": { + "tasks": { + "ingestion": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-ingestion" + }, + "inputs": { + "parameters": { + "input_location": { + "componentInputParameter": "input_location" } - }, - "taskInfo": { - "name": "ingestion" } }, - "trainer": { - "cachingOptions": { - "enableCache": true - }, - "componentRef": { - "name": "comp-trainer" - }, - "dependentTasks": [ - "ingestion" - ], - "inputs": { - "artifacts": { - "examples": { - "taskOutputArtifact": { - "outputArtifactKey": "examples", - "producerTask": "ingestion" - } - } - }, - "parameters": { - "n_epochs": { - "componentInputParameter": "n_epochs" - }, - "optimizer": { - "componentInputParameter": "optimizer" + "taskInfo": { + "name": "ingestion" + } + }, + "trainer": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-trainer" + }, + "dependentTasks": [ + "ingestion" + ], + "inputs": { + "artifacts": { + "examples": { + "taskOutputArtifact": { + "outputArtifactKey": "examples", + "producerTask": "ingestion" } } }, - "taskInfo": { - "name": "trainer" + "parameters": { + "n_epochs": { + "componentInputParameter": "n_epochs" + }, + "optimizer": { + "componentInputParameter": "optimizer" + } } - } - } - }, - "inputDefinitions": { - "parameters": { - "input_location": { - "parameterType": "STRING" - }, - "n_epochs": { - "parameterType": "NUMBER_INTEGER" }, - "optimizer": { - "parameterType": "STRING" + "taskInfo": { + "name": "trainer" } } } }, - "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.6" - }, - "runtimeConfig": { - "gcsOutputDirectory": "dummy_root", - "parameterValues": { - "input_location": "gs://test-bucket/pipeline_root", - "n_epochs": 200.0, - "optimizer": "sgd" + "inputDefinitions": { + "parameters": { + "input_location": { + "defaultValue": "gs://test-bucket/pipeline_root", + "parameterType": "STRING" + }, + "n_epochs": { + "defaultValue": 200.0, + "parameterType": "NUMBER_INTEGER" + }, + "optimizer": { + "defaultValue": "sgd", + "parameterType": "STRING" + } + } } - } + }, + "schemaVersion": "2.1.0", + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.py b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.py index 67e9852f00f7..2ecce9fbdeb1 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.py +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.py @@ -39,9 +39,7 @@ def my_pipeline(input_location: str = 'gs://test-bucket/pipeline_root', examples=ingestor.outputs['examples'], optimizer=optimizer, n_epochs=n_epochs).set_cpu_limit('4').set_memory_limit( - '14Gi').add_node_selector_constraint( - 'cloud.google.com/gke-accelerator', - 'tpu-v3').set_gpu_limit(1)) + '14Gi').add_node_selector_constraint('tpu-v3').set_gpu_limit('1')) if __name__ == '__main__':