From 30490391f5149b1d6e66c50b07db259a432ee936 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Fri, 2 Jul 2021 16:20:23 -0700 Subject: [PATCH] Do not add downstream links for sub-branch-nodes (#285) Signed-off-by: Haytham Abuelfutuh --- pkg/compiler/test/compiler_test.go | 24 +- .../5_myapp.workflows.cereal.mycereal_2.json | 547 +++++++++++++ .../5_myapp.workflows.cereal.mycereal_2.json | 717 ++++++++++++++++++ .../mycereal_condition_has_no_deps.json | 717 ++++++++++++++++++ .../testdata/branch/compiled/success_3.json | 3 +- .../testdata/branch/compiled/success_4.json | 6 +- .../testdata/branch/compiled/success_5.json | 3 +- .../5_myapp.workflows.cereal.mycereal_2.json | 651 ++++++++++++++++ .../k8s/mycereal_condition_has_no_deps.json | 655 ++++++++++++++++ .../test/testdata/branch/k8s/success_3.json | 6 +- .../test/testdata/branch/k8s/success_4.json | 12 +- .../test/testdata/branch/k8s/success_5.json | 6 +- .../mycereal_condition_has_no_deps.json | 547 +++++++++++++ pkg/compiler/workflow_compiler.go | 5 +- 14 files changed, 3869 insertions(+), 30 deletions(-) create mode 100755 pkg/compiler/test/testdata/branch/5_myapp.workflows.cereal.mycereal_2.json create mode 100755 pkg/compiler/test/testdata/branch/compiled/5_myapp.workflows.cereal.mycereal_2.json create mode 100755 pkg/compiler/test/testdata/branch/compiled/mycereal_condition_has_no_deps.json create mode 100755 pkg/compiler/test/testdata/branch/k8s/5_myapp.workflows.cereal.mycereal_2.json create mode 100755 pkg/compiler/test/testdata/branch/k8s/mycereal_condition_has_no_deps.json create mode 100755 pkg/compiler/test/testdata/branch/mycereal_condition_has_no_deps.json diff --git a/pkg/compiler/test/compiler_test.go b/pkg/compiler/test/compiler_test.go index 3b7249f64c..0eedb14a4c 100644 --- a/pkg/compiler/test/compiler_test.go +++ b/pkg/compiler/test/compiler_test.go @@ -10,6 +10,8 @@ import ( "strings" "testing" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" + "k8s.io/apimachinery/pkg/util/sets" "github.com/go-test/deep" @@ -304,7 +306,7 @@ func TestBranches(t *testing.T) { t.Run(path, func(t *testing.T) { // If you want to debug a single use-case. Uncomment this line. - //if !strings.HasSuffix(path, "success_8_nested.json") { + //if !strings.HasSuffix(path, "mycereal_condition_has_no_deps.json") { // t.SkipNow() //} @@ -317,14 +319,26 @@ func TestBranches(t *testing.T) { t.FailNow() } } else if filepath.Ext(path) == ".pb" { + m := &jsonpb.Marshaler{ + Indent: " ", + } + err = proto.Unmarshal(raw, wf) if !assert.NoError(t, err) { - t.FailNow() - } + tsk := &admin.TaskSpec{} + if !assert.NoError(t, proto.Unmarshal(raw, tsk)) { + t.FailNow() + } - m := &jsonpb.Marshaler{ - Indent: " ", + raw, _ := m.MarshalToString(tsk) + err = ioutil.WriteFile(strings.TrimSuffix(path, filepath.Ext(path))+"_task.json", []byte(raw), os.ModePerm) + if !assert.NoError(t, err) { + t.FailNow() + } + + return } + raw, err := m.MarshalToString(wf) if !assert.NoError(t, err) { t.FailNow() diff --git a/pkg/compiler/test/testdata/branch/5_myapp.workflows.cereal.mycereal_2.json b/pkg/compiler/test/testdata/branch/5_myapp.workflows.cereal.mycereal_2.json new file mode 100755 index 0000000000..62187e7ab7 --- /dev/null +++ b/pkg/compiler/test/testdata/branch/5_myapp.workflows.cereal.mycereal_2.json @@ -0,0 +1,547 @@ +{ + "workflow": { + "id": { + "resourceType": "WORKFLOW", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.mycereal", + "version": "{{ registration.version }}" + }, + "metadata": { + }, + "interface": { + "inputs": { + "variables": { + "cereal_path": { + "type": { + "simple": "STRING" + }, + "description": "cereal_path" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.load_cereal", + "retries": { + } + }, + "inputs": [ + { + "var": "filepath", + "binding": { + "promise": { + "var": "cereal_path" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.load_cereal", + "version": "{{ registration.version }}" + }, + "overrides": { + } + } + }, + { + "id": "n1", + "metadata": { + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.is_list_empty", + "retries": { + } + }, + "inputs": [ + { + "var": "lst", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.is_list_empty", + "version": "{{ registration.version }}" + }, + "overrides": { + } + } + }, + { + "id": "n2", + "metadata": { + "name": "is_lst_empty", + "retries": { + } + }, + "inputs": [ + { + "var": "n1.o0", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n1" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "n1.o0" + }, + "rightValue": { + "primitive": { + "boolean": false + } + } + } + }, + "thenNode": { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.cereal_name_avg", + "retries": { + } + }, + "inputs": [ + { + "var": "cereals", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.cereal_name_avg", + "version": "{{ registration.version }}" + }, + "overrides": { + } + } + } + }, + "error": { + "failedNodeId": "is_lst_empty", + "message": "Must specify cereals" + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ], + "metadataDefaults": { + } + }, + "tasks": [ + { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.load_cereal", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + } + }, + "interface": { + "inputs": { + "variables": { + "filepath": { + "type": { + "simple": "STRING" + }, + "description": "filepath" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "load_cereal" + ], + "resources": { + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ] + } + }, + { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.is_list_empty", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + } + }, + "interface": { + "inputs": { + "variables": { + "lst": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "lst" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "BOOLEAN" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "is_list_empty" + ], + "resources": { + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ] + } + }, + { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.name_length", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + } + }, + "interface": { + "inputs": { + "variables": { + "cereal": { + "type": { + "mapValueType": { + "simple": "STRING" + } + }, + "description": "cereal" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "name_length" + ], + "resources": { + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ] + } + }, + { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.average", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + } + }, + "interface": { + "inputs": { + "variables": { + "lst": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "lst" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "average" + ], + "resources": { + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ] + } + }, + { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.cereal_name_avg", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + } + }, + "interface": { + "inputs": { + "variables": { + "cereals": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "cereals" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "cereal_name_avg" + ], + "resources": { + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/5_myapp.workflows.cereal.mycereal_2.json b/pkg/compiler/test/testdata/branch/compiled/5_myapp.workflows.cereal.mycereal_2.json new file mode 100755 index 0000000000..21f3391fe0 --- /dev/null +++ b/pkg/compiler/test/testdata/branch/compiled/5_myapp.workflows.cereal.mycereal_2.json @@ -0,0 +1,717 @@ +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.mycereal", + "version": "{{ registration.version }}" + }, + "metadata": { + + }, + "interface": { + "inputs": { + "variables": { + "cereal_path": { + "type": { + "simple": "STRING" + }, + "description": "cereal_path" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ] + }, + { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.load_cereal", + "retries": { + + } + }, + "inputs": [ + { + "var": "filepath", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "cereal_path" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.load_cereal", + "version": "{{ registration.version }}" + }, + "overrides": { + + } + } + }, + { + "id": "n1", + "metadata": { + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.is_list_empty", + "retries": { + + } + }, + "inputs": [ + { + "var": "lst", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.is_list_empty", + "version": "{{ registration.version }}" + }, + "overrides": { + + } + } + }, + { + "id": "n2", + "metadata": { + "name": "is_lst_empty", + "retries": { + + } + }, + "inputs": [ + { + "var": "n1.o0", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n1" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "n1.o0" + }, + "rightValue": { + "primitive": { + "boolean": false + } + } + } + }, + "thenNode": { + "id": "n2-n0", + "metadata": { + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.cereal_name_avg", + "retries": { + + } + }, + "inputs": [ + { + "var": "cereals", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.cereal_name_avg", + "version": "{{ registration.version }}" + }, + "overrides": { + + } + } + } + }, + "error": { + "failedNodeId": "is_lst_empty", + "message": "Must specify cereals" + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ], + "metadataDefaults": { + + } + }, + "connections": { + "downstream": { + "n0": { + "ids": [ + "n1" + ] + }, + "n1": { + "ids": [ + "n2" + ] + }, + "n2": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "n0" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "n2" + ] + }, + "n0": { + "ids": [ + "start-node" + ] + }, + "n1": { + "ids": [ + "n0" + ] + }, + "n2": { + "ids": [ + "n1" + ] + }, + "n2-n0": { + "ids": [ + "n0" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.load_cereal", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + + } + }, + "interface": { + "inputs": { + "variables": { + "filepath": { + "type": { + "simple": "STRING" + }, + "description": "filepath" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "load_cereal" + ], + "resources": { + + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.is_list_empty", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + + } + }, + "interface": { + "inputs": { + "variables": { + "lst": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "lst" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "BOOLEAN" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "is_list_empty" + ], + "resources": { + + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.name_length", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + + } + }, + "interface": { + "inputs": { + "variables": { + "cereal": { + "type": { + "mapValueType": { + "simple": "STRING" + } + }, + "description": "cereal" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "name_length" + ], + "resources": { + + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.average", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + + } + }, + "interface": { + "inputs": { + "variables": { + "lst": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "lst" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "average" + ], + "resources": { + + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.cereal_name_avg", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + + } + }, + "interface": { + "inputs": { + "variables": { + "cereals": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "cereals" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "cereal_name_avg" + ], + "resources": { + + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/mycereal_condition_has_no_deps.json b/pkg/compiler/test/testdata/branch/compiled/mycereal_condition_has_no_deps.json new file mode 100755 index 0000000000..063b3bb637 --- /dev/null +++ b/pkg/compiler/test/testdata/branch/compiled/mycereal_condition_has_no_deps.json @@ -0,0 +1,717 @@ +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.mycereal", + "version": "{{ registration.version }}" + }, + "metadata": { + + }, + "interface": { + "inputs": { + "variables": { + "cereal_path": { + "type": { + "simple": "STRING" + }, + "description": "cereal_path" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ] + }, + { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.load_cereal", + "retries": { + + } + }, + "inputs": [ + { + "var": "filepath", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "cereal_path" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.load_cereal", + "version": "{{ registration.version }}" + }, + "overrides": { + + } + } + }, + { + "id": "n1", + "metadata": { + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.is_list_empty", + "retries": { + + } + }, + "inputs": [ + { + "var": "lst", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.is_list_empty", + "version": "{{ registration.version }}" + }, + "overrides": { + + } + } + }, + { + "id": "n2", + "metadata": { + "name": "is_lst_empty", + "retries": { + + } + }, + "inputs": [ + { + "var": "n1.o0", + "binding": { + "scalar": { + "primitive": { + "boolean": true + } + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "n1.o0" + }, + "rightValue": { + "primitive": { + "boolean": false + } + } + } + }, + "thenNode": { + "id": "n2-n0", + "metadata": { + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.cereal_name_avg", + "retries": { + + } + }, + "inputs": [ + { + "var": "cereals", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.cereal_name_avg", + "version": "{{ registration.version }}" + }, + "overrides": { + + } + } + } + }, + "error": { + "failedNodeId": "is_lst_empty", + "message": "Must specify cereals" + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ], + "metadataDefaults": { + + } + }, + "connections": { + "downstream": { + "n0": { + "ids": [ + "n1" + ] + }, + "n1": { + "ids": [ + "end-node" + ] + }, + "n2": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "n0", + "n2" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "n1", + "n2" + ] + }, + "n0": { + "ids": [ + "start-node" + ] + }, + "n1": { + "ids": [ + "n0" + ] + }, + "n2": { + "ids": [ + "start-node" + ] + }, + "n2-n0": { + "ids": [ + "n0" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.load_cereal", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + + } + }, + "interface": { + "inputs": { + "variables": { + "filepath": { + "type": { + "simple": "STRING" + }, + "description": "filepath" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "load_cereal" + ], + "resources": { + + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.is_list_empty", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + + } + }, + "interface": { + "inputs": { + "variables": { + "lst": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "lst" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "BOOLEAN" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "is_list_empty" + ], + "resources": { + + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.name_length", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + + } + }, + "interface": { + "inputs": { + "variables": { + "cereal": { + "type": { + "mapValueType": { + "simple": "STRING" + } + }, + "description": "cereal" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "name_length" + ], + "resources": { + + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.average", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + + } + }, + "interface": { + "inputs": { + "variables": { + "lst": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "lst" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "average" + ], + "resources": { + + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.cereal_name_avg", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + + } + }, + "interface": { + "inputs": { + "variables": { + "cereals": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "cereals" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "cereal_name_avg" + ], + "resources": { + + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_3.json b/pkg/compiler/test/testdata/branch/compiled/success_3.json index 3578ec5758..adf5b7070d 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_3.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_3.json @@ -187,8 +187,7 @@ "downstream": { "node-0": { "ids": [ - "node-1", - "node-1-branchnode-0" + "node-1" ] }, "node-1": { diff --git a/pkg/compiler/test/testdata/branch/compiled/success_4.json b/pkg/compiler/test/testdata/branch/compiled/success_4.json index 6f5431d98f..694a110700 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_4.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_4.json @@ -273,14 +273,12 @@ "downstream": { "node-0": { "ids": [ - "node-1", - "node-1-branchnode-0" + "node-1" ] }, "node-1": { "ids": [ - "node-2", - "node-2-branchnode-0" + "node-2" ] }, "node-2": { diff --git a/pkg/compiler/test/testdata/branch/compiled/success_5.json b/pkg/compiler/test/testdata/branch/compiled/success_5.json index b8ced620c5..76d477b6ee 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_5.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_5.json @@ -362,8 +362,7 @@ "node-0": { "ids": [ "end-node", - "node-1", - "node-1-branchnode-1" + "node-1" ] }, "node-1": { diff --git a/pkg/compiler/test/testdata/branch/k8s/5_myapp.workflows.cereal.mycereal_2.json b/pkg/compiler/test/testdata/branch/k8s/5_myapp.workflows.cereal.mycereal_2.json new file mode 100755 index 0000000000..a51a60cb55 --- /dev/null +++ b/pkg/compiler/test/testdata/branch/k8s/5_myapp.workflows.cereal.mycereal_2.json @@ -0,0 +1,651 @@ +{ + "kind": "flyteworkflow", + "apiVersion": "flyte.lyft.com/v1alpha1", + "metadata": { + "name": "name", + "namespace": "namespace", + "creationTimestamp": null, + "labels": { + "execution-id": "name", + "workflow-name": "myapp-workflows-cereal-mycereal" + } + }, + "spec": { + "id": "{{ registration.project }}:{{ registration.domain }}:myapp.workflows.cereal.mycereal", + "nodes": { + "end-node": { + "id": "end-node", + "resources": {}, + "kind": "end", + "inputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ] + }, + "n0": { + "id": "n0", + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.load_cereal", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.load_cereal\" version:\"{{ registration.version }}\" ", + "inputBindings": [ + { + "var": "filepath", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "cereal_path" + } + } + } + ], + "retry": { + "minAttempts": 1 + } + }, + "n1": { + "id": "n1", + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.is_list_empty", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.is_list_empty\" version:\"{{ registration.version }}\" ", + "inputBindings": [ + { + "var": "lst", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "retry": { + "minAttempts": 1 + } + }, + "n2": { + "id": "n2", + "name": "is_lst_empty", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "leftValue": { + "var": "n1.o0" + }, + "rightValue": { + "primitive": { + "boolean": false + } + } + } + }, + "then": "n2-n0" + }, + "elseFail": { + "failedNodeId": "is_lst_empty", + "message": "Must specify cereals" + } + }, + "inputBindings": [ + { + "var": "n1.o0", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + } + ], + "retry": { + "minAttempts": 1 + } + }, + "n2-n0": { + "id": "n2-n0", + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.cereal_name_avg", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.cereal_name_avg\" version:\"{{ registration.version }}\" ", + "inputBindings": [ + { + "var": "cereals", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "retry": { + "minAttempts": 1 + } + }, + "start-node": { + "id": "start-node", + "resources": {}, + "kind": "start" + } + }, + "connections": { + "n0": [ + "n1" + ], + "n1": [ + "n2" + ], + "n2": [ + "end-node" + ], + "start-node": [ + "n0" + ] + }, + "edges": { + "downstream": { + "n0": [ + "n1" + ], + "n1": [ + "n2" + ], + "n2": [ + "end-node" + ], + "start-node": [ + "n0" + ] + }, + "upstream": { + "end-node": [ + "n2" + ], + "n0": [ + "start-node" + ], + "n1": [ + "n0" + ], + "n2": [ + "n1" + ], + "n2-n0": [ + "n0" + ] + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + }, + "outputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ] + }, + "inputs": { + "literals": { + "cereal_path": { + "scalar": { + "primitive": { + "stringValue": "" + } + } + } + } + }, + "executionId": {}, + "tasks": { + "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.average\" version:\"{{ registration.version }}\" ": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.average", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {} + }, + "interface": { + "inputs": { + "variables": { + "lst": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "lst" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "average" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.cereal_name_avg\" version:\"{{ registration.version }}\" ": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.cereal_name_avg", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {} + }, + "interface": { + "inputs": { + "variables": { + "cereals": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "cereals" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "cereal_name_avg" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.is_list_empty\" version:\"{{ registration.version }}\" ": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.is_list_empty", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {} + }, + "interface": { + "inputs": { + "variables": { + "lst": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "lst" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "BOOLEAN" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "is_list_empty" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.load_cereal\" version:\"{{ registration.version }}\" ": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.load_cereal", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {} + }, + "interface": { + "inputs": { + "variables": { + "filepath": { + "type": { + "simple": "STRING" + }, + "description": "filepath" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "load_cereal" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.name_length\" version:\"{{ registration.version }}\" ": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.name_length", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {} + }, + "interface": { + "inputs": { + "variables": { + "cereal": { + "type": { + "mapValueType": { + "simple": "STRING" + } + }, + "description": "cereal" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "name_length" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + "node-defaults": {}, + "securityContext": {}, + "status": { + "phase": 0 + }, + "rawOutputDataConfig": {}, + "executionConfig": { + "TaskPluginImpls": null, + "MaxParallelism": 0 + } +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/mycereal_condition_has_no_deps.json b/pkg/compiler/test/testdata/branch/k8s/mycereal_condition_has_no_deps.json new file mode 100755 index 0000000000..08c19b90fe --- /dev/null +++ b/pkg/compiler/test/testdata/branch/k8s/mycereal_condition_has_no_deps.json @@ -0,0 +1,655 @@ +{ + "kind": "flyteworkflow", + "apiVersion": "flyte.lyft.com/v1alpha1", + "metadata": { + "name": "name", + "namespace": "namespace", + "creationTimestamp": null, + "labels": { + "execution-id": "name", + "workflow-name": "myapp-workflows-cereal-mycereal" + } + }, + "spec": { + "id": "{{ registration.project }}:{{ registration.domain }}:myapp.workflows.cereal.mycereal", + "nodes": { + "end-node": { + "id": "end-node", + "resources": {}, + "kind": "end", + "inputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ] + }, + "n0": { + "id": "n0", + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.load_cereal", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.load_cereal\" version:\"{{ registration.version }}\" ", + "inputBindings": [ + { + "var": "filepath", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "cereal_path" + } + } + } + ], + "retry": { + "minAttempts": 1 + } + }, + "n1": { + "id": "n1", + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.is_list_empty", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.is_list_empty\" version:\"{{ registration.version }}\" ", + "inputBindings": [ + { + "var": "lst", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "retry": { + "minAttempts": 1 + } + }, + "n2": { + "id": "n2", + "name": "is_lst_empty", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "leftValue": { + "var": "n1.o0" + }, + "rightValue": { + "primitive": { + "boolean": false + } + } + } + }, + "then": "n2-n0" + }, + "elseFail": { + "failedNodeId": "is_lst_empty", + "message": "Must specify cereals" + } + }, + "inputBindings": [ + { + "var": "n1.o0", + "binding": { + "scalar": { + "primitive": { + "boolean": true + } + } + } + } + ], + "retry": { + "minAttempts": 1 + } + }, + "n2-n0": { + "id": "n2-n0", + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.cereal_name_avg", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.cereal_name_avg\" version:\"{{ registration.version }}\" ", + "inputBindings": [ + { + "var": "cereals", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "retry": { + "minAttempts": 1 + } + }, + "start-node": { + "id": "start-node", + "resources": {}, + "kind": "start" + } + }, + "connections": { + "n0": [ + "n1" + ], + "n1": [ + "end-node" + ], + "n2": [ + "end-node" + ], + "start-node": [ + "n0", + "n2" + ] + }, + "edges": { + "downstream": { + "n0": [ + "n1" + ], + "n1": [ + "end-node" + ], + "n2": [ + "end-node" + ], + "start-node": [ + "n0", + "n2" + ] + }, + "upstream": { + "end-node": [ + "n1", + "n2" + ], + "n0": [ + "start-node" + ], + "n1": [ + "n0" + ], + "n2": [ + "start-node" + ], + "n2-n0": [ + "n0" + ] + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + }, + "outputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ] + }, + "inputs": { + "literals": { + "cereal_path": { + "scalar": { + "primitive": { + "stringValue": "" + } + } + } + } + }, + "executionId": {}, + "tasks": { + "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.average\" version:\"{{ registration.version }}\" ": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.average", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {} + }, + "interface": { + "inputs": { + "variables": { + "lst": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "lst" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "average" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.cereal_name_avg\" version:\"{{ registration.version }}\" ": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.cereal_name_avg", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {} + }, + "interface": { + "inputs": { + "variables": { + "cereals": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "cereals" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "cereal_name_avg" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.is_list_empty\" version:\"{{ registration.version }}\" ": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.is_list_empty", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {} + }, + "interface": { + "inputs": { + "variables": { + "lst": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "lst" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "BOOLEAN" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "is_list_empty" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.load_cereal\" version:\"{{ registration.version }}\" ": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.load_cereal", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {} + }, + "interface": { + "inputs": { + "variables": { + "filepath": { + "type": { + "simple": "STRING" + }, + "description": "filepath" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "load_cereal" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"{{ registration.project }}\" domain:\"{{ registration.domain }}\" name:\"myapp.workflows.cereal.name_length\" version:\"{{ registration.version }}\" ": { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.name_length", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": {} + }, + "interface": { + "inputs": { + "variables": { + "cereal": { + "type": { + "mapValueType": { + "simple": "STRING" + } + }, + "description": "cereal" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "name_length" + ], + "resources": {}, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ], + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + "node-defaults": {}, + "securityContext": {}, + "status": { + "phase": 0 + }, + "rawOutputDataConfig": {}, + "executionConfig": { + "TaskPluginImpls": null, + "MaxParallelism": 0 + } +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_3.json b/pkg/compiler/test/testdata/branch/k8s/success_3.json index bffd2c5f72..05c6d31ad4 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_3.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_3.json @@ -130,8 +130,7 @@ }, "connections": { "node-0": [ - "node-1", - "node-1-branchnode-0" + "node-1" ], "node-1": [ "end-node" @@ -144,8 +143,7 @@ "edges": { "downstream": { "node-0": [ - "node-1", - "node-1-branchnode-0" + "node-1" ], "node-1": [ "end-node" diff --git a/pkg/compiler/test/testdata/branch/k8s/success_4.json b/pkg/compiler/test/testdata/branch/k8s/success_4.json index 3a0fe4aed0..67153f43e6 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_4.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_4.json @@ -201,12 +201,10 @@ }, "connections": { "node-0": [ - "node-1", - "node-1-branchnode-0" + "node-1" ], "node-1": [ - "node-2", - "node-2-branchnode-0" + "node-2" ], "node-2": [ "end-node" @@ -220,12 +218,10 @@ "edges": { "downstream": { "node-0": [ - "node-1", - "node-1-branchnode-0" + "node-1" ], "node-1": [ - "node-2", - "node-2-branchnode-0" + "node-2" ], "node-2": [ "end-node" diff --git a/pkg/compiler/test/testdata/branch/k8s/success_5.json b/pkg/compiler/test/testdata/branch/k8s/success_5.json index fd0c0fe55c..e894306b0d 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_5.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_5.json @@ -258,8 +258,7 @@ "connections": { "node-0": [ "end-node", - "node-1", - "node-1-branchnode-1" + "node-1" ], "node-1": [ "node-2" @@ -275,8 +274,7 @@ "downstream": { "node-0": [ "end-node", - "node-1", - "node-1-branchnode-1" + "node-1" ], "node-1": [ "node-2" diff --git a/pkg/compiler/test/testdata/branch/mycereal_condition_has_no_deps.json b/pkg/compiler/test/testdata/branch/mycereal_condition_has_no_deps.json new file mode 100755 index 0000000000..6fc249a91f --- /dev/null +++ b/pkg/compiler/test/testdata/branch/mycereal_condition_has_no_deps.json @@ -0,0 +1,547 @@ +{ + "workflow": { + "id": { + "resourceType": "WORKFLOW", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.mycereal", + "version": "{{ registration.version }}" + }, + "metadata": { + }, + "interface": { + "inputs": { + "variables": { + "cereal_path": { + "type": { + "simple": "STRING" + }, + "description": "cereal_path" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.load_cereal", + "retries": { + } + }, + "inputs": [ + { + "var": "filepath", + "binding": { + "promise": { + "var": "cereal_path" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.load_cereal", + "version": "{{ registration.version }}" + }, + "overrides": { + } + } + }, + { + "id": "n1", + "metadata": { + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.is_list_empty", + "retries": { + } + }, + "inputs": [ + { + "var": "lst", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.is_list_empty", + "version": "{{ registration.version }}" + }, + "overrides": { + } + } + }, + { + "id": "n2", + "metadata": { + "name": "is_lst_empty", + "retries": { + } + }, + "inputs": [ + { + "var": "n1.o0", + "binding": { + "scalar": { + "primitive": { + "boolean": true + } + } + } + } + ], + "upstreamNodeIds": [ + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "n1.o0" + }, + "rightValue": { + "primitive": { + "boolean": false + } + } + } + }, + "thenNode": { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.myapp.workflows.cereal.cereal_name_avg", + "retries": { + } + }, + "inputs": [ + { + "var": "cereals", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.cereal_name_avg", + "version": "{{ registration.version }}" + }, + "overrides": { + } + } + } + }, + "error": { + "failedNodeId": "is_lst_empty", + "message": "Must specify cereals" + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ], + "metadataDefaults": { + } + }, + "tasks": [ + { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.load_cereal", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + } + }, + "interface": { + "inputs": { + "variables": { + "filepath": { + "type": { + "simple": "STRING" + }, + "description": "filepath" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "load_cereal" + ], + "resources": { + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ] + } + }, + { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.is_list_empty", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + } + }, + "interface": { + "inputs": { + "variables": { + "lst": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "lst" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "BOOLEAN" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "is_list_empty" + ], + "resources": { + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ] + } + }, + { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.name_length", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + } + }, + "interface": { + "inputs": { + "variables": { + "cereal": { + "type": { + "mapValueType": { + "simple": "STRING" + } + }, + "description": "cereal" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "name_length" + ], + "resources": { + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ] + } + }, + { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.average", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + } + }, + "interface": { + "inputs": { + "variables": { + "lst": { + "type": { + "collectionType": { + "simple": "INTEGER" + } + }, + "description": "lst" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "average" + ], + "resources": { + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ] + } + }, + { + "id": { + "resourceType": "TASK", + "project": "{{ registration.project }}", + "domain": "{{ registration.domain }}", + "name": "myapp.workflows.cereal.cereal_name_avg", + "version": "{{ registration.version }}" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.0.0+develop", + "flavor": "python" + }, + "retries": { + } + }, + "interface": { + "inputs": { + "variables": { + "cereals": { + "type": { + "collectionType": { + "mapValueType": { + "simple": "STRING" + } + } + }, + "description": "cereals" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1", + "args": [ + "pyflyte-execute", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}", + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "myapp.workflows.cereal", + "task-name", + "cereal_name_avg" + ], + "resources": { + }, + "env": [ + { + "key": "FLYTE_INTERNAL_IMAGE", + "value": "590375264460.dkr.ecr.us-east-2.amazonaws.com/flytekit-python-template:v1" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/workflow_compiler.go b/pkg/compiler/workflow_compiler.go index f0828a40ce..9b0c49dc79 100755 --- a/pkg/compiler/workflow_compiler.go +++ b/pkg/compiler/workflow_compiler.go @@ -150,7 +150,10 @@ func (w workflowBuilder) AddEdges(n c.NodeBuilder, edgeDirection c.EdgeDirection } // Add explicitly declared edges - if n.GetUpstreamNodeIds() != nil { + switch edgeDirection { + case c.EdgeDirectionDownstream: + fallthrough + case c.EdgeDirectionBidirectional: for _, upNode := range n.GetUpstreamNodeIds() { w.AddExecutionEdge(upNode, n.GetId()) }