From 4615e62546d672bda3c7f2c940458a713595567b Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Fri, 28 May 2021 09:43:35 -0700 Subject: [PATCH 01/14] Bump to pick up latest plugins (#267) Signed-off-by: Katrina Rogan Signed-off-by: Haytham Abuelfutuh --- go.mod | 4 ++-- go.sum | 22 +++++++++------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index e55e11b53..e941cf7fa 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/DiSiqueira/GoTree v1.0.1-0.20180907134536-53a8e837f295 github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1 github.com/fatih/color v1.10.0 - github.com/flyteorg/flyteidl v0.18.41 - github.com/flyteorg/flyteplugins v0.5.42 + github.com/flyteorg/flyteidl v0.18.50 + github.com/flyteorg/flyteplugins v0.5.49 github.com/flyteorg/flytestdlib v0.3.17 github.com/ghodss/yaml v1.0.0 github.com/go-redis/redis v6.15.7+incompatible diff --git a/go.sum b/go.sum index b9635f525..d54acfc96 100644 --- a/go.sum +++ b/go.sum @@ -106,7 +106,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -229,11 +228,11 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/flyteorg/flyteidl v0.18.25/go.mod h1:b5Fq4Z8a5b0mF6pEwTd48ufvikUGVkWSjZiMT0ZtqKI= -github.com/flyteorg/flyteidl v0.18.41 h1:aVDpCIhzab5iPpJZXofvQLKJvpLnZdWl/zdkU0SiQXI= -github.com/flyteorg/flyteidl v0.18.41/go.mod h1:IJD02cc/95QMkGDBJNibsr5aWd6V7TlQiJ8Iz5mVZ28= -github.com/flyteorg/flyteplugins v0.5.42 h1:G4DRR2r8LlmkV+orXloDi1ly+M5WuvAaNlWFgGGyy3A= -github.com/flyteorg/flyteplugins v0.5.42/go.mod h1:ireF+bYk8xjw9BfcMbPN/hN5aZeBJpP0CoQYHkSRL+w= +github.com/flyteorg/flyteidl v0.18.48/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= +github.com/flyteorg/flyteidl v0.18.50 h1:L1fMj6QEXoKin+cPQn9sfwJ1x14tlChdz1mG1WaaIW4= +github.com/flyteorg/flyteidl v0.18.50/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= +github.com/flyteorg/flyteplugins v0.5.49 h1:jqmNrsTQ2+m+vYKqDVNO3CYy9q3XYTms3XzOr3roAT0= +github.com/flyteorg/flyteplugins v0.5.49/go.mod h1:567WIA0Rr6QjmXsqvGsU+Cyb57Ia6qzddxIw//RPwYk= github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220= github.com/flyteorg/flytestdlib v0.3.17 h1:7OexDLAjTBzJNGMmKKFmUTkss0I9IFo1LdTMpvH4qqA= github.com/flyteorg/flytestdlib v0.3.17/go.mod h1:VlbQuHTE+z2N5qusfwi+6WEkeJoqr8Q0E4NtBAsdwkU= @@ -430,8 +429,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.12.2 h1:D0EVSTwQoQOyfY35QNSuPJA4jpZRtkoGYWQMB7XNg5o= -github.com/grpc-ecosystem/grpc-gateway v1.12.2/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -624,7 +623,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= @@ -858,7 +856,6 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1111,7 +1108,6 @@ google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dT google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -1127,6 +1123,7 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -1155,7 +1152,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1166,6 +1162,7 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= @@ -1200,7 +1197,6 @@ gopkg.in/kothar/go-backblaze.v0 v0.0.0-20190520213052-702d4e7eb465/go.mod h1:zJ2 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= From 1c142741fd26cc0a6f67526c741cf5f3e69b0b61 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Fri, 28 May 2021 20:24:07 -0700 Subject: [PATCH 02/14] Reflect node id change in upstream and downstream connections Signed-off-by: Haytham Abuelfutuh --- pkg/compiler/common/builder.go | 1 + pkg/compiler/common/mocks/workflow_builder.go | 5 +++ pkg/compiler/test/compiler_test.go | 6 ++-- .../branch/compiled/success_7_nested.json | 2 +- .../testdata/branch/k8s/success_7_nested.json | 2 +- pkg/compiler/validators/branch.go | 3 -- pkg/compiler/validators/node.go | 3 +- pkg/compiler/workflow_compiler.go | 33 +++++++++++++++++++ 8 files changed, 45 insertions(+), 10 deletions(-) diff --git a/pkg/compiler/common/builder.go b/pkg/compiler/common/builder.go index be65dd2de..41ba0dc2a 100644 --- a/pkg/compiler/common/builder.go +++ b/pkg/compiler/common/builder.go @@ -21,6 +21,7 @@ type WorkflowBuilder interface { AddUpstreamEdge(nodeProvider, nodeDependent NodeID) AddDownstreamEdge(nodeProvider, nodeDependent NodeID) AddNode(n NodeBuilder, errs errors.CompileErrors) (node NodeBuilder, ok bool) + ReplaceNodeId(oldID, newID string) ValidateWorkflow(fg *core.CompiledWorkflow, errs errors.CompileErrors) (Workflow, bool) NewNodeBuilder(n *core.Node) NodeBuilder } diff --git a/pkg/compiler/common/mocks/workflow_builder.go b/pkg/compiler/common/mocks/workflow_builder.go index d2deaa10e..eaaeb98ce 100644 --- a/pkg/compiler/common/mocks/workflow_builder.go +++ b/pkg/compiler/common/mocks/workflow_builder.go @@ -515,6 +515,11 @@ func (_m *WorkflowBuilder) NewNodeBuilder(n *core.Node) common.NodeBuilder { return r0 } +// ReplaceNodeId provides a mock function with given fields: oldID, newID +func (_m *WorkflowBuilder) ReplaceNodeId(oldID string, newID string) { + _m.Called(oldID, newID) +} + // StoreCompiledSubWorkflow provides a mock function with given fields: id, compiledWorkflow func (_m *WorkflowBuilder) StoreCompiledSubWorkflow(id core.Identifier, compiledWorkflow *core.CompiledWorkflow) { _m.Called(id, compiledWorkflow) diff --git a/pkg/compiler/test/compiler_test.go b/pkg/compiler/test/compiler_test.go index 0806f1ceb..e3bb6410f 100644 --- a/pkg/compiler/test/compiler_test.go +++ b/pkg/compiler/test/compiler_test.go @@ -195,9 +195,9 @@ func TestBranches(t *testing.T) { t.Run(path, func(t *testing.T) { // If you want to debug a single use-case. Uncomment this line. - //if !strings.HasSuffix(path, "success_1.json") { - // t.SkipNow() - //} + if !strings.HasSuffix(path, "success_7_nested.json") { + t.SkipNow() + } raw, err := ioutil.ReadFile(path) assert.NoError(t, err) diff --git a/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json b/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json index 25844a542..9db0acf27 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json @@ -1 +1 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.so_nested","version":"abc"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},{"id":"branchnode-2","metadata":{"name":"fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"branchnode-2-branchbranchnode-1","metadata":{"name":"inner_fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"thenNode":{"id":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"}}}},"error":{"failedNodeId":"inner_fractions","message":"Only \u003c0.5 allowed"}}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"thenNode":{"id":"branchnode-2-branchn1","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"}}}}],"error":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"branchnode-2":{"ids":["end-node"]},"start-node":{"ids":["branchnode-2"]}},"upstream":{"branchbranchn0":{"ids":["start-node"]},"branchbranchnode-1":{"ids":["start-node"]},"branchbranchnode-1-branchbranchn0":{"ids":["start-node"]},"branchn1":{"ids":["start-node"]},"branchnode-2":{"ids":["start-node"]},"branchnode-2-branchbranchnode-1":{"ids":["start-node"]},"end-node":{"ids":["branchnode-2"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.so_nested","version":"abc"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},{"id":"branchnode-2","metadata":{"name":"fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"branchnode-2-branchbranchnode-1","metadata":{"name":"inner_fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"thenNode":{"id":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"}}}},"error":{"failedNodeId":"inner_fractions","message":"Only \u003c0.5 allowed"}}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"thenNode":{"id":"branchnode-2-branchn1","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"}}}}],"error":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"branchnode-2":{"ids":["end-node"]},"start-node":{"ids":["branchnode-2"]}},"upstream":{"branchnode-2":{"ids":["start-node"]},"branchnode-2-branchbranchnode-1":{"ids":["start-node"]},"end-node":{"ids":["branchnode-2"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json b/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json index 4dc8500dd..f534a7f2b 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json @@ -1 +1 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"core-control-flow-run-conditions-so-nested"}},"spec":{"id":"test_proj:test_domain:core.control_flow.run_conditions.so_nested","nodes":{"branchnode-2":{"id":"branchnode-2","name":"fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"then":"branchnode-2-branchbranchnode-1"},"elseIf":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"then":"branchnode-2-branchn1"}],"elseFail":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchbranchnode-1":{"id":"branchnode-2-branchbranchnode-1","name":"inner_fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"then":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0"},"elseFail":{"failedNodeId":"inner_fractions","message":"Only \u003c0.5 allowed"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0":{"id":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchn1":{"id":"branchnode-2-branchn1","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"branchnode-2":["end-node"],"start-node":["branchnode-2"]},"edges":{"downstream":{"branchnode-2":["end-node"],"start-node":["branchnode-2"]},"upstream":{"branchbranchn0":["start-node"],"branchbranchnode-1":["start-node"],"branchbranchnode-1-branchbranchn0":["start-node"],"branchn1":["start-node"],"branchnode-2":["start-node"],"branchnode-2-branchbranchnode-1":["start-node"],"end-node":["branchnode-2"]}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}},"outputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},"inputs":{"literals":{"my_input":{"scalar":{"primitive":{"floatValue":0}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"core-control-flow-run-conditions-so-nested"}},"spec":{"id":"test_proj:test_domain:core.control_flow.run_conditions.so_nested","nodes":{"branchnode-2":{"id":"branchnode-2","name":"fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"then":"branchnode-2-branchbranchnode-1"},"elseIf":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"then":"branchnode-2-branchn1"}],"elseFail":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchbranchnode-1":{"id":"branchnode-2-branchbranchnode-1","name":"inner_fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"then":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0"},"elseFail":{"failedNodeId":"inner_fractions","message":"Only \u003c0.5 allowed"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0":{"id":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchn1":{"id":"branchnode-2-branchn1","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"branchnode-2":["end-node"],"start-node":["branchnode-2"]},"edges":{"downstream":{"branchnode-2":["end-node"],"start-node":["branchnode-2"]},"upstream":{"branchnode-2":["start-node"],"branchnode-2-branchbranchnode-1":["start-node"],"end-node":["branchnode-2"]}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}},"outputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},"inputs":{"literals":{"my_input":{"scalar":{"primitive":{"floatValue":0}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/pkg/compiler/validators/branch.go b/pkg/compiler/validators/branch.go index 66084b556..b3107b092 100644 --- a/pkg/compiler/validators/branch.go +++ b/pkg/compiler/validators/branch.go @@ -67,9 +67,6 @@ func validateBranchInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs error continue } - ValidateBindings(w, n, n.GetInputs(), &flyte.VariableMap{Variables: map[string]*flyte.Variable{}}, - false, EdgeDirectionUpstream, errs.NewScope()) - // Clear out the Inputs. We do not care if the inputs of each of the underlying nodes // match. We will pull the inputs needed for the underlying branch node at runtime. iface2 = &flyte.TypedInterface{ diff --git a/pkg/compiler/validators/node.go b/pkg/compiler/validators/node.go index 14d4e7aa5..5a01cb106 100644 --- a/pkg/compiler/validators/node.go +++ b/pkg/compiler/validators/node.go @@ -123,11 +123,10 @@ func ValidateNode(w c.WorkflowBuilder, n c.NodeBuilder, validateConditionTypes b // Validate branch node conditions and inner nodes. if n.GetBranchNode() != nil { if nodes, ok := ValidateBranchNode(w, n, validateConditionTypes, errs.NewScope()); ok { - renamedNodes := make(map[c.NodeID]c.NodeID, len(nodes)) for _, subNode := range nodes { oldID := subNode.GetId() subNode.SetID(branchNodeIDFormatter(n.GetId(), subNode.GetId())) - renamedNodes[oldID] = subNode.GetId() + w.ReplaceNodeId(oldID, subNode.GetId()) } } } else if workflowN := n.GetWorkflowNode(); workflowN != nil && workflowN.GetSubWorkflowRef() != nil { diff --git a/pkg/compiler/workflow_compiler.go b/pkg/compiler/workflow_compiler.go index 68c1e98aa..cf17cf4d3 100755 --- a/pkg/compiler/workflow_compiler.go +++ b/pkg/compiler/workflow_compiler.go @@ -124,6 +124,39 @@ func (w workflowBuilder) AddUpstreamEdge(nodeProvider, nodeDependent c.NodeID) { w.CoreWorkflow.Connections.Upstream[nodeDependent].Ids = w.upstreamNodes[nodeDependent].List() } +func (w workflowBuilder) ReplaceNodeId(oldID, newID string) { + if existingVal, found := w.upstreamNodes[oldID]; found { + w.upstreamNodes[newID] = existingVal + delete(w.upstreamNodes, oldID) + delete(w.CoreWorkflow.Connections.Upstream, oldID) + w.CoreWorkflow.Connections.Upstream[newID] = &core.ConnectionSet_IdList{ + Ids: make([]string, 1), + } + } + + for key, val := range w.upstreamNodes { + if key != oldID && val.Has(oldID) { + val.Delete(oldID) + val.Insert(newID) + w.CoreWorkflow.Connections.Upstream[key].Ids = val.List() + } + } + + if existingVal, found := w.downstreamNodes[oldID]; found { + w.downstreamNodes[newID] = existingVal + delete(w.CoreWorkflow.Connections.Downstream, oldID) + delete(w.downstreamNodes, oldID) + } + + for key, val := range w.downstreamNodes { + if key != oldID && val.Has(oldID) { + val.Delete(oldID) + val.Insert(newID) + w.CoreWorkflow.Connections.Downstream[key].Ids = val.List() + } + } +} + func (w workflowBuilder) AddDownstreamEdge(nodeProvider, nodeDependent c.NodeID) { if nodeProvider == "" { nodeProvider = c.StartNodeID From 7bd4a45acb1a0d4f97c9a8dbc1f11e1afd44740c Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 1 Jun 2021 12:22:04 -0700 Subject: [PATCH 03/14] wip Signed-off-by: Haytham Abuelfutuh --- config.yaml | 6 +- pkg/compiler/test/compiler_test.go | 30 +- .../testdata/branch/compiled/success_3.json | 2 +- .../testdata/branch/compiled/success_4.json | 2 +- .../testdata/branch/compiled/success_5.json | 2 +- .../testdata/branch/compiled/success_6.json | 2 +- .../branch/compiled/success_7_nested.json | 2 +- .../branch/compiled/success_8_nested.json | 1 + .../test/testdata/branch/k8s/success_3.json | 2 +- .../test/testdata/branch/k8s/success_4.json | 2 +- .../test/testdata/branch/k8s/success_5.json | 2 +- .../test/testdata/branch/k8s/success_6.json | 2 +- .../testdata/branch/k8s/success_7_nested.json | 2 +- .../testdata/branch/k8s/success_8_nested.json | 1 + .../testdata/branch/success_8_nested.json | 441 ++++++++++++++++++ pkg/compiler/validators/branch.go | 3 + 16 files changed, 483 insertions(+), 19 deletions(-) create mode 100755 pkg/compiler/test/testdata/branch/compiled/success_8_nested.json create mode 100755 pkg/compiler/test/testdata/branch/k8s/success_8_nested.json create mode 100755 pkg/compiler/test/testdata/branch/success_8_nested.json diff --git a/config.yaml b/config.yaml index 01bb7501b..0bac6f161 100644 --- a/config.yaml +++ b/config.yaml @@ -1,7 +1,7 @@ # This is a sample configuration file. # Real configuration when running inside K8s (local or otherwise) lives in a ConfigMap propeller: - rawoutput-prefix: s3://my-container/test/ + rawoutput-prefix: s3://flyte/test/ metadata-prefix: metadata/propeller/sandbox workers: 4 workflow-reeval-duration: 10s @@ -23,7 +23,7 @@ propeller: rate: 100 capacity: 1000 # This config assumes using `make start` in flytesnacks repo to startup a DinD k3s container - kube-config: "$HOME/kubeconfig/k3s/k3s.yaml" + kube-config: "$HOME/.kube/config" publish-k8s-events: true workflowStore: policy: "ResourceVersionCache" @@ -85,7 +85,7 @@ storage: access-key: minio auth-type: accesskey disable-ssl: true - endpoint: http://localhost:30084 + endpoint: http://localhost:9000 region: us-east-1 secret-key: miniostorage type: minio diff --git a/pkg/compiler/test/compiler_test.go b/pkg/compiler/test/compiler_test.go index e3bb6410f..490092f0f 100644 --- a/pkg/compiler/test/compiler_test.go +++ b/pkg/compiler/test/compiler_test.go @@ -195,16 +195,34 @@ func TestBranches(t *testing.T) { t.Run(path, func(t *testing.T) { // If you want to debug a single use-case. Uncomment this line. - if !strings.HasSuffix(path, "success_7_nested.json") { - t.SkipNow() - } + //if !strings.HasSuffix(path, "success_7_nested.json") { + // t.SkipNow() + //} raw, err := ioutil.ReadFile(path) assert.NoError(t, err) wf := &core.WorkflowClosure{} - err = jsonpb.UnmarshalString(string(raw), wf) - if !assert.NoError(t, err) { - t.FailNow() + if filepath.Ext(path) == ".json" { + err = jsonpb.UnmarshalString(string(raw), wf) + if !assert.NoError(t, err) { + t.FailNow() + } + } else if filepath.Ext(path) == ".pb" { + err = proto.Unmarshal(raw, wf) + if !assert.NoError(t, err) { + t.FailNow() + } + + m := &jsonpb.Marshaler{} + raw, err := m.MarshalToString(wf) + if !assert.NoError(t, err) { + t.FailNow() + } + + err = ioutil.WriteFile(strings.TrimSuffix(path, filepath.Ext(path))+".json", []byte(raw), os.ModePerm) + if !assert.NoError(t, err) { + t.FailNow() + } } t.Log("Compiling Workflow") diff --git a/pkg/compiler/test/testdata/branch/compiled/success_3.json b/pkg/compiler/test/testdata/branch/compiled/success_3.json index de85ced03..7b5c490af 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_3.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_3.json @@ -1 +1 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}}},"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["node-1"]},"node-1":{"ids":["end-node"]},"start-node":{"ids":["node-0","node-1"]}},"upstream":{"branchnode-0":{"ids":["node-0"]},"end-node":{"ids":["node-1"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0","start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}}},"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["node-1"]},"node-1":{"ids":["end-node"]},"start-node":{"ids":["node-0","node-1"]}},"upstream":{"end-node":{"ids":["node-1"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0","start-node"]},"node-1-branchnode-0":{"ids":[""]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_4.json b/pkg/compiler/test/testdata/branch/compiled/success_4.json index 76e1c6bdc..5e1c36c02 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_4.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_4.json @@ -1 +1 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}}},"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}},{"id":"node-2","metadata":{"name":"test2","retries":{},"interruptible":false},"inputs":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-1"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"var":".a"}}},"thenNode":{"id":"node-2-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-1"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}}},"error":{"failedNodeId":"test2","message":"Unable to choose a branch"}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["node-1"]},"node-1":{"ids":["node-2"]},"node-2":{"ids":["end-node"]},"start-node":{"ids":["node-0","node-1","node-2"]}},"upstream":{"branchnode-0":{"ids":["node-0","node-1"]},"end-node":{"ids":["node-2"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0","start-node"]},"node-2":{"ids":["node-1","start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}}},"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}},{"id":"node-2","metadata":{"name":"test2","retries":{},"interruptible":false},"inputs":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-1"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"var":".a"}}},"thenNode":{"id":"node-2-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-1"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}}},"error":{"failedNodeId":"test2","message":"Unable to choose a branch"}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["node-1"]},"node-1":{"ids":["node-2"]},"node-2":{"ids":["end-node"]},"start-node":{"ids":["node-0","node-1","node-2"]}},"upstream":{"end-node":{"ids":["node-2"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0","start-node"]},"node-1-branchnode-0":{"ids":[""]},"node-2":{"ids":["node-1","start-node"]},"node-2-branchnode-0":{"ids":[""]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_5.json b/pkg/compiler/test/testdata/branch/compiled/success_5.json index 3a5cdd1a1..7c8cf9dd5 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_5.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_5.json @@ -1 +1 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"},"b":{"type":{"simple":"STRING"},"description":"b"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"},"out_1":{"type":{"simple":"STRING"},"description":"out_1"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t1","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":"node-0.t1_int_output","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"4"}}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}},"other":[{"condition":{"comparison":{"operator":"GTE","leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"5"}}}},"thenNode":{"id":"node-1-branchnode-1","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"c"}}}],"upstreamNodeIds":["node-0"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}}],"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}},{"id":"node-2","metadata":{"name":"test2","retries":{},"interruptible":false},"inputs":[{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-1"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"primitive":{"stringValue":"hello "}}}},"thenNode":{"id":"node-2-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"It is hello"}}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}},"elseNode":{"id":"node-2-branchnode-1","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"Not Hello!"}}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["end-node","node-1"]},"node-1":{"ids":["node-2"]},"node-2":{"ids":["end-node"]},"start-node":{"ids":["node-0"]}},"upstream":{"branchnode-0":{"ids":["start-node"]},"branchnode-1":{"ids":["node-0"]},"end-node":{"ids":["node-0","node-2"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0"]},"node-2":{"ids":["node-1"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"},"t1_int_output":{"type":{"simple":"INTEGER"},"description":"t1_int_output"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"STRING"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"},"b":{"type":{"simple":"STRING"},"description":"b"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"},"out_1":{"type":{"simple":"STRING"},"description":"out_1"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t1","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":"node-0.t1_int_output","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"4"}}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}},"other":[{"condition":{"comparison":{"operator":"GTE","leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"5"}}}},"thenNode":{"id":"node-1-branchnode-1","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"c"}}}],"upstreamNodeIds":["node-0"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}}],"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}},{"id":"node-2","metadata":{"name":"test2","retries":{},"interruptible":false},"inputs":[{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-1"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"primitive":{"stringValue":"hello "}}}},"thenNode":{"id":"node-2-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"It is hello"}}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}},"elseNode":{"id":"node-2-branchnode-1","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"Not Hello!"}}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["end-node","node-1"]},"node-1":{"ids":["node-2"]},"node-2":{"ids":["end-node"]},"start-node":{"ids":["node-0"]}},"upstream":{"end-node":{"ids":["node-0","node-2"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0"]},"node-1-branchnode-0":{"ids":[""]},"node-1-branchnode-1":{"ids":[""]},"node-2":{"ids":["node-1"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"},"t1_int_output":{"type":{"simple":"INTEGER"},"description":"t1_int_output"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"STRING"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_6.json b/pkg/compiler/test/testdata/branch/compiled/success_6.json index fd7e99c8c..8ac74d67b 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_6.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_6.json @@ -1 +1 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.multiplier","version":"abc"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}]},{"id":"node-0","metadata":{"name":"fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"node-0-branchn0","metadata":{"name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.double","version":"abc"}}}},"elseNode":{"id":"node-0-branchn1","metadata":{"name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.square","version":"abc"}}}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["end-node"]},"start-node":{"ids":["node-0"]}},"upstream":{"branchn0":{"ids":["start-node"]},"branchn1":{"ids":["start-node"]},"end-node":{"ids":["node-0"]},"node-0":{"ids":["start-node"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.multiplier","version":"abc"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}]},{"id":"node-0","metadata":{"name":"fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"node-0-branchn0","metadata":{"name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.double","version":"abc"}}}},"elseNode":{"id":"node-0-branchn1","metadata":{"name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.square","version":"abc"}}}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["end-node"]},"start-node":{"ids":["node-0"]}},"upstream":{"end-node":{"ids":["node-0"]},"node-0":{"ids":["start-node"]},"node-0-branchn0":{"ids":[""]},"node-0-branchn1":{"ids":[""]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json b/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json index 9db0acf27..ed1fe739b 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json @@ -1 +1 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.so_nested","version":"abc"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},{"id":"branchnode-2","metadata":{"name":"fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"branchnode-2-branchbranchnode-1","metadata":{"name":"inner_fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"thenNode":{"id":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"}}}},"error":{"failedNodeId":"inner_fractions","message":"Only \u003c0.5 allowed"}}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"thenNode":{"id":"branchnode-2-branchn1","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"}}}}],"error":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"branchnode-2":{"ids":["end-node"]},"start-node":{"ids":["branchnode-2"]}},"upstream":{"branchnode-2":{"ids":["start-node"]},"branchnode-2-branchbranchnode-1":{"ids":["start-node"]},"end-node":{"ids":["branchnode-2"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.so_nested","version":"abc"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},{"id":"branchnode-2","metadata":{"name":"fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"branchnode-2-branchbranchnode-1","metadata":{"name":"inner_fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"thenNode":{"id":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"}}}},"error":{"failedNodeId":"inner_fractions","message":"Only \u003c0.5 allowed"}}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"thenNode":{"id":"branchnode-2-branchn1","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"}}}}],"error":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"branchnode-2":{"ids":["end-node"]},"start-node":{"ids":["branchnode-2"]}},"upstream":{"branchnode-2":{"ids":["start-node"]},"branchnode-2-branchbranchnode-1":{"ids":["start-node"]},"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0":{"ids":[""]},"branchnode-2-branchn1":{"ids":[""]},"end-node":{"ids":["branchnode-2"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json b/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json new file mode 100755 index 000000000..13af089e0 --- /dev/null +++ b/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json @@ -0,0 +1 @@ +{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.so_nested","version":"abc"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"n0-n0","metadata":{"name":"inner_fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"thenNode":{"id":"n0-n0-n0-n0","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.7}}}}}},"thenNode":{"id":"n0-n0-n0-n1","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"}}}}],"error":{"failedNodeId":"inner_fractions","message":"Only \u003c0.7 allowed"}}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"thenNode":{"id":"n0-n1","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"}}}}],"elseNode":{"id":"n0-n2","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"}}}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]},"n0-n0-n0-n0":{"ids":[""]},"n0-n0-n0-n1":{"ids":[""]},"n0-n2":{"ids":[""]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_3.json b/pkg/compiler/test/testdata/branch/k8s/success_3.json index f66f7260b..852bfa407 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_3.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_3.json @@ -1 +1 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"then":"node-1-branchnode-0"},"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["node-1"],"node-1":["end-node"],"start-node":["node-0","node-1"]},"edges":{"downstream":{"node-0":["node-1"],"node-1":["end-node"],"start-node":["node-0","node-1"]},"upstream":{"branchnode-0":["node-0"],"end-node":["node-1"],"node-0":["start-node"],"node-1":["node-0","start-node"]}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"then":"node-1-branchnode-0"},"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["node-1"],"node-1":["end-node"],"start-node":["node-0","node-1"]},"edges":{"downstream":{"node-0":["node-1"],"node-1":["end-node"],"start-node":["node-0","node-1"]},"upstream":{"end-node":["node-1"],"node-0":["start-node"],"node-1":["node-0","start-node"],"node-1-branchnode-0":[""]}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_4.json b/pkg/compiler/test/testdata/branch/k8s/success_4.json index 1be4946c2..e1c1379cb 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_4.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_4.json @@ -1 +1 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"then":"node-1-branchnode-0"},"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2":{"id":"node-2","name":"test2","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"var":".a"}}},"then":"node-2-branchnode-0"},"elseFail":{"failedNodeId":"test2","message":"Unable to choose a branch"}},"inputBindings":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-0":{"id":"node-2-branchnode-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0","node-1","node-2"]},"edges":{"downstream":{"node-0":["node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0","node-1","node-2"]},"upstream":{"branchnode-0":["node-0","node-1"],"end-node":["node-2"],"node-0":["start-node"],"node-1":["node-0","start-node"],"node-2":["node-1","start-node"]}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"then":"node-1-branchnode-0"},"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2":{"id":"node-2","name":"test2","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"var":".a"}}},"then":"node-2-branchnode-0"},"elseFail":{"failedNodeId":"test2","message":"Unable to choose a branch"}},"inputBindings":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-0":{"id":"node-2-branchnode-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0","node-1","node-2"]},"edges":{"downstream":{"node-0":["node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0","node-1","node-2"]},"upstream":{"end-node":["node-2"],"node-0":["start-node"],"node-1":["node-0","start-node"],"node-1-branchnode-0":[""],"node-2":["node-1","start-node"],"node-2-branchnode-0":[""]}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_5.json b/pkg/compiler/test/testdata/branch/k8s/success_5.json index 281d01a27..9782fd559 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_5.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_5.json @@ -1 +1 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t1","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"4"}}}},"then":"node-1-branchnode-0"},"elseIf":[{"condition":{"comparison":{"operator":"GTE","leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"5"}}}},"then":"node-1-branchnode-1"}],"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":"node-0.t1_int_output","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-1":{"id":"node-1-branchnode-1","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"c"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2":{"id":"node-2","name":"test2","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"primitive":{"stringValue":"hello "}}}},"then":"node-2-branchnode-0"},"else":"node-2-branchnode-1"},"inputBindings":[{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-0":{"id":"node-2-branchnode-0","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"It is hello"}}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-1":{"id":"node-2-branchnode-1","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"Not Hello!"}}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["end-node","node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0"]},"edges":{"downstream":{"node-0":["end-node","node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0"]},"upstream":{"branchnode-0":["start-node"],"branchnode-1":["node-0"],"end-node":["node-0","node-2"],"node-0":["start-node"],"node-1":["node-0"],"node-2":["node-1"]}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"},"out_1":{"type":{"simple":"STRING"},"description":"out_1"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}},"b":{"scalar":{"primitive":{"stringValue":""}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"},"t1_int_output":{"type":{"simple":"INTEGER"},"description":"t1_int_output"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"STRING"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t1","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"4"}}}},"then":"node-1-branchnode-0"},"elseIf":[{"condition":{"comparison":{"operator":"GTE","leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"5"}}}},"then":"node-1-branchnode-1"}],"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":"node-0.t1_int_output","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-1":{"id":"node-1-branchnode-1","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"c"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2":{"id":"node-2","name":"test2","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"primitive":{"stringValue":"hello "}}}},"then":"node-2-branchnode-0"},"else":"node-2-branchnode-1"},"inputBindings":[{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-0":{"id":"node-2-branchnode-0","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"It is hello"}}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-1":{"id":"node-2-branchnode-1","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"Not Hello!"}}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["end-node","node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0"]},"edges":{"downstream":{"node-0":["end-node","node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0"]},"upstream":{"end-node":["node-0","node-2"],"node-0":["start-node"],"node-1":["node-0"],"node-1-branchnode-0":[""],"node-1-branchnode-1":[""],"node-2":["node-1"]}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"},"out_1":{"type":{"simple":"STRING"},"description":"out_1"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}},"b":{"scalar":{"primitive":{"stringValue":""}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"},"t1_int_output":{"type":{"simple":"INTEGER"},"description":"t1_int_output"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"STRING"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_6.json b/pkg/compiler/test/testdata/branch/k8s/success_6.json index 961211bcb..e9ee61c4c 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_6.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_6.json @@ -1 +1 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"recipes-02-intermediate-run-conditions-multiplier"}},"spec":{"id":"test_proj:test_domain:recipes.02_intermediate.run_conditions.multiplier","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}]},"node-0":{"id":"node-0","name":"fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"then":"node-0-branchn0"},"else":"node-0-branchn1"},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-0-branchn0":{"id":"node-0-branchn0","name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-0-branchn1":{"id":"node-0-branchn1","name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["end-node"],"start-node":["node-0"]},"edges":{"downstream":{"node-0":["end-node"],"start-node":["node-0"]},"upstream":{"branchn0":["start-node"],"branchn1":["start-node"],"end-node":["node-0"],"node-0":["start-node"]}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}},"outputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}]},"inputs":{"literals":{"my_input":{"scalar":{"primitive":{"floatValue":0}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.double\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.square\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"recipes-02-intermediate-run-conditions-multiplier"}},"spec":{"id":"test_proj:test_domain:recipes.02_intermediate.run_conditions.multiplier","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}]},"node-0":{"id":"node-0","name":"fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"then":"node-0-branchn0"},"else":"node-0-branchn1"},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-0-branchn0":{"id":"node-0-branchn0","name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-0-branchn1":{"id":"node-0-branchn1","name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["end-node"],"start-node":["node-0"]},"edges":{"downstream":{"node-0":["end-node"],"start-node":["node-0"]},"upstream":{"end-node":["node-0"],"node-0":["start-node"],"node-0-branchn0":[""],"node-0-branchn1":[""]}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}},"outputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}]},"inputs":{"literals":{"my_input":{"scalar":{"primitive":{"floatValue":0}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.double\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.square\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json b/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json index f534a7f2b..b3dc85aca 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json @@ -1 +1 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"core-control-flow-run-conditions-so-nested"}},"spec":{"id":"test_proj:test_domain:core.control_flow.run_conditions.so_nested","nodes":{"branchnode-2":{"id":"branchnode-2","name":"fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"then":"branchnode-2-branchbranchnode-1"},"elseIf":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"then":"branchnode-2-branchn1"}],"elseFail":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchbranchnode-1":{"id":"branchnode-2-branchbranchnode-1","name":"inner_fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"then":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0"},"elseFail":{"failedNodeId":"inner_fractions","message":"Only \u003c0.5 allowed"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0":{"id":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchn1":{"id":"branchnode-2-branchn1","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"branchnode-2":["end-node"],"start-node":["branchnode-2"]},"edges":{"downstream":{"branchnode-2":["end-node"],"start-node":["branchnode-2"]},"upstream":{"branchnode-2":["start-node"],"branchnode-2-branchbranchnode-1":["start-node"],"end-node":["branchnode-2"]}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}},"outputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},"inputs":{"literals":{"my_input":{"scalar":{"primitive":{"floatValue":0}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"core-control-flow-run-conditions-so-nested"}},"spec":{"id":"test_proj:test_domain:core.control_flow.run_conditions.so_nested","nodes":{"branchnode-2":{"id":"branchnode-2","name":"fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"then":"branchnode-2-branchbranchnode-1"},"elseIf":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"then":"branchnode-2-branchn1"}],"elseFail":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchbranchnode-1":{"id":"branchnode-2-branchbranchnode-1","name":"inner_fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"then":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0"},"elseFail":{"failedNodeId":"inner_fractions","message":"Only \u003c0.5 allowed"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0":{"id":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchn1":{"id":"branchnode-2-branchn1","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"branchnode-2":["end-node"],"start-node":["branchnode-2"]},"edges":{"downstream":{"branchnode-2":["end-node"],"start-node":["branchnode-2"]},"upstream":{"branchnode-2":["start-node"],"branchnode-2-branchbranchnode-1":["start-node"],"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0":[""],"branchnode-2-branchn1":[""],"end-node":["branchnode-2"]}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}},"outputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},"inputs":{"literals":{"my_input":{"scalar":{"primitive":{"floatValue":0}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json b/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json new file mode 100755 index 000000000..b0c7c11f4 --- /dev/null +++ b/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json @@ -0,0 +1 @@ +{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"core-control-flow-run-conditions-so-nested"}},"spec":{"id":"test_proj:test_domain:core.control_flow.run_conditions.so_nested","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},"n0":{"id":"n0","name":"fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"then":"n0-n0"},"elseIf":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"then":"n0-n1"}],"else":"n0-n2"},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"n0-n0":{"id":"n0-n0","name":"inner_fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"then":"n0-n0-n0-n0"},"elseIf":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.7}}}}}},"then":"n0-n0-n0-n1"}],"elseFail":{"failedNodeId":"inner_fractions","message":"Only \u003c0.7 allowed"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"n0-n0-n0-n0":{"id":"n0-n0-n0-n0","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"n0-n0-n0-n1":{"id":"n0-n0-n0-n1","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"n0-n1":{"id":"n0-n1","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"n0-n2":{"id":"n0-n2","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"n0":["end-node"],"start-node":["n0"]},"edges":{"downstream":{"n0":["end-node"],"start-node":["n0"]},"upstream":{"end-node":["n0"],"n0":["start-node"],"n0-n0-n0-n0":[""],"n0-n0-n0-n1":[""],"n0-n2":[""]}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}},"outputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},"inputs":{"literals":{"my_input":{"scalar":{"primitive":{"floatValue":0}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/success_8_nested.json b/pkg/compiler/test/testdata/branch/success_8_nested.json new file mode 100755 index 000000000..d9d8af261 --- /dev/null +++ b/pkg/compiler/test/testdata/branch/success_8_nested.json @@ -0,0 +1,441 @@ +{ + "workflow": { + "id": { + "resourceType": "WORKFLOW", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.so_nested", + "version": "abc" + }, + "metadata": {}, + "interface": { + "inputs": { + "variables": { + "my_input": { + "type": { + "simple": "FLOAT" + }, + "description": "my_input" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "n0", + "metadata": { + "name": "fractions", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + } + } + }, + "thenNode": { + "id": "n0", + "metadata": { + "name": "inner_fractions", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "thenNode": { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + } + } + } + }, + "other": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.7 + } + } + } + } + } + }, + "thenNode": { + "id": "n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + } + } + } + } + ], + "error": { + "failedNodeId": "inner_fractions", + "message": "Only \u003c0.7 allowed" + } + } + } + } + }, + "other": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 10 + } + } + } + } + } + }, + "thenNode": { + "id": "n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + } + } + } + } + ], + "elseNode": { + "id": "n2", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "retries": {}, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + } + } + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "metadataDefaults": {} + }, + "tasks": [ + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + }, + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/validators/branch.go b/pkg/compiler/validators/branch.go index b3107b092..66084b556 100644 --- a/pkg/compiler/validators/branch.go +++ b/pkg/compiler/validators/branch.go @@ -67,6 +67,9 @@ func validateBranchInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs error continue } + ValidateBindings(w, n, n.GetInputs(), &flyte.VariableMap{Variables: map[string]*flyte.Variable{}}, + false, EdgeDirectionUpstream, errs.NewScope()) + // Clear out the Inputs. We do not care if the inputs of each of the underlying nodes // match. We will pull the inputs needed for the underlying branch node at runtime. iface2 = &flyte.TypedInterface{ From 038c2792f9453e7da36064a9ecb509655d19685d Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Sat, 29 May 2021 12:33:14 -0700 Subject: [PATCH 04/14] fix lint issue Signed-off-by: Ketan Umare Signed-off-by: Haytham Abuelfutuh --- pkg/compiler/common/builder.go | 2 +- pkg/compiler/common/mocks/workflow_builder.go | 2 +- pkg/compiler/validators/node.go | 2 +- pkg/compiler/workflow_compiler.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/compiler/common/builder.go b/pkg/compiler/common/builder.go index 41ba0dc2a..8f6b71c55 100644 --- a/pkg/compiler/common/builder.go +++ b/pkg/compiler/common/builder.go @@ -21,7 +21,7 @@ type WorkflowBuilder interface { AddUpstreamEdge(nodeProvider, nodeDependent NodeID) AddDownstreamEdge(nodeProvider, nodeDependent NodeID) AddNode(n NodeBuilder, errs errors.CompileErrors) (node NodeBuilder, ok bool) - ReplaceNodeId(oldID, newID string) + ReplaceNodeID(oldID, newID string) ValidateWorkflow(fg *core.CompiledWorkflow, errs errors.CompileErrors) (Workflow, bool) NewNodeBuilder(n *core.Node) NodeBuilder } diff --git a/pkg/compiler/common/mocks/workflow_builder.go b/pkg/compiler/common/mocks/workflow_builder.go index eaaeb98ce..4504a4eb0 100644 --- a/pkg/compiler/common/mocks/workflow_builder.go +++ b/pkg/compiler/common/mocks/workflow_builder.go @@ -516,7 +516,7 @@ func (_m *WorkflowBuilder) NewNodeBuilder(n *core.Node) common.NodeBuilder { } // ReplaceNodeId provides a mock function with given fields: oldID, newID -func (_m *WorkflowBuilder) ReplaceNodeId(oldID string, newID string) { +func (_m *WorkflowBuilder) ReplaceNodeID(oldID string, newID string) { _m.Called(oldID, newID) } diff --git a/pkg/compiler/validators/node.go b/pkg/compiler/validators/node.go index 5a01cb106..ccff53733 100644 --- a/pkg/compiler/validators/node.go +++ b/pkg/compiler/validators/node.go @@ -126,7 +126,7 @@ func ValidateNode(w c.WorkflowBuilder, n c.NodeBuilder, validateConditionTypes b for _, subNode := range nodes { oldID := subNode.GetId() subNode.SetID(branchNodeIDFormatter(n.GetId(), subNode.GetId())) - w.ReplaceNodeId(oldID, subNode.GetId()) + w.ReplaceNodeID(oldID, subNode.GetId()) } } } else if workflowN := n.GetWorkflowNode(); workflowN != nil && workflowN.GetSubWorkflowRef() != nil { diff --git a/pkg/compiler/workflow_compiler.go b/pkg/compiler/workflow_compiler.go index cf17cf4d3..bf25812c8 100755 --- a/pkg/compiler/workflow_compiler.go +++ b/pkg/compiler/workflow_compiler.go @@ -124,7 +124,7 @@ func (w workflowBuilder) AddUpstreamEdge(nodeProvider, nodeDependent c.NodeID) { w.CoreWorkflow.Connections.Upstream[nodeDependent].Ids = w.upstreamNodes[nodeDependent].List() } -func (w workflowBuilder) ReplaceNodeId(oldID, newID string) { +func (w workflowBuilder) ReplaceNodeID(oldID, newID string) { if existingVal, found := w.upstreamNodes[oldID]; found { w.upstreamNodes[newID] = existingVal delete(w.upstreamNodes, oldID) From 5d820769c6c5fe92fc6a8ecf9db21f83a5469823 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 1 Jun 2021 15:07:16 -0700 Subject: [PATCH 05/14] Fix connections for branch nodes Signed-off-by: Haytham Abuelfutuh --- pkg/compiler/common/builder.go | 9 + pkg/compiler/common/mocks/workflow_builder.go | 34 +- pkg/compiler/test/compiler_test.go | 119 +++- .../testdata/branch/compiled/success_1.json | 425 ++++++++++++- .../testdata/branch/compiled/success_2.json | 475 +++++++++++++- .../testdata/branch/compiled/success_3.json | 454 +++++++++++++- .../testdata/branch/compiled/success_4.json | 558 ++++++++++++++++- .../testdata/branch/compiled/success_5.json | 585 +++++++++++++++++- .../testdata/branch/compiled/success_6.json | 383 +++++++++++- .../branch/compiled/success_7_nested.json | 470 +++++++++++++- .../branch/compiled/success_8_nested.json | 570 ++++++++++++++++- .../test/testdata/branch/k8s/success_1.json | 393 +++++++++++- .../test/testdata/branch/k8s/success_2.json | 434 ++++++++++++- .../test/testdata/branch/k8s/success_3.json | 419 ++++++++++++- .../test/testdata/branch/k8s/success_4.json | 507 ++++++++++++++- .../test/testdata/branch/k8s/success_5.json | 532 +++++++++++++++- .../test/testdata/branch/k8s/success_6.json | 362 ++++++++++- .../testdata/branch/k8s/success_7_nested.json | 446 ++++++++++++- .../testdata/branch/k8s/success_8_nested.json | 528 +++++++++++++++- pkg/compiler/validators/bindings.go | 16 +- pkg/compiler/validators/bindings_test.go | 14 +- pkg/compiler/validators/branch.go | 7 +- pkg/compiler/validators/node.go | 10 +- pkg/compiler/workflow_compiler.go | 42 +- 24 files changed, 7723 insertions(+), 69 deletions(-) diff --git a/pkg/compiler/common/builder.go b/pkg/compiler/common/builder.go index 8f6b71c55..fe8743c66 100644 --- a/pkg/compiler/common/builder.go +++ b/pkg/compiler/common/builder.go @@ -11,6 +11,14 @@ const ( EndNodeID = "end-node" ) +type EdgeDirection uint8 + +const ( + EdgeDirectionBidirectional EdgeDirection = iota + EdgeDirectionDownstream + EdgeDirectionUpstream +) + //go:generate mockery -all -output=mocks -case=underscore // A mutable workflow used during the build of the intermediate layer. @@ -21,6 +29,7 @@ type WorkflowBuilder interface { AddUpstreamEdge(nodeProvider, nodeDependent NodeID) AddDownstreamEdge(nodeProvider, nodeDependent NodeID) AddNode(n NodeBuilder, errs errors.CompileErrors) (node NodeBuilder, ok bool) + AddEdges(n NodeBuilder, edgeDirection EdgeDirection, errs errors.CompileErrors) (ok bool) ReplaceNodeID(oldID, newID string) ValidateWorkflow(fg *core.CompiledWorkflow, errs errors.CompileErrors) (Workflow, bool) NewNodeBuilder(n *core.Node) NodeBuilder diff --git a/pkg/compiler/common/mocks/workflow_builder.go b/pkg/compiler/common/mocks/workflow_builder.go index 4504a4eb0..1c1b6afe7 100644 --- a/pkg/compiler/common/mocks/workflow_builder.go +++ b/pkg/compiler/common/mocks/workflow_builder.go @@ -21,6 +21,38 @@ func (_m *WorkflowBuilder) AddDownstreamEdge(nodeProvider string, nodeDependent _m.Called(nodeProvider, nodeDependent) } +type WorkflowBuilder_AddEdges struct { + *mock.Call +} + +func (_m WorkflowBuilder_AddEdges) Return(ok bool) *WorkflowBuilder_AddEdges { + return &WorkflowBuilder_AddEdges{Call: _m.Call.Return(ok)} +} + +func (_m *WorkflowBuilder) OnAddEdges(n common.NodeBuilder, edgeDirection common.EdgeDirection, errs errors.CompileErrors) *WorkflowBuilder_AddEdges { + c := _m.On("AddEdges", n, edgeDirection, errs) + return &WorkflowBuilder_AddEdges{Call: c} +} + +func (_m *WorkflowBuilder) OnAddEdgesMatch(matchers ...interface{}) *WorkflowBuilder_AddEdges { + c := _m.On("AddEdges", matchers...) + return &WorkflowBuilder_AddEdges{Call: c} +} + +// AddEdges provides a mock function with given fields: n, edgeDirection, errs +func (_m *WorkflowBuilder) AddEdges(n common.NodeBuilder, edgeDirection common.EdgeDirection, errs errors.CompileErrors) bool { + ret := _m.Called(n, edgeDirection, errs) + + var r0 bool + if rf, ok := ret.Get(0).(func(common.NodeBuilder, common.EdgeDirection, errors.CompileErrors) bool); ok { + r0 = rf(n, edgeDirection, errs) + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + // AddExecutionEdge provides a mock function with given fields: nodeFrom, nodeTo func (_m *WorkflowBuilder) AddExecutionEdge(nodeFrom string, nodeTo string) { _m.Called(nodeFrom, nodeTo) @@ -515,7 +547,7 @@ func (_m *WorkflowBuilder) NewNodeBuilder(n *core.Node) common.NodeBuilder { return r0 } -// ReplaceNodeId provides a mock function with given fields: oldID, newID +// ReplaceNodeID provides a mock function with given fields: oldID, newID func (_m *WorkflowBuilder) ReplaceNodeID(oldID string, newID string) { _m.Called(oldID, newID) } diff --git a/pkg/compiler/test/compiler_test.go b/pkg/compiler/test/compiler_test.go index 490092f0f..a4d428a2e 100644 --- a/pkg/compiler/test/compiler_test.go +++ b/pkg/compiler/test/compiler_test.go @@ -10,6 +10,8 @@ import ( "strings" "testing" + "k8s.io/apimachinery/pkg/util/sets" + "github.com/go-test/deep" "github.com/ghodss/yaml" @@ -182,6 +184,104 @@ func TestDynamic(t *testing.T) { })) } +func getAllSubNodeIDs(n *core.Node) sets.String { + res := sets.NewString() + if branchNode := n.GetBranchNode(); branchNode != nil { + thenNode := branchNode.IfElse.Case.ThenNode + if hasPromiseInputs(thenNode.GetInputs()) { + res.Insert(thenNode.GetId()) + } + + res = res.Union(getAllSubNodeIDs(thenNode)) + + for _, other := range branchNode.IfElse.Other { + if hasPromiseInputs(other.ThenNode.GetInputs()) { + res.Insert(other.ThenNode.GetId()) + } + + res = res.Union(getAllSubNodeIDs(other.ThenNode)) + } + + if elseNode := branchNode.IfElse.GetElseNode(); elseNode != nil { + if hasPromiseInputs(elseNode.GetInputs()) { + res.Insert(elseNode.GetId()) + } + + res = res.Union(getAllSubNodeIDs(elseNode)) + } + } + + // TODO: Support Sub workflow + + return res +} + +func getAllNodeIDsWithPromiseInputs(wf *core.CompiledWorkflow) sets.String { + s := sets.NewString() + for _, n := range wf.Template.Nodes { + if hasPromiseInputs(n.GetInputs()) { + s.Insert(n.GetId()) + } + + s = s.Union(getAllSubNodeIDs(n)) + } + + return s +} + +func bindingHasPromiseInputs(binding *core.BindingData) bool { + switch v := binding.GetValue().(type) { + case *core.BindingData_Collection: + for _, d := range v.Collection.Bindings { + if bindingHasPromiseInputs(d) { + return true + } + } + case *core.BindingData_Map: + for _, d := range v.Map.Bindings { + if bindingHasPromiseInputs(d) { + return true + } + } + case *core.BindingData_Promise: + return true + } + + return false +} + +func hasPromiseInputs(bindings []*core.Binding) bool { + for _, b := range bindings { + if bindingHasPromiseInputs(b.Binding) { + return true + } + } + + return false +} + +func assertNotIDsInConnections(t testing.TB, expectedNodeIDs sets.String, connections *core.ConnectionSet) bool { + actualNodeIDs := sets.NewString() + for id, lst := range connections.Downstream { + actualNodeIDs.Insert(id) + actualNodeIDs.Insert(lst.Ids...) + } + + for id, lst := range connections.Upstream { + actualNodeIDs.Insert(id) + actualNodeIDs.Insert(lst.Ids...) + } + + notFoundInConnections := expectedNodeIDs.Difference(actualNodeIDs) + correct := assert.Empty(t, notFoundInConnections, "All nodes must appear in connections") + + notFoundInNodes := actualNodeIDs.Difference(expectedNodeIDs) + notFoundInNodes.Delete("start-node") + correct = correct && assert.Empty(t, notFoundInNodes, "All connections must correspond to existing nodes") + + return correct +} + func TestBranches(t *testing.T) { errors.SetConfig(errors.Config{IncludeSource: true}) assert.NoError(t, filepath.Walk("testdata/branch", func(path string, info os.FileInfo, err error) error { @@ -195,7 +295,7 @@ func TestBranches(t *testing.T) { t.Run(path, func(t *testing.T) { // If you want to debug a single use-case. Uncomment this line. - //if !strings.HasSuffix(path, "success_7_nested.json") { + //if !strings.HasSuffix(path, "success_8_nested.json") { // t.SkipNow() //} @@ -213,7 +313,9 @@ func TestBranches(t *testing.T) { t.FailNow() } - m := &jsonpb.Marshaler{} + m := &jsonpb.Marshaler{ + Indent: " ", + } raw, err := m.MarshalToString(wf) if !assert.NoError(t, err) { t.FailNow() @@ -233,8 +335,10 @@ func TestBranches(t *testing.T) { t.FailNow() } - marshaler := jsonpb.Marshaler{} - rawStr, err := marshaler.MarshalToString(compiledWfc) + m := &jsonpb.Marshaler{ + Indent: " ", + } + rawStr, err := m.MarshalToString(compiledWfc) if !assert.NoError(t, err) { t.Fail() } @@ -256,6 +360,11 @@ func TestBranches(t *testing.T) { } } + nodeIDs := getAllNodeIDsWithPromiseInputs(compiledWfc.Primary) + if !assertNotIDsInConnections(t, nodeIDs, compiledWfc.Primary.Connections) { + t.FailNow() + } + inputs := map[string]interface{}{} for varName, v := range compiledWfc.Primary.Template.Interface.Inputs.Variables { inputs[varName] = coreutils.MustMakeDefaultLiteralForType(v.Type) @@ -270,7 +379,7 @@ func TestBranches(t *testing.T) { }, "namespace") if assert.NoError(t, err) { - raw, err := json.Marshal(flyteWf) + raw, err := json.MarshalIndent(flyteWf, "", " ") if assert.NoError(t, err) { assert.NotEmpty(t, raw) } diff --git a/pkg/compiler/test/testdata/branch/compiled/success_1.json b/pkg/compiler/test/testdata/branch/compiled/success_1.json index e9fd78dd4..c8a5e8118 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_1.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_1.json @@ -1 +1,424 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"out_0"},"rightValue":{"primitive":{"integer":"1"}}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"test_serialization.t2","retries":{},"interruptible":false},"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}},"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["node-1"]},"node-1":{"ids":["end-node"]},"start-node":{"ids":["node-0"]}},"upstream":{"end-node":{"ids":["node-1"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "project", + "domain": "domain", + "name": "test_serialization.my_wf", + "version": "version" + }, + "metadata": { + + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ] + }, + { + "id": "node-0", + "metadata": { + "name": "test_serialization.t3", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + } + } + }, + { + "id": "node-1", + "metadata": { + "name": "test1", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "out_0" + } + } + } + ], + "upstreamNodeIds": [ + "node-0" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "out_0" + }, + "rightValue": { + "primitive": { + "integer": "1" + } + } + } + }, + "thenNode": { + "id": "node-1-branchnode-0", + "metadata": { + "name": "test_serialization.t2", + "retries": { + + }, + "interruptible": false + }, + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + } + } + } + }, + "error": { + "failedNodeId": "test1", + "message": "Unable to choose branch" + } + } + } + } + ], + "outputs": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ], + "metadataDefaults": { + + } + }, + "connections": { + "downstream": { + "node-0": { + "ids": [ + "node-1" + ] + }, + "node-1": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "node-0" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "node-1" + ] + }, + "node-0": { + "ids": [ + "start-node" + ] + }, + "node-1": { + "ids": [ + "node-0" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t1", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t1", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t2", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t3", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_2.json b/pkg/compiler/test/testdata/branch/compiled/success_2.json index 48820fa41..5e1479146 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_2.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_2.json @@ -1 +1,474 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-1","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-2","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}},{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-0","node-1"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":"node-1.out_0"}}},"thenNode":{"id":"node-2-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}},"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["node-2"]},"node-1":{"ids":["node-2"]},"node-2":{"ids":["end-node"]},"start-node":{"ids":["node-0","node-1"]}},"upstream":{"end-node":{"ids":["node-2"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["start-node"]},"node-2":{"ids":["node-0","node-1"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "project", + "domain": "domain", + "name": "test_serialization.my_wf", + "version": "version" + }, + "metadata": { + + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-2", + "var": "out_0" + } + } + } + ] + }, + { + "id": "node-0", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t3", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + } + } + }, + { + "id": "node-1", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t3", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + } + } + }, + { + "id": "node-2", + "metadata": { + "name": "test1", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "node-0.out_0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "out_0" + } + } + }, + { + "var": "node-1.out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ], + "upstreamNodeIds": [ + "node-0", + "node-1" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "node-0.out_0" + }, + "rightValue": { + "var": "node-1.out_0" + } + } + }, + "thenNode": { + "id": "node-2-branchnode-0", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t2", + "retries": { + + }, + "interruptible": false + }, + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + } + } + } + }, + "error": { + "failedNodeId": "test1", + "message": "Unable to choose branch" + } + } + } + } + ], + "outputs": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-2", + "var": "out_0" + } + } + } + ], + "metadataDefaults": { + + } + }, + "connections": { + "downstream": { + "node-0": { + "ids": [ + "node-2" + ] + }, + "node-1": { + "ids": [ + "node-2" + ] + }, + "node-2": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "node-0", + "node-1" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "node-2" + ] + }, + "node-0": { + "ids": [ + "start-node" + ] + }, + "node-1": { + "ids": [ + "start-node" + ] + }, + "node-2": { + "ids": [ + "node-0", + "node-1" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t1", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t1", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t2", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t3", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_3.json b/pkg/compiler/test/testdata/branch/compiled/success_3.json index 7b5c490af..3578ec575 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_3.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_3.json @@ -1 +1,453 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}}},"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["node-1"]},"node-1":{"ids":["end-node"]},"start-node":{"ids":["node-0","node-1"]}},"upstream":{"end-node":{"ids":["node-1"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0","start-node"]},"node-1-branchnode-0":{"ids":[""]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "project", + "domain": "domain", + "name": "test_serialization.my_wf", + "version": "version" + }, + "metadata": { + + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ] + }, + { + "id": "node-0", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t3", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + } + } + }, + { + "id": "node-1", + "metadata": { + "name": "test1", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": ".a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + }, + { + "var": "node-0.out_0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "out_0" + } + } + } + ], + "upstreamNodeIds": [ + "node-0" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "node-0.out_0" + }, + "rightValue": { + "var": ".a" + } + } + }, + "thenNode": { + "id": "node-1-branchnode-0", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t3", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "out_0" + } + } + } + ], + "upstreamNodeIds": [ + "node-0" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + } + } + } + }, + "error": { + "failedNodeId": "test1", + "message": "Unable to choose branch" + } + } + } + } + ], + "outputs": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ], + "metadataDefaults": { + + } + }, + "connections": { + "downstream": { + "node-0": { + "ids": [ + "node-1", + "node-1-branchnode-0" + ] + }, + "node-1": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "node-0", + "node-1" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "node-1" + ] + }, + "node-0": { + "ids": [ + "start-node" + ] + }, + "node-1": { + "ids": [ + "node-0", + "start-node" + ] + }, + "node-1-branchnode-0": { + "ids": [ + "node-0" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t1", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t1", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t2", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t3", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_4.json b/pkg/compiler/test/testdata/branch/compiled/success_4.json index 5e1c36c02..6f5431d98 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_4.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_4.json @@ -1 +1,557 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"upstreamNodeIds":["node-0"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}}},"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}},{"id":"node-2","metadata":{"name":"test2","retries":{},"interruptible":false},"inputs":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-1"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"var":".a"}}},"thenNode":{"id":"node-2-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t3","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-1"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"}}}},"error":{"failedNodeId":"test2","message":"Unable to choose a branch"}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["node-1"]},"node-1":{"ids":["node-2"]},"node-2":{"ids":["end-node"]},"start-node":{"ids":["node-0","node-1","node-2"]}},"upstream":{"end-node":{"ids":["node-2"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0","start-node"]},"node-1-branchnode-0":{"ids":[""]},"node-2":{"ids":["node-1","start-node"]},"node-2-branchnode-0":{"ids":[""]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "project", + "domain": "domain", + "name": "test_serialization.my_wf", + "version": "version" + }, + "metadata": { + + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-2", + "var": "out_0" + } + } + } + ] + }, + { + "id": "node-0", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t3", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + } + } + }, + { + "id": "node-1", + "metadata": { + "name": "test1", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": ".a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + }, + { + "var": "node-0.out_0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "out_0" + } + } + } + ], + "upstreamNodeIds": [ + "node-0" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "node-0.out_0" + }, + "rightValue": { + "var": ".a" + } + } + }, + "thenNode": { + "id": "node-1-branchnode-0", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t3", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "out_0" + } + } + } + ], + "upstreamNodeIds": [ + "node-0" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + } + } + } + }, + "error": { + "failedNodeId": "test1", + "message": "Unable to choose branch" + } + } + } + }, + { + "id": "node-2", + "metadata": { + "name": "test2", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": ".a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + }, + { + "var": "node-1.out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ], + "upstreamNodeIds": [ + "node-1" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "node-1.out_0" + }, + "rightValue": { + "var": ".a" + } + } + }, + "thenNode": { + "id": "node-2-branchnode-0", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t3", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ], + "upstreamNodeIds": [ + "node-1" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + } + } + } + }, + "error": { + "failedNodeId": "test2", + "message": "Unable to choose a branch" + } + } + } + } + ], + "outputs": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-2", + "var": "out_0" + } + } + } + ], + "metadataDefaults": { + + } + }, + "connections": { + "downstream": { + "node-0": { + "ids": [ + "node-1", + "node-1-branchnode-0" + ] + }, + "node-1": { + "ids": [ + "node-2", + "node-2-branchnode-0" + ] + }, + "node-2": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "node-0", + "node-1", + "node-2" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "node-2" + ] + }, + "node-0": { + "ids": [ + "start-node" + ] + }, + "node-1": { + "ids": [ + "node-0", + "start-node" + ] + }, + "node-1-branchnode-0": { + "ids": [ + "node-0" + ] + }, + "node-2": { + "ids": [ + "node-1", + "start-node" + ] + }, + "node-2-branchnode-0": { + "ids": [ + "node-1" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t1", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t1", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t2", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t3", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_5.json b/pkg/compiler/test/testdata/branch/compiled/success_5.json index 7c8cf9dd5..b8ced620c 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_5.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_5.json @@ -1 +1,584 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"project","domain":"domain","name":"test_serialization.my_wf","version":"version"},"metadata":{},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"},"b":{"type":{"simple":"STRING"},"description":"b"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"},"out_1":{"type":{"simple":"STRING"},"description":"out_1"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},{"id":"node-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t1","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"}}},{"id":"node-1","metadata":{"name":"test1","retries":{},"interruptible":false},"inputs":[{"var":"node-0.t1_int_output","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}}],"upstreamNodeIds":["node-0"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"4"}}}},"thenNode":{"id":"node-1-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}},"other":[{"condition":{"comparison":{"operator":"GTE","leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"5"}}}},"thenNode":{"id":"node-1-branchnode-1","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"c"}}}],"upstreamNodeIds":["node-0"],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}}],"error":{"failedNodeId":"test1","message":"Unable to choose branch"}}}},{"id":"node-2","metadata":{"name":"test2","retries":{},"interruptible":false},"inputs":[{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"upstreamNodeIds":["node-1"],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"primitive":{"stringValue":"hello "}}}},"thenNode":{"id":"node-2-branchnode-0","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"It is hello"}}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}},"elseNode":{"id":"node-2-branchnode-1","metadata":{"name":"flytekit.annotated.task.test_serialization.t2","retries":{},"interruptible":false},"inputs":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"Not Hello!"}}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"}}}}}}],"outputs":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["end-node","node-1"]},"node-1":{"ids":["node-2"]},"node-2":{"ids":["end-node"]},"start-node":{"ids":["node-0"]}},"upstream":{"end-node":{"ids":["node-0","node-2"]},"node-0":{"ids":["start-node"]},"node-1":{"ids":["node-0"]},"node-1-branchnode-0":{"ids":[""]},"node-1-branchnode-1":{"ids":[""]},"node-2":{"ids":["node-1"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"},"t1_int_output":{"type":{"simple":"INTEGER"},"description":"t1_int_output"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"STRING"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "project", + "domain": "domain", + "name": "test_serialization.my_wf", + "version": "version" + }, + "metadata": { + + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + }, + "b": { + "type": { + "simple": "STRING" + }, + "description": "b" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + }, + "out_1": { + "type": { + "simple": "STRING" + }, + "description": "out_1" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "t1_int_output" + } + } + }, + { + "var": "out_1", + "binding": { + "promise": { + "nodeId": "node-2", + "var": "out_0" + } + } + } + ] + }, + { + "id": "node-0", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t1", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t1", + "version": "version" + } + } + }, + { + "id": "node-1", + "metadata": { + "name": "test1", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "node-0.t1_int_output", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "t1_int_output" + } + } + } + ], + "upstreamNodeIds": [ + "node-0" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "node-0.t1_int_output" + }, + "rightValue": { + "primitive": { + "integer": "4" + } + } + } + }, + "thenNode": { + "id": "node-1-branchnode-0", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t2", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "b" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + } + } + } + }, + "other": [ + { + "condition": { + "comparison": { + "operator": "GTE", + "leftValue": { + "var": "node-0.t1_int_output" + }, + "rightValue": { + "primitive": { + "integer": "5" + } + } + } + }, + "thenNode": { + "id": "node-1-branchnode-1", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t2", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "c" + } + } + } + ], + "upstreamNodeIds": [ + "node-0" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + } + } + } + } + ], + "error": { + "failedNodeId": "test1", + "message": "Unable to choose branch" + } + } + } + }, + { + "id": "node-2", + "metadata": { + "name": "test2", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "node-1.out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ], + "upstreamNodeIds": [ + "node-1" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "node-1.out_0" + }, + "rightValue": { + "primitive": { + "stringValue": "hello " + } + } + } + }, + "thenNode": { + "id": "node-2-branchnode-0", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t2", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "scalar": { + "primitive": { + "stringValue": "It is hello" + } + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + } + } + } + }, + "elseNode": { + "id": "node-2-branchnode-1", + "metadata": { + "name": "flytekit.annotated.task.test_serialization.t2", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "scalar": { + "primitive": { + "stringValue": "Not Hello!" + } + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + } + } + } + } + } + } + ], + "outputs": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "t1_int_output" + } + } + }, + { + "var": "out_1", + "binding": { + "promise": { + "nodeId": "node-2", + "var": "out_0" + } + } + } + ], + "metadataDefaults": { + + } + }, + "connections": { + "downstream": { + "node-0": { + "ids": [ + "end-node", + "node-1", + "node-1-branchnode-1" + ] + }, + "node-1": { + "ids": [ + "node-2" + ] + }, + "node-2": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "node-0" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "node-0", + "node-2" + ] + }, + "node-0": { + "ids": [ + "start-node" + ] + }, + "node-1": { + "ids": [ + "node-0" + ] + }, + "node-1-branchnode-0": { + "ids": [ + "start-node" + ] + }, + "node-1-branchnode-1": { + "ids": [ + "node-0" + ] + }, + "node-2": { + "ids": [ + "node-1" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t1", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "c": { + "type": { + "simple": "STRING" + }, + "description": "c" + }, + "t1_int_output": { + "type": { + "simple": "INTEGER" + }, + "description": "t1_int_output" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t1", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "STRING" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t2", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_6.json b/pkg/compiler/test/testdata/branch/compiled/success_6.json index 8ac74d67b..84e6f4a24 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_6.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_6.json @@ -1 +1,382 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.multiplier","version":"abc"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}]},{"id":"node-0","metadata":{"name":"fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"node-0-branchn0","metadata":{"name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.double","version":"abc"}}}},"elseNode":{"id":"node-0-branchn1","metadata":{"name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.square","version":"abc"}}}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"node-0":{"ids":["end-node"]},"start-node":{"ids":["node-0"]}},"upstream":{"end-node":{"ids":["node-0"]},"node-0":{"ids":["start-node"]},"node-0-branchn0":{"ids":[""]},"node-0-branchn1":{"ids":[""]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "test_proj", + "domain": "test_domain", + "name": "recipes.02_intermediate.run_conditions.multiplier", + "version": "abc" + }, + "metadata": { + + }, + "interface": { + "inputs": { + "variables": { + "my_input": { + "type": { + "simple": "FLOAT" + }, + "description": "my_input" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "o0" + } + } + } + ] + }, + { + "id": "node-0", + "metadata": { + "name": "fractions", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GTE", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LTE", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + } + } + }, + "thenNode": { + "id": "node-0-branchn0", + "metadata": { + "name": "flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.double", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "recipes.02_intermediate.run_conditions.double", + "version": "abc" + } + } + } + }, + "elseNode": { + "id": "node-0-branchn1", + "metadata": { + "name": "flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.square", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "recipes.02_intermediate.run_conditions.square", + "version": "abc" + } + } + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "o0" + } + } + } + ], + "metadataDefaults": { + + } + }, + "connections": { + "downstream": { + "node-0": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "node-0" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "node-0" + ] + }, + "node-0": { + "ids": [ + "start-node" + ] + }, + "node-0-branchn0": { + "ids": [ + "start-node" + ] + }, + "node-0-branchn1": { + "ids": [ + "start-node" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "recipes.02_intermediate.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "recipes.02_intermediate.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "recipes.02_intermediate.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "recipes.02_intermediate.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json b/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json index ed1fe739b..ff3768e94 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json @@ -1 +1,469 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.so_nested","version":"abc"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},{"id":"branchnode-2","metadata":{"name":"fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"branchnode-2-branchbranchnode-1","metadata":{"name":"inner_fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"thenNode":{"id":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"}}}},"error":{"failedNodeId":"inner_fractions","message":"Only \u003c0.5 allowed"}}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"thenNode":{"id":"branchnode-2-branchn1","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"}}}}],"error":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"branchnode-2":{"ids":["end-node"]},"start-node":{"ids":["branchnode-2"]}},"upstream":{"branchnode-2":{"ids":["start-node"]},"branchnode-2-branchbranchnode-1":{"ids":["start-node"]},"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0":{"ids":[""]},"branchnode-2-branchn1":{"ids":[""]},"end-node":{"ids":["branchnode-2"]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.so_nested", + "version": "abc" + }, + "metadata": { + + }, + "interface": { + "inputs": { + "variables": { + "my_input": { + "type": { + "simple": "FLOAT" + }, + "description": "my_input" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "branchnode-2", + "var": "o0" + } + } + } + ] + }, + { + "id": "branchnode-2", + "metadata": { + "name": "fractions", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + } + } + }, + "thenNode": { + "id": "branchnode-2-branchbranchnode-1", + "metadata": { + "name": "inner_fractions", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "thenNode": { + "id": "branchnode-2-branchbranchnode-1-branchbranchn0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + } + } + } + }, + "error": { + "failedNodeId": "inner_fractions", + "message": "Only \u003c0.5 allowed" + } + } + } + } + }, + "other": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 10 + } + } + } + } + } + }, + "thenNode": { + "id": "branchnode-2-branchn1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + } + } + } + } + ], + "error": { + "failedNodeId": "fractions", + "message": "The input must be between 0 and 10" + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "branchnode-2", + "var": "o0" + } + } + } + ], + "metadataDefaults": { + + } + }, + "connections": { + "downstream": { + "branchnode-2": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "branchnode-2" + ] + } + }, + "upstream": { + "branchnode-2": { + "ids": [ + "start-node" + ] + }, + "branchnode-2-branchbranchnode-1": { + "ids": [ + "start-node" + ] + }, + "branchnode-2-branchbranchnode-1-branchbranchn0": { + "ids": [ + "start-node" + ] + }, + "branchnode-2-branchn1": { + "ids": [ + "start-node" + ] + }, + "end-node": { + "ids": [ + "branchnode-2" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json b/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json index 13af089e0..e828c65a6 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json @@ -1 +1,569 @@ -{"primary":{"template":{"id":{"resourceType":"WORKFLOW","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.so_nested","version":"abc"},"metadata":{},"interface":{"inputs":{"variables":{"my_input":{"type":{"simple":"FLOAT"},"description":"my_input"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"nodes":[{"id":"start-node"},{"id":"end-node","inputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},{"id":"n0","metadata":{"name":"fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"thenNode":{"id":"n0-n0","metadata":{"name":"inner_fractions","retries":{},"interruptible":false},"inputs":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"branchNode":{"ifElse":{"case":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"thenNode":{"id":"n0-n0-n0-n0","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.7}}}}}},"thenNode":{"id":"n0-n0-n0-n1","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"}}}}],"error":{"failedNodeId":"inner_fractions","message":"Only \u003c0.7 allowed"}}}}},"other":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"thenNode":{"id":"n0-n1","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"}}}}],"elseNode":{"id":"n0-n2","metadata":{"name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","retries":{},"interruptible":false},"inputs":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"taskNode":{"referenceId":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"}}}}}}],"outputs":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}],"metadataDefaults":{}},"connections":{"downstream":{"n0":{"ids":["end-node"]},"start-node":{"ids":["n0"]}},"upstream":{"end-node":{"ids":["n0"]},"n0":{"ids":["start-node"]},"n0-n0-n0-n0":{"ids":[""]},"n0-n0-n0-n1":{"ids":[""]},"n0-n2":{"ids":[""]}}}},"tasks":[{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},{"template":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}}]} \ No newline at end of file +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.so_nested", + "version": "abc" + }, + "metadata": { + + }, + "interface": { + "inputs": { + "variables": { + "my_input": { + "type": { + "simple": "FLOAT" + }, + "description": "my_input" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ] + }, + { + "id": "n0", + "metadata": { + "name": "fractions", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + } + } + }, + "thenNode": { + "id": "n0-n0", + "metadata": { + "name": "inner_fractions", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "thenNode": { + "id": "n0-n0-n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + } + } + } + }, + "other": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.7 + } + } + } + } + } + }, + "thenNode": { + "id": "n0-n0-n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + } + } + } + } + ], + "error": { + "failedNodeId": "inner_fractions", + "message": "Only \u003c0.7 allowed" + } + } + } + } + }, + "other": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 10 + } + } + } + } + } + }, + "thenNode": { + "id": "n0-n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + } + } + } + } + ], + "elseNode": { + "id": "n0-n2", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + } + } + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "metadataDefaults": { + + } + }, + "connections": { + "downstream": { + "n0": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "n0" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "n0" + ] + }, + "n0": { + "ids": [ + "start-node" + ] + }, + "n0-n0": { + "ids": [ + "start-node" + ] + }, + "n0-n0-n0": { + "ids": [ + "start-node" + ] + }, + "n0-n0-n1": { + "ids": [ + "start-node" + ] + }, + "n0-n1": { + "ids": [ + "start-node" + ] + }, + "n0-n2": { + "ids": [ + "start-node" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_1.json b/pkg/compiler/test/testdata/branch/k8s/success_1.json index 183a121e0..44452b2dc 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_1.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_1.json @@ -1 +1,392 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"out_0"},"rightValue":{"primitive":{"integer":"1"}}}},"then":"node-1-branchnode-0"},"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["node-1"],"node-1":["end-node"],"start-node":["node-0"]},"edges":{"downstream":{"node-0":["node-1"],"node-1":["end-node"],"start-node":["node-0"]},"upstream":{"end-node":["node-1"],"node-0":["start-node"],"node-1":["node-0"]}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{ + "kind": "flyteworkflow", + "apiVersion": "flyte.lyft.com/v1alpha1", + "metadata": { + "name": "name", + "namespace": "namespace", + "creationTimestamp": null, + "labels": { + "execution-id": "name", + "workflow-name": "test-serialization-my-wf" + } + }, + "spec": { + "id": "project:domain:test_serialization.my_wf", + "nodes": { + "end-node": { + "id": "end-node", + "resources": {}, + "kind": "end", + "inputBindings": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ] + }, + "node-0": { + "id": "node-0", + "name": "test_serialization.t3", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-1": { + "id": "node-1", + "name": "test1", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "leftValue": { + "var": "out_0" + }, + "rightValue": { + "primitive": { + "integer": "1" + } + } + } + }, + "then": "node-1-branchnode-0" + }, + "elseFail": { + "failedNodeId": "test1", + "message": "Unable to choose branch" + } + }, + "inputBindings": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "out_0" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-1-branchnode-0": { + "id": "node-1-branchnode-0", + "name": "test_serialization.t2", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ", + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "start-node": { + "id": "start-node", + "resources": {}, + "kind": "start" + } + }, + "connections": { + "node-0": [ + "node-1" + ], + "node-1": [ + "end-node" + ], + "start-node": [ + "node-0" + ] + }, + "edges": { + "downstream": { + "node-0": [ + "node-1" + ], + "node-1": [ + "end-node" + ], + "start-node": [ + "node-0" + ] + }, + "upstream": { + "end-node": [ + "node-1" + ], + "node-0": [ + "start-node" + ], + "node-1": [ + "node-0" + ] + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + }, + "outputBindings": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ] + }, + "inputs": { + "literals": { + "a": { + "scalar": { + "primitive": { + "integer": "0" + } + } + } + } + }, + "executionId": {}, + "tasks": { + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t1", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": {}, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t1", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": {}, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t2", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t3", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + "node-defaults": {}, + "securityContext": {}, + "status": { + "phase": 0 + }, + "rawOutputDataConfig": {}, + "executionConfig": { + "TaskPluginImpls": null, + "MaxParallelism": 0 + } +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_2.json b/pkg/compiler/test/testdata/branch/k8s/success_2.json index 44afd3822..c27c891e1 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_2.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_2.json @@ -1 +1,433 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2":{"id":"node-2","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":"node-1.out_0"}}},"then":"node-2-branchnode-0"},"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}},{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-0":{"id":"node-2-branchnode-0","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["node-2"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0","node-1"]},"edges":{"downstream":{"node-0":["node-2"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0","node-1"]},"upstream":{"end-node":["node-2"],"node-0":["start-node"],"node-1":["start-node"],"node-2":["node-0","node-1"]}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{ + "kind": "flyteworkflow", + "apiVersion": "flyte.lyft.com/v1alpha1", + "metadata": { + "name": "name", + "namespace": "namespace", + "creationTimestamp": null, + "labels": { + "execution-id": "name", + "workflow-name": "test-serialization-my-wf" + } + }, + "spec": { + "id": "project:domain:test_serialization.my_wf", + "nodes": { + "end-node": { + "id": "end-node", + "resources": {}, + "kind": "end", + "inputBindings": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-2", + "var": "out_0" + } + } + } + ] + }, + "node-0": { + "id": "node-0", + "name": "flytekit.annotated.task.test_serialization.t3", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-1": { + "id": "node-1", + "name": "flytekit.annotated.task.test_serialization.t3", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-2": { + "id": "node-2", + "name": "test1", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "leftValue": { + "var": "node-0.out_0" + }, + "rightValue": { + "var": "node-1.out_0" + } + } + }, + "then": "node-2-branchnode-0" + }, + "elseFail": { + "failedNodeId": "test1", + "message": "Unable to choose branch" + } + }, + "inputBindings": [ + { + "var": "node-0.out_0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "out_0" + } + } + }, + { + "var": "node-1.out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-2-branchnode-0": { + "id": "node-2-branchnode-0", + "name": "flytekit.annotated.task.test_serialization.t2", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ", + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "start-node": { + "id": "start-node", + "resources": {}, + "kind": "start" + } + }, + "connections": { + "node-0": [ + "node-2" + ], + "node-1": [ + "node-2" + ], + "node-2": [ + "end-node" + ], + "start-node": [ + "node-0", + "node-1" + ] + }, + "edges": { + "downstream": { + "node-0": [ + "node-2" + ], + "node-1": [ + "node-2" + ], + "node-2": [ + "end-node" + ], + "start-node": [ + "node-0", + "node-1" + ] + }, + "upstream": { + "end-node": [ + "node-2" + ], + "node-0": [ + "start-node" + ], + "node-1": [ + "start-node" + ], + "node-2": [ + "node-0", + "node-1" + ] + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + }, + "outputBindings": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-2", + "var": "out_0" + } + } + } + ] + }, + "inputs": { + "literals": { + "a": { + "scalar": { + "primitive": { + "integer": "0" + } + } + } + } + }, + "executionId": {}, + "tasks": { + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t1", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": {}, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t1", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": {}, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t2", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t3", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + "node-defaults": {}, + "securityContext": {}, + "status": { + "phase": 0 + }, + "rawOutputDataConfig": {}, + "executionConfig": { + "TaskPluginImpls": null, + "MaxParallelism": 0 + } +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_3.json b/pkg/compiler/test/testdata/branch/k8s/success_3.json index 852bfa407..bffd2c5f7 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_3.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_3.json @@ -1 +1,418 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"then":"node-1-branchnode-0"},"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["node-1"],"node-1":["end-node"],"start-node":["node-0","node-1"]},"edges":{"downstream":{"node-0":["node-1"],"node-1":["end-node"],"start-node":["node-0","node-1"]},"upstream":{"end-node":["node-1"],"node-0":["start-node"],"node-1":["node-0","start-node"],"node-1-branchnode-0":[""]}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{ + "kind": "flyteworkflow", + "apiVersion": "flyte.lyft.com/v1alpha1", + "metadata": { + "name": "name", + "namespace": "namespace", + "creationTimestamp": null, + "labels": { + "execution-id": "name", + "workflow-name": "test-serialization-my-wf" + } + }, + "spec": { + "id": "project:domain:test_serialization.my_wf", + "nodes": { + "end-node": { + "id": "end-node", + "resources": {}, + "kind": "end", + "inputBindings": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ] + }, + "node-0": { + "id": "node-0", + "name": "flytekit.annotated.task.test_serialization.t3", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-1": { + "id": "node-1", + "name": "test1", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "leftValue": { + "var": "node-0.out_0" + }, + "rightValue": { + "var": ".a" + } + } + }, + "then": "node-1-branchnode-0" + }, + "elseFail": { + "failedNodeId": "test1", + "message": "Unable to choose branch" + } + }, + "inputBindings": [ + { + "var": ".a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + }, + { + "var": "node-0.out_0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "out_0" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-1-branchnode-0": { + "id": "node-1-branchnode-0", + "name": "flytekit.annotated.task.test_serialization.t3", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "out_0" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "start-node": { + "id": "start-node", + "resources": {}, + "kind": "start" + } + }, + "connections": { + "node-0": [ + "node-1", + "node-1-branchnode-0" + ], + "node-1": [ + "end-node" + ], + "start-node": [ + "node-0", + "node-1" + ] + }, + "edges": { + "downstream": { + "node-0": [ + "node-1", + "node-1-branchnode-0" + ], + "node-1": [ + "end-node" + ], + "start-node": [ + "node-0", + "node-1" + ] + }, + "upstream": { + "end-node": [ + "node-1" + ], + "node-0": [ + "start-node" + ], + "node-1": [ + "node-0", + "start-node" + ], + "node-1-branchnode-0": [ + "node-0" + ] + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + } + } + }, + "outputBindings": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ] + }, + "inputs": { + "literals": { + "a": { + "scalar": { + "primitive": { + "integer": "0" + } + } + } + } + }, + "executionId": {}, + "tasks": { + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t1", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": {}, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t1", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": {}, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t2", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t3", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + "node-defaults": {}, + "securityContext": {}, + "status": { + "phase": 0 + }, + "rawOutputDataConfig": {}, + "executionConfig": { + "TaskPluginImpls": null, + "MaxParallelism": 0 + } +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_4.json b/pkg/compiler/test/testdata/branch/k8s/success_4.json index e1c1379cb..3a0fe4aed 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_4.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_4.json @@ -1 +1,506 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.out_0"},"rightValue":{"var":".a"}}},"then":"node-1-branchnode-0"},"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-0.out_0","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2":{"id":"node-2","name":"test2","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"var":".a"}}},"then":"node-2-branchnode-0"},"elseFail":{"failedNodeId":"test2","message":"Unable to choose a branch"}},"inputBindings":[{"var":".a","binding":{"promise":{"nodeId":"start-node","var":"a"}}},{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-0":{"id":"node-2-branchnode-0","name":"flytekit.annotated.task.test_serialization.t3","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0","node-1","node-2"]},"edges":{"downstream":{"node-0":["node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0","node-1","node-2"]},"upstream":{"end-node":["node-2"],"node-0":["start-node"],"node-1":["node-0","start-node"],"node-1-branchnode-0":[""],"node-2":["node-1","start-node"],"node-2-branchnode-0":[""]}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t3","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t3","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{ + "kind": "flyteworkflow", + "apiVersion": "flyte.lyft.com/v1alpha1", + "metadata": { + "name": "name", + "namespace": "namespace", + "creationTimestamp": null, + "labels": { + "execution-id": "name", + "workflow-name": "test-serialization-my-wf" + } + }, + "spec": { + "id": "project:domain:test_serialization.my_wf", + "nodes": { + "end-node": { + "id": "end-node", + "resources": {}, + "kind": "end", + "inputBindings": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-2", + "var": "out_0" + } + } + } + ] + }, + "node-0": { + "id": "node-0", + "name": "flytekit.annotated.task.test_serialization.t3", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-1": { + "id": "node-1", + "name": "test1", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "leftValue": { + "var": "node-0.out_0" + }, + "rightValue": { + "var": ".a" + } + } + }, + "then": "node-1-branchnode-0" + }, + "elseFail": { + "failedNodeId": "test1", + "message": "Unable to choose branch" + } + }, + "inputBindings": [ + { + "var": ".a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + }, + { + "var": "node-0.out_0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "out_0" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-1-branchnode-0": { + "id": "node-1-branchnode-0", + "name": "flytekit.annotated.task.test_serialization.t3", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "out_0" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-2": { + "id": "node-2", + "name": "test2", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "leftValue": { + "var": "node-1.out_0" + }, + "rightValue": { + "var": ".a" + } + } + }, + "then": "node-2-branchnode-0" + }, + "elseFail": { + "failedNodeId": "test2", + "message": "Unable to choose a branch" + } + }, + "inputBindings": [ + { + "var": ".a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + }, + { + "var": "node-1.out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-2-branchnode-0": { + "id": "node-2-branchnode-0", + "name": "flytekit.annotated.task.test_serialization.t3", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "start-node": { + "id": "start-node", + "resources": {}, + "kind": "start" + } + }, + "connections": { + "node-0": [ + "node-1", + "node-1-branchnode-0" + ], + "node-1": [ + "node-2", + "node-2-branchnode-0" + ], + "node-2": [ + "end-node" + ], + "start-node": [ + "node-0", + "node-1", + "node-2" + ] + }, + "edges": { + "downstream": { + "node-0": [ + "node-1", + "node-1-branchnode-0" + ], + "node-1": [ + "node-2", + "node-2-branchnode-0" + ], + "node-2": [ + "end-node" + ], + "start-node": [ + "node-0", + "node-1", + "node-2" + ] + }, + "upstream": { + "end-node": [ + "node-2" + ], + "node-0": [ + "start-node" + ], + "node-1": [ + "node-0", + "start-node" + ], + "node-1-branchnode-0": [ + "node-0" + ], + "node-2": [ + "node-1", + "start-node" + ], + "node-2-branchnode-0": [ + "node-1" + ] + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + } + } + }, + "outputBindings": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-2", + "var": "out_0" + } + } + } + ] + }, + "inputs": { + "literals": { + "a": { + "scalar": { + "primitive": { + "integer": "0" + } + } + } + } + }, + "executionId": {}, + "tasks": { + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t1", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": {}, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t1", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": {}, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t2", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t3\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t3", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t3", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + "node-defaults": {}, + "securityContext": {}, + "status": { + "phase": 0 + }, + "rawOutputDataConfig": {}, + "executionConfig": { + "TaskPluginImpls": null, + "MaxParallelism": 0 + } +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_5.json b/pkg/compiler/test/testdata/branch/k8s/success_5.json index 9782fd559..fd0c0fe55 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_5.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_5.json @@ -1 +1,531 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"test-serialization-my-wf"}},"spec":{"id":"project:domain:test_serialization.my_wf","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"node-0":{"id":"node-0","name":"flytekit.annotated.task.test_serialization.t1","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"a"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1":{"id":"node-1","name":"test1","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"4"}}}},"then":"node-1-branchnode-0"},"elseIf":[{"condition":{"comparison":{"operator":"GTE","leftValue":{"var":"node-0.t1_int_output"},"rightValue":{"primitive":{"integer":"5"}}}},"then":"node-1-branchnode-1"}],"elseFail":{"failedNodeId":"test1","message":"Unable to choose branch"}},"inputBindings":[{"var":"node-0.t1_int_output","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-0":{"id":"node-1-branchnode-0","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"start-node","var":"b"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-1-branchnode-1":{"id":"node-1-branchnode-1","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"promise":{"nodeId":"node-0","var":"c"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2":{"id":"node-2","name":"test2","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"leftValue":{"var":"node-1.out_0"},"rightValue":{"primitive":{"stringValue":"hello "}}}},"then":"node-2-branchnode-0"},"else":"node-2-branchnode-1"},"inputBindings":[{"var":"node-1.out_0","binding":{"promise":{"nodeId":"node-1","var":"out_0"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-0":{"id":"node-2-branchnode-0","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"It is hello"}}}}],"retry":{"minAttempts":1},"interruptible":false},"node-2-branchnode-1":{"id":"node-2-branchnode-1","name":"flytekit.annotated.task.test_serialization.t2","resources":{},"kind":"task","task":"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ","inputBindings":[{"var":"a","binding":{"scalar":{"primitive":{"stringValue":"Not Hello!"}}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["end-node","node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0"]},"edges":{"downstream":{"node-0":["end-node","node-1"],"node-1":["node-2"],"node-2":["end-node"],"start-node":["node-0"]},"upstream":{"end-node":["node-0","node-2"],"node-0":["start-node"],"node-1":["node-0"],"node-1-branchnode-0":[""],"node-1-branchnode-1":[""],"node-2":["node-1"]}},"outputs":{"variables":{"out_0":{"type":{"simple":"INTEGER"},"description":"out_0"},"out_1":{"type":{"simple":"STRING"},"description":"out_1"}}},"outputBindings":[{"var":"out_0","binding":{"promise":{"nodeId":"node-0","var":"t1_int_output"}}},{"var":"out_1","binding":{"promise":{"nodeId":"node-2","var":"out_0"}}}]},"inputs":{"literals":{"a":{"scalar":{"primitive":{"integer":"0"}}},"b":{"scalar":{"primitive":{"stringValue":""}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t1","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"INTEGER"},"description":"a"}}},"outputs":{"variables":{"c":{"type":{"simple":"STRING"},"description":"c"},"t1_int_output":{"type":{"simple":"INTEGER"},"description":"t1_int_output"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t1","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ":{"id":{"resourceType":"TASK","project":"project","domain":"domain","name":"test_serialization.t2","version":"version"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"1.2.3","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"a":{"type":{"simple":"STRING"},"description":"a"}}},"outputs":{"variables":{"out_0":{"type":{"simple":"STRING"},"description":"out_0"}}}},"container":{"image":"image","args":["pyflyte-execute","--task-module","test_serialization","--task-name","t2","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{ + "kind": "flyteworkflow", + "apiVersion": "flyte.lyft.com/v1alpha1", + "metadata": { + "name": "name", + "namespace": "namespace", + "creationTimestamp": null, + "labels": { + "execution-id": "name", + "workflow-name": "test-serialization-my-wf" + } + }, + "spec": { + "id": "project:domain:test_serialization.my_wf", + "nodes": { + "end-node": { + "id": "end-node", + "resources": {}, + "kind": "end", + "inputBindings": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "t1_int_output" + } + } + }, + { + "var": "out_1", + "binding": { + "promise": { + "nodeId": "node-2", + "var": "out_0" + } + } + } + ] + }, + "node-0": { + "id": "node-0", + "name": "flytekit.annotated.task.test_serialization.t1", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "a" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-1": { + "id": "node-1", + "name": "test1", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "leftValue": { + "var": "node-0.t1_int_output" + }, + "rightValue": { + "primitive": { + "integer": "4" + } + } + } + }, + "then": "node-1-branchnode-0" + }, + "elseIf": [ + { + "condition": { + "comparison": { + "operator": "GTE", + "leftValue": { + "var": "node-0.t1_int_output" + }, + "rightValue": { + "primitive": { + "integer": "5" + } + } + } + }, + "then": "node-1-branchnode-1" + } + ], + "elseFail": { + "failedNodeId": "test1", + "message": "Unable to choose branch" + } + }, + "inputBindings": [ + { + "var": "node-0.t1_int_output", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "t1_int_output" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-1-branchnode-0": { + "id": "node-1-branchnode-0", + "name": "flytekit.annotated.task.test_serialization.t2", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "b" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-1-branchnode-1": { + "id": "node-1-branchnode-1", + "name": "flytekit.annotated.task.test_serialization.t2", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "c" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-2": { + "id": "node-2", + "name": "test2", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "leftValue": { + "var": "node-1.out_0" + }, + "rightValue": { + "primitive": { + "stringValue": "hello " + } + } + } + }, + "then": "node-2-branchnode-0" + }, + "else": "node-2-branchnode-1" + }, + "inputBindings": [ + { + "var": "node-1.out_0", + "binding": { + "promise": { + "nodeId": "node-1", + "var": "out_0" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-2-branchnode-0": { + "id": "node-2-branchnode-0", + "name": "flytekit.annotated.task.test_serialization.t2", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "scalar": { + "primitive": { + "stringValue": "It is hello" + } + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-2-branchnode-1": { + "id": "node-2-branchnode-1", + "name": "flytekit.annotated.task.test_serialization.t2", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "scalar": { + "primitive": { + "stringValue": "Not Hello!" + } + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "start-node": { + "id": "start-node", + "resources": {}, + "kind": "start" + } + }, + "connections": { + "node-0": [ + "end-node", + "node-1", + "node-1-branchnode-1" + ], + "node-1": [ + "node-2" + ], + "node-2": [ + "end-node" + ], + "start-node": [ + "node-0" + ] + }, + "edges": { + "downstream": { + "node-0": [ + "end-node", + "node-1", + "node-1-branchnode-1" + ], + "node-1": [ + "node-2" + ], + "node-2": [ + "end-node" + ], + "start-node": [ + "node-0" + ] + }, + "upstream": { + "end-node": [ + "node-0", + "node-2" + ], + "node-0": [ + "start-node" + ], + "node-1": [ + "node-0" + ], + "node-1-branchnode-0": [ + "start-node" + ], + "node-1-branchnode-1": [ + "node-0" + ], + "node-2": [ + "node-1" + ] + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "INTEGER" + }, + "description": "out_0" + }, + "out_1": { + "type": { + "simple": "STRING" + }, + "description": "out_1" + } + } + }, + "outputBindings": [ + { + "var": "out_0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "t1_int_output" + } + } + }, + { + "var": "out_1", + "binding": { + "promise": { + "nodeId": "node-2", + "var": "out_0" + } + } + } + ] + }, + "inputs": { + "literals": { + "a": { + "scalar": { + "primitive": { + "integer": "0" + } + } + }, + "b": { + "scalar": { + "primitive": { + "stringValue": "" + } + } + } + } + }, + "executionId": {}, + "tasks": { + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t1\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t1", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "INTEGER" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "c": { + "type": { + "simple": "STRING" + }, + "description": "c" + }, + "t1_int_output": { + "type": { + "simple": "INTEGER" + }, + "description": "t1_int_output" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t1", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"project\" domain:\"domain\" name:\"test_serialization.t2\" version:\"version\" ": { + "id": { + "resourceType": "TASK", + "project": "project", + "domain": "domain", + "name": "test_serialization.t2", + "version": "version" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "1.2.3", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "STRING" + }, + "description": "a" + } + } + }, + "outputs": { + "variables": { + "out_0": { + "type": { + "simple": "STRING" + }, + "description": "out_0" + } + } + } + }, + "container": { + "image": "image", + "args": [ + "pyflyte-execute", + "--task-module", + "test_serialization", + "--task-name", + "t2", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + "node-defaults": {}, + "securityContext": {}, + "status": { + "phase": 0 + }, + "rawOutputDataConfig": {}, + "executionConfig": { + "TaskPluginImpls": null, + "MaxParallelism": 0 + } +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_6.json b/pkg/compiler/test/testdata/branch/k8s/success_6.json index e9ee61c4c..0553ae0d4 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_6.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_6.json @@ -1 +1,361 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"recipes-02-intermediate-run-conditions-multiplier"}},"spec":{"id":"test_proj:test_domain:recipes.02_intermediate.run_conditions.multiplier","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}]},"node-0":{"id":"node-0","name":"fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LTE","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"then":"node-0-branchn0"},"else":"node-0-branchn1"},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-0-branchn0":{"id":"node-0-branchn0","name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"node-0-branchn1":{"id":"node-0-branchn1","name":"flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"node-0":["end-node"],"start-node":["node-0"]},"edges":{"downstream":{"node-0":["end-node"],"start-node":["node-0"]},"upstream":{"end-node":["node-0"],"node-0":["start-node"],"node-0-branchn0":[""],"node-0-branchn1":[""]}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}},"outputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"node-0","var":"o0"}}}]},"inputs":{"literals":{"my_input":{"scalar":{"primitive":{"floatValue":0}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.double\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.square\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"recipes.02_intermediate.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","recipes.02_intermediate.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{ + "kind": "flyteworkflow", + "apiVersion": "flyte.lyft.com/v1alpha1", + "metadata": { + "name": "name", + "namespace": "namespace", + "creationTimestamp": null, + "labels": { + "execution-id": "name", + "workflow-name": "recipes-02-intermediate-run-conditions-multiplier" + } + }, + "spec": { + "id": "test_proj:test_domain:recipes.02_intermediate.run_conditions.multiplier", + "nodes": { + "end-node": { + "id": "end-node", + "resources": {}, + "kind": "end", + "inputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "o0" + } + } + } + ] + }, + "node-0": { + "id": "node-0", + "name": "fractions", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GTE", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LTE", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + } + } + }, + "then": "node-0-branchn0" + }, + "else": "node-0-branchn1" + }, + "inputBindings": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-0-branchn0": { + "id": "node-0-branchn0", + "name": "flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.double", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.double\" version:\"abc\" ", + "inputBindings": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "node-0-branchn1": { + "id": "node-0-branchn1", + "name": "flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.square", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.square\" version:\"abc\" ", + "inputBindings": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "start-node": { + "id": "start-node", + "resources": {}, + "kind": "start" + } + }, + "connections": { + "node-0": [ + "end-node" + ], + "start-node": [ + "node-0" + ] + }, + "edges": { + "downstream": { + "node-0": [ + "end-node" + ], + "start-node": [ + "node-0" + ] + }, + "upstream": { + "end-node": [ + "node-0" + ], + "node-0": [ + "start-node" + ], + "node-0-branchn0": [ + "start-node" + ], + "node-0-branchn1": [ + "start-node" + ] + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + }, + "outputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "node-0", + "var": "o0" + } + } + } + ] + }, + "inputs": { + "literals": { + "my_input": { + "scalar": { + "primitive": { + "floatValue": 0 + } + } + } + } + }, + "executionId": {}, + "tasks": { + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.double\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "recipes.02_intermediate.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "recipes.02_intermediate.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"recipes.02_intermediate.run_conditions.square\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "recipes.02_intermediate.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "recipes.02_intermediate.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + "node-defaults": {}, + "securityContext": {}, + "status": { + "phase": 0 + }, + "rawOutputDataConfig": {}, + "executionConfig": { + "TaskPluginImpls": null, + "MaxParallelism": 0 + } +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json b/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json index b3dc85aca..57fefb0cc 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json @@ -1 +1,445 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"core-control-flow-run-conditions-so-nested"}},"spec":{"id":"test_proj:test_domain:core.control_flow.run_conditions.so_nested","nodes":{"branchnode-2":{"id":"branchnode-2","name":"fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"then":"branchnode-2-branchbranchnode-1"},"elseIf":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"then":"branchnode-2-branchn1"}],"elseFail":{"failedNodeId":"fractions","message":"The input must be between 0 and 10"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchbranchnode-1":{"id":"branchnode-2-branchbranchnode-1","name":"inner_fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"then":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0"},"elseFail":{"failedNodeId":"inner_fractions","message":"Only \u003c0.5 allowed"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0":{"id":"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"branchnode-2-branchn1":{"id":"branchnode-2-branchn1","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"branchnode-2":["end-node"],"start-node":["branchnode-2"]},"edges":{"downstream":{"branchnode-2":["end-node"],"start-node":["branchnode-2"]},"upstream":{"branchnode-2":["start-node"],"branchnode-2-branchbranchnode-1":["start-node"],"branchnode-2-branchbranchnode-1-branchbranchnode-1-branchbranchn0":[""],"branchnode-2-branchn1":[""],"end-node":["branchnode-2"]}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}},"outputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"branchnode-2","var":"o0"}}}]},"inputs":{"literals":{"my_input":{"scalar":{"primitive":{"floatValue":0}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{ + "kind": "flyteworkflow", + "apiVersion": "flyte.lyft.com/v1alpha1", + "metadata": { + "name": "name", + "namespace": "namespace", + "creationTimestamp": null, + "labels": { + "execution-id": "name", + "workflow-name": "core-control-flow-run-conditions-so-nested" + } + }, + "spec": { + "id": "test_proj:test_domain:core.control_flow.run_conditions.so_nested", + "nodes": { + "branchnode-2": { + "id": "branchnode-2", + "name": "fractions", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + } + } + }, + "then": "branchnode-2-branchbranchnode-1" + }, + "elseIf": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 10 + } + } + } + } + } + }, + "then": "branchnode-2-branchn1" + } + ], + "elseFail": { + "failedNodeId": "fractions", + "message": "The input must be between 0 and 10" + } + }, + "inputBindings": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "branchnode-2-branchbranchnode-1": { + "id": "branchnode-2-branchbranchnode-1", + "name": "inner_fractions", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "then": "branchnode-2-branchbranchnode-1-branchbranchn0" + }, + "elseFail": { + "failedNodeId": "inner_fractions", + "message": "Only \u003c0.5 allowed" + } + }, + "inputBindings": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "branchnode-2-branchbranchnode-1-branchbranchn0": { + "id": "branchnode-2-branchbranchnode-1-branchbranchn0", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ", + "inputBindings": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "branchnode-2-branchn1": { + "id": "branchnode-2-branchn1", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ", + "inputBindings": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "end-node": { + "id": "end-node", + "resources": {}, + "kind": "end", + "inputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "branchnode-2", + "var": "o0" + } + } + } + ] + }, + "start-node": { + "id": "start-node", + "resources": {}, + "kind": "start" + } + }, + "connections": { + "branchnode-2": [ + "end-node" + ], + "start-node": [ + "branchnode-2" + ] + }, + "edges": { + "downstream": { + "branchnode-2": [ + "end-node" + ], + "start-node": [ + "branchnode-2" + ] + }, + "upstream": { + "branchnode-2": [ + "start-node" + ], + "branchnode-2-branchbranchnode-1": [ + "start-node" + ], + "branchnode-2-branchbranchnode-1-branchbranchn0": [ + "start-node" + ], + "branchnode-2-branchn1": [ + "start-node" + ], + "end-node": [ + "branchnode-2" + ] + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + }, + "outputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "branchnode-2", + "var": "o0" + } + } + } + ] + }, + "inputs": { + "literals": { + "my_input": { + "scalar": { + "primitive": { + "floatValue": 0 + } + } + } + } + }, + "executionId": {}, + "tasks": { + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + "node-defaults": {}, + "securityContext": {}, + "status": { + "phase": 0 + }, + "rawOutputDataConfig": {}, + "executionConfig": { + "TaskPluginImpls": null, + "MaxParallelism": 0 + } +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json b/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json index b0c7c11f4..9290bf2c3 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json @@ -1 +1,527 @@ -{"kind":"flyteworkflow","apiVersion":"flyte.lyft.com/v1alpha1","metadata":{"name":"name","namespace":"namespace","creationTimestamp":null,"labels":{"execution-id":"name","workflow-name":"core-control-flow-run-conditions-so-nested"}},"spec":{"id":"test_proj:test_domain:core.control_flow.run_conditions.so_nested","nodes":{"end-node":{"id":"end-node","resources":{},"kind":"end","inputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},"n0":{"id":"n0","name":"fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}}}},"then":"n0-n0"},"elseIf":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":1}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":10}}}}}},"then":"n0-n1"}],"else":"n0-n2"},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"n0-n0":{"id":"n0-n0","name":"inner_fractions","resources":{},"kind":"branch","branch":{"if":{"condition":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"then":"n0-n0-n0-n0"},"elseIf":[{"condition":{"conjunction":{"leftExpression":{"comparison":{"operator":"GT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.5}}}},"rightExpression":{"comparison":{"operator":"LT","leftValue":{"var":".my_input"},"rightValue":{"primitive":{"floatValue":0.7}}}}}},"then":"n0-n0-n0-n1"}],"elseFail":{"failedNodeId":"inner_fractions","message":"Only \u003c0.7 allowed"}},"inputBindings":[{"var":".my_input","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"n0-n0-n0-n0":{"id":"n0-n0-n0-n0","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"n0-n0-n0-n1":{"id":"n0-n0-n0-n1","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"n0-n1":{"id":"n0-n1","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.square","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"n0-n2":{"id":"n0-n2","name":"flytekit.core.python_function_task.core.control_flow.run_conditions.double","resources":{},"kind":"task","task":"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ","inputBindings":[{"var":"n","binding":{"promise":{"nodeId":"start-node","var":"my_input"}}}],"retry":{"minAttempts":1},"interruptible":false},"start-node":{"id":"start-node","resources":{},"kind":"start"}},"connections":{"n0":["end-node"],"start-node":["n0"]},"edges":{"downstream":{"n0":["end-node"],"start-node":["n0"]},"upstream":{"end-node":["n0"],"n0":["start-node"],"n0-n0-n0-n0":[""],"n0-n0-n0-n1":[""],"n0-n2":[""]}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}},"outputBindings":[{"var":"o0","binding":{"promise":{"nodeId":"n0","var":"o0"}}}]},"inputs":{"literals":{"my_input":{"scalar":{"primitive":{"floatValue":0}}}}},"executionId":{},"tasks":{"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.double","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","double","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}},"resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ":{"id":{"resourceType":"TASK","project":"test_proj","domain":"test_domain","name":"core.control_flow.run_conditions.square","version":"abc"},"type":"python-task","metadata":{"runtime":{"type":"FLYTE_SDK","version":"0.16.0","flavor":"python"},"retries":{},"interruptible":false},"interface":{"inputs":{"variables":{"n":{"type":{"simple":"FLOAT"},"description":"n"}}},"outputs":{"variables":{"o0":{"type":{"simple":"FLOAT"},"description":"o0"}}}},"container":{"image":"image:name","args":["pyflyte-execute","--task-module","core.control_flow.run_conditions","--task-name","square","--inputs","{{.input}}","--output-prefix","{{.outputPrefix}}","--raw-output-data-prefix","{{.rawOutputDataPrefix}}"],"resources":{},"config":[{"key":"testKey1","value":"testValue1"},{"key":"testKey2","value":"testValue2"},{"key":"testKey3","value":"testValue3"}]}}},"node-defaults":{},"securityContext":{},"status":{"phase":0},"rawOutputDataConfig":{},"executionConfig":{"TaskPluginImpls":null,"MaxParallelism":0}} \ No newline at end of file +{ + "kind": "flyteworkflow", + "apiVersion": "flyte.lyft.com/v1alpha1", + "metadata": { + "name": "name", + "namespace": "namespace", + "creationTimestamp": null, + "labels": { + "execution-id": "name", + "workflow-name": "core-control-flow-run-conditions-so-nested" + } + }, + "spec": { + "id": "test_proj:test_domain:core.control_flow.run_conditions.so_nested", + "nodes": { + "end-node": { + "id": "end-node", + "resources": {}, + "kind": "end", + "inputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ] + }, + "n0": { + "id": "n0", + "name": "fractions", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + } + } + }, + "then": "n0-n0" + }, + "elseIf": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 1 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 10 + } + } + } + } + } + }, + "then": "n0-n1" + } + ], + "else": "n0-n2" + }, + "inputBindings": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "n0-n0": { + "id": "n0-n0", + "name": "inner_fractions", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "then": "n0-n0-n0" + }, + "elseIf": [ + { + "condition": { + "conjunction": { + "leftExpression": { + "comparison": { + "operator": "GT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.5 + } + } + } + }, + "rightExpression": { + "comparison": { + "operator": "LT", + "leftValue": { + "var": ".my_input" + }, + "rightValue": { + "primitive": { + "floatValue": 0.7 + } + } + } + } + } + }, + "then": "n0-n0-n1" + } + ], + "elseFail": { + "failedNodeId": "inner_fractions", + "message": "Only \u003c0.7 allowed" + } + }, + "inputBindings": [ + { + "var": ".my_input", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "n0-n0-n0": { + "id": "n0-n0-n0", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ", + "inputBindings": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "n0-n0-n1": { + "id": "n0-n0-n1", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ", + "inputBindings": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "n0-n1": { + "id": "n0-n1", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ", + "inputBindings": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "n0-n2": { + "id": "n0-n2", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ", + "inputBindings": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "start-node": { + "id": "start-node", + "resources": {}, + "kind": "start" + } + }, + "connections": { + "n0": [ + "end-node" + ], + "start-node": [ + "n0" + ] + }, + "edges": { + "downstream": { + "n0": [ + "end-node" + ], + "start-node": [ + "n0" + ] + }, + "upstream": { + "end-node": [ + "n0" + ], + "n0": [ + "start-node" + ], + "n0-n0": [ + "start-node" + ], + "n0-n0-n0": [ + "start-node" + ], + "n0-n0-n1": [ + "start-node" + ], + "n0-n1": [ + "start-node" + ], + "n0-n2": [ + "start-node" + ] + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + }, + "outputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ] + }, + "inputs": { + "literals": { + "my_input": { + "scalar": { + "primitive": { + "floatValue": 0 + } + } + } + } + }, + "executionId": {}, + "tasks": { + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + "node-defaults": {}, + "securityContext": {}, + "status": { + "phase": 0 + }, + "rawOutputDataConfig": {}, + "executionConfig": { + "TaskPluginImpls": null, + "MaxParallelism": 0 + } +} \ No newline at end of file diff --git a/pkg/compiler/validators/bindings.go b/pkg/compiler/validators/bindings.go index 3016838b9..c7cac6942 100644 --- a/pkg/compiler/validators/bindings.go +++ b/pkg/compiler/validators/bindings.go @@ -99,16 +99,8 @@ func validateBinding(w c.WorkflowBuilder, nodeID c.NodeID, nodeParam string, bin return nil, nil, !errs.HasErrors() } -type EdgeDirection uint8 - -const ( - EdgeDirectionBidirectional EdgeDirection = iota - EdgeDirectionDownstream - EdgeDirectionUpstream -) - func ValidateBindings(w c.WorkflowBuilder, node c.Node, bindings []*flyte.Binding, params *flyte.VariableMap, - validateParamTypes bool, edgeDirection EdgeDirection, errs errors.CompileErrors) (resolved *flyte.VariableMap, ok bool) { + validateParamTypes bool, edgeDirection c.EdgeDirection, errs errors.CompileErrors) (resolved *flyte.VariableMap, ok bool) { resolved = &flyte.VariableMap{ Variables: make(map[string]*flyte.Variable, len(bindings)), @@ -136,11 +128,11 @@ func ValidateBindings(w c.WorkflowBuilder, node c.Node, bindings []*flyte.Bindin for _, upNode := range upstreamNodes { // Add implicit Edges switch edgeDirection { - case EdgeDirectionBidirectional: + case c.EdgeDirectionBidirectional: w.AddExecutionEdge(upNode, node.GetId()) - case EdgeDirectionDownstream: + case c.EdgeDirectionDownstream: w.AddDownstreamEdge(upNode, node.GetId()) - case EdgeDirectionUpstream: + case c.EdgeDirectionUpstream: w.AddUpstreamEdge(upNode, node.GetId()) } } diff --git a/pkg/compiler/validators/bindings_test.go b/pkg/compiler/validators/bindings_test.go index 9c5f2bc74..181062d5e 100644 --- a/pkg/compiler/validators/bindings_test.go +++ b/pkg/compiler/validators/bindings_test.go @@ -3,6 +3,8 @@ package validators import ( "testing" + c "github.com/flyteorg/flytepropeller/pkg/compiler/common" + "github.com/flyteorg/flyteidl/clients/go/coreutils" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flytepropeller/pkg/compiler/common/mocks" @@ -19,7 +21,7 @@ func TestValidateBindings(t *testing.T) { bindings := []*core.Binding{} vars := &core.VariableMap{} compileErrors := compilerErrors.NewCompileErrors() - resolved, ok := ValidateBindings(wf, n, bindings, vars, true, EdgeDirectionBidirectional, compileErrors) + resolved, ok := ValidateBindings(wf, n, bindings, vars, true, c.EdgeDirectionBidirectional, compileErrors) assert.True(t, ok) assert.Empty(t, resolved.Variables) }) @@ -35,7 +37,7 @@ func TestValidateBindings(t *testing.T) { } vars := &core.VariableMap{} compileErrors := compilerErrors.NewCompileErrors() - _, ok := ValidateBindings(wf, n, bindings, vars, true, EdgeDirectionBidirectional, compileErrors) + _, ok := ValidateBindings(wf, n, bindings, vars, true, c.EdgeDirectionBidirectional, compileErrors) assert.False(t, ok) if !compileErrors.HasErrors() { assert.Error(t, compileErrors) @@ -67,7 +69,7 @@ func TestValidateBindings(t *testing.T) { } compileErrors := compilerErrors.NewCompileErrors() - _, ok := ValidateBindings(wf, n, bindings, vars, true, EdgeDirectionBidirectional, compileErrors) + _, ok := ValidateBindings(wf, n, bindings, vars, true, c.EdgeDirectionBidirectional, compileErrors) assert.False(t, ok) if !compileErrors.HasErrors() { assert.Error(t, compileErrors) @@ -96,7 +98,7 @@ func TestValidateBindings(t *testing.T) { } compileErrors := compilerErrors.NewCompileErrors() - _, ok := ValidateBindings(wf, n, bindings, vars, true, EdgeDirectionBidirectional, compileErrors) + _, ok := ValidateBindings(wf, n, bindings, vars, true, c.EdgeDirectionBidirectional, compileErrors) assert.True(t, ok) if compileErrors.HasErrors() { assert.NoError(t, compileErrors) @@ -130,7 +132,7 @@ func TestValidateBindings(t *testing.T) { } compileErrors := compilerErrors.NewCompileErrors() - _, ok := ValidateBindings(wf, n, bindings, vars, true, EdgeDirectionBidirectional, compileErrors) + _, ok := ValidateBindings(wf, n, bindings, vars, true, c.EdgeDirectionBidirectional, compileErrors) assert.True(t, ok) if compileErrors.HasErrors() { assert.NoError(t, compileErrors) @@ -192,7 +194,7 @@ func TestValidateBindings(t *testing.T) { } compileErrors := compilerErrors.NewCompileErrors() - _, ok := ValidateBindings(wf, n, bindings, vars, true, EdgeDirectionBidirectional, compileErrors) + _, ok := ValidateBindings(wf, n, bindings, vars, true, c.EdgeDirectionBidirectional, compileErrors) assert.True(t, ok) if compileErrors.HasErrors() { assert.NoError(t, compileErrors) diff --git a/pkg/compiler/validators/branch.go b/pkg/compiler/validators/branch.go index 66084b556..fcbb4e2a6 100644 --- a/pkg/compiler/validators/branch.go +++ b/pkg/compiler/validators/branch.go @@ -60,16 +60,13 @@ func validateBranchInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs error } for _, block := range cases { + // TODO: This should use a compiler type to carry over the already-computed interface in the previous pass. n := w.NewNodeBuilder(block) iface2, ok := ValidateUnderlyingInterface(w, n, errs.NewScope()) - if !ok { continue } - ValidateBindings(w, n, n.GetInputs(), &flyte.VariableMap{Variables: map[string]*flyte.Variable{}}, - false, EdgeDirectionUpstream, errs.NewScope()) - // Clear out the Inputs. We do not care if the inputs of each of the underlying nodes // match. We will pull the inputs needed for the underlying branch node at runtime. iface2 = &flyte.TypedInterface{ @@ -90,7 +87,7 @@ func validateBranchInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs error // When we come to validate the conditions themselves, we will look up these variables and fail if a variable is used // in a condition but doesn't have a node input binding. inputVarsFromBindings, _ := ValidateBindings(w, node, node.GetInputs(), &flyte.VariableMap{Variables: map[string]*flyte.Variable{}}, - false, EdgeDirectionUpstream, errs.NewScope()) + false, c.EdgeDirectionUpstream, errs.NewScope()) if !errs.HasErrors() && iface != nil { iface = &flyte.TypedInterface{ diff --git a/pkg/compiler/validators/node.go b/pkg/compiler/validators/node.go index ccff53733..3f9b61eb1 100644 --- a/pkg/compiler/validators/node.go +++ b/pkg/compiler/validators/node.go @@ -86,6 +86,9 @@ func ValidateBranchNode(w c.WorkflowBuilder, n c.NodeBuilder, requireParamType b } for _, wrapperNode := range subNodes { + oldID := wrapperNode.GetId() + wrapperNode.SetID(branchNodeIDFormatter(n.GetId(), wrapperNode.GetId())) + w.ReplaceNodeID(oldID, wrapperNode.GetId()) if ValidateNode(w, wrapperNode, requireParamType, errs.NewScope()) { // Add to the global nodes to be able to reference it later discoveredNodes = append(discoveredNodes, wrapperNode) @@ -124,9 +127,10 @@ func ValidateNode(w c.WorkflowBuilder, n c.NodeBuilder, validateConditionTypes b if n.GetBranchNode() != nil { if nodes, ok := ValidateBranchNode(w, n, validateConditionTypes, errs.NewScope()); ok { for _, subNode := range nodes { - oldID := subNode.GetId() - subNode.SetID(branchNodeIDFormatter(n.GetId(), subNode.GetId())) - w.ReplaceNodeID(oldID, subNode.GetId()) + w.AddEdges(subNode, c.EdgeDirectionUpstream, errs.NewScope()) + // oldID := subNode.GetId() + // subNode.SetID(branchNodeIDFormatter(n.GetId(), subNode.GetId())) + // w.ReplaceNodeID(oldID, subNode.GetId()) } } } else if workflowN := n.GetWorkflowNode(); workflowN != nil && workflowN.GetSubWorkflowRef() != nil { diff --git a/pkg/compiler/workflow_compiler.go b/pkg/compiler/workflow_compiler.go index bf25812c8..682fd2539 100755 --- a/pkg/compiler/workflow_compiler.go +++ b/pkg/compiler/workflow_compiler.go @@ -176,7 +176,7 @@ func (w workflowBuilder) AddExecutionEdge(nodeFrom, nodeTo c.NodeID) { w.AddUpstreamEdge(nodeFrom, nodeTo) } -func (w workflowBuilder) AddEdges(n c.NodeBuilder, errs errors.CompileErrors) (ok bool) { +func (w workflowBuilder) AddEdges(n c.NodeBuilder, edgeDirection c.EdgeDirection, errs errors.CompileErrors) (ok bool) { if n.GetInterface() == nil { // If there were errors computing node's interface, don't add any edges and just bail. return @@ -191,7 +191,7 @@ func (w workflowBuilder) AddEdges(n c.NodeBuilder, errs errors.CompileErrors) (o // Add implicit Edges _, ok = v.ValidateBindings(&w, n, n.GetInputs(), n.GetInterface().GetInputs(), - true /* validateParamTypes */, v.EdgeDirectionBidirectional, errs.NewScope()) + true /* validateParamTypes */, edgeDirection, errs.NewScope()) return } @@ -250,23 +250,23 @@ func (w workflowBuilder) ValidateWorkflow(fg *flyteWorkflow, errs errors.Compile // Because conditions in branch nodes do not carry type information with them for the variables involved (e.g. // if x == y), we need to wait till all nodes have populated their interfaces before we can resolve x and y to their // original types and then validate whether they are compatible for comparison. - if !errs.HasErrors() { - for _, n := range wf.Nodes { - if n.GetBranchNode() != nil { - if inputVars, ok := v.ValidateBindings(&wf, n, n.GetInputs(), n.GetInterface().GetInputs(), - false /* validateParamTypes */, v.EdgeDirectionUpstream, errs.NewScope()); ok { - merge, err := v.UnionDistinctVariableMaps(n.GetInterface().Inputs.Variables, inputVars.Variables) - if err != nil { - errs.Collect(errors.NewWorkflowBuildError(err)) - } - - n.GetInterface().Inputs = &core.VariableMap{Variables: merge} - - v.ValidateBranchNode(&wf, n, true /* validateConditionTypes */, errs.NewScope()) - } - } - } - } + //if !errs.HasErrors() { + // for _, n := range wf.Nodes { + // if n.GetBranchNode() != nil { + // if inputVars, ok := v.ValidateBindings(&wf, n, n.GetInputs(), n.GetInterface().GetInputs(), + // false /* validateParamTypes */, c.EdgeDirectionUpstream, errs.NewScope()); ok { + // merge, err := v.UnionDistinctVariableMaps(n.GetInterface().Inputs.Variables, inputVars.Variables) + // if err != nil { + // errs.Collect(errors.NewWorkflowBuildError(err)) + // } + // + // n.GetInterface().Inputs = &core.VariableMap{Variables: merge} + // + // v.ValidateBranchNode(&wf, n, true /* validateConditionTypes */, errs.NewScope()) + // } + // } + // } + //} // Add explicitly and implicitly declared edges for nodeID, n := range wf.Nodes { @@ -274,7 +274,7 @@ func (w workflowBuilder) ValidateWorkflow(fg *flyteWorkflow, errs errors.Compile continue } - wf.AddEdges(n, errs.NewScope()) + wf.AddEdges(n, c.EdgeDirectionBidirectional, errs.NewScope()) } // Add execution edges for orphan nodes that don't have any inward/outward edges. @@ -307,7 +307,7 @@ func (w workflowBuilder) ValidateWorkflow(fg *flyteWorkflow, errs errors.Compile if _, wfIfaceOk := v.ValidateInterface(globalOutputNode.GetId(), globalOutputNode.GetInterface(), errs.NewScope()); wfIfaceOk { v.ValidateBindings(&wf, globalOutputNode, globalOutputNode.GetInputs(), globalOutputNode.GetInterface().GetInputs(), true, /* validateParamTypes */ - v.EdgeDirectionBidirectional, errs.NewScope()) + c.EdgeDirectionBidirectional, errs.NewScope()) } // Validate no cycles are detected. From e215c83ef07b59da0a13de04258b46cd14e743c0 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 1 Jun 2021 15:52:29 -0700 Subject: [PATCH 06/14] Support mismatching interfaces for branches Signed-off-by: Haytham Abuelfutuh --- pkg/compiler/test/compiler_test.go | 28 +- .../branch/compiled/success_9_nested.json | 610 ++++++++++++++++++ .../testdata/branch/k8s/success_9_nested.json | 551 ++++++++++++++++ .../testdata/branch/success_9_nested.json | 453 +++++++++++++ pkg/compiler/validators/branch.go | 14 +- pkg/compiler/validators/vars.go | 23 - 6 files changed, 1641 insertions(+), 38 deletions(-) create mode 100755 pkg/compiler/test/testdata/branch/compiled/success_9_nested.json create mode 100755 pkg/compiler/test/testdata/branch/k8s/success_9_nested.json create mode 100755 pkg/compiler/test/testdata/branch/success_9_nested.json diff --git a/pkg/compiler/test/compiler_test.go b/pkg/compiler/test/compiler_test.go index a4d428a2e..772b7ef4d 100644 --- a/pkg/compiler/test/compiler_test.go +++ b/pkg/compiler/test/compiler_test.go @@ -216,10 +216,20 @@ func getAllSubNodeIDs(n *core.Node) sets.String { return res } -func getAllNodeIDsWithPromiseInputs(wf *core.CompiledWorkflow) sets.String { +type nodePredicate func(n *core.Node) bool + +var hasPromiseNodePredicate = func(n *core.Node) bool { + return hasPromiseInputs(n.GetInputs()) +} + +var allNodesPredicate = func(n *core.Node) bool { + return true +} + +func getAllMatchingNodes(wf *core.CompiledWorkflow, predicate nodePredicate) sets.String { s := sets.NewString() for _, n := range wf.Template.Nodes { - if hasPromiseInputs(n.GetInputs()) { + if predicate(n) { s.Insert(n.GetId()) } @@ -260,7 +270,7 @@ func hasPromiseInputs(bindings []*core.Binding) bool { return false } -func assertNotIDsInConnections(t testing.TB, expectedNodeIDs sets.String, connections *core.ConnectionSet) bool { +func assertNotIDsInConnections(t testing.TB, nodeIDsWithDeps, allNodeIDs sets.String, connections *core.ConnectionSet) bool { actualNodeIDs := sets.NewString() for id, lst := range connections.Downstream { actualNodeIDs.Insert(id) @@ -272,11 +282,10 @@ func assertNotIDsInConnections(t testing.TB, expectedNodeIDs sets.String, connec actualNodeIDs.Insert(lst.Ids...) } - notFoundInConnections := expectedNodeIDs.Difference(actualNodeIDs) + notFoundInConnections := nodeIDsWithDeps.Difference(actualNodeIDs) correct := assert.Empty(t, notFoundInConnections, "All nodes must appear in connections") - notFoundInNodes := actualNodeIDs.Difference(expectedNodeIDs) - notFoundInNodes.Delete("start-node") + notFoundInNodes := actualNodeIDs.Difference(allNodeIDs) correct = correct && assert.Empty(t, notFoundInNodes, "All connections must correspond to existing nodes") return correct @@ -295,7 +304,7 @@ func TestBranches(t *testing.T) { t.Run(path, func(t *testing.T) { // If you want to debug a single use-case. Uncomment this line. - //if !strings.HasSuffix(path, "success_8_nested.json") { + //if !strings.HasSuffix(path, "success_9_nested.json") { // t.SkipNow() //} @@ -360,8 +369,9 @@ func TestBranches(t *testing.T) { } } - nodeIDs := getAllNodeIDsWithPromiseInputs(compiledWfc.Primary) - if !assertNotIDsInConnections(t, nodeIDs, compiledWfc.Primary.Connections) { + allNodeIDs := getAllMatchingNodes(compiledWfc.Primary, allNodesPredicate) + nodeIDsWithDeps := getAllMatchingNodes(compiledWfc.Primary, hasPromiseNodePredicate) + if !assertNotIDsInConnections(t, nodeIDsWithDeps, allNodeIDs, compiledWfc.Primary.Connections) { t.FailNow() } diff --git a/pkg/compiler/test/testdata/branch/compiled/success_9_nested.json b/pkg/compiler/test/testdata/branch/compiled/success_9_nested.json new file mode 100755 index 000000000..9e8950285 --- /dev/null +++ b/pkg/compiler/test/testdata/branch/compiled/success_9_nested.json @@ -0,0 +1,610 @@ +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.consume_outputs", + "version": "abc" + }, + "metadata": { + + }, + "interface": { + "inputs": { + "variables": { + "my_input": { + "type": { + "simple": "FLOAT" + }, + "description": "my_input" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ] + }, + { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.coin_toss", + "retries": { + + }, + "interruptible": false + }, + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.coin_toss", + "version": "abc" + } + } + }, + { + "id": "n1", + "metadata": { + "name": "double_or_square", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n0.o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "n0.o0" + }, + "rightValue": { + "primitive": { + "boolean": true + } + } + } + }, + "thenNode": { + "id": "n1-n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + } + } + } + }, + "elseNode": { + "id": "n1-n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.sum_diff", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + }, + { + "var": "b", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.sum_diff", + "version": "abc" + } + } + } + } + } + }, + { + "id": "n2", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n1" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ], + "metadataDefaults": { + + } + }, + "connections": { + "downstream": { + "n0": { + "ids": [ + "n1" + ] + }, + "n1": { + "ids": [ + "n2" + ] + }, + "n2": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "n0" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "n2" + ] + }, + "n0": { + "ids": [ + "start-node" + ] + }, + "n1": { + "ids": [ + "n0" + ] + }, + "n1-n0": { + "ids": [ + "start-node" + ] + }, + "n1-n1": { + "ids": [ + "n1-n0", + "start-node" + ] + }, + "n2": { + "ids": [ + "n1" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.sum_diff", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + }, + "b": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + }, + "o1": { + "type": { + "simple": "FLOAT" + }, + "description": "o1" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.coin_toss", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "BOOLEAN" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_9_nested.json b/pkg/compiler/test/testdata/branch/k8s/success_9_nested.json new file mode 100755 index 000000000..bd3a006b1 --- /dev/null +++ b/pkg/compiler/test/testdata/branch/k8s/success_9_nested.json @@ -0,0 +1,551 @@ +{ + "kind": "flyteworkflow", + "apiVersion": "flyte.lyft.com/v1alpha1", + "metadata": { + "name": "name", + "namespace": "namespace", + "creationTimestamp": null, + "labels": { + "execution-id": "name", + "workflow-name": "core-control-flow-run-conditions-consume-outputs" + } + }, + "spec": { + "id": "test_proj:test_domain:core.control_flow.run_conditions.consume_outputs", + "nodes": { + "end-node": { + "id": "end-node", + "resources": {}, + "kind": "end", + "inputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ] + }, + "n0": { + "id": "n0", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.coin_toss", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.coin_toss\" version:\"abc\" ", + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "n1": { + "id": "n1", + "name": "double_or_square", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "leftValue": { + "var": "n0.o0" + }, + "rightValue": { + "primitive": { + "boolean": true + } + } + } + }, + "then": "n1-n0" + }, + "else": "n1-n1" + }, + "inputBindings": [ + { + "var": "n0.o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "n1-n0": { + "id": "n1-n0", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ", + "inputBindings": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "n1-n1": { + "id": "n1-n1", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.sum_diff", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.sum_diff\" version:\"abc\" ", + "inputBindings": [ + { + "var": "a", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + }, + { + "var": "b", + "binding": { + "promise": { + "nodeId": "start-node", + "var": "my_input" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "n2": { + "id": "n2", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ", + "inputBindings": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "start-node": { + "id": "start-node", + "resources": {}, + "kind": "start" + } + }, + "connections": { + "n0": [ + "n1" + ], + "n1": [ + "n2" + ], + "n2": [ + "end-node" + ], + "start-node": [ + "n0" + ] + }, + "edges": { + "downstream": { + "n0": [ + "n1" + ], + "n1": [ + "n2" + ], + "n2": [ + "end-node" + ], + "start-node": [ + "n0" + ] + }, + "upstream": { + "end-node": [ + "n2" + ], + "n0": [ + "start-node" + ], + "n1": [ + "n0" + ], + "n1-n0": [ + "start-node" + ], + "n1-n1": [ + "n1-n0", + "start-node" + ], + "n2": [ + "n1" + ] + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + }, + "outputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ] + }, + "inputs": { + "literals": { + "my_input": { + "scalar": { + "primitive": { + "floatValue": 0 + } + } + } + } + }, + "executionId": {}, + "tasks": { + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.coin_toss\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.coin_toss", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": {}, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "BOOLEAN" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.sum_diff\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.sum_diff", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + }, + "b": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + }, + "o1": { + "type": { + "simple": "FLOAT" + }, + "description": "o1" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + "node-defaults": {}, + "securityContext": {}, + "status": { + "phase": 0 + }, + "rawOutputDataConfig": {}, + "executionConfig": { + "TaskPluginImpls": null, + "MaxParallelism": 0 + } +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/success_9_nested.json b/pkg/compiler/test/testdata/branch/success_9_nested.json new file mode 100755 index 000000000..7b6a35b16 --- /dev/null +++ b/pkg/compiler/test/testdata/branch/success_9_nested.json @@ -0,0 +1,453 @@ +{ + "workflow": { + "id": { + "resourceType": "WORKFLOW", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.consume_outputs", + "version": "abc" + }, + "metadata": { + }, + "interface": { + "inputs": { + "variables": { + "my_input": { + "type": { + "simple": "FLOAT" + }, + "description": "my_input" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.coin_toss", + "retries": { + }, + "interruptible": false + }, + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.coin_toss", + "version": "abc" + } + } + }, + { + "id": "n1", + "metadata": { + "name": "double_or_square", + "retries": { + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n0.o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "n0.o0" + }, + "rightValue": { + "primitive": { + "boolean": true + } + } + } + }, + "thenNode": { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", + "retries": { + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + } + } + } + }, + "elseNode": { + "id": "n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.sum_diff", + "retries": { + }, + "interruptible": false + }, + "inputs": [ + { + "var": "a", + "binding": { + "promise": { + "var": "my_input" + } + } + }, + { + "var": "b", + "binding": { + "promise": { + "var": "my_input" + } + } + } + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.sum_diff", + "version": "abc" + } + } + } + } + } + }, + { + "id": "n2", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", + "retries": { + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n1" + ], + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n2", + "var": "o0" + } + } + } + ], + "metadataDefaults": { + } + }, + "tasks": [ + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + }, + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.sum_diff", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + }, + "b": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + }, + "o1": { + "type": { + "simple": "FLOAT" + }, + "description": "o1" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + }, + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.coin_toss", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "BOOLEAN" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + }, + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/validators/branch.go b/pkg/compiler/validators/branch.go index fcbb4e2a6..035274a72 100644 --- a/pkg/compiler/validators/branch.go +++ b/pkg/compiler/validators/branch.go @@ -28,15 +28,18 @@ func validateBranchInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs error return } - finalOutputParameterNames := sets.NewString() - var outputs map[string]*flyte.Variable - outputsSet := sets.NewString() + finalOutputParameterNames := sets.NewString() validateIfaceMatch := func(nodeId string, iface2 *flyte.TypedInterface, errsScope errors.CompileErrors) (match bool) { outputs2, outputs2Set := buildVariablesIndex(iface2.Outputs) - validateVarsSetMatch(nodeId, outputs, outputs2, outputsSet, outputs2Set, errsScope.NewScope()) + // Validate that parameters that exist in both interfaces have compatible types. finalOutputParameterNames = finalOutputParameterNames.Intersection(outputs2Set) + for paramName := range finalOutputParameterNames { + if validateVarType(nodeId, paramName, outputs[paramName], outputs2[paramName].Type, errs.NewScope()) { + validateVarType(nodeId, paramName, outputs2[paramName], outputs[paramName].Type, errs.NewScope()) + } + } return !errsScope.HasErrors() } @@ -76,8 +79,7 @@ func validateBranchInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs error if iface == nil { iface = iface2 - outputs, outputsSet = buildVariablesIndex(iface.Outputs) - finalOutputParameterNames = finalOutputParameterNames.Union(outputsSet) + outputs, finalOutputParameterNames = buildVariablesIndex(iface.Outputs) } else { validateIfaceMatch(n.GetId(), iface2, errs.NewScope()) } diff --git a/pkg/compiler/validators/vars.go b/pkg/compiler/validators/vars.go index a3f2ca61d..a01ab062c 100644 --- a/pkg/compiler/validators/vars.go +++ b/pkg/compiler/validators/vars.go @@ -4,7 +4,6 @@ import ( flyte "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" c "github.com/flyteorg/flytepropeller/pkg/compiler/common" "github.com/flyteorg/flytepropeller/pkg/compiler/errors" - "k8s.io/apimachinery/pkg/util/sets" ) func validateOutputVar(n c.NodeBuilder, paramName string, errs errors.CompileErrors) ( @@ -47,28 +46,6 @@ func validateVarType(nodeID c.NodeID, paramName string, param *flyte.Variable, return !errs.HasErrors() } -func validateVarsSetMatch(nodeID string, params1, params2 map[string]*flyte.Variable, - params1Set, params2Set sets.String, errs errors.CompileErrors) { - // Validate that parameters that exist in both interfaces have compatible types. - inBoth := params1Set.Intersection(params2Set) - for paramName := range inBoth { - if validateVarType(nodeID, paramName, params1[paramName], params2[paramName].Type, errs.NewScope()) { - validateVarType(nodeID, paramName, params2[paramName], params1[paramName].Type, errs.NewScope()) - } - } - - // All remaining params on either sides indicate errors - inLeftSide := params1Set.Difference(params2Set) - for range inLeftSide { - errs.Collect(errors.NewMismatchingInterfacesErr(nodeID, nodeID)) - } - - inRightSide := params2Set.Difference(params1Set) - for range inRightSide { - errs.Collect(errors.NewMismatchingInterfacesErr(nodeID, nodeID)) - } -} - // Validate parameters have their required attributes set func validateVariables(nodeID c.NodeID, params *flyte.VariableMap, errs errors.CompileErrors) { for paramName, param := range params.Variables { From b039d159974eece7ac3ca5a90ddcaf58a7e3cd10 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 1 Jun 2021 18:27:11 -0700 Subject: [PATCH 07/14] only fail adding nodes in compiler transformer when nodes are different Signed-off-by: Haytham Abuelfutuh --- pkg/compiler/transformers/k8s/node.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/compiler/transformers/k8s/node.go b/pkg/compiler/transformers/k8s/node.go index 3f5ef5312..6449be552 100644 --- a/pkg/compiler/transformers/k8s/node.go +++ b/pkg/compiler/transformers/k8s/node.go @@ -1,11 +1,14 @@ package k8s import ( + "strings" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flytepropeller/pkg/apis/flyteworkflow/v1alpha1" "github.com/flyteorg/flytepropeller/pkg/compiler/common" "github.com/flyteorg/flytepropeller/pkg/compiler/errors" "github.com/flyteorg/flytepropeller/pkg/utils" + "github.com/go-test/deep" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -187,8 +190,10 @@ func buildNodes(nodes []*core.Node, tasks []*core.CompiledTask, errs errors.Comp for _, nref := range nodeSpecs { n := nref - if _, exists := res[n.ID]; exists { - errs.Collect(errors.NewValueCollisionError(nodeBuilder.GetId(), "Id", n.ID)) + if existingNode, exists := res[n.ID]; exists { + if diff := deep.Equal(existingNode, n); diff != nil { + errs.Collect(errors.NewValueCollisionError(nodeBuilder.GetId(), strings.Join(diff, "\r\n"), n.ID)) + } } res[n.ID] = n From c696cdf814c12d40d1e93ae041f8b2cb14cb4845 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 1 Jun 2021 20:12:57 -0700 Subject: [PATCH 08/14] remove replaceNodeID Signed-off-by: Haytham Abuelfutuh --- pkg/compiler/common/builder.go | 1 - pkg/compiler/common/mocks/workflow_builder.go | 5 - pkg/compiler/test/compiler_test.go | 2 +- .../branch/compiled/success_10_simple.json | 660 ++++++++++++++++++ .../branch/compiled/success_7_nested.json | 4 +- .../branch/compiled/success_8_nested.json | 8 +- .../branch/compiled/success_9_nested.json | 1 - .../branch/k8s/success_10_simple.json | 598 ++++++++++++++++ .../testdata/branch/k8s/success_7_nested.json | 8 +- .../testdata/branch/k8s/success_8_nested.json | 16 +- .../testdata/branch/k8s/success_9_nested.json | 1 - .../testdata/branch/success_10_simple.json | 489 +++++++++++++ pkg/compiler/validators/branch.go | 1 + pkg/compiler/validators/node.go | 6 - pkg/compiler/workflow_compiler.go | 33 - 15 files changed, 1767 insertions(+), 66 deletions(-) create mode 100755 pkg/compiler/test/testdata/branch/compiled/success_10_simple.json create mode 100755 pkg/compiler/test/testdata/branch/k8s/success_10_simple.json create mode 100755 pkg/compiler/test/testdata/branch/success_10_simple.json diff --git a/pkg/compiler/common/builder.go b/pkg/compiler/common/builder.go index fe8743c66..f0bf9919b 100644 --- a/pkg/compiler/common/builder.go +++ b/pkg/compiler/common/builder.go @@ -30,7 +30,6 @@ type WorkflowBuilder interface { AddDownstreamEdge(nodeProvider, nodeDependent NodeID) AddNode(n NodeBuilder, errs errors.CompileErrors) (node NodeBuilder, ok bool) AddEdges(n NodeBuilder, edgeDirection EdgeDirection, errs errors.CompileErrors) (ok bool) - ReplaceNodeID(oldID, newID string) ValidateWorkflow(fg *core.CompiledWorkflow, errs errors.CompileErrors) (Workflow, bool) NewNodeBuilder(n *core.Node) NodeBuilder } diff --git a/pkg/compiler/common/mocks/workflow_builder.go b/pkg/compiler/common/mocks/workflow_builder.go index 1c1b6afe7..88885222c 100644 --- a/pkg/compiler/common/mocks/workflow_builder.go +++ b/pkg/compiler/common/mocks/workflow_builder.go @@ -547,11 +547,6 @@ func (_m *WorkflowBuilder) NewNodeBuilder(n *core.Node) common.NodeBuilder { return r0 } -// ReplaceNodeID provides a mock function with given fields: oldID, newID -func (_m *WorkflowBuilder) ReplaceNodeID(oldID string, newID string) { - _m.Called(oldID, newID) -} - // StoreCompiledSubWorkflow provides a mock function with given fields: id, compiledWorkflow func (_m *WorkflowBuilder) StoreCompiledSubWorkflow(id core.Identifier, compiledWorkflow *core.CompiledWorkflow) { _m.Called(id, compiledWorkflow) diff --git a/pkg/compiler/test/compiler_test.go b/pkg/compiler/test/compiler_test.go index 772b7ef4d..c8ec83750 100644 --- a/pkg/compiler/test/compiler_test.go +++ b/pkg/compiler/test/compiler_test.go @@ -304,7 +304,7 @@ func TestBranches(t *testing.T) { t.Run(path, func(t *testing.T) { // If you want to debug a single use-case. Uncomment this line. - //if !strings.HasSuffix(path, "success_9_nested.json") { + //if !strings.HasSuffix(path, "success_7_nested.json") { // t.SkipNow() //} diff --git a/pkg/compiler/test/testdata/branch/compiled/success_10_simple.json b/pkg/compiler/test/testdata/branch/compiled/success_10_simple.json new file mode 100755 index 000000000..b255580c6 --- /dev/null +++ b/pkg/compiler/test/testdata/branch/compiled/success_10_simple.json @@ -0,0 +1,660 @@ +{ + "primary": { + "template": { + "id": { + "resourceType": "WORKFLOW", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.basic_boolean_wf", + "version": "abc" + }, + "metadata": { + + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "start-node" + }, + { + "id": "end-node", + "inputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + } + ] + }, + { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.coin_toss", + "retries": { + + }, + "interruptible": false + }, + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.coin_toss", + "version": "abc" + } + } + }, + { + "id": "n1", + "metadata": { + "name": "test", + "retries": { + + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n0.o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "n0.o0" + }, + "rightValue": { + "primitive": { + "boolean": true + } + } + } + }, + "thenNode": { + "id": "n1-n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.success", + "retries": { + + }, + "interruptible": false + }, + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.success", + "version": "abc" + } + } + } + }, + "elseNode": { + "id": "n1-n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.failed", + "retries": { + + }, + "interruptible": false + }, + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.failed", + "version": "abc" + } + } + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + } + ], + "metadataDefaults": { + + } + }, + "connections": { + "downstream": { + "n0": { + "ids": [ + "n1" + ] + }, + "n1": { + "ids": [ + "end-node" + ] + }, + "start-node": { + "ids": [ + "n0" + ] + } + }, + "upstream": { + "end-node": { + "ids": [ + "n1" + ] + }, + "n0": { + "ids": [ + "start-node" + ] + }, + "n1": { + "ids": [ + "n0" + ] + } + } + } + }, + "tasks": [ + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.success", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.failed", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.sum_diff", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + }, + "b": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + }, + "o1": { + "type": { + "simple": "FLOAT" + }, + "description": "o1" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.coin_toss", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "BOOLEAN" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + { + "template": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + + }, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json b/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json index ff3768e94..00a5565f0 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json @@ -141,7 +141,7 @@ } }, "thenNode": { - "id": "branchnode-2-branchbranchnode-1-branchbranchn0", + "id": "branchnode-2-branchbranchnode-1-branchnode-2-branchbranchnode-1-branchbranchn0", "metadata": { "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", "retries": { @@ -290,7 +290,7 @@ "start-node" ] }, - "branchnode-2-branchbranchnode-1-branchbranchn0": { + "branchnode-2-branchbranchnode-1-branchnode-2-branchbranchnode-1-branchbranchn0": { "ids": [ "start-node" ] diff --git a/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json b/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json index e828c65a6..2c6a79b7e 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json @@ -141,7 +141,7 @@ } }, "thenNode": { - "id": "n0-n0-n0", + "id": "n0-n0-n0-n0-n0", "metadata": { "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", "retries": { @@ -204,7 +204,7 @@ } }, "thenNode": { - "id": "n0-n0-n1", + "id": "n0-n0-n0-n0-n1", "metadata": { "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", "retries": { @@ -385,12 +385,12 @@ "start-node" ] }, - "n0-n0-n0": { + "n0-n0-n0-n0-n0": { "ids": [ "start-node" ] }, - "n0-n0-n1": { + "n0-n0-n0-n0-n1": { "ids": [ "start-node" ] diff --git a/pkg/compiler/test/testdata/branch/compiled/success_9_nested.json b/pkg/compiler/test/testdata/branch/compiled/success_9_nested.json index 9e8950285..1ae442f02 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_9_nested.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_9_nested.json @@ -276,7 +276,6 @@ }, "n1-n1": { "ids": [ - "n1-n0", "start-node" ] }, diff --git a/pkg/compiler/test/testdata/branch/k8s/success_10_simple.json b/pkg/compiler/test/testdata/branch/k8s/success_10_simple.json new file mode 100755 index 000000000..5515aad3e --- /dev/null +++ b/pkg/compiler/test/testdata/branch/k8s/success_10_simple.json @@ -0,0 +1,598 @@ +{ + "kind": "flyteworkflow", + "apiVersion": "flyte.lyft.com/v1alpha1", + "metadata": { + "name": "name", + "namespace": "namespace", + "creationTimestamp": null, + "labels": { + "execution-id": "name", + "workflow-name": "core-control-flow-run-conditions-basic-boolean-wf" + } + }, + "spec": { + "id": "test_proj:test_domain:core.control_flow.run_conditions.basic_boolean_wf", + "nodes": { + "end-node": { + "id": "end-node", + "resources": {}, + "kind": "end", + "inputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + } + ] + }, + "n0": { + "id": "n0", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.coin_toss", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.coin_toss\" version:\"abc\" ", + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "n1": { + "id": "n1", + "name": "test", + "resources": {}, + "kind": "branch", + "branch": { + "if": { + "condition": { + "comparison": { + "leftValue": { + "var": "n0.o0" + }, + "rightValue": { + "primitive": { + "boolean": true + } + } + } + }, + "then": "n1-n0" + }, + "else": "n1-n1" + }, + "inputBindings": [ + { + "var": "n0.o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "n1-n0": { + "id": "n1-n0", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.success", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.success\" version:\"abc\" ", + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "n1-n1": { + "id": "n1-n1", + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.failed", + "resources": {}, + "kind": "task", + "task": "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.failed\" version:\"abc\" ", + "retry": { + "minAttempts": 1 + }, + "interruptible": false + }, + "start-node": { + "id": "start-node", + "resources": {}, + "kind": "start" + } + }, + "connections": { + "n0": [ + "n1" + ], + "n1": [ + "end-node" + ], + "start-node": [ + "n0" + ] + }, + "edges": { + "downstream": { + "n0": [ + "n1" + ], + "n1": [ + "end-node" + ], + "start-node": [ + "n0" + ] + }, + "upstream": { + "end-node": [ + "n1" + ], + "n0": [ + "start-node" + ], + "n1": [ + "n0" + ] + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + }, + "outputBindings": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + } + ] + }, + "inputs": {}, + "executionId": {}, + "tasks": { + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.coin_toss\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.coin_toss", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": {}, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "BOOLEAN" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.double\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.failed\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.failed", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": {}, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.square\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.success\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.success", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": {}, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + }, + "resource_type:TASK project:\"test_proj\" domain:\"test_domain\" name:\"core.control_flow.run_conditions.sum_diff\" version:\"abc\" ": { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.sum_diff", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": {}, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + }, + "b": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + }, + "o1": { + "type": { + "simple": "FLOAT" + }, + "description": "o1" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": {}, + "config": [ + { + "key": "testKey1", + "value": "testValue1" + }, + { + "key": "testKey2", + "value": "testValue2" + }, + { + "key": "testKey3", + "value": "testValue3" + } + ] + } + } + }, + "node-defaults": {}, + "securityContext": {}, + "status": { + "phase": 0 + }, + "rawOutputDataConfig": {}, + "executionConfig": { + "TaskPluginImpls": null, + "MaxParallelism": 0 + } +} \ No newline at end of file diff --git a/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json b/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json index 57fefb0cc..47277f845 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json @@ -128,7 +128,7 @@ } } }, - "then": "branchnode-2-branchbranchnode-1-branchbranchn0" + "then": "branchnode-2-branchbranchnode-1-branchnode-2-branchbranchnode-1-branchbranchn0" }, "elseFail": { "failedNodeId": "inner_fractions", @@ -151,8 +151,8 @@ }, "interruptible": false }, - "branchnode-2-branchbranchnode-1-branchbranchn0": { - "id": "branchnode-2-branchbranchnode-1-branchbranchn0", + "branchnode-2-branchbranchnode-1-branchnode-2-branchbranchnode-1-branchbranchn0": { + "id": "branchnode-2-branchbranchnode-1-branchnode-2-branchbranchnode-1-branchbranchn0", "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", "resources": {}, "kind": "task", @@ -241,7 +241,7 @@ "branchnode-2-branchbranchnode-1": [ "start-node" ], - "branchnode-2-branchbranchnode-1-branchbranchn0": [ + "branchnode-2-branchbranchnode-1-branchnode-2-branchbranchnode-1-branchbranchn0": [ "start-node" ], "branchnode-2-branchn1": [ diff --git a/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json b/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json index 9290bf2c3..8584a141e 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json @@ -141,7 +141,7 @@ } } }, - "then": "n0-n0-n0" + "then": "n0-n0-n0-n0-n0" }, "elseIf": [ { @@ -175,7 +175,7 @@ } } }, - "then": "n0-n0-n1" + "then": "n0-n0-n0-n0-n1" } ], "elseFail": { @@ -199,8 +199,8 @@ }, "interruptible": false }, - "n0-n0-n0": { - "id": "n0-n0-n0", + "n0-n0-n0-n0-n0": { + "id": "n0-n0-n0-n0-n0", "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", "resources": {}, "kind": "task", @@ -221,8 +221,8 @@ }, "interruptible": false }, - "n0-n0-n1": { - "id": "n0-n0-n1", + "n0-n0-n0-n0-n1": { + "id": "n0-n0-n0-n0-n1", "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", "resources": {}, "kind": "task", @@ -320,10 +320,10 @@ "n0-n0": [ "start-node" ], - "n0-n0-n0": [ + "n0-n0-n0-n0-n0": [ "start-node" ], - "n0-n0-n1": [ + "n0-n0-n0-n0-n1": [ "start-node" ], "n0-n1": [ diff --git a/pkg/compiler/test/testdata/branch/k8s/success_9_nested.json b/pkg/compiler/test/testdata/branch/k8s/success_9_nested.json index bd3a006b1..d88b95481 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_9_nested.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_9_nested.json @@ -203,7 +203,6 @@ "start-node" ], "n1-n1": [ - "n1-n0", "start-node" ], "n2": [ diff --git a/pkg/compiler/test/testdata/branch/success_10_simple.json b/pkg/compiler/test/testdata/branch/success_10_simple.json new file mode 100755 index 000000000..edda01a1e --- /dev/null +++ b/pkg/compiler/test/testdata/branch/success_10_simple.json @@ -0,0 +1,489 @@ +{ + "workflow": { + "id": { + "resourceType": "WORKFLOW", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.basic_boolean_wf", + "version": "abc" + }, + "metadata": { + }, + "interface": { + "inputs": { + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "nodes": [ + { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.coin_toss", + "retries": { + }, + "interruptible": false + }, + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.coin_toss", + "version": "abc" + } + } + }, + { + "id": "n1", + "metadata": { + "name": "test", + "retries": { + }, + "interruptible": false + }, + "inputs": [ + { + "var": "n0.o0", + "binding": { + "promise": { + "nodeId": "n0", + "var": "o0" + } + } + } + ], + "upstreamNodeIds": [ + "n0" + ], + "branchNode": { + "ifElse": { + "case": { + "condition": { + "comparison": { + "leftValue": { + "var": "n0.o0" + }, + "rightValue": { + "primitive": { + "boolean": true + } + } + } + }, + "thenNode": { + "id": "n0", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.success", + "retries": { + }, + "interruptible": false + }, + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.success", + "version": "abc" + } + } + } + }, + "elseNode": { + "id": "n1", + "metadata": { + "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.failed", + "retries": { + }, + "interruptible": false + }, + "taskNode": { + "referenceId": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.failed", + "version": "abc" + } + } + } + } + } + } + ], + "outputs": [ + { + "var": "o0", + "binding": { + "promise": { + "nodeId": "n1", + "var": "o0" + } + } + } + ], + "metadataDefaults": { + } + }, + "tasks": [ + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.double", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + }, + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.success", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + }, + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.failed", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "INTEGER" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + }, + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.sum_diff", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "a": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + }, + "b": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + }, + "o1": { + "type": { + "simple": "FLOAT" + }, + "description": "o1" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + }, + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.coin_toss", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "BOOLEAN" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "double", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + }, + { + "id": { + "resourceType": "TASK", + "project": "test_proj", + "domain": "test_domain", + "name": "core.control_flow.run_conditions.square", + "version": "abc" + }, + "type": "python-task", + "metadata": { + "runtime": { + "type": "FLYTE_SDK", + "version": "0.16.0", + "flavor": "python" + }, + "retries": { + }, + "interruptible": false + }, + "interface": { + "inputs": { + "variables": { + "n": { + "type": { + "simple": "FLOAT" + }, + "description": "n" + } + } + }, + "outputs": { + "variables": { + "o0": { + "type": { + "simple": "FLOAT" + }, + "description": "o0" + } + } + } + }, + "container": { + "image": "image:name", + "args": [ + "pyflyte-execute", + "--task-module", + "core.control_flow.run_conditions", + "--task-name", + "square", + "--inputs", + "{{.input}}", + "--output-prefix", + "{{.outputPrefix}}", + "--raw-output-data-prefix", + "{{.rawOutputDataPrefix}}" + ], + "resources": { + } + } + } + ] +} \ No newline at end of file diff --git a/pkg/compiler/validators/branch.go b/pkg/compiler/validators/branch.go index 035274a72..90d774c4d 100644 --- a/pkg/compiler/validators/branch.go +++ b/pkg/compiler/validators/branch.go @@ -65,6 +65,7 @@ func validateBranchInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs error for _, block := range cases { // TODO: This should use a compiler type to carry over the already-computed interface in the previous pass. n := w.NewNodeBuilder(block) + n.SetID(branchNodeIDFormatter(node.GetId(), n.GetId())) iface2, ok := ValidateUnderlyingInterface(w, n, errs.NewScope()) if !ok { continue diff --git a/pkg/compiler/validators/node.go b/pkg/compiler/validators/node.go index 3f9b61eb1..c0515bdbb 100644 --- a/pkg/compiler/validators/node.go +++ b/pkg/compiler/validators/node.go @@ -86,9 +86,6 @@ func ValidateBranchNode(w c.WorkflowBuilder, n c.NodeBuilder, requireParamType b } for _, wrapperNode := range subNodes { - oldID := wrapperNode.GetId() - wrapperNode.SetID(branchNodeIDFormatter(n.GetId(), wrapperNode.GetId())) - w.ReplaceNodeID(oldID, wrapperNode.GetId()) if ValidateNode(w, wrapperNode, requireParamType, errs.NewScope()) { // Add to the global nodes to be able to reference it later discoveredNodes = append(discoveredNodes, wrapperNode) @@ -128,9 +125,6 @@ func ValidateNode(w c.WorkflowBuilder, n c.NodeBuilder, validateConditionTypes b if nodes, ok := ValidateBranchNode(w, n, validateConditionTypes, errs.NewScope()); ok { for _, subNode := range nodes { w.AddEdges(subNode, c.EdgeDirectionUpstream, errs.NewScope()) - // oldID := subNode.GetId() - // subNode.SetID(branchNodeIDFormatter(n.GetId(), subNode.GetId())) - // w.ReplaceNodeID(oldID, subNode.GetId()) } } } else if workflowN := n.GetWorkflowNode(); workflowN != nil && workflowN.GetSubWorkflowRef() != nil { diff --git a/pkg/compiler/workflow_compiler.go b/pkg/compiler/workflow_compiler.go index 682fd2539..2ef92eabe 100755 --- a/pkg/compiler/workflow_compiler.go +++ b/pkg/compiler/workflow_compiler.go @@ -124,39 +124,6 @@ func (w workflowBuilder) AddUpstreamEdge(nodeProvider, nodeDependent c.NodeID) { w.CoreWorkflow.Connections.Upstream[nodeDependent].Ids = w.upstreamNodes[nodeDependent].List() } -func (w workflowBuilder) ReplaceNodeID(oldID, newID string) { - if existingVal, found := w.upstreamNodes[oldID]; found { - w.upstreamNodes[newID] = existingVal - delete(w.upstreamNodes, oldID) - delete(w.CoreWorkflow.Connections.Upstream, oldID) - w.CoreWorkflow.Connections.Upstream[newID] = &core.ConnectionSet_IdList{ - Ids: make([]string, 1), - } - } - - for key, val := range w.upstreamNodes { - if key != oldID && val.Has(oldID) { - val.Delete(oldID) - val.Insert(newID) - w.CoreWorkflow.Connections.Upstream[key].Ids = val.List() - } - } - - if existingVal, found := w.downstreamNodes[oldID]; found { - w.downstreamNodes[newID] = existingVal - delete(w.CoreWorkflow.Connections.Downstream, oldID) - delete(w.downstreamNodes, oldID) - } - - for key, val := range w.downstreamNodes { - if key != oldID && val.Has(oldID) { - val.Delete(oldID) - val.Insert(newID) - w.CoreWorkflow.Connections.Downstream[key].Ids = val.List() - } - } -} - func (w workflowBuilder) AddDownstreamEdge(nodeProvider, nodeDependent c.NodeID) { if nodeProvider == "" { nodeProvider = c.StartNodeID From 6364601832255390879b7a80173c898ee21e2a12 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 1 Jun 2021 20:46:25 -0700 Subject: [PATCH 09/14] Fix unit tests Signed-off-by: Haytham Abuelfutuh --- pkg/compiler/builders.go | 24 +++++++++++++------ pkg/compiler/test/compiler_test.go | 2 +- .../branch/compiled/success_7_nested.json | 4 ++-- .../branch/compiled/success_8_nested.json | 8 +++---- .../testdata/branch/k8s/success_7_nested.json | 8 +++---- .../testdata/branch/k8s/success_8_nested.json | 16 ++++++------- pkg/compiler/validators/branch.go | 4 ++++ pkg/compiler/validators/branch_test.go | 5 ++++ pkg/compiler/validators/interface.go | 4 ++++ pkg/compiler/validators/interface_test.go | 7 ++++-- pkg/compiler/workflow_compiler.go | 1 + pkg/compiler/workflow_compiler_test.go | 5 +++- 12 files changed, 59 insertions(+), 29 deletions(-) diff --git a/pkg/compiler/builders.go b/pkg/compiler/builders.go index 01b6457ea..9da7c177e 100755 --- a/pkg/compiler/builders.go +++ b/pkg/compiler/builders.go @@ -1,6 +1,8 @@ package compiler import ( + "fmt" + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" c "github.com/flyteorg/flytepropeller/pkg/compiler/common" ) @@ -12,12 +14,13 @@ type flyteNode = core.Node // A builder object for the Graph struct. This contains information the compiler uses while building the final Graph // struct. type workflowBuilder struct { - CoreWorkflow *flyteWorkflow - LaunchPlans map[c.WorkflowIDKey]c.InterfaceProvider - Tasks c.TaskIndex - downstreamNodes c.StringAdjacencyList - upstreamNodes c.StringAdjacencyList - Nodes c.NodeIndex + CoreWorkflow *flyteWorkflow + LaunchPlans map[c.WorkflowIDKey]c.InterfaceProvider + Tasks c.TaskIndex + downstreamNodes c.StringAdjacencyList + upstreamNodes c.StringAdjacencyList + Nodes c.NodeIndex + NodeBuilderIndex c.NodeIndex // These are references to all subgraphs and tasks passed to CompileWorkflow. They will be passed around but will // not show in their entirety in the final Graph. The required subset of these will be added to each subgraph as @@ -53,7 +56,14 @@ func (w workflowBuilder) GetUpstreamNodes() c.StringAdjacencyList { } func (w workflowBuilder) NewNodeBuilder(n *flyteNode) c.NodeBuilder { - return &nodeBuilder{flyteNode: n} + address := fmt.Sprintf("%p", n) + if existingBuilder, found := w.NodeBuilderIndex[address]; found { + return existingBuilder + } + + newObj := &nodeBuilder{flyteNode: n} + w.NodeBuilderIndex[address] = newObj + return newObj } func (w workflowBuilder) GetNode(id c.NodeID) (node c.NodeBuilder, found bool) { diff --git a/pkg/compiler/test/compiler_test.go b/pkg/compiler/test/compiler_test.go index c8ec83750..1c9b50ca2 100644 --- a/pkg/compiler/test/compiler_test.go +++ b/pkg/compiler/test/compiler_test.go @@ -304,7 +304,7 @@ func TestBranches(t *testing.T) { t.Run(path, func(t *testing.T) { // If you want to debug a single use-case. Uncomment this line. - //if !strings.HasSuffix(path, "success_7_nested.json") { + //if !strings.HasSuffix(path, "success_8_nested.json") { // t.SkipNow() //} diff --git a/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json b/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json index 00a5565f0..ff3768e94 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_7_nested.json @@ -141,7 +141,7 @@ } }, "thenNode": { - "id": "branchnode-2-branchbranchnode-1-branchnode-2-branchbranchnode-1-branchbranchn0", + "id": "branchnode-2-branchbranchnode-1-branchbranchn0", "metadata": { "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", "retries": { @@ -290,7 +290,7 @@ "start-node" ] }, - "branchnode-2-branchbranchnode-1-branchnode-2-branchbranchnode-1-branchbranchn0": { + "branchnode-2-branchbranchnode-1-branchbranchn0": { "ids": [ "start-node" ] diff --git a/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json b/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json index 2c6a79b7e..e828c65a6 100755 --- a/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json +++ b/pkg/compiler/test/testdata/branch/compiled/success_8_nested.json @@ -141,7 +141,7 @@ } }, "thenNode": { - "id": "n0-n0-n0-n0-n0", + "id": "n0-n0-n0", "metadata": { "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", "retries": { @@ -204,7 +204,7 @@ } }, "thenNode": { - "id": "n0-n0-n0-n0-n1", + "id": "n0-n0-n1", "metadata": { "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", "retries": { @@ -385,12 +385,12 @@ "start-node" ] }, - "n0-n0-n0-n0-n0": { + "n0-n0-n0": { "ids": [ "start-node" ] }, - "n0-n0-n0-n0-n1": { + "n0-n0-n1": { "ids": [ "start-node" ] diff --git a/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json b/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json index 47277f845..57fefb0cc 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_7_nested.json @@ -128,7 +128,7 @@ } } }, - "then": "branchnode-2-branchbranchnode-1-branchnode-2-branchbranchnode-1-branchbranchn0" + "then": "branchnode-2-branchbranchnode-1-branchbranchn0" }, "elseFail": { "failedNodeId": "inner_fractions", @@ -151,8 +151,8 @@ }, "interruptible": false }, - "branchnode-2-branchbranchnode-1-branchnode-2-branchbranchnode-1-branchbranchn0": { - "id": "branchnode-2-branchbranchnode-1-branchnode-2-branchbranchnode-1-branchbranchn0", + "branchnode-2-branchbranchnode-1-branchbranchn0": { + "id": "branchnode-2-branchbranchnode-1-branchbranchn0", "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", "resources": {}, "kind": "task", @@ -241,7 +241,7 @@ "branchnode-2-branchbranchnode-1": [ "start-node" ], - "branchnode-2-branchbranchnode-1-branchnode-2-branchbranchnode-1-branchbranchn0": [ + "branchnode-2-branchbranchnode-1-branchbranchn0": [ "start-node" ], "branchnode-2-branchn1": [ diff --git a/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json b/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json index 8584a141e..9290bf2c3 100755 --- a/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json +++ b/pkg/compiler/test/testdata/branch/k8s/success_8_nested.json @@ -141,7 +141,7 @@ } } }, - "then": "n0-n0-n0-n0-n0" + "then": "n0-n0-n0" }, "elseIf": [ { @@ -175,7 +175,7 @@ } } }, - "then": "n0-n0-n0-n0-n1" + "then": "n0-n0-n1" } ], "elseFail": { @@ -199,8 +199,8 @@ }, "interruptible": false }, - "n0-n0-n0-n0-n0": { - "id": "n0-n0-n0-n0-n0", + "n0-n0-n0": { + "id": "n0-n0-n0", "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.double", "resources": {}, "kind": "task", @@ -221,8 +221,8 @@ }, "interruptible": false }, - "n0-n0-n0-n0-n1": { - "id": "n0-n0-n0-n0-n1", + "n0-n0-n1": { + "id": "n0-n0-n1", "name": "flytekit.core.python_function_task.core.control_flow.run_conditions.square", "resources": {}, "kind": "task", @@ -320,10 +320,10 @@ "n0-n0": [ "start-node" ], - "n0-n0-n0-n0-n0": [ + "n0-n0-n0": [ "start-node" ], - "n0-n0-n0-n0-n1": [ + "n0-n0-n1": [ "start-node" ], "n0-n1": [ diff --git a/pkg/compiler/validators/branch.go b/pkg/compiler/validators/branch.go index 90d774c4d..c89473c6c 100644 --- a/pkg/compiler/validators/branch.go +++ b/pkg/compiler/validators/branch.go @@ -8,6 +8,10 @@ import ( ) func validateBranchInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs errors.CompileErrors) (iface *flyte.TypedInterface, ok bool) { + if node.GetInterface() != nil { + return node.GetInterface(), true + } + if branch := node.GetBranchNode(); branch == nil { errs.Collect(errors.NewValueRequiredErr(node.GetId(), "Branch")) return diff --git a/pkg/compiler/validators/branch_test.go b/pkg/compiler/validators/branch_test.go index 2c04aa110..594c68842 100644 --- a/pkg/compiler/validators/branch_test.go +++ b/pkg/compiler/validators/branch_test.go @@ -49,6 +49,8 @@ func Test_validateBranchInterface(t *testing.T) { n2.OnGetTaskNode().Return(taskNode) n2.On("SetInterface", mock.Anything) n2.OnGetInputs().Return([]*core.Binding{}) + n2.On("SetID", mock.Anything).Return() + n2.OnGetInterface().Return(nil) task := &mocks.Task{} task.OnGetInterface().Return(&core.TypedInterface{}) @@ -61,6 +63,8 @@ func Test_validateBranchInterface(t *testing.T) { t.Run("single branch", func(t *testing.T) { n := &mocks.NodeBuilder{} + n.OnGetInterface().Return(nil) + n.On("SetID", mock.Anything).Return() n.OnGetId().Return("n1") n.OnGetBranchNode().Return(&core.BranchNode{ IfElse: &core.IfElseBlock{ @@ -90,6 +94,7 @@ func Test_validateBranchInterface(t *testing.T) { t.Run("two conditions", func(t *testing.T) { n := &mocks.NodeBuilder{} n.OnGetId().Return("n1") + n.OnGetInterface().Return(nil) n.OnGetInputs().Return([]*core.Binding{}) n.OnGetBranchNode().Return(&core.BranchNode{ IfElse: &core.IfElseBlock{ diff --git a/pkg/compiler/validators/interface.go b/pkg/compiler/validators/interface.go index c82511d6b..aa0524319 100644 --- a/pkg/compiler/validators/interface.go +++ b/pkg/compiler/validators/interface.go @@ -34,6 +34,10 @@ func ValidateInterface(nodeID c.NodeID, iface *core.TypedInterface, errs errors. // Validates underlying interface of a node and returns the effective Typed Interface. func ValidateUnderlyingInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs errors.CompileErrors) (iface *core.TypedInterface, ok bool) { + if node.GetInterface() != nil { + return node.GetInterface(), true + } + switch node.GetCoreNode().GetTarget().(type) { case *core.Node_TaskNode: if node.GetTaskNode().GetReferenceId() == nil { diff --git a/pkg/compiler/validators/interface_test.go b/pkg/compiler/validators/interface_test.go index 70bef9832..2d4742f0f 100644 --- a/pkg/compiler/validators/interface_test.go +++ b/pkg/compiler/validators/interface_test.go @@ -72,8 +72,9 @@ func TestValidateUnderlyingInterface(t *testing.T) { t.Run("Invalid empty node", func(t *testing.T) { wfBuilder := mocks.WorkflowBuilder{} nodeBuilder := mocks.NodeBuilder{} - nodeBuilder.On("GetCoreNode").Return(&core.Node{}) - nodeBuilder.On("GetId").Return("node_1") + nodeBuilder.OnGetCoreNode().Return(&core.Node{}) + nodeBuilder.OnGetId().Return("node_1") + nodeBuilder.OnGetInterface().Return(nil) errs := errors.NewCompileErrors() iface, ifaceOk := ValidateUnderlyingInterface(&wfBuilder, &nodeBuilder, errs.NewScope()) assert.False(t, ifaceOk) @@ -106,6 +107,7 @@ func TestValidateUnderlyingInterface(t *testing.T) { TaskNode: taskNode, }, }) + nodeBuilder.OnGetInterface().Return(nil) nodeBuilder.On("GetTaskNode").Return(taskNode) nodeBuilder.On("GetId").Return("node_1") @@ -144,6 +146,7 @@ func TestValidateUnderlyingInterface(t *testing.T) { nodeBuilder.On("GetId").Return("node_1") nodeBuilder.On("SetInterface", mock.Anything).Return() nodeBuilder.On("GetInputs").Return([]*core.Binding{}) + nodeBuilder.OnGetInterface().Return(nil) t.Run("Self", func(t *testing.T) { errs := errors.NewCompileErrors() diff --git a/pkg/compiler/workflow_compiler.go b/pkg/compiler/workflow_compiler.go index 2ef92eabe..c1b1b6c29 100755 --- a/pkg/compiler/workflow_compiler.go +++ b/pkg/compiler/workflow_compiler.go @@ -383,6 +383,7 @@ func newWorkflowBuilder(fg *flyteWorkflow, wfIndex c.WorkflowIndex, tasks c.Task CoreWorkflow: fg, LaunchPlans: map[string]c.InterfaceProvider{}, Nodes: c.NewNodeIndex(), + NodeBuilderIndex: c.NewNodeIndex(), Tasks: c.NewTaskIndex(), downstreamNodes: c.StringAdjacencyList{}, upstreamNodes: c.StringAdjacencyList{}, diff --git a/pkg/compiler/workflow_compiler_test.go b/pkg/compiler/workflow_compiler_test.go index 3b4617fc2..68b7d6548 100755 --- a/pkg/compiler/workflow_compiler_test.go +++ b/pkg/compiler/workflow_compiler_test.go @@ -390,7 +390,10 @@ func newNodeIDSet(nodeIDs ...common.NodeID) sets.String { } func TestValidateReachable(t *testing.T) { - graph := &workflowBuilder{} + graph := &workflowBuilder{ + NodeBuilderIndex: common.NewNodeIndex(), + } + graph.downstreamNodes = map[string]sets.String{ v1alpha1.StartNodeID: newNodeIDSet("1"), "1": newNodeIDSet("5", "2"), From ea9288147bf19e32eac8815d0a08103205313d56 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 1 Jun 2021 21:05:52 -0700 Subject: [PATCH 10/14] Clean up commented code Signed-off-by: Haytham Abuelfutuh --- pkg/compiler/workflow_compiler.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/pkg/compiler/workflow_compiler.go b/pkg/compiler/workflow_compiler.go index c1b1b6c29..c1f2193f6 100755 --- a/pkg/compiler/workflow_compiler.go +++ b/pkg/compiler/workflow_compiler.go @@ -213,28 +213,6 @@ func (w workflowBuilder) ValidateWorkflow(fg *flyteWorkflow, errs errors.Compile } } - // At this point, all nodes except branch nodes have populated all their input and output interfaces, - // Because conditions in branch nodes do not carry type information with them for the variables involved (e.g. - // if x == y), we need to wait till all nodes have populated their interfaces before we can resolve x and y to their - // original types and then validate whether they are compatible for comparison. - //if !errs.HasErrors() { - // for _, n := range wf.Nodes { - // if n.GetBranchNode() != nil { - // if inputVars, ok := v.ValidateBindings(&wf, n, n.GetInputs(), n.GetInterface().GetInputs(), - // false /* validateParamTypes */, c.EdgeDirectionUpstream, errs.NewScope()); ok { - // merge, err := v.UnionDistinctVariableMaps(n.GetInterface().Inputs.Variables, inputVars.Variables) - // if err != nil { - // errs.Collect(errors.NewWorkflowBuildError(err)) - // } - // - // n.GetInterface().Inputs = &core.VariableMap{Variables: merge} - // - // v.ValidateBranchNode(&wf, n, true /* validateConditionTypes */, errs.NewScope()) - // } - // } - // } - //} - // Add explicitly and implicitly declared edges for nodeID, n := range wf.Nodes { if nodeID == c.StartNodeID { From 0b6a55f31e0615ea5d66625058889b5e462897a4 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Tue, 1 Jun 2021 21:06:36 -0700 Subject: [PATCH 11/14] bump DCO Signed-off-by: Haytham Abuelfutuh From 2242bd4e18df05599af7519854c422c638d3f9f7 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 2 Jun 2021 08:42:02 -0700 Subject: [PATCH 12/14] Cleanup Signed-off-by: Haytham Abuelfutuh --- pkg/compiler/builders.go | 4 ++-- pkg/compiler/common/builder.go | 2 +- pkg/compiler/common/mocks/workflow_builder.go | 22 +++++++++---------- pkg/compiler/validators/branch.go | 3 +-- pkg/compiler/validators/branch_test.go | 2 +- pkg/compiler/validators/node.go | 4 ++-- pkg/compiler/workflow_compiler.go | 6 ++--- pkg/compiler/workflow_compiler_test.go | 2 +- 8 files changed, 22 insertions(+), 23 deletions(-) diff --git a/pkg/compiler/builders.go b/pkg/compiler/builders.go index 9da7c177e..e71d2c8a6 100755 --- a/pkg/compiler/builders.go +++ b/pkg/compiler/builders.go @@ -33,7 +33,7 @@ type workflowBuilder struct { func (w workflowBuilder) GetFailureNode() c.Node { if w.GetCoreWorkflow() != nil && w.GetCoreWorkflow().GetTemplate() != nil && w.GetCoreWorkflow().GetTemplate().FailureNode != nil { - return w.NewNodeBuilder(w.GetCoreWorkflow().GetTemplate().FailureNode) + return w.GetOrCreateNodeBuilder(w.GetCoreWorkflow().GetTemplate().FailureNode) } return nil @@ -55,7 +55,7 @@ func (w workflowBuilder) GetUpstreamNodes() c.StringAdjacencyList { return w.upstreamNodes } -func (w workflowBuilder) NewNodeBuilder(n *flyteNode) c.NodeBuilder { +func (w workflowBuilder) GetOrCreateNodeBuilder(n *flyteNode) c.NodeBuilder { address := fmt.Sprintf("%p", n) if existingBuilder, found := w.NodeBuilderIndex[address]; found { return existingBuilder diff --git a/pkg/compiler/common/builder.go b/pkg/compiler/common/builder.go index f0bf9919b..84238db91 100644 --- a/pkg/compiler/common/builder.go +++ b/pkg/compiler/common/builder.go @@ -31,7 +31,7 @@ type WorkflowBuilder interface { AddNode(n NodeBuilder, errs errors.CompileErrors) (node NodeBuilder, ok bool) AddEdges(n NodeBuilder, edgeDirection EdgeDirection, errs errors.CompileErrors) (ok bool) ValidateWorkflow(fg *core.CompiledWorkflow, errs errors.CompileErrors) (Workflow, bool) - NewNodeBuilder(n *core.Node) NodeBuilder + GetOrCreateNodeBuilder(n *core.Node) NodeBuilder } // A mutable node used during the build of the intermediate layer. diff --git a/pkg/compiler/common/mocks/workflow_builder.go b/pkg/compiler/common/mocks/workflow_builder.go index 88885222c..806bf7789 100644 --- a/pkg/compiler/common/mocks/workflow_builder.go +++ b/pkg/compiler/common/mocks/workflow_builder.go @@ -513,26 +513,26 @@ func (_m *WorkflowBuilder) GetUpstreamNodes() common.StringAdjacencyList { return r0 } -type WorkflowBuilder_NewNodeBuilder struct { +type WorkflowBuilder_GetOrCreateNodeBuilder struct { *mock.Call } -func (_m WorkflowBuilder_NewNodeBuilder) Return(_a0 common.NodeBuilder) *WorkflowBuilder_NewNodeBuilder { - return &WorkflowBuilder_NewNodeBuilder{Call: _m.Call.Return(_a0)} +func (_m WorkflowBuilder_GetOrCreateNodeBuilder) Return(_a0 common.NodeBuilder) *WorkflowBuilder_GetOrCreateNodeBuilder { + return &WorkflowBuilder_GetOrCreateNodeBuilder{Call: _m.Call.Return(_a0)} } -func (_m *WorkflowBuilder) OnNewNodeBuilder(n *core.Node) *WorkflowBuilder_NewNodeBuilder { - c := _m.On("NewNodeBuilder", n) - return &WorkflowBuilder_NewNodeBuilder{Call: c} +func (_m *WorkflowBuilder) OnGetOrCreateNodeBuilder(n *core.Node) *WorkflowBuilder_GetOrCreateNodeBuilder { + c := _m.On("GetOrCreateNodeBuilder", n) + return &WorkflowBuilder_GetOrCreateNodeBuilder{Call: c} } -func (_m *WorkflowBuilder) OnNewNodeBuilderMatch(matchers ...interface{}) *WorkflowBuilder_NewNodeBuilder { - c := _m.On("NewNodeBuilder", matchers...) - return &WorkflowBuilder_NewNodeBuilder{Call: c} +func (_m *WorkflowBuilder) OnGetOrCreateNodeBuilderMatch(matchers ...interface{}) *WorkflowBuilder_GetOrCreateNodeBuilder { + c := _m.On("GetOrCreateNodeBuilder", matchers...) + return &WorkflowBuilder_GetOrCreateNodeBuilder{Call: c} } -// NewNodeBuilder provides a mock function with given fields: n -func (_m *WorkflowBuilder) NewNodeBuilder(n *core.Node) common.NodeBuilder { +// GetOrCreateNodeBuilder provides a mock function with given fields: n +func (_m *WorkflowBuilder) GetOrCreateNodeBuilder(n *core.Node) common.NodeBuilder { ret := _m.Called(n) var r0 common.NodeBuilder diff --git a/pkg/compiler/validators/branch.go b/pkg/compiler/validators/branch.go index c89473c6c..5d3849d8a 100644 --- a/pkg/compiler/validators/branch.go +++ b/pkg/compiler/validators/branch.go @@ -67,8 +67,7 @@ func validateBranchInterface(w c.WorkflowBuilder, node c.NodeBuilder, errs error } for _, block := range cases { - // TODO: This should use a compiler type to carry over the already-computed interface in the previous pass. - n := w.NewNodeBuilder(block) + n := w.GetOrCreateNodeBuilder(block) n.SetID(branchNodeIDFormatter(node.GetId(), n.GetId())) iface2, ok := ValidateUnderlyingInterface(w, n, errs.NewScope()) if !ok { diff --git a/pkg/compiler/validators/branch_test.go b/pkg/compiler/validators/branch_test.go index 594c68842..4c28e3fd3 100644 --- a/pkg/compiler/validators/branch_test.go +++ b/pkg/compiler/validators/branch_test.go @@ -59,7 +59,7 @@ func Test_validateBranchInterface(t *testing.T) { wf.OnGetTask(identifier).Return(task, true) errs := compilerErrors.NewCompileErrors() - wf.OnNewNodeBuilder(coreN2).Return(n2) + wf.OnGetOrCreateNodeBuilder(coreN2).Return(n2) t.Run("single branch", func(t *testing.T) { n := &mocks.NodeBuilder{} diff --git a/pkg/compiler/validators/node.go b/pkg/compiler/validators/node.go index c0515bdbb..e9a81f0a0 100644 --- a/pkg/compiler/validators/node.go +++ b/pkg/compiler/validators/node.go @@ -73,13 +73,13 @@ func ValidateBranchNode(w c.WorkflowBuilder, n c.NodeBuilder, requireParamType b if block.GetThenNode() == nil { errs.Collect(errors.NewBranchNodeNotSpecified(n.GetId())) } else { - wrapperNode := w.NewNodeBuilder(block.GetThenNode()) + wrapperNode := w.GetOrCreateNodeBuilder(block.GetThenNode()) subNodes = append(subNodes, wrapperNode) } } if elseNode := n.GetBranchNode().IfElse.GetElseNode(); elseNode != nil { - wrapperNode := w.NewNodeBuilder(elseNode) + wrapperNode := w.GetOrCreateNodeBuilder(elseNode) subNodes = append(subNodes, wrapperNode) } else if defaultElse := n.GetBranchNode().IfElse.GetDefault(); defaultElse == nil { errs.Collect(errors.NewBranchNodeHasNoDefault(n.GetId())) diff --git a/pkg/compiler/workflow_compiler.go b/pkg/compiler/workflow_compiler.go index c1f2193f6..f0828a40c 100755 --- a/pkg/compiler/workflow_compiler.go +++ b/pkg/compiler/workflow_compiler.go @@ -192,11 +192,11 @@ func (w workflowBuilder) ValidateWorkflow(fg *flyteWorkflow, errs errors.Compile Upstream: make(map[string]*core.ConnectionSet_IdList), } - globalInputNode, _ := wf.AddNode(wf.NewNodeBuilder(startNode), errs) + globalInputNode, _ := wf.AddNode(wf.GetOrCreateNodeBuilder(startNode), errs) globalInputNode.SetInterface(&core.TypedInterface{Outputs: wf.CoreWorkflow.Template.Interface.Inputs}) endNode := &core.Node{Id: c.EndNodeID} - globalOutputNode, _ := wf.AddNode(wf.NewNodeBuilder(endNode), errs) + globalOutputNode, _ := wf.AddNode(wf.GetOrCreateNodeBuilder(endNode), errs) globalOutputNode.SetInterface(&core.TypedInterface{Inputs: wf.CoreWorkflow.Template.Interface.Outputs}) globalOutputNode.SetInputs(wf.CoreWorkflow.Template.Outputs) @@ -208,7 +208,7 @@ func (w workflowBuilder) ValidateWorkflow(fg *flyteWorkflow, errs errors.Compile // Add and validate all other nodes for _, n := range checkpoint { topLevelNodes.Insert(n.Id) - if node, addOk := wf.AddNode(wf.NewNodeBuilder(n), errs.NewScope()); addOk { + if node, addOk := wf.AddNode(wf.GetOrCreateNodeBuilder(n), errs.NewScope()); addOk { v.ValidateNode(&wf, node, false /* validateConditionTypes */, errs.NewScope()) } } diff --git a/pkg/compiler/workflow_compiler_test.go b/pkg/compiler/workflow_compiler_test.go index 68b7d6548..272bba3cd 100755 --- a/pkg/compiler/workflow_compiler_test.go +++ b/pkg/compiler/workflow_compiler_test.go @@ -403,7 +403,7 @@ func TestValidateReachable(t *testing.T) { } for range graph.downstreamNodes { - graph.Nodes = common.NewNodeIndex(graph.NewNodeBuilder(nil)) + graph.Nodes = common.NewNodeIndex(graph.GetOrCreateNodeBuilder(nil)) } errs := errors.NewCompileErrors() From 86117e61948f55cf486809e5873e1fc1797c402f Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 2 Jun 2021 08:48:20 -0700 Subject: [PATCH 13/14] Revert config.yaml Signed-off-by: Haytham Abuelfutuh --- config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config.yaml b/config.yaml index 0bac6f161..01bb7501b 100644 --- a/config.yaml +++ b/config.yaml @@ -1,7 +1,7 @@ # This is a sample configuration file. # Real configuration when running inside K8s (local or otherwise) lives in a ConfigMap propeller: - rawoutput-prefix: s3://flyte/test/ + rawoutput-prefix: s3://my-container/test/ metadata-prefix: metadata/propeller/sandbox workers: 4 workflow-reeval-duration: 10s @@ -23,7 +23,7 @@ propeller: rate: 100 capacity: 1000 # This config assumes using `make start` in flytesnacks repo to startup a DinD k3s container - kube-config: "$HOME/.kube/config" + kube-config: "$HOME/kubeconfig/k3s/k3s.yaml" publish-k8s-events: true workflowStore: policy: "ResourceVersionCache" @@ -85,7 +85,7 @@ storage: access-key: minio auth-type: accesskey disable-ssl: true - endpoint: http://localhost:9000 + endpoint: http://localhost:30084 region: us-east-1 secret-key: miniostorage type: minio From 03ba50b2cbb25f3e7506d94453354b917a891d68 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Wed, 2 Jun 2021 13:03:26 -0700 Subject: [PATCH 14/14] PR Comments Signed-off-by: Haytham Abuelfutuh --- pkg/compiler/test/compiler_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/compiler/test/compiler_test.go b/pkg/compiler/test/compiler_test.go index 1c9b50ca2..3b7249f64 100644 --- a/pkg/compiler/test/compiler_test.go +++ b/pkg/compiler/test/compiler_test.go @@ -270,7 +270,7 @@ func hasPromiseInputs(bindings []*core.Binding) bool { return false } -func assertNotIDsInConnections(t testing.TB, nodeIDsWithDeps, allNodeIDs sets.String, connections *core.ConnectionSet) bool { +func assertNodeIDsInConnections(t testing.TB, nodeIDsWithDeps, allNodeIDs sets.String, connections *core.ConnectionSet) bool { actualNodeIDs := sets.NewString() for id, lst := range connections.Downstream { actualNodeIDs.Insert(id) @@ -371,7 +371,7 @@ func TestBranches(t *testing.T) { allNodeIDs := getAllMatchingNodes(compiledWfc.Primary, allNodesPredicate) nodeIDsWithDeps := getAllMatchingNodes(compiledWfc.Primary, hasPromiseNodePredicate) - if !assertNotIDsInConnections(t, nodeIDsWithDeps, allNodeIDs, compiledWfc.Primary.Connections) { + if !assertNodeIDsInConnections(t, nodeIDsWithDeps, allNodeIDs, compiledWfc.Primary.Connections) { t.FailNow() }