Adopt flyteidl's ordered variable map change (flyteorg#305)

eapolinario · Sep 2, 2021 · 23f809c · 23f809c
1 parent 6ae7428
commit 23f809c
Show file tree

Hide file tree

Showing 73 changed files with 3,818 additions and 2,783 deletions.
diff --git a/flytepropeller/cmd/kubectl-flyte/cmd/create_test.go b/flytepropeller/cmd/kubectl-flyte/cmd/create_test.go
@@ -27,12 +27,12 @@ func init() {
 
 func createEmptyVariableMap() *core.VariableMap {
 	res := &core.VariableMap{
-		Variables: map[string]*core.Variable{},
+		Variables: []*core.VariableMapEntry{},
 	}
 	return res
 }
 
-func createVariableMap(variableMap map[string]*core.Variable) *core.VariableMap {
+func createVariableMap(variableMap []*core.VariableMapEntry) *core.VariableMap {
 	res := &core.VariableMap{
 		Variables: variableMap,
 	}
@@ -138,18 +138,24 @@ func generateWorkflowWithInputs(t *testing.T) {
 		Workflow: &core.WorkflowTemplate{
 			Id: &core.Identifier{Name: "workflow-with-inputs"},
 			Interface: &core.TypedInterface{
-				Inputs: createVariableMap(map[string]*core.Variable{
-					"x": {
-						Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}},
+				Inputs: createVariableMap([]*core.VariableMapEntry{
+					{
+						Name: "x",
+						Var: &core.Variable{
+							Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}},
 					},
-					"y": {
-						Type: &core.LiteralType{
-							Type: &core.LiteralType_CollectionType{
-								CollectionType: &core.LiteralType{
-									Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING},
+					{
+						Name: "y",
+						Var: &core.Variable{
+							Type: &core.LiteralType{
+								Type: &core.LiteralType_CollectionType{
+									CollectionType: &core.LiteralType{
+										Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING},
+									},
 								},
 							},
-						}},
+						},
+					},
 				}),
 			},
 			Nodes: []*core.Node{
@@ -183,18 +189,25 @@ func generateWorkflowWithInputs(t *testing.T) {
 			{
 				Id: &core.Identifier{Name: "task-1"},
 				Interface: &core.TypedInterface{
-					Inputs: createVariableMap(map[string]*core.Variable{
-						"x": {
-							Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}},
+					Inputs: createVariableMap([]*core.VariableMapEntry{
+						{
+							Name: "x",
+							Var: &core.Variable{
+								Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}},
+							},
 						},
-						"y": {
-							Type: &core.LiteralType{
-								Type: &core.LiteralType_CollectionType{
-									CollectionType: &core.LiteralType{
-										Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING},
+						{
+							Name: "y",
+							Var: &core.Variable{
+								Type: &core.LiteralType{
+									Type: &core.LiteralType_CollectionType{
+										CollectionType: &core.LiteralType{
+											Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING},
+										},
 									},
 								},
-							}},
+							},
+						},
 					}),
 				},
 				Target: &core.TaskTemplate_Container{

diff --git a/flytepropeller/cmd/kubectl-flyte/cmd/testdata/workflow.json.golden b/flytepropeller/cmd/kubectl-flyte/cmd/testdata/workflow.json.golden
@@ -1 +1 @@
-{"workflow":{"id":{"name":"workflow-id-123"},"interface":{"inputs":{"variables":{}}},"nodes":[{"id":"node-1","taskNode":{"referenceId":{"name":"task-1"}}},{"id":"node-2","taskNode":{"referenceId":{"name":"task-2"}}}]},"tasks":[{"id":{"name":"task-1"},"interface":{"inputs":{"variables":{}}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"]}},{"id":{"name":"task-2"},"interface":{"inputs":{"variables":{}}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"]}}]}
+{"workflow":{"id":{"name":"workflow-id-123"},"interface":{"inputs":{}},"nodes":[{"id":"node-1","taskNode":{"referenceId":{"name":"task-1"}}},{"id":"node-2","taskNode":{"referenceId":{"name":"task-2"}}}]},"tasks":[{"id":{"name":"task-1"},"interface":{"inputs":{}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"]}},{"id":{"name":"task-2"},"interface":{"inputs":{}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"]}}]}
diff --git a/flytepropeller/cmd/kubectl-flyte/cmd/testdata/workflow.yaml.golden b/flytepropeller/cmd/kubectl-flyte/cmd/testdata/workflow.yaml.golden
@@ -8,8 +8,7 @@ tasks:
   id:
     name: task-1
   interface:
-    inputs:
-      variables: {}
+    inputs: {}
 - container:
     args:
     - testArg
@@ -19,14 +18,12 @@ tasks:
   id:
     name: task-2
   interface:
-    inputs:
-      variables: {}
+    inputs: {}
 workflow:
   id:
     name: workflow-id-123
   interface:
-    inputs:
-      variables: {}
+    inputs: {}
   nodes:
   - id: node-1
     taskNode:

diff --git a/flytepropeller/cmd/kubectl-flyte/cmd/testdata/workflow_w_inputs.json.golden b/flytepropeller/cmd/kubectl-flyte/cmd/testdata/workflow_w_inputs.json.golden
@@ -1 +1 @@
-{"workflow":{"id":{"name":"workflow-with-inputs"},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"INTEGER"}},"y":{"type":{"collectionType":{"simple":"STRING"}}}}}},"nodes":[{"id":"node-1","inputs":[{"var":"x","binding":{"promise":{"var":"x"}}},{"var":"y","binding":{"promise":{"var":"y"}}}],"taskNode":{"referenceId":{"name":"task-1"}}},{"id":"node-2","taskNode":{"referenceId":{"name":"task-2"}}}]},"tasks":[{"id":{"name":"task-1"},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"INTEGER"}},"y":{"type":{"collectionType":{"simple":"STRING"}}}}}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"],"resources":{"requests":[{"name":"CPU","value":"2"},{"name":"MEMORY","value":"2048Mi"}]}}},{"id":{"name":"task-2"},"interface":{"inputs":{"variables":{}}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"]}}]}
+{"workflow":{"id":{"name":"workflow-with-inputs"},"interface":{"inputs":{"variables":[{"key":"x","value":{"type":{"simple":"INTEGER"}}},{"key":"y","value":{"type":{"collectionType":{"simple":"STRING"}}}}]}},"nodes":[{"id":"node-1","inputs":[{"var":"x","binding":{"promise":{"var":"x"}}},{"var":"y","binding":{"promise":{"var":"y"}}}],"taskNode":{"referenceId":{"name":"task-1"}}},{"id":"node-2","taskNode":{"referenceId":{"name":"task-2"}}}]},"tasks":[{"id":{"name":"task-1"},"interface":{"inputs":{"variables":[{"key":"x","value":{"type":{"simple":"INTEGER"}}},{"key":"y","value":{"type":{"collectionType":{"simple":"STRING"}}}}]}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"],"resources":{"requests":[{"name":"CPU","value":"2"},{"name":"MEMORY","value":"2048Mi"}]}}},{"id":{"name":"task-2"},"interface":{"inputs":{}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"]}}]}
diff --git a/flytepropeller/cmd/kubectl-flyte/cmd/testdata/workflow_w_inputs.yaml.golden b/flytepropeller/cmd/kubectl-flyte/cmd/testdata/workflow_w_inputs.yaml.golden
@@ -16,10 +16,12 @@ tasks:
   interface:
     inputs:
       variables:
-        x:
+      - key: x
+        value:
           type:
             simple: INTEGER
-        "y":
+      - key: "y"
+        value:
           type:
             collectionType:
               simple: STRING
@@ -32,18 +34,19 @@ tasks:
   id:
     name: task-2
   interface:
-    inputs:
-      variables: {}
+    inputs: {}
 workflow:
   id:
     name: workflow-with-inputs
   interface:
     inputs:
       variables:
-        x:
+      - key: x
+        value:
           type:
             simple: INTEGER
-        "y":
+      - key: "y"
+        value:
           type:
             collectionType:
               simple: STRING

diff --git a/flytepropeller/go.mod b/flytepropeller/go.mod
@@ -6,8 +6,8 @@ require (
 	github.com/DiSiqueira/GoTree v1.0.1-0.20180907134536-53a8e837f295
 	github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1
 	github.com/fatih/color v1.10.0
-	github.com/flyteorg/flyteidl v0.19.22
-	github.com/flyteorg/flyteplugins v0.5.72
+	github.com/flyteorg/flyteidl v0.20.0
+	github.com/flyteorg/flyteplugins v0.5.73
 	github.com/flyteorg/flytestdlib v0.3.34
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-redis/redis v6.15.7+incompatible

diff --git a/flytepropeller/go.sum b/flytepropeller/go.sum
@@ -72,6 +72,7 @@ github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DiSiqueira/GoTree v1.0.1-0.20180907134536-53a8e837f295 h1:xJ0dAkuxJXfwdH7IaSzBEbSQxEDz36YUmt7+CB4zoNA=
@@ -229,11 +230,10 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
-github.com/flyteorg/flyteidl v0.19.2/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
-github.com/flyteorg/flyteidl v0.19.22 h1:e3M0Dob/r5n+AJfAByzad/svMAVes7XfZVxUNCi6AeQ=
-github.com/flyteorg/flyteidl v0.19.22/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
-github.com/flyteorg/flyteplugins v0.5.72 h1:mtbpn4nuFrhZ2DadUlLxjK2RQ9FBFqDf0LaXVpe2hyY=
-github.com/flyteorg/flyteplugins v0.5.72/go.mod h1:YjahYP+i4/Qn+dFvxMOGbhDtkQT4EiH4Kb88KNK505A=
+github.com/flyteorg/flyteidl v0.20.0 h1:g5xGayFfPSzFJxJedgL390WFSEbGYjFiPey+NXAB030=
+github.com/flyteorg/flyteidl v0.20.0/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
+github.com/flyteorg/flyteplugins v0.5.73 h1:xIF7aYPL0E22G0r59qdvfWHKsIT9150TB4sIu17vHNU=
+github.com/flyteorg/flyteplugins v0.5.73/go.mod h1:BvqwQPl77lRBuxX4FVI4v/CpftV87XAa2Ghqho3pXm8=
 github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220=
 github.com/flyteorg/flytestdlib v0.3.33/go.mod h1:7cDWkY3v7xsoesFcDdu6DSW5Q2U2W5KlHUbUHSwBG1Q=
 github.com/flyteorg/flytestdlib v0.3.34 h1:OOuV03X8c1AWInzBU6IRsqpEF6y8WDJngbPcdL4VktY=
@@ -1227,6 +1227,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 k8s.io/api v0.0.0-20210217171935-8e2decd92398/go.mod h1:60tmSUpHxGPFerNHbo/ayI2lKxvtrhbxFyXuEIWJd78=
 k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78=

diff --git a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/testdata/task.yaml b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/testdata/task.yaml
@@ -3,15 +3,20 @@ id:
 type: "demo"
 interface:
   inputs:
-    variables:
-      value_to_print:
-        type:
-          simple: INTEGER
+    variables: # not backwards compatible
+      -
+        name: value_to_print
+        var:
+          type:
+            simple: INTEGER
   outputs:
     variables:
-      out:
-        type:
-          simple: INTEGER
+      -
+        name: out
+        var:
+          type:
+            simple: INTEGER
+
 metadata:
   runtime:
     version: 1.19.0b7

diff --git a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/testdata/workflowspec.yaml b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/testdata/workflowspec.yaml
@@ -205,14 +205,16 @@ tasks:
     interface:
       inputs:
         variables:
-          value_to_print:
-            type:
-              simple: INTEGER
+          - name: value_to_print
+            var:
+              type:
+                simple: INTEGER
       outputs:
         variables:
-          out:
-            type:
-              simple: INTEGER
+          - name: out
+            var:
+              type:
+                simple: INTEGER
     metadata:
       runtime:
         version: 1.19.0b7

diff --git a/flytepropeller/pkg/compiler/admin_test.go b/flytepropeller/pkg/compiler/admin_test.go
@@ -18,21 +18,27 @@ var launchPlanIdentifier = core.Identifier{
 }
 
 var inputs = core.ParameterMap{
-	Parameters: map[string]*core.Parameter{
-		"foo": {
-			Var: &core.Variable{
-				Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}},
-			},
-			Behavior: &core.Parameter_Default{
-				Default: coreutils.MustMakeLiteral("foo-value"),
+	Parameters: []*core.ParameterMapEntry{
+		{
+			Name: "foo",
+			Parameter: &core.Parameter{
+				Var: &core.Variable{
+					Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}},
+				},
+				Behavior: &core.Parameter_Default{
+					Default: coreutils.MustMakeLiteral("foo-value"),
+				},
 			},
 		},
 	},
 }
 var outputs = core.VariableMap{
-	Variables: map[string]*core.Variable{
-		"foo": {
-			Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}},
+	Variables: []*core.VariableMapEntry{
+		{
+			Name: "foo",
+			Var: &core.Variable{
+				Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}},
+			},
 		},
 	},
 }
@@ -58,15 +64,18 @@ func TestGetId(t *testing.T) {
 func TestGetExpectedInputs(t *testing.T) {
 	launchPlan := getDummyLaunchPlan()
 	provider := NewLaunchPlanInterfaceProvider(launchPlan)
-	assert.Contains(t, (*provider.GetExpectedInputs()).Parameters, "foo")
-	assert.NotNil(t, (*provider.GetExpectedInputs()).Parameters["foo"].Var.Type.GetSimple())
-	assert.EqualValues(t, "STRING", (*provider.GetExpectedInputs()).Parameters["foo"].Var.Type.GetSimple().String())
-	assert.NotNil(t, (*provider.GetExpectedInputs()).Parameters["foo"].GetDefault())
+	assert.Equal(t, 1, len((*provider.GetExpectedInputs()).Parameters))
+	assert.Equal(t, "foo", (*provider.GetExpectedInputs()).Parameters[0].GetName())
+	assert.NotNil(t, (*provider.GetExpectedInputs()).Parameters[0].GetParameter().Var.Type.GetSimple())
+	assert.EqualValues(t, "STRING", (*provider.GetExpectedInputs()).Parameters[0].GetParameter().Var.Type.GetSimple().String())
+	assert.NotNil(t, (*provider.GetExpectedInputs()).Parameters[0].GetParameter().GetDefault())
 }
 
 func TestGetExpectedOutputs(t *testing.T) {
 	launchPlan := getDummyLaunchPlan()
 	provider := NewLaunchPlanInterfaceProvider(launchPlan)
-	assert.EqualValues(t, outputs.Variables["foo"].GetType().GetType(),
-		provider.GetExpectedOutputs().Variables["foo"].GetType().GetType())
+	assert.Equal(t, 1, len(outputs.Variables))
+	assert.Equal(t, "foo", outputs.Variables[0].GetName())
+	assert.EqualValues(t, outputs.Variables[0].GetVar().GetType().GetType(),
+		provider.GetExpectedOutputs().Variables[0].GetVar().GetType().GetType())
 }
diff --git a/flytepropeller/pkg/compiler/task_compiler_test.go b/flytepropeller/pkg/compiler/task_compiler_test.go
@@ -23,8 +23,11 @@ func TestValidateContainerCommand(t *testing.T) {
 	task := core.TaskTemplate{
 		Id: &core.Identifier{Name: "task_123"},
 		Interface: &core.TypedInterface{
-			Inputs: createVariableMap(map[string]*core.Variable{
-				"foo": {},
+			Inputs: createVariableMap([]*core.VariableMapEntry{
+				{
+					Name: "foo",
+					Var:  &core.Variable{},
+				},
 			}),
 			Outputs: createEmptyVariableMap(),
 		},
@@ -59,8 +62,11 @@ func TestValidateK8sPod(t *testing.T) {
 	task := core.TaskTemplate{
 		Id: &core.Identifier{Name: "task_123"},
 		Interface: &core.TypedInterface{
-			Inputs: createVariableMap(map[string]*core.Variable{
-				"foo": {},
+			Inputs: createVariableMap([]*core.VariableMapEntry{
+				{
+					Name: "foo",
+					Var:  &core.Variable{},
+				},
 			}),
 			Outputs: createEmptyVariableMap(),
 		},