Skip to content

Commit

Permalink
Adopt flyteidl's ordered variable map change (flyteorg#305)
Browse files Browse the repository at this point in the history
  • Loading branch information
mayitbeegh authored Sep 2, 2021
1 parent 6ae7428 commit 23f809c
Show file tree
Hide file tree
Showing 73 changed files with 3,818 additions and 2,783 deletions.
53 changes: 33 additions & 20 deletions flytepropeller/cmd/kubectl-flyte/cmd/create_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ func init() {

func createEmptyVariableMap() *core.VariableMap {
res := &core.VariableMap{
Variables: map[string]*core.Variable{},
Variables: []*core.VariableMapEntry{},
}
return res
}

func createVariableMap(variableMap map[string]*core.Variable) *core.VariableMap {
func createVariableMap(variableMap []*core.VariableMapEntry) *core.VariableMap {
res := &core.VariableMap{
Variables: variableMap,
}
Expand Down Expand Up @@ -138,18 +138,24 @@ func generateWorkflowWithInputs(t *testing.T) {
Workflow: &core.WorkflowTemplate{
Id: &core.Identifier{Name: "workflow-with-inputs"},
Interface: &core.TypedInterface{
Inputs: createVariableMap(map[string]*core.Variable{
"x": {
Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}},
Inputs: createVariableMap([]*core.VariableMapEntry{
{
Name: "x",
Var: &core.Variable{
Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}},
},
"y": {
Type: &core.LiteralType{
Type: &core.LiteralType_CollectionType{
CollectionType: &core.LiteralType{
Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING},
{
Name: "y",
Var: &core.Variable{
Type: &core.LiteralType{
Type: &core.LiteralType_CollectionType{
CollectionType: &core.LiteralType{
Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING},
},
},
},
}},
},
},
}),
},
Nodes: []*core.Node{
Expand Down Expand Up @@ -183,18 +189,25 @@ func generateWorkflowWithInputs(t *testing.T) {
{
Id: &core.Identifier{Name: "task-1"},
Interface: &core.TypedInterface{
Inputs: createVariableMap(map[string]*core.Variable{
"x": {
Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}},
Inputs: createVariableMap([]*core.VariableMapEntry{
{
Name: "x",
Var: &core.Variable{
Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}},
},
},
"y": {
Type: &core.LiteralType{
Type: &core.LiteralType_CollectionType{
CollectionType: &core.LiteralType{
Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING},
{
Name: "y",
Var: &core.Variable{
Type: &core.LiteralType{
Type: &core.LiteralType_CollectionType{
CollectionType: &core.LiteralType{
Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING},
},
},
},
}},
},
},
}),
},
Target: &core.TaskTemplate_Container{
Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"workflow":{"id":{"name":"workflow-id-123"},"interface":{"inputs":{"variables":{}}},"nodes":[{"id":"node-1","taskNode":{"referenceId":{"name":"task-1"}}},{"id":"node-2","taskNode":{"referenceId":{"name":"task-2"}}}]},"tasks":[{"id":{"name":"task-1"},"interface":{"inputs":{"variables":{}}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"]}},{"id":{"name":"task-2"},"interface":{"inputs":{"variables":{}}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"]}}]}
{"workflow":{"id":{"name":"workflow-id-123"},"interface":{"inputs":{}},"nodes":[{"id":"node-1","taskNode":{"referenceId":{"name":"task-1"}}},{"id":"node-2","taskNode":{"referenceId":{"name":"task-2"}}}]},"tasks":[{"id":{"name":"task-1"},"interface":{"inputs":{}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"]}},{"id":{"name":"task-2"},"interface":{"inputs":{}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"]}}]}
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,7 @@ tasks:
id:
name: task-1
interface:
inputs:
variables: {}
inputs: {}
- container:
args:
- testArg
Expand All @@ -19,14 +18,12 @@ tasks:
id:
name: task-2
interface:
inputs:
variables: {}
inputs: {}
workflow:
id:
name: workflow-id-123
interface:
inputs:
variables: {}
inputs: {}
nodes:
- id: node-1
taskNode:
Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"workflow":{"id":{"name":"workflow-with-inputs"},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"INTEGER"}},"y":{"type":{"collectionType":{"simple":"STRING"}}}}}},"nodes":[{"id":"node-1","inputs":[{"var":"x","binding":{"promise":{"var":"x"}}},{"var":"y","binding":{"promise":{"var":"y"}}}],"taskNode":{"referenceId":{"name":"task-1"}}},{"id":"node-2","taskNode":{"referenceId":{"name":"task-2"}}}]},"tasks":[{"id":{"name":"task-1"},"interface":{"inputs":{"variables":{"x":{"type":{"simple":"INTEGER"}},"y":{"type":{"collectionType":{"simple":"STRING"}}}}}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"],"resources":{"requests":[{"name":"CPU","value":"2"},{"name":"MEMORY","value":"2048Mi"}]}}},{"id":{"name":"task-2"},"interface":{"inputs":{"variables":{}}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"]}}]}
{"workflow":{"id":{"name":"workflow-with-inputs"},"interface":{"inputs":{"variables":[{"key":"x","value":{"type":{"simple":"INTEGER"}}},{"key":"y","value":{"type":{"collectionType":{"simple":"STRING"}}}}]}},"nodes":[{"id":"node-1","inputs":[{"var":"x","binding":{"promise":{"var":"x"}}},{"var":"y","binding":{"promise":{"var":"y"}}}],"taskNode":{"referenceId":{"name":"task-1"}}},{"id":"node-2","taskNode":{"referenceId":{"name":"task-2"}}}]},"tasks":[{"id":{"name":"task-1"},"interface":{"inputs":{"variables":[{"key":"x","value":{"type":{"simple":"INTEGER"}}},{"key":"y","value":{"type":{"collectionType":{"simple":"STRING"}}}}]}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"],"resources":{"requests":[{"name":"CPU","value":"2"},{"name":"MEMORY","value":"2048Mi"}]}}},{"id":{"name":"task-2"},"interface":{"inputs":{}},"container":{"image":"myflyteimage:latest","command":["execute-task"],"args":["testArg"]}}]}
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,12 @@ tasks:
interface:
inputs:
variables:
x:
- key: x
value:
type:
simple: INTEGER
"y":
- key: "y"
value:
type:
collectionType:
simple: STRING
Expand All @@ -32,18 +34,19 @@ tasks:
id:
name: task-2
interface:
inputs:
variables: {}
inputs: {}
workflow:
id:
name: workflow-with-inputs
interface:
inputs:
variables:
x:
- key: x
value:
type:
simple: INTEGER
"y":
- key: "y"
value:
type:
collectionType:
simple: STRING
Expand Down
4 changes: 2 additions & 2 deletions flytepropeller/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ require (
github.com/DiSiqueira/GoTree v1.0.1-0.20180907134536-53a8e837f295
github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1
github.com/fatih/color v1.10.0
github.com/flyteorg/flyteidl v0.19.22
github.com/flyteorg/flyteplugins v0.5.72
github.com/flyteorg/flyteidl v0.20.0
github.com/flyteorg/flyteplugins v0.5.73
github.com/flyteorg/flytestdlib v0.3.34
github.com/ghodss/yaml v1.0.0
github.com/go-redis/redis v6.15.7+incompatible
Expand Down
11 changes: 6 additions & 5 deletions flytepropeller/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DiSiqueira/GoTree v1.0.1-0.20180907134536-53a8e837f295 h1:xJ0dAkuxJXfwdH7IaSzBEbSQxEDz36YUmt7+CB4zoNA=
Expand Down Expand Up @@ -229,11 +230,10 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
github.com/flyteorg/flyteidl v0.19.2/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
github.com/flyteorg/flyteidl v0.19.22 h1:e3M0Dob/r5n+AJfAByzad/svMAVes7XfZVxUNCi6AeQ=
github.com/flyteorg/flyteidl v0.19.22/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
github.com/flyteorg/flyteplugins v0.5.72 h1:mtbpn4nuFrhZ2DadUlLxjK2RQ9FBFqDf0LaXVpe2hyY=
github.com/flyteorg/flyteplugins v0.5.72/go.mod h1:YjahYP+i4/Qn+dFvxMOGbhDtkQT4EiH4Kb88KNK505A=
github.com/flyteorg/flyteidl v0.20.0 h1:g5xGayFfPSzFJxJedgL390WFSEbGYjFiPey+NXAB030=
github.com/flyteorg/flyteidl v0.20.0/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
github.com/flyteorg/flyteplugins v0.5.73 h1:xIF7aYPL0E22G0r59qdvfWHKsIT9150TB4sIu17vHNU=
github.com/flyteorg/flyteplugins v0.5.73/go.mod h1:BvqwQPl77lRBuxX4FVI4v/CpftV87XAa2Ghqho3pXm8=
github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220=
github.com/flyteorg/flytestdlib v0.3.33/go.mod h1:7cDWkY3v7xsoesFcDdu6DSW5Q2U2W5KlHUbUHSwBG1Q=
github.com/flyteorg/flytestdlib v0.3.34 h1:OOuV03X8c1AWInzBU6IRsqpEF6y8WDJngbPcdL4VktY=
Expand Down Expand Up @@ -1227,6 +1227,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.0.0-20210217171935-8e2decd92398/go.mod h1:60tmSUpHxGPFerNHbo/ayI2lKxvtrhbxFyXuEIWJd78=
k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78=
Expand Down
19 changes: 12 additions & 7 deletions flytepropeller/pkg/apis/flyteworkflow/v1alpha1/testdata/task.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,20 @@ id:
type: "demo"
interface:
inputs:
variables:
value_to_print:
type:
simple: INTEGER
variables: # not backwards compatible
-
name: value_to_print
var:
type:
simple: INTEGER
outputs:
variables:
out:
type:
simple: INTEGER
-
name: out
var:
type:
simple: INTEGER

metadata:
runtime:
version: 1.19.0b7
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -205,14 +205,16 @@ tasks:
interface:
inputs:
variables:
value_to_print:
type:
simple: INTEGER
- name: value_to_print
var:
type:
simple: INTEGER
outputs:
variables:
out:
type:
simple: INTEGER
- name: out
var:
type:
simple: INTEGER
metadata:
runtime:
version: 1.19.0b7
Expand Down
41 changes: 25 additions & 16 deletions flytepropeller/pkg/compiler/admin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,27 @@ var launchPlanIdentifier = core.Identifier{
}

var inputs = core.ParameterMap{
Parameters: map[string]*core.Parameter{
"foo": {
Var: &core.Variable{
Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}},
},
Behavior: &core.Parameter_Default{
Default: coreutils.MustMakeLiteral("foo-value"),
Parameters: []*core.ParameterMapEntry{
{
Name: "foo",
Parameter: &core.Parameter{
Var: &core.Variable{
Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}},
},
Behavior: &core.Parameter_Default{
Default: coreutils.MustMakeLiteral("foo-value"),
},
},
},
},
}
var outputs = core.VariableMap{
Variables: map[string]*core.Variable{
"foo": {
Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}},
Variables: []*core.VariableMapEntry{
{
Name: "foo",
Var: &core.Variable{
Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}},
},
},
},
}
Expand All @@ -58,15 +64,18 @@ func TestGetId(t *testing.T) {
func TestGetExpectedInputs(t *testing.T) {
launchPlan := getDummyLaunchPlan()
provider := NewLaunchPlanInterfaceProvider(launchPlan)
assert.Contains(t, (*provider.GetExpectedInputs()).Parameters, "foo")
assert.NotNil(t, (*provider.GetExpectedInputs()).Parameters["foo"].Var.Type.GetSimple())
assert.EqualValues(t, "STRING", (*provider.GetExpectedInputs()).Parameters["foo"].Var.Type.GetSimple().String())
assert.NotNil(t, (*provider.GetExpectedInputs()).Parameters["foo"].GetDefault())
assert.Equal(t, 1, len((*provider.GetExpectedInputs()).Parameters))
assert.Equal(t, "foo", (*provider.GetExpectedInputs()).Parameters[0].GetName())
assert.NotNil(t, (*provider.GetExpectedInputs()).Parameters[0].GetParameter().Var.Type.GetSimple())
assert.EqualValues(t, "STRING", (*provider.GetExpectedInputs()).Parameters[0].GetParameter().Var.Type.GetSimple().String())
assert.NotNil(t, (*provider.GetExpectedInputs()).Parameters[0].GetParameter().GetDefault())
}

func TestGetExpectedOutputs(t *testing.T) {
launchPlan := getDummyLaunchPlan()
provider := NewLaunchPlanInterfaceProvider(launchPlan)
assert.EqualValues(t, outputs.Variables["foo"].GetType().GetType(),
provider.GetExpectedOutputs().Variables["foo"].GetType().GetType())
assert.Equal(t, 1, len(outputs.Variables))
assert.Equal(t, "foo", outputs.Variables[0].GetName())
assert.EqualValues(t, outputs.Variables[0].GetVar().GetType().GetType(),
provider.GetExpectedOutputs().Variables[0].GetVar().GetType().GetType())
}
14 changes: 10 additions & 4 deletions flytepropeller/pkg/compiler/task_compiler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,11 @@ func TestValidateContainerCommand(t *testing.T) {
task := core.TaskTemplate{
Id: &core.Identifier{Name: "task_123"},
Interface: &core.TypedInterface{
Inputs: createVariableMap(map[string]*core.Variable{
"foo": {},
Inputs: createVariableMap([]*core.VariableMapEntry{
{
Name: "foo",
Var: &core.Variable{},
},
}),
Outputs: createEmptyVariableMap(),
},
Expand Down Expand Up @@ -59,8 +62,11 @@ func TestValidateK8sPod(t *testing.T) {
task := core.TaskTemplate{
Id: &core.Identifier{Name: "task_123"},
Interface: &core.TypedInterface{
Inputs: createVariableMap(map[string]*core.Variable{
"foo": {},
Inputs: createVariableMap([]*core.VariableMapEntry{
{
Name: "foo",
Var: &core.Variable{},
},
}),
Outputs: createEmptyVariableMap(),
},
Expand Down
Loading

0 comments on commit 23f809c

Please sign in to comment.