Skip to content

Commit

Permalink
Literal Transformations from a string or an interface based on litera…
Browse files Browse the repository at this point in the history 
…lTypes (flyteorg#120)

* wip: more transformations

Signed-off-by: Ketan Umare <[email protected]>

* Changed the String formatter from %s to %v and added more unit tests (flyteorg#122)

Signed-off-by: pmahindrakar-oss <[email protected]>

Co-authored-by: pmahindrakar-oss <[email protected]>
Signed-off-by: Ketan Umare <[email protected]>

* Added default condition for unsupported type while creating literal (flyteorg#124)

* Changed the String formatter from %s to %v and added more unit tests

Signed-off-by: pmahindrakar-oss <[email protected]>

* Added unsupported type default condition

Signed-off-by: pmahindrakar-oss <[email protected]>

Co-authored-by: pmahindrakar-oss <[email protected]>
Signed-off-by: pmahindrakar-oss <[email protected]>

Co-authored-by: pmahindrakar-oss <[email protected]>
Co-authored-by: pmahindrakar-oss <[email protected]>
  • Loading branch information
@kumare3 @pmahindrakar-oss
3 people authored Mar 17, 2021
1 parent 8fad9ca commit d514c56
Show file tree
Hide file tree
Showing 3 changed files with 217 additions and 222 deletions.
57 changes: 57 additions & 0 deletions clients/go/coreutils/literals.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -437,3 +437,60 @@ func MakeLiteralForBlob(path storage.DataReference, isDir bool, format string) *
},
}
}

func MakeLiteralForType(t *core.LiteralType, v interface{}) (*core.Literal, error) {
l := &core.Literal{}
switch t.Type.(type) {
case *core.LiteralType_MapValueType:
newT := t.Type.(*core.LiteralType_MapValueType)
newV, ok := v.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("map value types can only be of type map[string]interface{}, but found %v", reflect.TypeOf(v))
}

literals := make(map[string]*core.Literal, len(newV))
for key, val := range newV {
lv, err := MakeLiteralForType(newT.MapValueType, val)
if err != nil {
return nil, err
}
literals[key] = lv
}
l.Value = &core.Literal_Map{
Map: &core.LiteralMap{
Literals: literals,
},
}
case *core.LiteralType_CollectionType:
newT := t.Type.(*core.LiteralType_CollectionType)
newV, ok := v.([]interface{})
if !ok {
return nil, fmt.Errorf("collection type expected but found %v", reflect.TypeOf(v))
}

literals := make([]*core.Literal, 0, len(newV))
for _, val := range newV {
lv, err := MakeLiteralForType(newT.CollectionType, val)
if err != nil {
return nil, err
}
literals = append(literals, lv)
}
l.Value = &core.Literal_Collection{
Collection: &core.LiteralCollection{
Literals: literals,
},
}
case *core.LiteralType_Simple:
newT := t.Type.(*core.LiteralType_Simple)
lv, err := MakeLiteralForSimpleType(newT.Simple, fmt.Sprintf("%v", v))
if err != nil {
return nil, err
}
return lv, nil
default:
return nil, fmt.Errorf("unsupported type %s", t.String())
}

return l, nil
}
160 changes: 160 additions & 0 deletions clients/go/coreutils/literals_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package coreutils

import (
"fmt"
"reflect"
"testing"
"time"
Expand Down Expand Up @@ -366,3 +367,162 @@ func TestMakeLiteralForBlob(t *testing.T) {
})
}
}

func TestMakeLiteralForType(t *testing.T) {
t.Run("SimpleInteger", func(t *testing.T) {
var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}
val, err := MakeLiteralForType(literalType, 1)
assert.NoError(t, err)
literalVal := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_Integer{Integer: 1}}}}}}
expectedVal, _ := ExtractFromLiteral(literalVal)
actualVal, _ := ExtractFromLiteral(val)
assert.Equal(t, expectedVal, actualVal)
})

t.Run("SimpleFloat", func(t *testing.T) {
var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}}
val, err := MakeLiteralForType(literalType, 1)
assert.NoError(t, err)
literalVal := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_FloatValue{FloatValue: 1.0}}}}}}
expectedVal, _ := ExtractFromLiteral(literalVal)
actualVal, _ := ExtractFromLiteral(val)
assert.Equal(t, expectedVal, actualVal)
})

t.Run("ArrayStrings", func(t *testing.T) {
var literalType = &core.LiteralType{Type: &core.LiteralType_CollectionType{
CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}}}
strArray := []interface{}{"hello", "world"}
val, err := MakeLiteralForType(literalType, strArray)
assert.NoError(t, err)
literalVal1 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "hello"}}}}}}
literalVal2 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world"}}}}}}
literalCollection := []*core.Literal{literalVal1, literalVal2}
literalVal := &core.Literal{Value: &core.Literal_Collection{Collection: &core.LiteralCollection{Literals: literalCollection}}}
expectedVal, _ := ExtractFromLiteral(literalVal)
actualVal, _ := ExtractFromLiteral(val)
assert.Equal(t, expectedVal, actualVal)
})

t.Run("ArrayOfArrayStringsNotSupported", func(t *testing.T) {
var literalType = &core.LiteralType{Type: &core.LiteralType_CollectionType{
CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}}}
strArrayOfArray := [][]interface{}{{"hello1", "world1"}, {"hello2", "world2"}}
_, err := MakeLiteralForType(literalType, strArrayOfArray)
expectedErrorf := fmt.Errorf("collection type expected but found [][]interface {}")
assert.Equal(t, expectedErrorf, err)
})

t.Run("ArrayOfArrayStringsTypeErasure", func(t *testing.T) {
var collectionType = &core.LiteralType{Type: &core.LiteralType_CollectionType{
CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}}}
var literalType = &core.LiteralType{Type: &core.LiteralType_CollectionType{
CollectionType: collectionType}}

createList1 := func() interface{} {
return []interface{}{"hello1", "world1"}
}
createList2 := func() interface{} {
return []interface{}{"hello2", "world2"}
}
createNestedList := func() interface{} {
return []interface{}{createList1(), createList2()}
}
var strArrayOfArray = createNestedList()
val, err := MakeLiteralForType(literalType, strArrayOfArray)
assert.NoError(t, err)
literalVal11 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "hello1"}}}}}}
literalVal12 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world1"}}}}}}
literalCollection1Val := []*core.Literal{literalVal11, literalVal12}

literalCollection1 := &core.Literal{Value: &core.Literal_Collection{Collection: &core.LiteralCollection{Literals: literalCollection1Val}}}

literalVal21 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "hello2"}}}}}}
literalVal22 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world2"}}}}}}
literalCollection2Val := []*core.Literal{literalVal21, literalVal22}
literalCollection2 := &core.Literal{Value: &core.Literal_Collection{Collection: &core.LiteralCollection{Literals: literalCollection2Val}}}
literalCollection := []*core.Literal{literalCollection1, literalCollection2}

literalVal := &core.Literal{Value: &core.Literal_Collection{Collection: &core.LiteralCollection{Literals: literalCollection}}}
expectedVal, _ := ExtractFromLiteral(literalVal)
actualVal, _ := ExtractFromLiteral(val)
assert.Equal(t, expectedVal, actualVal)
})

t.Run("MapStrings", func(t *testing.T) {
var literalType = &core.LiteralType{Type: &core.LiteralType_MapValueType{
MapValueType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}}}
mapVal := map[string]interface{}{"hello1": "world1", "hello2": "world2"}
val, err := MakeLiteralForType(literalType, mapVal)
assert.NoError(t, err)
literalVal1 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world1"}}}}}}
literalVal2 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world2"}}}}}}
literalMapVal := map[string]*core.Literal{"hello1": literalVal1, "hello2": literalVal2}
literalVal := &core.Literal{Value: &core.Literal_Map{Map: &core.LiteralMap{Literals: literalMapVal}}}
expectedVal, _ := ExtractFromLiteral(literalVal)
actualVal, _ := ExtractFromLiteral(val)
assert.Equal(t, expectedVal, actualVal)
})

t.Run("MapArrayOfStringsFail", func(t *testing.T) {
var literalType = &core.LiteralType{Type: &core.LiteralType_MapValueType{
MapValueType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}}}
strArray := map[string][]interface{}{"hello1": {"world11", "world12"}, "hello2": {"world21", "world22"}}
_, err := MakeLiteralForType(literalType, strArray)
expectedErrorf := fmt.Errorf("map value types can only be of type map[string]interface{}, but found map[string][]interface {}")
assert.Equal(t, expectedErrorf, err)
})

t.Run("MapArrayOfStringsTypeErasure", func(t *testing.T) {
var collectionType = &core.LiteralType{Type: &core.LiteralType_CollectionType{
CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}}}
var literalType = &core.LiteralType{Type: &core.LiteralType_MapValueType{
MapValueType: collectionType}}
createList1 := func() interface{} {
return []interface{}{"world11", "world12"}
}
createList2 := func() interface{} {
return []interface{}{"world21", "world22"}
}
strArray := map[string]interface{}{"hello1": createList1(), "hello2": createList2()}
val, err := MakeLiteralForType(literalType, strArray)
assert.NoError(t, err)
literalVal11 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world11"}}}}}}
literalVal12 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world12"}}}}}}
literalCollection1 := []*core.Literal{literalVal11, literalVal12}
literalVal1 := &core.Literal{Value: &core.Literal_Collection{Collection: &core.LiteralCollection{Literals: literalCollection1}}}
literalVal21 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world21"}}}}}}
literalVal22 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world22"}}}}}}
literalCollection2 := []*core.Literal{literalVal21, literalVal22}
literalVal2 := &core.Literal{Value: &core.Literal_Collection{Collection: &core.LiteralCollection{Literals: literalCollection2}}}
literalMapVal := map[string]*core.Literal{"hello1": literalVal1, "hello2": literalVal2}
literalVal := &core.Literal{Value: &core.Literal_Map{Map: &core.LiteralMap{Literals: literalMapVal}}}
expectedVal, _ := ExtractFromLiteral(literalVal)
actualVal, _ := ExtractFromLiteral(val)
assert.Equal(t, expectedVal, actualVal)
})

t.Run("UnsupportedType", func(t *testing.T) {
var literalType = &core.LiteralType{Type: &core.LiteralType_Schema{
Schema: &core.SchemaType{}}}
var schema interface{}
_, err := MakeLiteralForType(literalType, schema)
expectedErrorf := fmt.Errorf("unsupported type schema:<> ")
assert.Equal(t, expectedErrorf, err)

})
}
Loading

0 comments on commit d514c56

Please sign in to comment.