From 66699ed14edcd67fee382b8ad0c5b503c476a237 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Mon, 19 Feb 2024 20:10:34 +1100 Subject: [PATCH] feat: ingress and encoder will omit some empty fields (#957) Specifically Any, Unit, Option, maps, and arrays. Also added some tests for BuildRequestBody(). --- backend/controller/ingress/alias.go | 11 +- backend/controller/ingress/ingress_test.go | 85 ++++++-- backend/controller/ingress/request.go | 16 +- backend/controller/ingress/request_test.go | 190 ++++++++++++++++++ .../xyz/block/ftl/v1/schema/schema.pb.go | 58 +++--- .../xyz/block/ftl/v1/schema/schema.proto | 4 +- backend/schema/parser.go | 5 +- backend/schema/validate.go | 3 + examples/go/echo/go.mod | 1 + examples/go/echo/go.sum | 8 +- .../xyz/block/ftl/v1/schema/schema_pb.ts | 16 +- go-runtime/encoding/encoding.go | 34 +++- go-runtime/ftl/types.go | 3 - go.mod | 4 +- go.sum | 8 +- 15 files changed, 366 insertions(+), 80 deletions(-) create mode 100644 backend/controller/ingress/request_test.go diff --git a/backend/controller/ingress/alias.go b/backend/controller/ingress/alias.go index 9d1206d2d0..de90d0499c 100644 --- a/backend/controller/ingress/alias.go +++ b/backend/controller/ingress/alias.go @@ -7,15 +7,18 @@ import ( ) func transformAliasedFields(sch *schema.Schema, t schema.Type, obj any, aliaser func(obj map[string]any, field *schema.Field) string) error { + if obj == nil { + return nil + } switch t := t.(type) { case *schema.DataRef: data, err := sch.ResolveDataRefMonomorphised(t) if err != nil { - return fmt.Errorf("failed to resolve data type: %w", err) + return fmt.Errorf("%s: failed to resolve data type: %w", t.Pos, err) } m, ok := obj.(map[string]any) if !ok { - return fmt.Errorf("expected map, got %T", obj) + return fmt.Errorf("%s: expected map, got %T", t.Pos, obj) } for _, field := range data.Fields { name := aliaser(m, field) @@ -27,7 +30,7 @@ func transformAliasedFields(sch *schema.Schema, t schema.Type, obj any, aliaser case *schema.Array: a, ok := obj.([]any) if !ok { - return fmt.Errorf("expected array, got %T", obj) + return fmt.Errorf("%s: expected array, got %T", t.Pos, obj) } for _, elem := range a { if err := transformAliasedFields(sch, t.Element, elem, aliaser); err != nil { @@ -38,7 +41,7 @@ func transformAliasedFields(sch *schema.Schema, t schema.Type, obj any, aliaser case *schema.Map: m, ok := obj.(map[string]any) if !ok { - return fmt.Errorf("expected map, got %T", obj) + return fmt.Errorf("%s: expected map, got %T", t.Pos, obj) } for key, value := range m { if err := transformAliasedFields(sch, t.Key, key, aliaser); err != nil { diff --git a/backend/controller/ingress/ingress_test.go b/backend/controller/ingress/ingress_test.go index 3fecb93c22..1e827bd3db 100644 --- a/backend/controller/ingress/ingress_test.go +++ b/backend/controller/ingress/ingress_test.go @@ -49,22 +49,71 @@ func TestValidation(t *testing.T) { name string schema string request obj + err string }{ - {name: "int", schema: `module test { data Test { intValue Int } }`, request: obj{"intValue": 10.0}}, - {name: "float", schema: `module test { data Test { floatValue Float } }`, request: obj{"floatValue": 10.0}}, - {name: "string", schema: `module test { data Test { stringValue String } }`, request: obj{"stringValue": "test"}}, - {name: "bool", schema: `module test { data Test { boolValue Bool } }`, request: obj{"boolValue": true}}, - {name: "intString", schema: `module test { data Test { intValue Int } }`, request: obj{"intValue": "10"}}, - {name: "floatString", schema: `module test { data Test { floatValue Float } }`, request: obj{"floatValue": "10.0"}}, - {name: "boolString", schema: `module test { data Test { boolValue Bool } }`, request: obj{"boolValue": "true"}}, - {name: "array", schema: `module test { data Test { arrayValue [String] } }`, request: obj{"arrayValue": []any{"test1", "test2"}}}, - {name: "map", schema: `module test { data Test { mapValue {String: String} } }`, request: obj{"mapValue": obj{"key1": "value1", "key2": "value2"}}}, - {name: "dataRef", schema: `module test { data Nested { intValue Int } data Test { dataRef Nested } }`, request: obj{"dataRef": obj{"intValue": 10.0}}}, - {name: "optional", schema: `module test { data Test { intValue Int? } }`, request: obj{}}, - {name: "optionalProvided", schema: `module test { data Test { intValue Int? } }`, request: obj{"intValue": 10.0}}, - {name: "arrayDataRef", schema: `module test { data Nested { intValue Int } data Test { arrayValue [Nested] } }`, request: obj{"arrayValue": []any{obj{"intValue": 10.0}, obj{"intValue": 20.0}}}}, - {name: "mapDataRef", schema: `module test { data Nested { intValue Int } data Test { mapValue {String: Nested} } }`, request: obj{"mapValue": obj{"key1": obj{"intValue": 10.0}, "key2": obj{"intValue": 20.0}}}}, - {name: "otherModuleRef", schema: `module other { data Other { intValue Int } } module test { data Test { otherRef other.Other } }`, request: obj{"otherRef": obj{"intValue": 10.0}}}, + {name: "Int", + schema: `module test { data Test { intValue Int } }`, + request: obj{"intValue": 10.0}}, + {name: "Float", + schema: `module test { data Test { floatValue Float } }`, + request: obj{"floatValue": 10.0}}, + {name: "String", + schema: `module test { data Test { stringValue String } }`, + request: obj{"stringValue": "test"}}, + {name: "Bool", + schema: `module test { data Test { boolValue Bool } }`, + request: obj{"boolValue": true}}, + {name: "IntString", + schema: `module test { data Test { intValue Int } }`, + request: obj{"intValue": "10"}}, + {name: "FloatString", + schema: `module test { data Test { floatValue Float } }`, + request: obj{"floatValue": "10.0"}}, + {name: "BoolString", + schema: `module test { data Test { boolValue Bool } }`, + request: obj{"boolValue": "true"}}, + {name: "Array", + schema: `module test { data Test { arrayValue [String] } }`, + request: obj{"arrayValue": []any{"test1", "test2"}}}, + {name: "Map", + schema: `module test { data Test { mapValue {String: String} } }`, + request: obj{"mapValue": obj{"key1": "value1", "key2": "value2"}}}, + {name: "DataRef", + schema: `module test { data Nested { intValue Int } data Test { dataRef Nested } }`, + request: obj{"dataRef": obj{"intValue": 10.0}}}, + {name: "Optional", + schema: `module test { data Test { intValue Int? } }`, + request: obj{}}, + {name: "OptionalProvided", + schema: `module test { data Test { intValue Int? } }`, + request: obj{"intValue": 10.0}}, + {name: "ArrayDataRef", + schema: `module test { data Nested { intValue Int } data Test { arrayValue [Nested] } }`, + request: obj{"arrayValue": []any{obj{"intValue": 10.0}, obj{"intValue": 20.0}}}}, + {name: "MapDataRef", + schema: `module test { data Nested { intValue Int } data Test { mapValue {String: Nested} } }`, + request: obj{"mapValue": obj{"key1": obj{"intValue": 10.0}, "key2": obj{"intValue": 20.0}}}}, + {name: "OtherModuleRef", + schema: `module other { data Other { intValue Int } } module test { data Test { otherRef other.Other } }`, + request: obj{"otherRef": obj{"intValue": 10.0}}}, + {name: "AllowedMissingFieldTypes", + schema: ` + module test { + data Test { + array [Int] + map {String: Int} + any Any + bytes Bytes + unit Unit + } + }`, + request: obj{}}, + {name: "RequiredFields", + schema: `module test { data Test { int Int } }`, + request: obj{}, + err: "int is required", + }, + // TODO: More tests for invalid data. } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -72,7 +121,11 @@ func TestValidation(t *testing.T) { assert.NoError(t, err) err = validateRequestMap(&schema.DataRef{Module: "test", Name: "Test"}, nil, test.request, sch) - assert.NoError(t, err, "%v", test.name) + if test.err != "" { + assert.EqualError(t, err, test.err) + } else { + assert.NoError(t, err) + } }) } } diff --git a/backend/controller/ingress/request.go b/backend/controller/ingress/request.go index 5d55faae07..a39a9f9d22 100644 --- a/backend/controller/ingress/request.go +++ b/backend/controller/ingress/request.go @@ -18,7 +18,7 @@ import ( func BuildRequestBody(route *dal.IngressRoute, r *http.Request, sch *schema.Schema) ([]byte, error) { verb := sch.ResolveVerbRef(&schema.VerbRef{Name: route.Verb, Module: route.Module}) if verb == nil { - return nil, fmt.Errorf("unknown verb %s", route.Verb) + return nil, fmt.Errorf("unknown verb %q", route.Verb) } request, ok := verb.Request.(*schema.DataRef) @@ -241,9 +241,8 @@ func validateRequestMap(dataRef *schema.DataRef, path path, request map[string]a for _, field := range data.Fields { fieldPath := append(path, "."+field.Name) //nolint:gocritic - _, isOptional := field.Type.(*schema.Optional) value, haveValue := request[field.Name] - if !isOptional && !haveValue { + if !haveValue && !allowMissingField(field) { errs = append(errs, fmt.Errorf("%s is required", fieldPath)) continue } @@ -260,6 +259,17 @@ func validateRequestMap(dataRef *schema.DataRef, path path, request map[string]a return errors.Join(errs...) } +// Fields of these types can be omitted from the JSON representation. +func allowMissingField(field *schema.Field) bool { + switch field.Type.(type) { + case *schema.Optional, *schema.Any, *schema.Array, *schema.Map, *schema.Bytes, *schema.Unit: + return true + + case *schema.Bool, *schema.DataRef, *schema.Float, *schema.Int, *schema.String, *schema.Time: + } + return false +} + func parseQueryParams(values url.Values, data *schema.Data) (map[string]any, error) { if jsonStr, ok := values["@json"]; ok { if len(values) > 1 { diff --git a/backend/controller/ingress/request_test.go b/backend/controller/ingress/request_test.go new file mode 100644 index 0000000000..fe0ca39da8 --- /dev/null +++ b/backend/controller/ingress/request_test.go @@ -0,0 +1,190 @@ +package ingress + +import ( + "bytes" + "encoding/json" + "net/http" + "net/url" + "reflect" + "testing" + + "github.com/alecthomas/assert/v2" + + "github.com/TBD54566975/ftl/backend/controller/dal" + "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/go-runtime/encoding" + "github.com/TBD54566975/ftl/go-runtime/ftl" +) + +type AliasRequest struct { + Aliased string `json:"alias"` +} + +type PathParameterRequest struct { + Username string +} + +type MissingTypes struct { + Optional ftl.Option[string] `json:"optional,omitempty"` + Array []string `json:"array,omitempty"` + Map map[string]string `json:"map,omitempty"` + Any any `json:"any,omitempty"` + Unit ftl.Unit `json:"unit,omitempty"` +} + +type PostJSONPayload struct { + Foo string +} + +// HTTPRequest mirrors builtin.HttpRequest. +type HTTPRequest[Body any] struct { + Body Body + Headers map[string][]string `json:"headers,omitempty"` + Method string + Path string + PathParameters map[string]string `json:"pathParameters,omitempty"` + Query map[string][]string `json:"query,omitempty"` +} + +func TestBuildRequestBody(t *testing.T) { + sch, err := schema.ParseString("test", ` + module test { + data AliasRequest { + aliased String alias json "alias" + } + + data PathParameterRequest { + username String + } + + data MissingTypes { + optional String? + array [String] + map {String: String} + any Any + unit Unit + } + + data JsonPayload { + foo String + } + + verb getAlias(HttpRequest) HttpResponse + ingress http GET /getAlias + + verb getPath(HttpRequest) HttpResponse + ingress http GET /getPath/{username} + + verb postMissingTypes(HttpRequest) HttpResponse + ingress http POST /postMissingTypes + + verb postJsonPayload(HttpRequest) HttpResponse + ingress http POST /postJsonPayload + } + `) + assert.NoError(t, err) + for _, test := range []struct { + name string + verb string + method string + path string + query url.Values + body obj + expected any + err string + }{ + {name: "UnknownVerb", + verb: "unknown", + err: `unknown verb "unknown"`}, + {name: "UnknownModule", + verb: "unknown", + err: `unknown verb "unknown"`}, + //FIXME: Query parameter decoding doesn't work? + // + // {name: "QueryParameterDecoding", + // verb: "getAlias", + // method: "GET", + // path: "/getAlias", + // query: map[string][]string{ + // "alias": {"value"}, + // }, + // expected: HTTPRequest[AliasRequest]{ + // Method: "GET", + // Path: "/getAlias", + // Query: map[string][]string{ + // "alias": {"value"}, + // }, + // Body: AliasRequest{ + // Aliased: "value", + // }, + // }, + // }, + {name: "AllowMissingFieldTypes", + verb: "postMissingTypes", + method: "POST", + path: "/postMissingTypes", + expected: HTTPRequest[MissingTypes]{ + Method: "POST", + Path: "/postMissingTypes", + Body: MissingTypes{}, + }, + }, + {name: "JSONPayload", + verb: "postJsonPayload", + method: "POST", + path: "/postJsonPayload", + body: obj{"foo": "bar"}, + expected: HTTPRequest[PostJSONPayload]{ + Method: "POST", + Path: "/postJsonPayload", + Body: PostJSONPayload{Foo: "bar"}, + }, + }, + // FIXME: Path parameters don't seem to be interpolated into body fields? + // + // {name: "PathParameterDecoding", + // verb: "getPath", + // method: "GET", + // path: "/getPath/bob", + // expected: HTTPRequest[PathParameterRequest]{ + // Method: "GET", + // Path: "/getPath/bob", + // PathParameters: map[string]string{ + // "username": "bob", + // }, + // Body: PathParameterRequest{ + // Username: "bob", + // }, + // }, + // }, + } { + t.Run(test.name, func(t *testing.T) { + if test.body == nil { + test.body = obj{} + } + body, err := encoding.Marshal(test.body) + assert.NoError(t, err) + requestURL := "http://127.0.0.1" + test.path + if test.query != nil { + requestURL += "?" + test.query.Encode() + } + r, err := http.NewRequest(test.method, requestURL, bytes.NewReader(body)) //nolint:noctx + assert.NoError(t, err) + requestBody, err := BuildRequestBody(&dal.IngressRoute{ + Path: test.path, + Module: "test", + Verb: test.verb, + }, r, sch) + if test.err != "" { + assert.EqualError(t, err, test.err) + return + } + assert.NoError(t, err) + actualrv := reflect.New(reflect.TypeOf(test.expected)) + actual := actualrv.Interface() + err = json.Unmarshal(requestBody, actual) + assert.NoError(t, err) + assert.Equal(t, test.expected, actualrv.Elem().Interface(), assert.OmitEmpty()) + }) + } +} diff --git a/backend/protos/xyz/block/ftl/v1/schema/schema.pb.go b/backend/protos/xyz/block/ftl/v1/schema/schema.pb.go index 5b3e81cc03..f952487959 100644 --- a/backend/protos/xyz/block/ftl/v1/schema/schema.pb.go +++ b/backend/protos/xyz/block/ftl/v1/schema/schema.pb.go @@ -1795,9 +1795,9 @@ type Type struct { // *Type_Time // *Type_Array // *Type_Map - // *Type_DataRef - // *Type_Unit // *Type_Any + // *Type_Unit + // *Type_DataRef // *Type_Optional Value isType_Value `protobuf_oneof:"value"` } @@ -1897,9 +1897,9 @@ func (x *Type) GetMap() *Map { return nil } -func (x *Type) GetDataRef() *DataRef { - if x, ok := x.GetValue().(*Type_DataRef); ok { - return x.DataRef +func (x *Type) GetAny() *Any { + if x, ok := x.GetValue().(*Type_Any); ok { + return x.Any } return nil } @@ -1911,9 +1911,9 @@ func (x *Type) GetUnit() *Unit { return nil } -func (x *Type) GetAny() *Any { - if x, ok := x.GetValue().(*Type_Any); ok { - return x.Any +func (x *Type) GetDataRef() *DataRef { + if x, ok := x.GetValue().(*Type_DataRef); ok { + return x.DataRef } return nil } @@ -1961,16 +1961,16 @@ type Type_Map struct { Map *Map `protobuf:"bytes,8,opt,name=map,proto3,oneof"` } -type Type_DataRef struct { - DataRef *DataRef `protobuf:"bytes,9,opt,name=dataRef,proto3,oneof"` +type Type_Any struct { + Any *Any `protobuf:"bytes,9,opt,name=any,proto3,oneof"` } type Type_Unit struct { Unit *Unit `protobuf:"bytes,10,opt,name=unit,proto3,oneof"` } -type Type_Any struct { - Any *Any `protobuf:"bytes,11,opt,name=any,proto3,oneof"` +type Type_DataRef struct { + DataRef *DataRef `protobuf:"bytes,11,opt,name=dataRef,proto3,oneof"` } type Type_Optional struct { @@ -1993,11 +1993,11 @@ func (*Type_Array) isType_Value() {} func (*Type_Map) isType_Value() {} -func (*Type_DataRef) isType_Value() {} +func (*Type_Any) isType_Value() {} func (*Type_Unit) isType_Value() {} -func (*Type_Any) isType_Value() {} +func (*Type_DataRef) isType_Value() {} func (*Type_Optional) isType_Value() {} @@ -2487,18 +2487,18 @@ var file_xyz_block_ftl_v1_schema_schema_proto_rawDesc = []byte{ 0x79, 0x48, 0x00, 0x52, 0x05, 0x61, 0x72, 0x72, 0x61, 0x79, 0x12, 0x30, 0x0a, 0x03, 0x6d, 0x61, 0x70, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x2e, 0x4d, 0x61, 0x70, 0x48, 0x00, 0x52, 0x03, 0x6d, 0x61, 0x70, 0x12, 0x3c, 0x0a, 0x07, - 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x66, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, - 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, - 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x66, 0x48, - 0x00, 0x52, 0x07, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x66, 0x12, 0x33, 0x0a, 0x04, 0x75, 0x6e, - 0x69, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x2e, 0x55, 0x6e, 0x69, 0x74, 0x48, 0x00, 0x52, 0x04, 0x75, 0x6e, 0x69, 0x74, 0x12, - 0x30, 0x0a, 0x03, 0x61, 0x6e, 0x79, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x78, + 0x61, 0x2e, 0x4d, 0x61, 0x70, 0x48, 0x00, 0x52, 0x03, 0x6d, 0x61, 0x70, 0x12, 0x30, 0x0a, 0x03, + 0x61, 0x6e, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x2e, 0x41, 0x6e, 0x79, 0x48, 0x00, 0x52, 0x03, 0x61, 0x6e, 0x79, 0x12, 0x33, + 0x0a, 0x04, 0x75, 0x6e, 0x69, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, - 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x41, 0x6e, 0x79, 0x48, 0x00, 0x52, 0x03, 0x61, 0x6e, - 0x79, 0x12, 0x3f, 0x0a, 0x08, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x18, 0x0c, 0x20, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x55, 0x6e, 0x69, 0x74, 0x48, 0x00, 0x52, 0x04, 0x75, + 0x6e, 0x69, 0x74, 0x12, 0x3c, 0x0a, 0x07, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x66, 0x18, 0x0b, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x44, + 0x61, 0x74, 0x61, 0x52, 0x65, 0x66, 0x48, 0x00, 0x52, 0x07, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, + 0x66, 0x12, 0x3f, 0x0a, 0x08, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x48, 0x00, 0x52, 0x08, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, @@ -2651,9 +2651,9 @@ var file_xyz_block_ftl_v1_schema_schema_proto_depIdxs = []int32{ 27, // 52: xyz.block.ftl.v1.schema.Type.time:type_name -> xyz.block.ftl.v1.schema.Time 4, // 53: xyz.block.ftl.v1.schema.Type.array:type_name -> xyz.block.ftl.v1.schema.Array 17, // 54: xyz.block.ftl.v1.schema.Type.map:type_name -> xyz.block.ftl.v1.schema.Map - 8, // 55: xyz.block.ftl.v1.schema.Type.dataRef:type_name -> xyz.block.ftl.v1.schema.DataRef + 3, // 55: xyz.block.ftl.v1.schema.Type.any:type_name -> xyz.block.ftl.v1.schema.Any 30, // 56: xyz.block.ftl.v1.schema.Type.unit:type_name -> xyz.block.ftl.v1.schema.Unit - 3, // 57: xyz.block.ftl.v1.schema.Type.any:type_name -> xyz.block.ftl.v1.schema.Any + 8, // 57: xyz.block.ftl.v1.schema.Type.dataRef:type_name -> xyz.block.ftl.v1.schema.DataRef 23, // 58: xyz.block.ftl.v1.schema.Type.optional:type_name -> xyz.block.ftl.v1.schema.Optional 24, // 59: xyz.block.ftl.v1.schema.TypeParameter.pos:type_name -> xyz.block.ftl.v1.schema.Position 24, // 60: xyz.block.ftl.v1.schema.Unit.pos:type_name -> xyz.block.ftl.v1.schema.Position @@ -3108,9 +3108,9 @@ func file_xyz_block_ftl_v1_schema_schema_proto_init() { (*Type_Time)(nil), (*Type_Array)(nil), (*Type_Map)(nil), - (*Type_DataRef)(nil), - (*Type_Unit)(nil), (*Type_Any)(nil), + (*Type_Unit)(nil), + (*Type_DataRef)(nil), (*Type_Optional)(nil), } file_xyz_block_ftl_v1_schema_schema_proto_msgTypes[29].OneofWrappers = []interface{}{} diff --git a/backend/protos/xyz/block/ftl/v1/schema/schema.proto b/backend/protos/xyz/block/ftl/v1/schema/schema.proto index bc0c830b96..315b9ed356 100644 --- a/backend/protos/xyz/block/ftl/v1/schema/schema.proto +++ b/backend/protos/xyz/block/ftl/v1/schema/schema.proto @@ -181,9 +181,9 @@ message Type { Time time = 6; Array array = 7; Map map = 8; - DataRef dataRef = 9; + Any any = 9; Unit unit = 10; - Any any = 11; + DataRef dataRef = 11; Optional optional = 12; } } diff --git a/backend/schema/parser.go b/backend/schema/parser.go index 30880774f5..a481f69c78 100644 --- a/backend/schema/parser.go +++ b/backend/schema/parser.go @@ -15,7 +15,10 @@ var ( declUnion = []Decl{&Data{}, &Verb{}, &Database{}} nonOptionalTypeUnion = []Type{ &Int{}, &Float{}, &String{}, &Bytes{}, &Bool{}, &Time{}, &Array{}, - &Map{}, &DataRef{}, &Unit{}, &Any{}, + &Map{}, &Any{}, &Unit{}, + // Note: any types resolved by identifier (eg. "Any", "Unit", etc.) must + // be prior to DataRef. + &DataRef{}, } typeUnion = append(nonOptionalTypeUnion, &Optional{}) metadataUnion = []Metadata{&MetadataCalls{}, &MetadataIngress{}, &MetadataDatabases{}} diff --git a/backend/schema/validate.go b/backend/schema/validate.go index 5c8dc76813..5d18d4a6b3 100644 --- a/backend/schema/validate.go +++ b/backend/schema/validate.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/alecthomas/participle/v2" + "github.com/alecthomas/repr" "golang.design/x/reflect" "golang.org/x/exp/maps" @@ -217,6 +218,8 @@ func ValidateModule(module *Module) error { if n.Module == "" { merr = append(merr, fmt.Errorf("%s: unqualified reference to invalid data structure %q", n.Pos, n)) } + repr.Println(n) + repr.Println(mdecl) n.Module = mdecl.Module.Name } } else if n.Module == "" || n.Module == module.Name { // Don't report errors for external modules. diff --git a/examples/go/echo/go.mod b/examples/go/echo/go.mod index 825474c6b5..be2025ac15 100644 --- a/examples/go/echo/go.mod +++ b/examples/go/echo/go.mod @@ -12,6 +12,7 @@ require ( connectrpc.com/otelconnect v0.7.0 // indirect github.com/alecthomas/concurrency v0.0.2 // indirect github.com/alecthomas/participle/v2 v2.1.1 // indirect + github.com/alecthomas/repr v0.4.0 // indirect github.com/alecthomas/types v0.10.1 // indirect github.com/alessio/shellescape v1.4.2 // indirect github.com/danieljoos/wincred v1.2.0 // indirect diff --git a/examples/go/echo/go.sum b/examples/go/echo/go.sum index b8979128d1..d6053afb6b 100644 --- a/examples/go/echo/go.sum +++ b/examples/go/echo/go.sum @@ -4,14 +4,14 @@ connectrpc.com/grpcreflect v1.2.0 h1:Q6og1S7HinmtbEuBvARLNwYmTbhEGRpHDhqrPNlmK+U connectrpc.com/grpcreflect v1.2.0/go.mod h1:nwSOKmE8nU5u/CidgHtPYk1PFI3U9ignz7iDMxOYkSY= connectrpc.com/otelconnect v0.7.0 h1:ZH55ZZtcJOTKWWLy3qmL4Pam4RzRWBJFOqTPyAqCXkY= connectrpc.com/otelconnect v0.7.0/go.mod h1:Bt2ivBymHZHqxvo4HkJ0EwHuUzQN6k2l0oH+mp/8nwc= -github.com/alecthomas/assert/v2 v2.5.0 h1:OJKYg53BQx06/bMRBSPDCO49CbCDNiUQXwdoNrt6x5w= -github.com/alecthomas/assert/v2 v2.5.0/go.mod h1:fw5suVxB+wfYJ3291t0hRTqtGzFYdSwstnRQdaQx2DM= +github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU= +github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/concurrency v0.0.2 h1:Q3kGPtLbleMbH9lHX5OBFvJygfyFw29bXZKBg+IEVuo= github.com/alecthomas/concurrency v0.0.2/go.mod h1:GmuQb/iHX7mbNtPlC/WDzEFxDMB0HYFer2Qda9QTs7w= github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8= github.com/alecthomas/participle/v2 v2.1.1/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c= -github.com/alecthomas/repr v0.3.0 h1:NeYzUPfjjlqHY4KtzgKJiWd6sVq2eNUPTi34PiFGjY8= -github.com/alecthomas/repr v0.3.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/types v0.10.1 h1:PuBMoHpFL2jaW3VgPDRhCk1oKoBCzfbsL5sAxEc3U3A= github.com/alecthomas/types v0.10.1/go.mod h1:fIOGnLeeUJXe1AAVofQmMaEMWLxY9bK4QxTLGIo30PA= github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= diff --git a/frontend/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts b/frontend/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts index 2fe14ca519..6754093f79 100644 --- a/frontend/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts +++ b/frontend/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts @@ -1389,10 +1389,10 @@ export class Type extends Message { case: "map"; } | { /** - * @generated from field: xyz.block.ftl.v1.schema.DataRef dataRef = 9; + * @generated from field: xyz.block.ftl.v1.schema.Any any = 9; */ - value: DataRef; - case: "dataRef"; + value: Any; + case: "any"; } | { /** * @generated from field: xyz.block.ftl.v1.schema.Unit unit = 10; @@ -1401,10 +1401,10 @@ export class Type extends Message { case: "unit"; } | { /** - * @generated from field: xyz.block.ftl.v1.schema.Any any = 11; + * @generated from field: xyz.block.ftl.v1.schema.DataRef dataRef = 11; */ - value: Any; - case: "any"; + value: DataRef; + case: "dataRef"; } | { /** * @generated from field: xyz.block.ftl.v1.schema.Optional optional = 12; @@ -1429,9 +1429,9 @@ export class Type extends Message { { no: 6, name: "time", kind: "message", T: Time, oneof: "value" }, { no: 7, name: "array", kind: "message", T: Array, oneof: "value" }, { no: 8, name: "map", kind: "message", T: Map, oneof: "value" }, - { no: 9, name: "dataRef", kind: "message", T: DataRef, oneof: "value" }, + { no: 9, name: "any", kind: "message", T: Any, oneof: "value" }, { no: 10, name: "unit", kind: "message", T: Unit, oneof: "value" }, - { no: 11, name: "any", kind: "message", T: Any, oneof: "value" }, + { no: 11, name: "dataRef", kind: "message", T: DataRef, oneof: "value" }, { no: 12, name: "optional", kind: "message", T: Optional, oneof: "value" }, ]); diff --git a/go-runtime/encoding/encoding.go b/go-runtime/encoding/encoding.go index 43852b0860..4cba75fe38 100644 --- a/go-runtime/encoding/encoding.go +++ b/go-runtime/encoding/encoding.go @@ -9,6 +9,9 @@ import ( "encoding/json" "fmt" "reflect" + "strings" + + "github.com/alecthomas/repr" "github.com/TBD54566975/ftl/backend/schema/strcase" ) @@ -25,6 +28,10 @@ func Marshal(v any) ([]byte, error) { } func encodeValue(v reflect.Value, w *bytes.Buffer) error { + if !v.IsValid() { + w.WriteString("null") + return nil + } t := v.Type() switch { case t.Kind() == reflect.Ptr && t.Elem().Implements(jsonUnmarshaler): @@ -90,6 +97,12 @@ func encodeValue(v reflect.Value, w *bytes.Buffer) error { case reflect.Bool: return encodeBool(v, w) + case reflect.Interface: // any + if t != reflect.TypeOf((*any)(nil)).Elem() { + return fmt.Errorf("the only interface type supported is any, not %s", t) + } + return encodeValue(v.Elem(), w) + default: panic(fmt.Sprintf("unsupported type: %s", v.Type())) } @@ -97,13 +110,26 @@ func encodeValue(v reflect.Value, w *bytes.Buffer) error { func encodeStruct(v reflect.Value, w *bytes.Buffer) error { w.WriteRune('{') + afterFirst := false for i := 0; i < v.NumField(); i++ { - if i > 0 { + ft := v.Type().Field(i) + t := ft.Type + fv := v.Field(i) + // Types that can be skipped if they're zero. + if (t.Kind() == reflect.Slice && fv.Len() == 0) || + (t.Kind() == reflect.Map && fv.Len() == 0) || + (t.String() == "ftl.Unit" && fv.IsZero()) || + (strings.HasPrefix(t.String(), "ftl.Option[") && fv.IsZero()) || + (t == reflect.TypeOf((*any)(nil)).Elem() && fv.IsZero()) { + repr.Println(ft.Name, fv.Interface()) + continue + } + if afterFirst { w.WriteRune(',') } - field := v.Type().Field(i) - w.WriteString(`"` + strcase.ToLowerCamel(field.Name) + `":`) - if err := encodeValue(v.Field(i), w); err != nil { + afterFirst = true + w.WriteString(`"` + strcase.ToLowerCamel(ft.Name) + `":`) + if err := encodeValue(fv, w); err != nil { return err } } diff --git a/go-runtime/ftl/types.go b/go-runtime/ftl/types.go index 623bf84af8..2e849c4b29 100644 --- a/go-runtime/ftl/types.go +++ b/go-runtime/ftl/types.go @@ -9,9 +9,6 @@ import ( "github.com/TBD54566975/ftl/backend/schema" ) -// Any is a generic type that can hold any value. -type Any = any - // Unit is a type that has no value. // // It can be used as a parameter or return value to indicate that a function diff --git a/go.mod b/go.mod index 2fba933057..7a9bed1e03 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/TBD54566975/scaffolder v0.8.0 github.com/TBD54566975/scaffolder/extensions/javascript v0.8.0 - github.com/alecthomas/assert/v2 v2.5.0 + github.com/alecthomas/assert/v2 v2.6.0 github.com/alecthomas/atomic v0.1.0-alpha2 github.com/alecthomas/concurrency v0.0.2 github.com/alecthomas/kong v0.8.1 @@ -65,7 +65,7 @@ require ( ) require ( - github.com/alecthomas/repr v0.3.0 + github.com/alecthomas/repr v0.4.0 github.com/alessio/shellescape v1.4.2 // indirect github.com/benbjohnson/clock v1.3.5 github.com/cenkalti/backoff/v4 v4.2.1 // indirect diff --git a/go.sum b/go.sum index 969891a280..838d52e4ee 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/TBD54566975/scaffolder v0.8.0 h1:DWl1K3dWcLsOPAYGQGPQXtffrml6XCB0tF05 github.com/TBD54566975/scaffolder v0.8.0/go.mod h1:Ab/jbQ4q8EloYL0nbkdh2DVvkGc4nxr1OcIbdMpTxxg= github.com/TBD54566975/scaffolder/extensions/javascript v0.8.0 h1:FvsUx2k5WhPwuSW2V8WAoZeOv+rps78NGHHKHHx0Fpg= github.com/TBD54566975/scaffolder/extensions/javascript v0.8.0/go.mod h1:eFaN113dwz5MO58yycBPGEyPO7/l9NhDtHPjA2ZuKPg= -github.com/alecthomas/assert/v2 v2.5.0 h1:OJKYg53BQx06/bMRBSPDCO49CbCDNiUQXwdoNrt6x5w= -github.com/alecthomas/assert/v2 v2.5.0/go.mod h1:fw5suVxB+wfYJ3291t0hRTqtGzFYdSwstnRQdaQx2DM= +github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU= +github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/atomic v0.1.0-alpha2 h1:dqwXmax66gXvHhsOS4pGPZKqYOlTkapELkLb3MNdlH8= github.com/alecthomas/atomic v0.1.0-alpha2/go.mod h1:zD6QGEyw49HIq19caJDc2NMXAy8rNi9ROrxtMXATfyI= github.com/alecthomas/concurrency v0.0.2 h1:Q3kGPtLbleMbH9lHX5OBFvJygfyFw29bXZKBg+IEVuo= @@ -22,8 +22,8 @@ github.com/alecthomas/kong-toml v0.1.0 h1:jKrdGj/G0mkHGbOV5a+Cok3cTDZ+Qxa5CBq177 github.com/alecthomas/kong-toml v0.1.0/go.mod h1:aDIxp+T6kJZY9zLThZX6qI9xxUlQgYlvqmw7PI//7/Y= github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8= github.com/alecthomas/participle/v2 v2.1.1/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c= -github.com/alecthomas/repr v0.3.0 h1:NeYzUPfjjlqHY4KtzgKJiWd6sVq2eNUPTi34PiFGjY8= -github.com/alecthomas/repr v0.3.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/types v0.10.1 h1:PuBMoHpFL2jaW3VgPDRhCk1oKoBCzfbsL5sAxEc3U3A= github.com/alecthomas/types v0.10.1/go.mod h1:fIOGnLeeUJXe1AAVofQmMaEMWLxY9bK4QxTLGIo30PA= github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0=