From 50794bd11de2ec81cb5e3939a1a407d46a21067a Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Tue, 17 Sep 2024 09:12:59 +0800 Subject: [PATCH 1/2] [Flyte][1][IDL] Binary IDL With MessagePack Signed-off-by: Future-Outlier --- flyteidl/clients/go/assets/admin.swagger.json | 38 +++++++++++++------ .../gen/pb-es/flyteidl/core/literals_pb.ts | 4 ++ .../gen/pb-go/flyteidl/core/literals.pb.go | 4 +- .../cacheservice/cacheservice.swagger.json | 6 ++- .../datacatalog/datacatalog.swagger.json | 6 ++- .../flyteidl/service/admin.swagger.json | 6 ++- .../flyteidl/service/agent.swagger.json | 6 ++- .../flyteidl/service/dataproxy.swagger.json | 6 ++- .../external_plugin_service.swagger.json | 6 ++- .../flyteidl/service/signal.swagger.json | 6 ++- flyteidl/gen/pb_rust/flyteidl.core.rs | 2 + flyteidl/protos/flyteidl/core/literals.proto | 4 +- 12 files changed, 65 insertions(+), 29 deletions(-) diff --git a/flyteidl/clients/go/assets/admin.swagger.json b/flyteidl/clients/go/assets/admin.swagger.json index 6ebfd70f8d..241baeb53c 100644 --- a/flyteidl/clients/go/assets/admin.swagger.json +++ b/flyteidl/clients/go/assets/admin.swagger.json @@ -6661,10 +6661,12 @@ "properties": { "value": { "type": "string", - "format": "byte" + "format": "byte", + "description": "Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict." }, "tag": { - "type": "string" + "type": "string", + "description": "The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization." } }, "description": "A simple byte array with a tag to help different parts of the system communicate about what is in the byte array.\nIt's strongly advisable that consumers of this type define a unique tag and validate the tag before parsing the data." @@ -7418,6 +7420,10 @@ "$ref": "#/definitions/coreLiteralMap", "description": "A map of strings to literals." }, + "offloaded_metadata": { + "$ref": "#/definitions/coreLiteralOffloadedMetadata", + "description": "Offloaded literal metadata\nWhen you deserialize the offloaded metadata, it would be of Literal and its type would be defined by LiteralType stored in offloaded_metadata." + }, "hash": { "type": "string", "title": "A hash representing this literal.\nThis is used for caching purposes. For more details refer to RFC 1893\n(https://github.com/flyteorg/flyte/blob/master/rfc/system/1893-caching-of-offloaded-objects.md)" @@ -7428,15 +7434,6 @@ "type": "string" }, "description": "Additional metadata for literals." - }, - "uri": { - "type": "string", - "description": "If this literal is offloaded, this field will contain metadata including the offload location." - }, - "size_bytes": { - "type": "string", - "format": "uint64", - "description": "Includes information about the size of the literal." } }, "description": "A simple value. This supports any level of nesting (e.g. array of array of array of Blobs) as well as simple primitives." @@ -7466,6 +7463,25 @@ }, "description": "A map of literals. This is a workaround since oneofs in proto messages cannot contain a repeated field." }, + "coreLiteralOffloadedMetadata": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "description": "The location of the offloaded core.Literal." + }, + "size_bytes": { + "type": "string", + "format": "uint64", + "description": "The size of the offloaded data." + }, + "inferred_type": { + "$ref": "#/definitions/coreLiteralType", + "description": "The inferred literal type of the offloaded data." + } + }, + "description": "A message that contains the metadata of the offloaded data." + }, "coreLiteralType": { "type": "object", "properties": { diff --git a/flyteidl/gen/pb-es/flyteidl/core/literals_pb.ts b/flyteidl/gen/pb-es/flyteidl/core/literals_pb.ts index 95ebbd9de9..69fcde4375 100644 --- a/flyteidl/gen/pb-es/flyteidl/core/literals_pb.ts +++ b/flyteidl/gen/pb-es/flyteidl/core/literals_pb.ts @@ -215,11 +215,15 @@ export class BlobMetadata extends Message { */ export class Binary extends Message { /** + * Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict. + * * @generated from field: bytes value = 1; */ value = new Uint8Array(0); /** + * The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization. + * * @generated from field: string tag = 2; */ tag = ""; diff --git a/flyteidl/gen/pb-go/flyteidl/core/literals.pb.go b/flyteidl/gen/pb-go/flyteidl/core/literals.pb.go index 3f6e223749..2225e74077 100644 --- a/flyteidl/gen/pb-go/flyteidl/core/literals.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/core/literals.pb.go @@ -315,8 +315,8 @@ type Binary struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Value []byte `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` - Tag string `protobuf:"bytes,2,opt,name=tag,proto3" json:"tag,omitempty"` + Value []byte `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` // Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict. + Tag string `protobuf:"bytes,2,opt,name=tag,proto3" json:"tag,omitempty"` // The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization. } func (x *Binary) Reset() { diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/cacheservice/cacheservice.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/cacheservice/cacheservice.swagger.json index c30c350754..204e9e7122 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/cacheservice/cacheservice.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/cacheservice/cacheservice.swagger.json @@ -117,10 +117,12 @@ "properties": { "value": { "type": "string", - "format": "byte" + "format": "byte", + "description": "Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict." }, "tag": { - "type": "string" + "type": "string", + "description": "The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization." } }, "description": "A simple byte array with a tag to help different parts of the system communicate about what is in the byte array.\nIt's strongly advisable that consumers of this type define a unique tag and validate the tag before parsing the data." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/datacatalog/datacatalog.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/datacatalog/datacatalog.swagger.json index dbfcc5b85e..990cc1ec4a 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/datacatalog/datacatalog.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/datacatalog/datacatalog.swagger.json @@ -91,10 +91,12 @@ "properties": { "value": { "type": "string", - "format": "byte" + "format": "byte", + "description": "Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict." }, "tag": { - "type": "string" + "type": "string", + "description": "The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization." } }, "description": "A simple byte array with a tag to help different parts of the system communicate about what is in the byte array.\nIt's strongly advisable that consumers of this type define a unique tag and validate the tag before parsing the data." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json index ef81380d1e..241baeb53c 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json @@ -6661,10 +6661,12 @@ "properties": { "value": { "type": "string", - "format": "byte" + "format": "byte", + "description": "Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict." }, "tag": { - "type": "string" + "type": "string", + "description": "The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization." } }, "description": "A simple byte array with a tag to help different parts of the system communicate about what is in the byte array.\nIt's strongly advisable that consumers of this type define a unique tag and validate the tag before parsing the data." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json index 373b9c4c3d..070b6a8c60 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json @@ -903,10 +903,12 @@ "properties": { "value": { "type": "string", - "format": "byte" + "format": "byte", + "description": "Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict." }, "tag": { - "type": "string" + "type": "string", + "description": "The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization." } }, "description": "A simple byte array with a tag to help different parts of the system communicate about what is in the byte array.\nIt's strongly advisable that consumers of this type define a unique tag and validate the tag before parsing the data." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/dataproxy.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/dataproxy.swagger.json index bff6ca737a..20f32b743d 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/dataproxy.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/dataproxy.swagger.json @@ -207,10 +207,12 @@ "properties": { "value": { "type": "string", - "format": "byte" + "format": "byte", + "description": "Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict." }, "tag": { - "type": "string" + "type": "string", + "description": "The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization." } }, "description": "A simple byte array with a tag to help different parts of the system communicate about what is in the byte array.\nIt's strongly advisable that consumers of this type define a unique tag and validate the tag before parsing the data." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json index 029c42ffd3..e690cc556c 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json @@ -233,10 +233,12 @@ "properties": { "value": { "type": "string", - "format": "byte" + "format": "byte", + "description": "Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict." }, "tag": { - "type": "string" + "type": "string", + "description": "The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization." } }, "description": "A simple byte array with a tag to help different parts of the system communicate about what is in the byte array.\nIt's strongly advisable that consumers of this type define a unique tag and validate the tag before parsing the data." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/signal.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/signal.swagger.json index d325ed4764..841cb04f26 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/signal.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/signal.swagger.json @@ -283,10 +283,12 @@ "properties": { "value": { "type": "string", - "format": "byte" + "format": "byte", + "description": "Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict." }, "tag": { - "type": "string" + "type": "string", + "description": "The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization." } }, "description": "A simple byte array with a tag to help different parts of the system communicate about what is in the byte array.\nIt's strongly advisable that consumers of this type define a unique tag and validate the tag before parsing the data." diff --git a/flyteidl/gen/pb_rust/flyteidl.core.rs b/flyteidl/gen/pb_rust/flyteidl.core.rs index 441609be89..bfbf82203d 100644 --- a/flyteidl/gen/pb_rust/flyteidl.core.rs +++ b/flyteidl/gen/pb_rust/flyteidl.core.rs @@ -400,8 +400,10 @@ pub struct BlobMetadata { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Binary { + /// Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict. #[prost(bytes="vec", tag="1")] pub value: ::prost::alloc::vec::Vec, + /// The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization. #[prost(string, tag="2")] pub tag: ::prost::alloc::string::String, } diff --git a/flyteidl/protos/flyteidl/core/literals.proto b/flyteidl/protos/flyteidl/core/literals.proto index 1eb004482c..66e4821867 100644 --- a/flyteidl/protos/flyteidl/core/literals.proto +++ b/flyteidl/protos/flyteidl/core/literals.proto @@ -42,8 +42,8 @@ message BlobMetadata { // A simple byte array with a tag to help different parts of the system communicate about what is in the byte array. // It's strongly advisable that consumers of this type define a unique tag and validate the tag before parsing the data. message Binary { - bytes value = 1; - string tag = 2; + bytes value = 1; // Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict. + string tag = 2; // The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization. } // A strongly typed schema that defines the interface of data retrieved from the underlying storage medium. From c94c3fa51d1df6d98f117d1cf6173c31467eb399 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Thu, 19 Sep 2024 00:46:24 +0800 Subject: [PATCH 2/2] add comments in compiler Signed-off-by: Future-Outlier --- flytepropeller/pkg/compiler/validators/bindings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytepropeller/pkg/compiler/validators/bindings.go b/flytepropeller/pkg/compiler/validators/bindings.go index 337d04966d..53535ba260 100644 --- a/flytepropeller/pkg/compiler/validators/bindings.go +++ b/flytepropeller/pkg/compiler/validators/bindings.go @@ -147,7 +147,7 @@ func validateBinding(w c.WorkflowBuilder, node c.Node, nodeParam string, binding } else if sourceType.GetMapValueType() != nil { sourceType = sourceType.GetMapValueType() } else if sourceType.GetStructure() != nil && sourceType.GetStructure().GetDataclassType() != nil { - + // This is for retrieving the literal type of an attribute in a dataclass or Pydantic BaseModel tmpType, exist = sourceType.GetStructure().GetDataclassType()[attr.GetStringValue()] if !exist {