diff --git a/disperser/api/grpc/encoder/encoder.pb.go b/disperser/api/grpc/encoder/encoder.pb.go index 908a5691ea..e7baba1a31 100644 --- a/disperser/api/grpc/encoder/encoder.pb.go +++ b/disperser/api/grpc/encoder/encoder.pb.go @@ -20,6 +20,55 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type ChunkEncodingFormat int32 + +const ( + ChunkEncodingFormat_UNKNOWN ChunkEncodingFormat = 0 + ChunkEncodingFormat_GNARK ChunkEncodingFormat = 1 + ChunkEncodingFormat_GOB ChunkEncodingFormat = 2 +) + +// Enum value maps for ChunkEncodingFormat. +var ( + ChunkEncodingFormat_name = map[int32]string{ + 0: "UNKNOWN", + 1: "GNARK", + 2: "GOB", + } + ChunkEncodingFormat_value = map[string]int32{ + "UNKNOWN": 0, + "GNARK": 1, + "GOB": 2, + } +) + +func (x ChunkEncodingFormat) Enum() *ChunkEncodingFormat { + p := new(ChunkEncodingFormat) + *p = x + return p +} + +func (x ChunkEncodingFormat) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ChunkEncodingFormat) Descriptor() protoreflect.EnumDescriptor { + return file_encoder_encoder_proto_enumTypes[0].Descriptor() +} + +func (ChunkEncodingFormat) Type() protoreflect.EnumType { + return &file_encoder_encoder_proto_enumTypes[0] +} + +func (x ChunkEncodingFormat) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ChunkEncodingFormat.Descriptor instead. +func (ChunkEncodingFormat) EnumDescriptor() ([]byte, []int) { + return file_encoder_encoder_proto_rawDescGZIP(), []int{0} +} + // BlomCommitments contains the blob's commitment, degree proof, and the actual degree type BlobCommitment struct { state protoimpl.MessageState @@ -213,6 +262,8 @@ type EncodeBlobReply struct { Commitment *BlobCommitment `protobuf:"bytes,1,opt,name=commitment,proto3" json:"commitment,omitempty"` Chunks [][]byte `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks,omitempty"` + // How the above chunks are encoded. + ChunkEncodingFormat ChunkEncodingFormat `protobuf:"varint,3,opt,name=chunk_encoding_format,json=chunkEncodingFormat,proto3,enum=encoder.ChunkEncodingFormat" json:"chunk_encoding_format,omitempty"` } func (x *EncodeBlobReply) Reset() { @@ -261,6 +312,13 @@ func (x *EncodeBlobReply) GetChunks() [][]byte { return nil } +func (x *EncodeBlobReply) GetChunkEncodingFormat() ChunkEncodingFormat { + if x != nil { + return x.ChunkEncodingFormat + } + return ChunkEncodingFormat_UNKNOWN +} + var File_encoder_encoder_proto protoreflect.FileDescriptor var file_encoder_encoder_proto_rawDesc = []byte{ @@ -287,22 +345,31 @@ var file_encoder_encoder_proto_rawDesc = []byte{ 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x52, 0x0e, 0x65, 0x6e, 0x63, 0x6f, - 0x64, 0x69, 0x6e, 0x67, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0x62, 0x0a, 0x0f, 0x45, 0x6e, - 0x63, 0x6f, 0x64, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x37, 0x0a, - 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x17, 0x2e, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x42, 0x6c, 0x6f, 0x62, - 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, - 0x69, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x32, 0x4f, - 0x0a, 0x07, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x12, 0x44, 0x0a, 0x0a, 0x45, 0x6e, 0x63, - 0x6f, 0x64, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x65, - 0x72, 0x2e, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x45, 0x6e, - 0x63, 0x6f, 0x64, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, - 0x2f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x4c, 0x61, - 0x79, 0x72, 0x2d, 0x4c, 0x61, 0x62, 0x73, 0x2f, 0x65, 0x69, 0x67, 0x65, 0x6e, 0x64, 0x61, 0x2f, - 0x61, 0x70, 0x69, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x72, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x64, 0x69, 0x6e, 0x67, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0xb4, 0x01, 0x0a, 0x0f, 0x45, + 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x37, + 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x42, 0x6c, 0x6f, + 0x62, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0a, 0x63, 0x6f, 0x6d, + 0x6d, 0x69, 0x74, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x12, + 0x50, 0x0a, 0x15, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, + 0x67, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, + 0x2e, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x45, 0x6e, + 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x13, 0x63, 0x68, + 0x75, 0x6e, 0x6b, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x2a, 0x36, 0x0a, 0x13, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, + 0x6e, 0x67, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, + 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x47, 0x4e, 0x41, 0x52, 0x4b, 0x10, 0x01, + 0x12, 0x07, 0x0a, 0x03, 0x47, 0x4f, 0x42, 0x10, 0x02, 0x32, 0x4f, 0x0a, 0x07, 0x45, 0x6e, 0x63, + 0x6f, 0x64, 0x65, 0x72, 0x12, 0x44, 0x0a, 0x0a, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x42, 0x6c, + 0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x45, 0x6e, 0x63, + 0x6f, 0x64, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, + 0x2e, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2e, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x42, + 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x2f, 0x5a, 0x2d, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x4c, 0x61, 0x79, 0x72, 0x2d, 0x4c, 0x61, + 0x62, 0x73, 0x2f, 0x65, 0x69, 0x67, 0x65, 0x6e, 0x64, 0x61, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, + 0x72, 0x70, 0x63, 0x2f, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -317,23 +384,26 @@ func file_encoder_encoder_proto_rawDescGZIP() []byte { return file_encoder_encoder_proto_rawDescData } +var file_encoder_encoder_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_encoder_encoder_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_encoder_encoder_proto_goTypes = []interface{}{ - (*BlobCommitment)(nil), // 0: encoder.BlobCommitment - (*EncodingParams)(nil), // 1: encoder.EncodingParams - (*EncodeBlobRequest)(nil), // 2: encoder.EncodeBlobRequest - (*EncodeBlobReply)(nil), // 3: encoder.EncodeBlobReply + (ChunkEncodingFormat)(0), // 0: encoder.ChunkEncodingFormat + (*BlobCommitment)(nil), // 1: encoder.BlobCommitment + (*EncodingParams)(nil), // 2: encoder.EncodingParams + (*EncodeBlobRequest)(nil), // 3: encoder.EncodeBlobRequest + (*EncodeBlobReply)(nil), // 4: encoder.EncodeBlobReply } var file_encoder_encoder_proto_depIdxs = []int32{ - 1, // 0: encoder.EncodeBlobRequest.encoding_params:type_name -> encoder.EncodingParams - 0, // 1: encoder.EncodeBlobReply.commitment:type_name -> encoder.BlobCommitment - 2, // 2: encoder.Encoder.EncodeBlob:input_type -> encoder.EncodeBlobRequest - 3, // 3: encoder.Encoder.EncodeBlob:output_type -> encoder.EncodeBlobReply - 3, // [3:4] is the sub-list for method output_type - 2, // [2:3] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 2, // 0: encoder.EncodeBlobRequest.encoding_params:type_name -> encoder.EncodingParams + 1, // 1: encoder.EncodeBlobReply.commitment:type_name -> encoder.BlobCommitment + 0, // 2: encoder.EncodeBlobReply.chunk_encoding_format:type_name -> encoder.ChunkEncodingFormat + 3, // 3: encoder.Encoder.EncodeBlob:input_type -> encoder.EncodeBlobRequest + 4, // 4: encoder.Encoder.EncodeBlob:output_type -> encoder.EncodeBlobReply + 4, // [4:5] is the sub-list for method output_type + 3, // [3:4] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_encoder_encoder_proto_init() } @@ -396,13 +466,14 @@ func file_encoder_encoder_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_encoder_encoder_proto_rawDesc, - NumEnums: 0, + NumEnums: 1, NumMessages: 4, NumExtensions: 0, NumServices: 1, }, GoTypes: file_encoder_encoder_proto_goTypes, DependencyIndexes: file_encoder_encoder_proto_depIdxs, + EnumInfos: file_encoder_encoder_proto_enumTypes, MessageInfos: file_encoder_encoder_proto_msgTypes, }.Build() File_encoder_encoder_proto = out.File diff --git a/disperser/api/proto/encoder/encoder.proto b/disperser/api/proto/encoder/encoder.proto index 7a67e6a4a4..f5813f9563 100644 --- a/disperser/api/proto/encoder/encoder.proto +++ b/disperser/api/proto/encoder/encoder.proto @@ -27,9 +27,17 @@ message EncodeBlobRequest { EncodingParams encoding_params = 2; } -// EncodeBlobReply returns all encoded chunks along with BlobCommitment for the same, +enum ChunkEncodingFormat { + UNKNOWN = 0; + GNARK = 1; + GOB = 2; +} + +// EncodeBlobReply returns all encoded chunks along with BlobCommitment for the same, // where Chunk is the smallest unit that is distributed to DA nodes message EncodeBlobReply { BlobCommitment commitment = 1; repeated bytes chunks = 2; + // How the above chunks are encoded. + ChunkEncodingFormat chunk_encoding_format = 3; } diff --git a/disperser/cmd/encoder/config.go b/disperser/cmd/encoder/config.go index d1d4717ca9..513a970182 100644 --- a/disperser/cmd/encoder/config.go +++ b/disperser/cmd/encoder/config.go @@ -24,9 +24,10 @@ func NewConfig(ctx *cli.Context) (Config, error) { EncoderConfig: kzg.ReadCLIConfig(ctx), LoggerConfig: *loggerConfig, ServerConfig: &encoder.ServerConfig{ - GrpcPort: ctx.GlobalString(flags.GrpcPortFlag.Name), - MaxConcurrentRequests: ctx.GlobalInt(flags.MaxConcurrentRequestsFlag.Name), - RequestPoolSize: ctx.GlobalInt(flags.RequestPoolSizeFlag.Name), + GrpcPort: ctx.GlobalString(flags.GrpcPortFlag.Name), + MaxConcurrentRequests: ctx.GlobalInt(flags.MaxConcurrentRequestsFlag.Name), + RequestPoolSize: ctx.GlobalInt(flags.RequestPoolSizeFlag.Name), + EnableGnarkChunkEncoding: ctx.Bool(flags.EnableGnarkChunkEncodingFlag.Name), }, MetricsConfig: encoder.MetrisConfig{ HTTPPort: ctx.GlobalString(flags.MetricsHTTPPort.Name), diff --git a/disperser/cmd/encoder/flags/flags.go b/disperser/cmd/encoder/flags/flags.go index db560bbaa2..27a1bec0c8 100644 --- a/disperser/cmd/encoder/flags/flags.go +++ b/disperser/cmd/encoder/flags/flags.go @@ -47,6 +47,12 @@ var ( Value: 32, EnvVar: common.PrefixEnvVar(envVarPrefix, "REQUEST_POOL_SIZE"), } + EnableGnarkChunkEncodingFlag = cli.BoolFlag{ + Name: common.PrefixFlag(FlagPrefix, "enable-gnark-chunk-encoding"), + Usage: "if true, will produce chunks in Gnark, instead of Gob", + Required: false, + EnvVar: common.PrefixEnvVar(envVarPrefix, "ENABLE_GNARK_CHUNK_ENCODING"), + } ) var requiredFlags = []cli.Flag{ @@ -58,6 +64,7 @@ var optionalFlags = []cli.Flag{ EnableMetrics, MaxConcurrentRequestsFlag, RequestPoolSizeFlag, + EnableGnarkChunkEncodingFlag, } // Flags contains the list of configuration options available to the binary. diff --git a/disperser/encoder/client.go b/disperser/encoder/client.go index 8a72b08c85..6052bcf68c 100644 --- a/disperser/encoder/client.go +++ b/disperser/encoder/client.go @@ -60,11 +60,18 @@ func (c client) EncodeBlob(ctx context.Context, data []byte, encodingParams enco if err != nil { return nil, nil, err } + var format core.ChunkEncodingFormat + switch reply.GetChunkEncodingFormat() { + case pb.ChunkEncodingFormat_GNARK: + format = core.GnarkChunkEncodingFormat + case pb.ChunkEncodingFormat_GOB: + format = core.GobChunkEncodingFormat + case pb.ChunkEncodingFormat_UNKNOWN: + format = core.GobChunkEncodingFormat + } chunksData := &core.ChunksData{ - Chunks: reply.GetChunks(), - // TODO(jianoaix): plumb the encoding format for the encoder server. For now it's fine - // as it's hard coded using Gob at Encoder server. - Format: core.GobChunkEncodingFormat, + Chunks: reply.GetChunks(), + Format: format, ChunkLen: int(encodingParams.ChunkLength), } return &encoding.BlobCommitments{ diff --git a/disperser/encoder/config.go b/disperser/encoder/config.go index 9a0c8309e2..dcb2311584 100644 --- a/disperser/encoder/config.go +++ b/disperser/encoder/config.go @@ -5,7 +5,8 @@ const ( ) type ServerConfig struct { - GrpcPort string - MaxConcurrentRequests int - RequestPoolSize int + GrpcPort string + MaxConcurrentRequests int + RequestPoolSize int + EnableGnarkChunkEncoding bool } diff --git a/disperser/encoder/server.go b/disperser/encoder/server.go index 4b0f940c70..c4ab0f1219 100644 --- a/disperser/encoder/server.go +++ b/disperser/encoder/server.go @@ -116,8 +116,20 @@ func (s *Server) handleEncoding(ctx context.Context, req *pb.EncodeBlobRequest) var chunksData [][]byte + var format pb.ChunkEncodingFormat + if s.config.EnableGnarkChunkEncoding { + format = pb.ChunkEncodingFormat_GNARK + } else { + format = pb.ChunkEncodingFormat_GOB + } + for _, chunk := range chunks { - chunkSerialized, err := chunk.Serialize() + var chunkSerialized []byte + if s.config.EnableGnarkChunkEncoding { + chunkSerialized, err = chunk.SerializeGnark() + } else { + chunkSerialized, err = chunk.Serialize() + } if err != nil { return nil, err } @@ -135,7 +147,8 @@ func (s *Server) handleEncoding(ctx context.Context, req *pb.EncodeBlobRequest) LengthProof: lengthProofData, Length: uint32(commits.Length), }, - Chunks: chunksData, + Chunks: chunksData, + ChunkEncodingFormat: format, }, nil }