Merge pull request #578 from semyon-dev/master

New implementation for HTTP service type

.gitignore

@@ -1,4 +1,7 @@
+# idea cache
 .idea/
+
+# npm cache and configs
 resources/blockchain/node_modules/
 resources/blockchain/build/
 snetd.db
@@ -13,6 +16,10 @@ blockchain/multi_party_escrow.go
 blockchain/singularity_net_token.go
 *.pb.go
 
+# service proto
+service.proto
+service.proto-tmp.pb
+
 # Binaries for programs and plugins
 *.exe
 *.exe~

README.md

@@ -24,20 +24,17 @@ These instructions are intended to facilitate the development and testing of Sin
 deploying SingularityNET services using SingularityNET Daemon should install the appropriate binary as
 [released](#release).
 
-### Prerequisites
+### Prerequisites and dependencies
 
 * [Go 1.21+](https://golang.org/dl/)
 * [NodeJS 15+ w/npm](https://nodejs.org/en/download/)
+* [Protoc v25.0+](https://github.com/protocolbuffers/protobuf/releases)
 
-### Dependencies
-
-* install [Protoc v25.0+](https://github.com/protocolbuffers/protobuf/releases)
-
-* If you want to cross-compile you will also need Docker
+Protoc, nodejs, go and go/bin should be in environment variables.
 
 ### Installing
 
-* Clone the git repository to the following path $GOPATH/src/github.com/singnet/
+* Clone the git repository (for example $GOPATH/src/github.com/singnet/)
 
 ```bash
 $ git clone [email protected]:singnet/snet-daemon.git
@@ -50,12 +47,16 @@ $ cd snet-daemon
 $ ./scripts/install
 ```
 
-* Build snet-daemon (on Linux amd64 platform), see below section if you want to cross compile instead.
-  Please note using ldflags, the latest tagged version , sha1 revision and the build time are set as part of the build.
+* Build snet-daemon. Please note using ldflags, the latest tagged version , sha1 revision and the build time are set as part of the build.
   You need to pass the version as shown in the example below
 
 ```bash
-$ ./scripts/build linux amd64 <version>
+$ ./scripts/build <linux/windows/darwin> <amd64/arm/arm64> <version>
+```
+
+Example:
+```bash
+$ ./scripts/build linux amd64 v5.1.2
 ```
 
 * Generate default config file snet-daemon (on Linux amd64 platform)
@@ -66,20 +67,12 @@ $ ./build/snetd-linux-amd64 init
 
 **** Please update the registry address in daemon config based on the test network used
 
-#### Cross-compiling
+#### Multi-compiling
 
-If you want to build snetd for platforms other than the one you are on, run `./scripts/build-xgo` instead
+If you want to build snetd for several platforms, run `./scripts/build-all <version>` instead
 of `./scripts/build`.
 
-You can edit the script to choose a specific platform, but by default it will build for Linux, OSX, and Windows (amd64
-for all, except Linux which will also build for arm6)
-
-Please note using ldflags the latest tagged version (passed as the first parameter to the script) , sha1 revision and
-the build time are set as part of the build.
-
-```bash
-$ ./scripts/build-xgo <version>
-```
+You can edit the script to choose a specific platforms, but by default it will build for Linux, OSX, and Windows
 
 #### Run Deamon
 
@@ -159,24 +152,29 @@ These properties you should usually change before starting daemon for the first
 time.
 
 * **blockchain_network_selected**  (required)
-  Name of the network to be used for Daemon possible values are one of (kovan,ropsten,main,local or rinkeby).
+  Name of the network to be used for Daemon possible values are one of (goerli, sepolia, main, local).
   Daemon will automatically read the Registry address associated with this network For local network ( you can also
   specify the registry address manually),see the blockchain_network_config.json
 
-* **daemon_type** (required;) -
-  Defines the type of service. Available values :grpc,
-  jsonrpc, [http](https://dev.singularitynet.io/docs/ai-developers/service-type-http/), process.
-
-* **service_credentials** (required if daemon_type is http):
+* **service_credentials** (optional, for service_type http only):
   Array of credentials, example:
 
 ```
-[{"key": "X-Banana-API-Key",
-"value": "546bd7d4-d3e1-46ba-b752-bc45e4dc5b39",
-"location": "header"}]
+"service_credentials": [
+    {
+      "key": "example_body_param",
+      "value": 12345,
+      "location": "body"
+    },
+    {
+      "key": "X-API-Key",
+      "value": "546bd7d4-d3e1-46ba-b752-bc45e4dc5b39",
+      "location": "header"
+    }
+  ],
 ```
 
-Location can be: query or header
+Location can be: query, header or body. Query and header values must be string.
 
 * **daemon_end_point** (required;) -
   Defines the ip and the port on which the daemon listens to.

blockchain/serviceMetadata.go

@@ -3,14 +3,21 @@ package blockchain
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/emicklei/proto"
+	pproto "github.com/emicklei/proto"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/pkg/errors"
 	"github.com/singnet/snet-daemon/config"
 	"github.com/singnet/snet-daemon/ipfsutils"
 	log "github.com/sirupsen/logrus"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protodesc"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/reflect/protoregistry"
+	"google.golang.org/protobuf/types/descriptorpb"
 	"math/big"
 	"os"
+	"os/exec"
+	"path"
 	"strings"
 )
 
@@ -137,7 +144,10 @@ import (
 	        ]
 	}
 */
-const IpfsPrefix = "ipfs://"
+const (
+	IpfsPrefix   = "ipfs://"
+	serviceProto = "service.proto"
+)
 
 type ServiceMetadata struct {
 	Version       int                 `json:"version"`
@@ -156,8 +166,9 @@ type ServiceMetadata struct {
 	freeCallSignerAddress     common.Address
 	isfreeCallAllowed         bool
 	freeCallsAllowed          int
-	dynamicPriceMethodMapping map[string]string `json:"dynamicpricing"`
-	trainingMethods           []string          `json:"training_methods"`
+	DynamicPriceMethodMapping map[string]string           `json:"dynamicpricing"`
+	TrainingMethods           []string                    `json:"training_methods"`
+	ProtoFile                 protoreflect.FileDescriptor `json:"-"`
 }
 type Tiers struct {
 	Tiers Tier `json:"tier"`
@@ -270,10 +281,7 @@ func ReadServiceMetaDataFromLocalFile(filename string) (*ServiceMetadata, error)
 func getRegistryCaller() (reg *RegistryCaller) {
 	ethClient, err := GetEthereumClient()
 	if err != nil {
-
-		log.WithError(err).
-			Panic("Unable to get Blockchain client ")
-
+		log.WithError(err).Panic("Unable to get Blockchain client ")
 	}
 	defer ethClient.Close()
 	registryContractAddress := getRegistryAddressKey()
@@ -320,25 +328,24 @@ func InitServiceMetaDataFromJson(jsonData string) (*ServiceMetadata, error) {
 	if err := setFreeCallData(metaData); err != nil {
 		return nil, err
 	}
-	//If Dynamic pricing is enabled ,there will be mandatory checks on the service proto
-	//this is to ensure that the standards on how one defines the methods to invoke is followed
-	if config.GetBool(config.EnableDynamicPricing) {
-		if err := setServiceProto(metaData); err != nil {
-			return nil, err
-		}
+
+	if err := setServiceProto(metaData); err != nil {
+		return nil, err
 	}
-	e, err := json.Marshal(metaData.dynamicPriceMethodMapping)
+	dynamicPriceMethodMappingJson, err := json.Marshal(metaData.DynamicPriceMethodMapping)
 	if err != nil {
 		log.Println(err)
 	}
 
-	log.Println(string(e))
-	e1, err := json.Marshal(metaData.trainingMethods)
+	log.Debugln("dynamicPriceMethodMappingJson: ", string(dynamicPriceMethodMappingJson))
+
+	trainingMethodsJson, err := json.Marshal(metaData.TrainingMethods)
 	if err != nil {
 		log.Println(err)
 	}
 
-	log.Println(string(e1))
+	log.Debugln("trainingMethodsJson: ", string(trainingMethodsJson))
+
 	return metaData, err
 }
 
@@ -445,7 +452,7 @@ func (metaData *ServiceMetadata) GetDynamicPricingMethodAssociated(methodFullNam
 	if !config.GetBool(config.EnableDynamicPricing) {
 		return
 	}
-	pricingMethod = metaData.dynamicPriceMethodMapping[methodFullName]
+	pricingMethod = metaData.DynamicPriceMethodMapping[methodFullName]
 	if strings.Compare("", pricingMethod) == 0 {
 		isDynamicPricingEligible = false
 	} else {
@@ -460,9 +467,10 @@ func (metaData *ServiceMetadata) IsModelTraining(methodFullName string) (useMode
 	if !config.GetBool(config.ModelTrainingEnabled) {
 		return false
 	}
-	useModelTrainingEndPoint = isElementInArray(methodFullName, metaData.trainingMethods)
+	useModelTrainingEndPoint = isElementInArray(methodFullName, metaData.TrainingMethods)
 	return
 }
+
 func isElementInArray(a string, list []string) bool {
 	for _, b := range list {
 		if b == a {
@@ -471,52 +479,137 @@ func isElementInArray(a string, list []string) bool {
 	}
 	return false
 }
+
+func createProtoRegistry(srcDir string, filename string) (*protoregistry.Files, error) {
+	// Create descriptors using the protoc binary.
+	// Imported dependencies are included so that the descriptors are self-contained.
+	tmpFile := filename + "-tmp.pb"
+	cmd := exec.Command("protoc",
+		"--include_imports",
+		"--descriptor_set_out="+tmpFile,
+		"-I"+srcDir,
+		path.Join(srcDir, filename))
+
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return nil, err
+	}
+	//defer os.Remove(tmpFile)
+
+	marshalledDescriptorSet, err := os.ReadFile(tmpFile)
+	if err != nil {
+		return nil, err
+	}
+	descriptorSet := descriptorpb.FileDescriptorSet{}
+	err = proto.Unmarshal(marshalledDescriptorSet, &descriptorSet)
+	if err != nil {
+		log.Println("can't unmarshal: ", err)
+		return nil, err
+	}
+
+	files, err := protodesc.NewFiles(&descriptorSet)
+	if err != nil {
+		log.Println(err)
+		return nil, err
+	}
+
+	return files, nil
+}
+
 func setServiceProto(metaData *ServiceMetadata) (err error) {
-	metaData.dynamicPriceMethodMapping = make(map[string]string, 0)
-	metaData.trainingMethods = make([]string, 0)
+	metaData.DynamicPriceMethodMapping = make(map[string]string, 0)
+	metaData.TrainingMethods = make([]string, 0)
 	//This is to handler the scenario where there could be multiple protos associated with the service proto
 	protoFiles, err := ipfsutils.ReadFilesCompressed(ipfsutils.GetIpfsFile(metaData.ModelIpfsHash))
+	if err != nil {
+		return err
+	}
+
+	if metaData.ServiceType == "http" {
+		if len(protoFiles) > 1 {
+			log.Fatalln("daemon support only one proto file for HTTP services!")
+		}
+	}
+
 	for _, file := range protoFiles {
-		if srvProto, err := parseServiceProto(file); err != nil {
-			return err
-		} else {
-			dynamicMethodMap, trainingMethodMap, err := buildDynamicPricingMethodsMap(srvProto)
+		log.Debugln("Protofile: ", file)
+
+		if metaData.ServiceType == "http" {
+			_, err = os.Create(serviceProto)
 			if err != nil {
+				log.Fatalln("Can't create proto file: ", err)
+			}
+
+			err = os.WriteFile(serviceProto, []byte(file), 0666)
+			if err != nil {
+				log.Fatalln("Can't write to proto file: ", err)
+			}
+		}
+
+		//If Dynamic pricing is enabled ,there will be mandatory checks on the service proto
+		//this is to ensure that the standards on how one defines the methods to invoke is followed
+		if config.GetBool(config.EnableDynamicPricing) {
+			if srvProto, err := parseServiceProto(file); err != nil {
 				return err
+			} else {
+				dynamicMethodMap, trainingMethodMap, err := buildDynamicPricingMethodsMap(srvProto)
+				if err != nil {
+					return err
+				}
+				metaData.DynamicPriceMethodMapping = dynamicMethodMap
+				metaData.TrainingMethods = trainingMethodMap
 			}
-			metaData.dynamicPriceMethodMapping = dynamicMethodMap
-			metaData.trainingMethods = trainingMethodMap
 		}
 	}
 
+	if metaData.ServiceType == "http" {
+		files, err := createProtoRegistry(".", serviceProto)
+		if err != nil {
+			log.Println("createProtoRegistry: ", err)
+		}
+
+		protoFile, err := files.FindFileByPath(serviceProto)
+		if err != nil {
+			log.Println("files.FindFileByPat: ", err)
+		}
+
+		err = files.RegisterFile(protoFile)
+		if err != nil {
+			log.Println("files.RegisterFile(desc): ", err)
+		}
+
+		metaData.ProtoFile = protoFile
+	}
+
 	return nil
 }
 
-func parseServiceProto(serviceProtoFile string) (*proto.Proto, error) {
+func parseServiceProto(serviceProtoFile string) (*pproto.Proto, error) {
 	reader := strings.NewReader(serviceProtoFile)
-	parser := proto.NewParser(reader)
+	parser := pproto.NewParser(reader)
 	parsedProto, err := parser.Parse()
 	if err != nil {
 		return nil, err
 	}
 	return parsedProto, nil
 }
 
-func buildDynamicPricingMethodsMap(serviceProto *proto.Proto) (dynamicPricingMethodMapping map[string]string,
+func buildDynamicPricingMethodsMap(serviceProto *pproto.Proto) (dynamicPricingMethodMapping map[string]string,
 	trainingMethodPricing []string, err error) {
 	dynamicPricingMethodMapping = make(map[string]string, 0)
 	trainingMethodPricing = make([]string, 0)
 	var pkgName, serviceName, methodName string
 	for _, elem := range serviceProto.Elements {
 		//package is parsed earlier than service ( per documentation)
-		if pkg, ok := elem.(*proto.Package); ok {
+		if pkg, ok := elem.(*pproto.Package); ok {
 			pkgName = pkg.Name
 		}
-
-		if service, ok := elem.(*proto.Service); ok {
+		if service, ok := elem.(*pproto.Service); ok {
 			serviceName = service.Name
 			for _, serviceElements := range service.Elements {
-				if rpcMethod, ok := serviceElements.(*proto.RPC); ok {
+				if rpcMethod, ok := serviceElements.(*pproto.RPC); ok {
 					methodName = rpcMethod.Name
 					for _, methodOption := range rpcMethod.Options {
 						if strings.Compare(methodOption.Name, "(pricing.my_method_option).estimatePriceMethod") == 0 {