Executor

examples/models/mean_classifier/Makefile

@@ -12,3 +12,9 @@ build_grpc:
 
 push_grpc:
 	docker push ${IMAGE_BASE}_grpc:${VERSION}
+
+run_rest_local:
+	export PREDICTIVE_UNIT_SERVICE_PORT=9000 && seldon-core-microservice --service-type MODEL MeanClassifier REST
+
+run_grpc_local:
+	export PREDICTIVE_UNIT_SERVICE_PORT=9000 && seldon-core-microservice --service-type MODEL MeanClassifier GRPC

executor/.gitignore

@@ -0,0 +1,8 @@
+./executor
+./vendor
+./tensorflow
+./serving
+./bin
+./cover.out
+./executor.tar
+

executor/Dockerfile

@@ -0,0 +1,27 @@
+# Build the manager binary
+FROM golang:1.12.5 as builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+COPY proto/ proto/
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the go source
+COPY main.go main.go
+COPY api/ api/
+COPY client/ client/
+COPY predictor/ predictor/
+
+# Build
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o executor main.go
+
+# Use distroless as minimal base image to package the manager binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM gcr.io/distroless/static:latest
+WORKDIR /
+COPY --from=builder /workspace/executor .
+ENTRYPOINT ["/executor"]

executor/Makefile

@@ -0,0 +1,79 @@
+VERSION := $(shell cat ../version.txt)
+# Image URL to use all building/pushing image targets
+IMG ?= seldonio/seldon-core-executor:${VERSION}
+
+# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+ifeq (,$(shell go env GOBIN))
+GOBIN=$(shell go env GOPATH)/bin
+else
+GOBIN=$(shell go env GOBIN)
+endif
+
+# Run go fmt against code
+fmt:
+	go fmt ./...
+
+# Run go vet against code
+vet:
+	go vet ./...
+
+
+# Build manager binary
+executor: fmt vet
+	go build -o executor main.go
+
+
+.PHONY: copy_protos
+copy_protos:
+	cp -r ../proto/tensorflow/tensorflow/** proto/tensorflow
+
+
+.PHONY: compile_seldon_proto
+compile_seldon_proto:
+	cp ../proto/prediction.proto api/grpc
+	cd api/grpc && protoc -I. -I${GOPATH}/src/github.com/tensorflow/tensorflow --go_out=paths=source_relative,plugins=grpc:. prediction.proto
+	rm api/grpc/prediction.proto
+
+# https://github.com/tensorflow/serving/issues/1365#issuecomment-525351995
+.PHONY: compile_tensorflow_proto
+compile_tensorflow_proto:
+	git clone -b r1.15 https://github.com/tensorflow/tensorflow.git
+	git clone -b r1.14 https://github.com/tensorflow/serving.git
+	go run protoc.go
+	go mod edit -replace=github.com/tensorflow/tensorflow/tensorflow/go/core=./proto/tensorflow/core
+	cd proto/tensorflow/core && go mod init github.com/tensorflow/tensorflow/tensorflow/go/core && cd -
+	go build ./proto/tensorflow/serving
+
+# Run tests
+test: fmt vet
+	go test ./api/... ./predictor/...  -coverprofile cover.out
+
+# Build the docker image
+docker-build: test
+	docker build . -t ${IMG}
+
+# Push the docker image
+docker-push:
+	docker push ${IMG}
+
+
+kind-image-install: docker-build
+	docker save ${IMG} > executor.tar
+	kind load image-archive executor.tar --loglevel trace
+
+
+.PHONY: test_grpc_local
+test_grpc_local:
+	cd proto && grpcurl -plaintext -proto ./prediction.proto  0.0.0.0:8000 seldon.protos.Seldon/Predict
+
+.PHONY: clean
+clean:
+	rm -rf vendor
+	rm -rf tensorflow
+	rm -rf serving
+
+start_test_grpc_server:
+	./executor --sdep seldon-model --namespace default --predictor example --file samples/model_test_grpc.yaml --http_port 8000 -- grpc_port 5000 
+
+test_grpc_local:
+	cd proto && grpcurl -d '{"data":{"ndarray":[[1.0,2.0]]}}' -plaintext -proto ./prediction.proto  0.0.0.0:5000 seldon.protos.Seldon/Predict

executor/README.md

@@ -0,0 +1,65 @@
+# Seldon Executor
+
+This Go project replaces the Seldon Java Engine.  It is presently in development.
+
+** Do not use in production **
+
+
+## Functionality
+
+The focus is to provide a smaller more efficient graph orchestror. It itends t provide:
+
+ * http REST Seldon protocol server for Seldon graphs.
+ * grpc Seldon protocol server for Seldon graphs.
+ * Ability to handle other protocols in future, e.g. Tensorflow or NVIDIA TensorRT Server.
+
+The REST and gRPC server are more restricted in that:
+
+ * Only 1 is active and it assume your entire graph made up of REST or gRPC components. Mixing is not allowed.
+
+The executor at present will not include functionality presently in the Java engine which will need to be provided elsewhere. Specifically:
+
+ * No request logging
+   * The roadmap will be to use the kfserving InferenceLogger we are proposing in that project and update the Seldon Deployment schema to allow that to be injected as needd.
+ * No graph metrics
+   * The roadmap will be to assume each graph component exposes their own Prometheus metrics
+ * No meta data additions
+   * It will be assumed this will be done by graph components.
+
+To realise some of the above the Seldon Wrappers will need to be extended.
+
+## Testing
+
+You can choose to use this executor by adding the annotation `seldon.io/executor: "true"`. This annotation will be active until this project progresses from incubating status.
+
+An example is shown below:
+
+```JSON
+apiVersion: machinelearning.seldon.io/v1alpha2
+kind: SeldonDeployment
+metadata:
+  labels:
+    app: seldon
+  name: seldon-model
+spec:
+  annotations:
+    seldon.io/executor: "true"
+  name: test-deployment
+  predictors:
+  - componentSpecs:
+    - spec:
+        containers:
+        - image: seldonio/mock_classifier_rest:1.3
+          name: classifier
+    graph:
+      children: []
+      endpoint:
+        type: REST
+      name: classifier
+      type: MODEL
+    labels:
+      version: v1
+    name: example
+    replicas: 1
+
+```

executor/api/client/client.go

@@ -0,0 +1,39 @@
+package client
+
+import (
+	"fmt"
+	"github.com/seldonio/seldon-core/executor/api/payload"
+	"golang.org/x/xerrors"
+	"io"
+)
+
+type SeldonApiClient interface {
+	Predict(host string, port int32, msg payload.SeldonPayload) (payload.SeldonPayload, error)
+	TransformInput(host string, port int32, msg payload.SeldonPayload) (payload.SeldonPayload, error)
+	Route(host string, port int32, msg payload.SeldonPayload) (int, error)
+	Combine(host string, port int32, msgs []payload.SeldonPayload) (payload.SeldonPayload, error)
+	TransformOutput(host string, port int32, msg payload.SeldonPayload) (payload.SeldonPayload, error)
+	Unmarshall(msg []byte) (payload.SeldonPayload, error)
+	Marshall(out io.Writer, msg payload.SeldonPayload) error
+	CreateErrorPayload(err error) payload.SeldonPayload
+}
+
+type SeldonApiError struct {
+	Message string
+	Code    int
+	frame   xerrors.Frame
+}
+
+func (se SeldonApiError) FormatError(p xerrors.Printer) error {
+	p.Printf("%d %s", se.Code, se.Message)
+	se.frame.Format(p)
+	return nil
+}
+
+func (se SeldonApiError) Format(f fmt.State, c rune) {
+	xerrors.FormatError(se, f, c)
+}
+
+func (se SeldonApiError) Error() string {
+	return fmt.Sprint(se)
+}

executor/api/client/seldondeployment_client.go

@@ -0,0 +1,66 @@
+package client
+
+import (
+	"fmt"
+	"github.com/go-logr/logr"
+	"github.com/seldonio/seldon-core/executor/api/machinelearning/v1alpha2"
+	clientset "github.com/seldonio/seldon-core/executor/client/clientset/versioned/typed/machinelearning/v1alpha2"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/homedir"
+	"path/filepath"
+	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+)
+
+type SeldonDeploymentClient struct {
+	client *clientset.MachinelearningV1alpha2Client
+	Log    logr.Logger
+}
+
+func NewSeldonDeploymentClient(path *string) *SeldonDeploymentClient {
+
+	var config *rest.Config
+	var err error
+	if path != nil && *path != "" {
+		config, err = clientcmd.BuildConfigFromFlags("", *path)
+		if err != nil {
+			panic(err.Error())
+		}
+	} else {
+		config, err = rest.InClusterConfig()
+		if err != nil {
+			if home := homedir.HomeDir(); home != "" {
+				homepath := filepath.Join(home, ".kube", "config")
+				config, err = clientcmd.BuildConfigFromFlags("", homepath)
+				if err != nil {
+					panic(err.Error())
+				}
+			}
+		}
+	}
+
+	kubeClientset, err := clientset.NewForConfig(config)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	return &SeldonDeploymentClient{
+		kubeClientset,
+		logf.Log.WithName("SeldonRestApi"),
+	}
+}
+
+func (sd *SeldonDeploymentClient) GetPredcitor(sdepName string, namespace string, predictorName string) (*v1alpha2.PredictorSpec, error) {
+	sdep, err := sd.client.SeldonDeployments(namespace).Get(sdepName, v1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	for _, predictor := range sdep.Spec.Predictors {
+		if predictor.Name == predictorName {
+			return &predictor, nil
+		}
+	}
+
+	return nil, fmt.Errorf("Failed to find predictor with name %s", predictorName)
+}