rclone-based storage initializer - first steps

ci_build_and_push_images.sh

@@ -139,6 +139,14 @@ function build_push_storage_initializer {
     STORAGE_INITIALIZER_EXIT_VALUE=$?
 }
 
+function build_push_rclone_storage_initializer {
+    make \
+    -C components/rclone-storage-initializer \
+        docker-build \
+    docker-push
+    RCLONE_STORAGE_INITIALIZER_EXIT_VALUE=$?
+}
+
 function build_push_mab {
     make \
 	-C components/routers/epsilon-greedy \
@@ -161,6 +169,7 @@ build_push_xgboostserver
 build_push_tfproxy
 build_push_alibi_explainer
 build_push_storage_initializer
+build_push_rclone_storage_initializer
 build_push_mab
 
 #######################################
@@ -180,6 +189,7 @@ echo "Mock model exit value: $MOCK_MODEL_EXIT_VALUE"
 echo "Alibi Detect exit value: $ALIBI_DETECT_EXIT_VALUE"
 echo "Request Logger exit value: $LOGGER_EXIT_VALUE"
 echo "Tensorflow Proxy exit value: $TFPROXY_EXIT_VALUE"
+echo "Rclone Storage Initializer exit value: $RCLONE_STORAGE_INITIALIZER_EXIT_VALUE"
 echo "MAB exit value: $MAB_EXIT_VALUE"
 
 exit $((${PYTHON_EXIT_VALUE} \
@@ -194,7 +204,6 @@ exit $((${PYTHON_EXIT_VALUE} \
     + ${XGBOOST_EXIT_VALUE} \
     + ${TFPROXY_EXIT_VALUE} \
     + ${STORAGE_INITIALIZER_EXIT_VALUE} \
+    + ${RCLONE_STORAGE_INITIALIZER_EXIT_VALUE} \
     + ${MAB_EXIT_VALUE} \
     + ${EXPLAIN_EXIT_VALUE}))
-
-

components/rclone-storage-initializer/Dockerfile

@@ -0,0 +1,9 @@
+FROM rclone/rclone:1.54.0
+LABEL name="Storage Initializer (rclone based)" \
+      vendor="Seldon Technologies" \
+      version="1.8.0-dev" \
+      release="1" \
+      summary="Storage Initializer for Seldon Core" \
+      description="Allows Seldon Core to download artifacts from cloud and local storage to a local volume"
+
+ENTRYPOINT ["rclone", "copy"]

components/rclone-storage-initializer/Makefile

@@ -0,0 +1,14 @@
+SHELL := /bin/bash
+IMAGE := rclone-storage-initializer
+VERSION := $(shell cat ../../version.txt)
+
+KIND_NAME ?= kind
+
+docker-build:
+	docker build --file=Dockerfile --force-rm=true -t seldonio/${IMAGE}:${VERSION} .
+
+docker-push:
+	docker push seldonio/${IMAGE}:${VERSION}
+
+kind-load: docker-build
+	kind load docker-image seldonio/${IMAGE}:${VERSION} --name ${KIND_NAME}

components/rclone-storage-initializer/readme.md

@@ -0,0 +1,102 @@
+# Rclone Based Storage Initializer
+
+
+
+## Rclone configuration
+
+### General notes
+
+Rclone remotes can be configured using the environmental variables:
+```
+RCLONE_CONFIG_<remote name>_<config variable>: <config value>
+```
+
+Note: multiple remotes can be configured simultaneously.
+
+Once the remote is configured the modelUri that is compatible with `rclone` takes form
+```
+modelUri: <remote>:<bucket name>
+```
+for example `modelUri: s3:sklearn/iris`.
+
+### Helm values
+
+To configure rclone-based storage initializer with your Seldon Core installation create
+the `seldon-rclone-secret` using one of the configurations bellow and use following helm values:
+
+
+```yaml
+storageInitializer:
+  image: seldonio/rclone-storage-initializer:1.8.0-dev
+
+predictiveUnit:
+  defaultEnvSecretRefName: seldon-rclone-secret
+```
+
+
+### Example for public GCS configuration
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: seldon-rclone-secret
+type: Opaque
+stringData:
+  RCLONE_CONFIG_GS_TYPE: google cloud storage
+  RCLONE_CONFIG_GS_ANONYMOUS: "true"
+```
+
+Example deployment
+
+```yaml
+apiVersion: machinelearning.seldon.io/v1
+kind: SeldonDeployment
+metadata:
+  name: rclone-sklearn-gs
+spec:
+  predictors:
+  - name: default
+    replicas: 1
+    graph:
+      name: classifier
+      implementation: SKLEARN_SERVER
+      modelUri: gs:seldon-models/sklearn/iris
+      envSecretRefName: seldon-rclone-secret
+```
+
+
+### Example minio configuration
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: seldon-rclone-secret
+type: Opaque
+stringData:
+  RCLONE_CONFIG_S3_TYPE: s3
+  RCLONE_CONFIG_S3_PROVIDER: minio
+  RCLONE_CONFIG_S3_ENV_AUTH: "false"
+  RCLONE_CONFIG_S3_ACCESS_KEY_ID: minioadmin
+  RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: minioadmin
+  RCLONE_CONFIG_S3_ENDPOINT: http://minio.minio-system.svc.cluster.local:9000
+```
+
+
+
+### Example S3 with IAM roles configuration
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: seldon-rclone-secret
+type: Opaque
+stringData:
+  RCLONE_CONFIG_S3_TYPE: s3
+  RCLONE_CONFIG_S3_PROVIDER: aws
+  RCLONE_CONFIG_S3_ACCESS_KEY_ID: ""
+  RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: ""
+  RCLONE_CONFIG_S3_ENV_AUTH: "true"
+```