feat(template): added secret template function (roboll#1221)

* feat(tmpl): added fetchSecretValue template function

This adds a tmpl `fetchSecretValue` and `expandSecretRefs` function by:
- Adding:
    - `expandSecretRefs` function in tmpl package that uses vals
    package to fetch secrets
    - `fetchSecretValue` function in tmpl package like below but for
    single string value
    - gomock for tests purpose
- Changing:
    - move init of vals package to function (so the same instance can be used for template values and rendering the whole template)

* doc(secret): added doc how to use new tmpl methods

Added example usage of `fetchSecretValue` and `expandSecretRefs`

docs/remote-secrets.md

@@ -0,0 +1,58 @@
+# Secrets 
+
+helmfile can handle secrets using [helm-secrets](https://github.com/zendesk/helm-secrets) plugin or using remote secrets storage 
+(everything that package [vals](https://github.com/variantdev/vals) can handle vault, AWS SSM etc)  
+This section will describe the second use case. 
+
+# Remote secrets 
+
+This paragraph will describe how to use remote secrets storage (vault, SSM etc) in helmfile 
+
+## Fetching single key
+
+To fetch single key from remote secret storage you can use `fetchSecretValue` template function example below
+
+```yaml 
+# helmfile.yaml 
+
+repositories: 
+  - name: stable 
+    url: https://kubernetes-charts.storage.googleapis.com 
+
+environments: 
+  default: 
+    values:
+      - service:
+          password: ref+vault://svc/#pass
+          login: ref+vault://svc/#login
+releases:
+  - name: service 
+    namespace: default
+    labels:
+      cluster: services
+      secrets: vault
+    chart: stable/svc
+    version: 0.1.0
+    values:
+      - service:
+          login: {{ .Values.service.login | fetchSecretValue }} # this will resolve ref+vault://svc/#pass and fetch secret from vault
+          password: {{ .Values.service.password | fetchSecretValue | quote }}
+      # - values/service.yaml.gotmpl   # alternatively 
+```
+## Fetching multiple keys
+Alternatively you can use `expandSecretRefs` to fetch a map of secrets 
+```yaml
+# values/service.yaml.gotmpl
+service:
+{{ .Values.service | expandSecretRefs | toYaml | nindent 2 }}
+```
+
+This will produce
+```yaml
+# values/service.yaml
+service:
+  login: svc-login # fetched from vault
+  password: pass
+
+```
+

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a
 	github.com/go-test/deep v1.0.3
+	github.com/golang/mock v1.4.3
 	github.com/google/go-cmp v0.4.0
 	github.com/gosuri/uitable v0.0.3
 	github.com/hashicorp/go-getter v1.3.0
@@ -29,9 +30,8 @@ require (
 	go.uber.org/zap v1.9.1
 	golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf // indirect
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
-	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
-	google.golang.org/appengine v1.6.5 // indirect
 	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v2 v2.2.4
+	gotest.tools v2.2.0+incompatible
 	gotest.tools/v3 v3.0.3-0.20200410202438-4e4a41b7851a
 )

pkg/app/app.go

@@ -16,17 +16,13 @@ import (
 
 	"github.com/roboll/helmfile/pkg/argparser"
 	"github.com/roboll/helmfile/pkg/helmexec"
+	"github.com/roboll/helmfile/pkg/plugins"
 	"github.com/roboll/helmfile/pkg/remote"
 	"github.com/roboll/helmfile/pkg/state"
 	"github.com/variantdev/vals"
 	"go.uber.org/zap"
 )
 
-const (
-	// cache size for improving performance of ref+.* secrets rendering
-	valsCacheSize = 512
-)
-
 type App struct {
 	OverrideKubeContext string
 	OverrideHelmBinary  string
@@ -95,7 +91,7 @@ func Init(app *App) *App {
 	app.directoryExistsAt = directoryExistsAt
 
 	var err error
-	app.valsRuntime, err = vals.New(vals.Options{CacheSize: valsCacheSize})
+	app.valsRuntime, err = plugins.ValsInstance()
 	if err != nil {
 		panic(fmt.Sprintf("Failed to initialize vals runtime: %v", err))
 	}

pkg/plugins/vals.go

@@ -0,0 +1,23 @@
+package plugins
+
+import (
+	"github.com/variantdev/vals"
+	"sync"
+)
+
+const (
+	// cache size for improving performance of ref+.* secrets rendering
+	valsCacheSize = 512
+)
+
+var instance *vals.Runtime
+var once sync.Once
+
+func ValsInstance() (*vals.Runtime, error) {
+	var err error
+	once.Do(func() {
+		instance, err = vals.New(vals.Options{CacheSize: valsCacheSize})
+	})
+
+	return instance, err
+}

pkg/plugins/vals_test.go

@@ -0,0 +1,17 @@
+package plugins
+
+import "testing"
+
+func TestValsInstance(t *testing.T) {
+	i, err := ValsInstance()
+
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+
+	i2, _ := ValsInstance()
+
+	if i != i2 {
+		t.Error("Instances should be equal")
+	}
+}

pkg/tmpl/context_funcs.go

@@ -17,16 +17,18 @@ type Values = map[string]interface{}
 
 func (c *Context) createFuncMap() template.FuncMap {
 	funcMap := template.FuncMap{
-		"exec":           c.Exec,
-		"readFile":       c.ReadFile,
-		"toYaml":         ToYaml,
-		"fromYaml":       FromYaml,
-		"setValueAtPath": SetValueAtPath,
-		"requiredEnv":    RequiredEnv,
-		"get":            get,
-		"getOrNil":       getOrNil,
-		"tpl":            c.Tpl,
-		"required":       Required,
+		"exec":             c.Exec,
+		"readFile":         c.ReadFile,
+		"toYaml":           ToYaml,
+		"fromYaml":         FromYaml,
+		"setValueAtPath":   SetValueAtPath,
+		"requiredEnv":      RequiredEnv,
+		"get":              get,
+		"getOrNil":         getOrNil,
+		"tpl":              c.Tpl,
+		"required":         Required,
+		"fetchSecretValue": fetchSecretValue,
+		"expandSecretRefs": fetchSecretValues,
 	}
 	if c.preRender {
 		// disable potential side-effect template calls

pkg/tmpl/expand_secret_ref.go

@@ -0,0 +1,63 @@
+package tmpl
+
+import (
+	"errors"
+	"fmt"
+	"github.com/roboll/helmfile/pkg/plugins"
+	"github.com/variantdev/vals"
+	"sync"
+)
+
+//to generate mock run mockgen -source=expand_secret_ref.go -destination=expand_secrets_mock.go -package=tmpl
+type valClient interface {
+	Eval(template map[string]interface{}) (map[string]interface{}, error)
+}
+
+var once sync.Once
+var secretsClient valClient
+
+func fetchSecretValue(path string) (string, error) {
+	tmpMap := make(map[string]interface{})
+	tmpMap["key"] = path
+	resultMap, err := fetchSecretValues(tmpMap)
+	if err != nil {
+		return "", err
+	}
+
+	rendered, ok := resultMap["key"]
+	if !ok {
+		return "", errors.New(fmt.Sprintf("unexpected error occurred, %v doesn't have 'key' key", resultMap))
+	}
+
+	result, ok := rendered.(string)
+	if !ok {
+		return "", errors.New(fmt.Sprintf("expected %v to be string", rendered))
+	}
+
+	return result, nil
+}
+
+func fetchSecretValues(values map[string]interface{}) (map[string]interface{}, error) {
+	var err error
+	// below lines are for tests
+	once.Do(func() {
+		var valRuntime *vals.Runtime
+		if secretsClient == nil {
+			valRuntime, err = plugins.ValsInstance()
+			if err != nil {
+				return
+			}
+			secretsClient = valRuntime
+		}
+	})
+	if secretsClient == nil {
+		return nil, err
+	}
+
+	resultMap, err := secretsClient.Eval(values)
+	if err != nil {
+		return nil, err
+	}
+
+	return resultMap, nil
+}

pkg/tmpl/expand_secret_ref_test.go

@@ -0,0 +1,78 @@
+package tmpl
+
+import (
+	"errors"
+	"github.com/golang/mock/gomock"
+	"gotest.tools/assert"
+	"testing"
+)
+
+func Test_fetchSecretValue(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+	c := NewMockvalClient(controller)
+	secretsClient = c
+
+	secretPath := "ref+vault://key/#path"
+	expectArg := make(map[string]interface{})
+	expectArg["key"] = secretPath
+
+	valsResult := make(map[string]interface{})
+	valsResult["key"] = "key_value"
+	c.EXPECT().Eval(expectArg).Return(valsResult, nil)
+	result, err := fetchSecretValue(secretPath)
+	assert.NilError(t, err)
+	assert.Equal(t, result, "key_value")
+}
+
+func Test_fetchSecretValue_error(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+	c := NewMockvalClient(controller)
+	secretsClient = c
+
+	secretPath := "ref+vault://key/#path"
+	expectArg := make(map[string]interface{})
+	expectArg["key"] = secretPath
+
+	expectedErr := errors.New("some error")
+	c.EXPECT().Eval(expectArg).Return(nil, expectedErr)
+	result, err := fetchSecretValue(secretPath)
+	assert.Equal(t, err, expectedErr)
+	assert.Equal(t, result, "")
+}
+
+func Test_fetchSecretValue_no_key(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+	c := NewMockvalClient(controller)
+	secretsClient = c
+
+	secretPath := "ref+vault://key/#path"
+	expectArg := make(map[string]interface{})
+	expectArg["key"] = secretPath
+
+	valsResult := make(map[string]interface{})
+	c.EXPECT().Eval(expectArg).Return(valsResult, nil)
+	result, err := fetchSecretValue(secretPath)
+	assert.Error(t, err, "unexpected error occurred, map[] doesn't have 'key' key")
+	assert.Equal(t, result, "")
+}
+
+func Test_fetchSecretValue_invalid_type(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+	c := NewMockvalClient(controller)
+	secretsClient = c
+
+	secretPath := "ref+vault://key/#path"
+	expectArg := make(map[string]interface{})
+	expectArg["key"] = secretPath
+
+	valsResult := make(map[string]interface{})
+	valsResult["key"] = 10
+	c.EXPECT().Eval(expectArg).Return(valsResult, nil)
+	result, err := fetchSecretValue(secretPath)
+	assert.Error(t, err, "expected 10 to be string")
+	assert.Equal(t, result, "")
+}