Provisioning: Interpolate env vars in provisioning files

pkg/services/provisioning/dashboards/types.go

@@ -1,6 +1,7 @@
 package dashboards
 
 import (
+	"github.com/grafana/grafana/pkg/services/provisioning/values"
 	"time"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
@@ -40,14 +41,14 @@ type DashboardAsConfigV1 struct {
 }
 
 type DashboardProviderConfigs struct {
-	Name                  string                 `json:"name" yaml:"name"`
-	Type                  string                 `json:"type" yaml:"type"`
-	OrgId                 int64                  `json:"orgId" yaml:"orgId"`
-	Folder                string                 `json:"folder" yaml:"folder"`
-	Editable              bool                   `json:"editable" yaml:"editable"`
-	Options               map[string]interface{} `json:"options" yaml:"options"`
-	DisableDeletion       bool                   `json:"disableDeletion" yaml:"disableDeletion"`
-	UpdateIntervalSeconds int64                  `json:"updateIntervalSeconds" yaml:"updateIntervalSeconds"`
+	Name                  values.StringValue `json:"name" yaml:"name"`
+	Type                  values.StringValue `json:"type" yaml:"type"`
+	OrgId                 values.Int64Value  `json:"orgId" yaml:"orgId"`
+	Folder                values.StringValue `json:"folder" yaml:"folder"`
+	Editable              values.BoolValue   `json:"editable" yaml:"editable"`
+	Options               values.JSONValue   `json:"options" yaml:"options"`
+	DisableDeletion       values.BoolValue   `json:"disableDeletion" yaml:"disableDeletion"`
+	UpdateIntervalSeconds values.Int64Value  `json:"updateIntervalSeconds" yaml:"updateIntervalSeconds"`
 }
 
 func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardDTO, error) {
@@ -90,14 +91,14 @@ func (dc *DashboardAsConfigV1) mapToDashboardAsConfig() []*DashboardsAsConfig {
 
 	for _, v := range dc.Providers {
 		r = append(r, &DashboardsAsConfig{
-			Name:                  v.Name,
-			Type:                  v.Type,
-			OrgId:                 v.OrgId,
-			Folder:                v.Folder,
-			Editable:              v.Editable,
-			Options:               v.Options,
-			DisableDeletion:       v.DisableDeletion,
-			UpdateIntervalSeconds: v.UpdateIntervalSeconds,
+			Name:                  v.Name.Value(),
+			Type:                  v.Type.Value(),
+			OrgId:                 v.OrgId.Value(),
+			Folder:                v.Folder.Value(),
+			Editable:              v.Editable.Value(),
+			Options:               v.Options.Value(),
+			DisableDeletion:       v.DisableDeletion.Value(),
+			UpdateIntervalSeconds: v.UpdateIntervalSeconds.Value(),
 		})
 	}
 

pkg/services/provisioning/datasources/types.go

@@ -2,6 +2,7 @@ package datasources
 
 import (
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/provisioning/values"
 )
 import "github.com/grafana/grafana/pkg/components/simplejson"
 
@@ -87,23 +88,23 @@ type DataSourceFromConfigV0 struct {
 }
 
 type DataSourceFromConfigV1 struct {
-	OrgId             int64                  `json:"orgId" yaml:"orgId"`
-	Version           int                    `json:"version" yaml:"version"`
-	Name              string                 `json:"name" yaml:"name"`
-	Type              string                 `json:"type" yaml:"type"`
-	Access            string                 `json:"access" yaml:"access"`
-	Url               string                 `json:"url" yaml:"url"`
-	Password          string                 `json:"password" yaml:"password"`
-	User              string                 `json:"user" yaml:"user"`
-	Database          string                 `json:"database" yaml:"database"`
-	BasicAuth         bool                   `json:"basicAuth" yaml:"basicAuth"`
-	BasicAuthUser     string                 `json:"basicAuthUser" yaml:"basicAuthUser"`
-	BasicAuthPassword string                 `json:"basicAuthPassword" yaml:"basicAuthPassword"`
-	WithCredentials   bool                   `json:"withCredentials" yaml:"withCredentials"`
-	IsDefault         bool                   `json:"isDefault" yaml:"isDefault"`
-	JsonData          map[string]interface{} `json:"jsonData" yaml:"jsonData"`
-	SecureJsonData    map[string]string      `json:"secureJsonData" yaml:"secureJsonData"`
-	Editable          bool                   `json:"editable" yaml:"editable"`
+	OrgId             values.Int64Value     `json:"orgId" yaml:"orgId"`
+	Version           values.IntValue       `json:"version" yaml:"version"`
+	Name              values.StringValue    `json:"name" yaml:"name"`
+	Type              values.StringValue    `json:"type" yaml:"type"`
+	Access            values.StringValue    `json:"access" yaml:"access"`
+	Url               values.StringValue    `json:"url" yaml:"url"`
+	Password          values.StringValue    `json:"password" yaml:"password"`
+	User              values.StringValue    `json:"user" yaml:"user"`
+	Database          values.StringValue    `json:"database" yaml:"database"`
+	BasicAuth         values.BoolValue      `json:"basicAuth" yaml:"basicAuth"`
+	BasicAuthUser     values.StringValue    `json:"basicAuthUser" yaml:"basicAuthUser"`
+	BasicAuthPassword values.StringValue    `json:"basicAuthPassword" yaml:"basicAuthPassword"`
+	WithCredentials   values.BoolValue      `json:"withCredentials" yaml:"withCredentials"`
+	IsDefault         values.BoolValue      `json:"isDefault" yaml:"isDefault"`
+	JsonData          values.JSONValue      `json:"jsonData" yaml:"jsonData"`
+	SecureJsonData    values.StringMapValue `json:"secureJsonData" yaml:"secureJsonData"`
+	Editable          values.BoolValue      `json:"editable" yaml:"editable"`
 }
 
 func (cfg *DatasourcesAsConfigV1) mapToDatasourceFromConfig(apiVersion int64) *DatasourcesAsConfig {
@@ -117,23 +118,23 @@ func (cfg *DatasourcesAsConfigV1) mapToDatasourceFromConfig(apiVersion int64) *D
 
 	for _, ds := range cfg.Datasources {
 		r.Datasources = append(r.Datasources, &DataSourceFromConfig{
-			OrgId:             ds.OrgId,
-			Name:              ds.Name,
-			Type:              ds.Type,
-			Access:            ds.Access,
-			Url:               ds.Url,
-			Password:          ds.Password,
-			User:              ds.User,
-			Database:          ds.Database,
-			BasicAuth:         ds.BasicAuth,
-			BasicAuthUser:     ds.BasicAuthUser,
-			BasicAuthPassword: ds.BasicAuthPassword,
-			WithCredentials:   ds.WithCredentials,
-			IsDefault:         ds.IsDefault,
-			JsonData:          ds.JsonData,
-			SecureJsonData:    ds.SecureJsonData,
-			Editable:          ds.Editable,
-			Version:           ds.Version,
+			OrgId:             ds.OrgId.Value(),
+			Name:              ds.Name.Value(),
+			Type:              ds.Type.Value(),
+			Access:            ds.Access.Value(),
+			Url:               ds.Url.Value(),
+			Password:          ds.Password.Value(),
+			User:              ds.User.Value(),
+			Database:          ds.Database.Value(),
+			BasicAuth:         ds.BasicAuth.Value(),
+			BasicAuthUser:     ds.BasicAuthUser.Value(),
+			BasicAuthPassword: ds.BasicAuthPassword.Value(),
+			WithCredentials:   ds.WithCredentials.Value(),
+			IsDefault:         ds.IsDefault.Value(),
+			JsonData:          ds.JsonData.Value(),
+			SecureJsonData:    ds.SecureJsonData.Value(),
+			Editable:          ds.Editable.Value(),
+			Version:           ds.Version.Value(),
 		})
 	}
 

pkg/services/provisioning/values/values.go

@@ -0,0 +1,178 @@
+// A set of value types to use in provisioning. They add custom unmarshaling logic that puts the string values
+// through os.ExpandEnv.
+package values
+
+import (
+	"github.com/pkg/errors"
+	"os"
+	"reflect"
+	"strconv"
+)
+
+// Can be changed for testing
+var getEnv = os.Getenv
+
+type IntValue struct {
+	value int
+}
+
+func (val *IntValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	strValue, err := getInterpolated(unmarshal)
+	if err != nil {
+		return err
+	}
+	if len(strValue) == 0 {
+		// To keep the same behaviour as the yaml lib which just does not set the value if it is empty.
+		return nil
+	}
+	val.value, err = strconv.Atoi(strValue)
+	return errors.Wrap(err, "cannot convert value int")
+}
+
+func (val *IntValue) Value() int {
+	return val.value
+}
+
+type Int64Value struct {
+	value int64
+}
+
+func (val *Int64Value) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	strValue, err := getInterpolated(unmarshal)
+	if err != nil {
+		return err
+	}
+	if len(strValue) == 0 {
+		// To keep the same behaviour as the yaml lib which just does not set the value if it is empty.
+		return nil
+	}
+	val.value, err = strconv.ParseInt(strValue, 10, 64)
+	return err
+}
+
+func (val *Int64Value) Value() int64 {
+	return val.value
+}
+
+type StringValue struct {
+	value string
+}
+
+func (val *StringValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	strValue, err := getInterpolated(unmarshal)
+	val.value = strValue
+	return err
+}
+
+func (val *StringValue) Value() string {
+	return val.value
+}
+
+type BoolValue struct {
+	value bool
+}
+
+func (val *BoolValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	strValue, err := getInterpolated(unmarshal)
+	if err != nil {
+		return err
+	}
+	val.value, err = strconv.ParseBool(strValue)
+	return err
+}
+
+func (val *BoolValue) Value() bool {
+	return val.value
+}
+
+type JSONValue struct {
+	value map[string]interface{}
+}
+
+func (val *JSONValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	d := make(map[string]interface{})
+	err := unmarshal(d)
+	if err != nil {
+		return err
+	}
+	for key, val := range d {
+		d[key] = tranformInterface(val)
+	}
+	val.value = d
+	return err
+}
+
+func (val *JSONValue) Value() map[string]interface{} {
+	return val.value
+}
+
+type StringMapValue struct {
+	value map[string]string
+}
+
+func (val *StringMapValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	d := make(map[string]string)
+	err := unmarshal(d)
+	if err != nil {
+		return err
+	}
+	for key, val := range d {
+		d[key] = interpolateValue(val)
+	}
+	val.value = d
+	return err
+}
+
+func (val *StringMapValue) Value() map[string]string {
+	return val.value
+}
+
+// tranformInterface tries to transform any interface type into proper value with env expansion. It travers maps and
+// slices and the actual interpolation is done on all simple string values in the structure.
+func tranformInterface(i interface{}) interface{} {
+	switch reflect.TypeOf(i).Kind() {
+	case reflect.Slice:
+		return transformSlice(i.([]interface{}))
+	case reflect.Map:
+		return transformMap(i.(map[interface{}]interface{}))
+	case reflect.String:
+		return interpolateValue(i.(string))
+	default:
+		// Was int, float or some other value that we do not need to do any transform on.
+		return i
+	}
+}
+
+func transformSlice(i []interface{}) interface{} {
+	var transformed []interface{}
+	for _, val := range i {
+		transformed = append(transformed, tranformInterface(val))
+	}
+	return transformed
+}
+
+func transformMap(i map[interface{}]interface{}) interface{} {
+	transformed := make(map[interface{}]interface{})
+	for key, val := range i {
+		transformed[key] = tranformInterface(val)
+	}
+	return transformed
+}
+
+// interpolateValue returns final value after interpolation. At the moment only env var interpolation is done
+// here but in the future something like interpolation from file could be also done here.
+func interpolateValue(val string) string {
+	return os.Expand(val, getEnv)
+}
+
+// getInterpolated unmarshals the value as string and runs interpolation on it. It is the responsibility of each
+// value type to convert this string value to appropriate type.
+func getInterpolated(unmarshal func(interface{}) error) (string, error) {
+	var d string
+	err := unmarshal(&d)
+	if err != nil {
+		return "", err
+	}
+	d = interpolateValue(d)
+	return d, nil
+}