feat(pkger): add jsonnet support for package files

CHANGELOG.md

@@ -38,6 +38,7 @@
 1. [16260](https://github.com/influxdata/influxdb/pull/16260): Capture User-Agent header as query source for logging purposes
 1. [16469](https://github.com/influxdata/influxdb/pull/16469): Add support for configurable max batch size in points write handler
 1. [16509](https://github.com/influxdata/influxdb/pull/16509): Add support for applying an influx package via a public facing URL
+1. [16511](https://github.com/influxdata/influxdb/pull/16511): Add jsonnet support for influx packages
 
 ### Bug Fixes
 

cmd/influx/pkg.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -95,7 +94,7 @@ func (b *cmdPkgBuilder) cmdPkgApply() *cobra.Command {
 	cmd.Short = "Apply a pkg to create resources"
 
 	cmd.Flags().StringVarP(&b.file, "file", "f", "", "Path to package file")
-	cmd.MarkFlagFilename("file", "yaml", "yml", "json")
+	cmd.MarkFlagFilename("file", "yaml", "yml", "json", "jsonnet")
 	cmd.Flags().BoolVarP(&b.quiet, "quiet", "q", false, "disable output printing")
 	cmd.Flags().StringVar(&b.applyOpts.force, "force", "", `TTY input, if package will have destructive changes, proceed if set "true"`)
 	cmd.Flags().StringVarP(&b.applyOpts.url, "url", "u", "", "URL to retrieve a package.")
@@ -551,14 +550,7 @@ func pkgFromReader(stdin io.Reader) (*pkger.Pkg, error) {
 		return nil, err
 	}
 
-	var enc pkger.Encoding
-	switch http.DetectContentType(b[0:512]) {
-	case "application/json":
-		enc = pkger.EncodingJSON
-	default:
-		enc = pkger.EncodingYAML
-	}
-	return pkger.Parse(enc, pkger.FromString(string(b)))
+	return pkger.Parse(pkger.EncodingSource, pkger.FromString(string(b)))
 }
 
 func pkgFromFile(path string) (*pkger.Pkg, error) {
@@ -568,6 +560,8 @@ func pkgFromFile(path string) (*pkger.Pkg, error) {
 		enc = pkger.EncodingYAML
 	case ".json":
 		enc = pkger.EncodingJSON
+	case ".jsonnet":
+		enc = pkger.EncodingJsonnet
 	default:
 		return nil, errors.New("file provided must be one of yaml/yml/json extension but got: " + ext)
 	}

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/google/btree v1.0.0
 	github.com/google/go-cmp v0.3.1
 	github.com/google/go-github v17.0.0+incompatible
+	github.com/google/go-jsonnet v0.14.0
 	github.com/goreleaser/goreleaser v0.97.0
 	github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c // indirect
 	github.com/hashicorp/raft v1.0.0 // indirect

go.sum

@@ -166,6 +166,8 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-jsonnet v0.14.0 h1:as/sAfmjOHqY/OMBR4mv9I8ZY0/jNuqN3u44AicwxPs=
+github.com/google/go-jsonnet v0.14.0/go.mod h1:zPGC9lj/TbjkBtUACIvYR/ILHrFqKRhxeEA+bLyeMnY=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -288,11 +290,14 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4=
@@ -539,6 +544,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

http/pkger_http_server.go

@@ -12,6 +12,7 @@ import (
 	"github.com/influxdata/influxdb"
 	pctx "github.com/influxdata/influxdb/context"
 	"github.com/influxdata/influxdb/pkg/httpc"
+	"github.com/influxdata/influxdb/pkg/jsonnet"
 	"github.com/influxdata/influxdb/pkger"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v3"
@@ -232,6 +233,9 @@ func decodeWithEncoding(r *http.Request, v interface{}) (pkger.Encoding, error)
 		dec      interface{ Decode(interface{}) error }
 	)
 	switch contentType := r.Header.Get("Content-Type"); contentType {
+	case "application/x-jsonnet":
+		encoding = pkger.EncodingJsonnet
+		dec = jsonnet.NewDecoder(r.Body)
 	case "text/yml", "application/x-yaml":
 		encoding = pkger.EncodingYAML
 		dec = yaml.NewDecoder(r.Body)

http/pkger_http_server_test.go

@@ -103,6 +103,15 @@ func TestPkgerHTTPServer(t *testing.T) {
 						URL:    "https://gist.githubusercontent.com/jsteenb2/3a3b2b5fcbd6179b2494c2b54aa2feb0/raw/1717709ffadbeed5dfc88ff4cac5bf912c6930bf/bucket_pkg_json",
 					},
 				},
+				{
+					name:        "app jsonnet",
+					contentType: "application/x-jsonnet",
+					reqBody: fluxTTP.ReqApplyPkg{
+						DryRun: true,
+						OrgID:  influxdb.ID(9000).String(),
+						Pkg:    bucketPkg(t, pkger.EncodingJsonnet),
+					},
+				},
 			}
 
 			for _, tt := range tests {
@@ -257,6 +266,29 @@ func bucketPkg(t *testing.T, encoding pkger.Encoding) *pkger.Pkg {
 
 	var pkgStr string
 	switch encoding {
+	case pkger.EncodingJsonnet:
+		pkgStr = `
+local Bucket(name, desc) = {
+    kind: 'Bucket',
+    name: name,
+    description: desc,
+};
+
+{
+   apiVersion: "0.1.0",
+   kind: "Package",
+   meta: {
+     pkgName: "pkg_name",
+     pkgVersion: "1",
+     description: "pack description"
+   },
+   spec: {
+     resources: [
+        Bucket(name="rucket_1", desc="bucket 1 description"),
+     ]
+   }
+}
+`
 	case pkger.EncodingJSON:
 		pkgStr = `
 {

http/swagger.yml

@@ -4319,6 +4319,9 @@ paths:
           application/json:
             schema:
               $ref: "#/components/schemas/PkgApply"
+          application/x-jsonnet:
+            schema:
+              $ref: "#/components/schemas/PkgApply"
           text/yml:
             schema:
               $ref: "#/components/schemas/PkgApply"

pkg/jsonnet/decode.go

@@ -0,0 +1,34 @@
+package jsonnet
+
+import (
+	"encoding/json"
+	"io"
+	"io/ioutil"
+
+	"github.com/google/go-jsonnet"
+)
+
+// Decoder type can decoce a jsonnet stream into the given output.
+type Decoder struct {
+	r io.Reader
+}
+
+// NewDecoder creates a new decoder.
+func NewDecoder(r io.Reader) *Decoder {
+	return &Decoder{r: r}
+}
+
+// Decode decodes the stream into the provide value.
+func (d *Decoder) Decode(v interface{}) error {
+	b, err := ioutil.ReadAll(d.r)
+	if err != nil {
+		return err
+	}
+
+	vm := jsonnet.MakeVM()
+	jsonStr, err := vm.EvaluateSnippet("memory", string(b))
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal([]byte(jsonStr), &v)
+}

pkg/jsonnet/decode_test.go

@@ -0,0 +1,48 @@
+package jsonnet_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/influxdata/influxdb/pkg/jsonnet"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDecoder(t *testing.T) {
+	type (
+		person struct {
+			Name    string `json:"name"`
+			Welcome string `json:"welcome"`
+		}
+
+		persons struct {
+			Person1 person `json:"person1"`
+			Person2 person `json:"person2"`
+		}
+	)
+
+	const entry = `{
+  person1: {
+    name: "Alice",
+    welcome: "Hello " + self.name + "!",
+  },
+  person2: self.person1 { name: "Bob" },
+}`
+
+	var out persons
+	require.NoError(t, jsonnet.NewDecoder(strings.NewReader(entry)).Decode(&out))
+
+	expected := persons{
+		Person1: person{
+			Name:    "Alice",
+			Welcome: "Hello Alice!",
+		},
+		Person2: person{
+			Name:    "Bob",
+			Welcome: "Hello Bob!",
+		},
+	}
+	assert.Equal(t, expected, out)
+}

pkger/parser.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/influxdata/influxdb"
+	"github.com/influxdata/influxdb/pkg/jsonnet"
 	"gopkg.in/yaml.v3"
 )
 
@@ -29,6 +30,7 @@ type Encoding int
 const (
 	EncodingUnknown Encoding = iota
 	EncodingJSON
+	EncodingJsonnet
 	EncodingSource // EncodingSource draws the encoding type by inferring it from the source.
 	EncodingYAML
 )
@@ -38,6 +40,8 @@ func (e Encoding) String() string {
 	switch e {
 	case EncodingJSON:
 		return "json"
+	case EncodingJsonnet:
+		return "jsonnet"
 	case EncodingSource:
 		return "source"
 	case EncodingYAML:
@@ -61,6 +65,8 @@ func Parse(encoding Encoding, readerFn ReaderFn, opts ...ValidateOptFn) (*Pkg, e
 	switch encoding {
 	case EncodingJSON:
 		return parseJSON(r, opts...)
+	case EncodingJsonnet:
+		return parseJsonnet(r, opts...)
 	case EncodingSource:
 		return parseSource(r, opts...)
 	case EncodingYAML:
@@ -122,6 +128,10 @@ func parseJSON(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {
 	return parse(json.NewDecoder(r), opts...)
 }
 
+func parseJsonnet(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {
+	return parse(jsonnet.NewDecoder(r), opts...)
+}
+
 func parseSource(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {
 	var b []byte
 	if byter, ok := r.(interface{ Bytes() []byte }); ok {
@@ -136,10 +146,16 @@ func parseSource(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) {
 
 	contentType := http.DetectContentType(b)
 	switch {
+	case strings.Contains(contentType, "jsonnet"):
+		// highly unlikely to fall in here with supported content type detection as is
+		return parseJsonnet(bytes.NewReader(b), opts...)
 	case strings.Contains(contentType, "json"):
 		return parseJSON(bytes.NewReader(b), opts...)
-	default:
+	case strings.Contains(contentType, "yaml"),
+		strings.Contains(contentType, "yml"):
 		return parseYAML(bytes.NewReader(b), opts...)
+	default:
+		return parseJsonnet(r, opts...)
 	}
 }
 

pkger/parser_test.go

@@ -4338,6 +4338,52 @@ spec:
 			require.True(t, ok)
 		})
 	})
+
+	t.Run("jsonnet support", func(t *testing.T) {
+		pkg := validParsedPkg(t, "testdata/bucket_associates_labels.jsonnet", EncodingJsonnet, baseAsserts{
+			version:     "0.1.0",
+			kind:        KindPackage,
+			description: "pack description",
+			metaName:    "pkg_name",
+			metaVersion: "1",
+		})
+
+		sum := pkg.Summary()
+
+		labels := []SummaryLabel{
+			{
+				Name: "label_1",
+				Properties: struct {
+					Color       string `json:"color"`
+					Description string `json:"description"`
+				}{Color: "#eee888", Description: "desc_1"},
+			},
+		}
+		assert.Equal(t, labels, sum.Labels)
+
+		bkts := []SummaryBucket{
+			{
+				Name:              "rucket_1",
+				Description:       "desc_1",
+				RetentionPeriod:   10000 * time.Second,
+				LabelAssociations: labels,
+			},
+			{
+				Name:              "rucket_2",
+				Description:       "desc_2",
+				RetentionPeriod:   20000 * time.Second,
+				LabelAssociations: labels,
+			},
+			{
+				Name:              "rucket_3",
+				Description:       "desc_3",
+				RetentionPeriod:   30000 * time.Second,
+				LabelAssociations: labels,
+			},
+		}
+		assert.Equal(t, bkts, sum.Buckets)
+	})
+
 }
 
 func Test_IsParseError(t *testing.T) {