Merge pull request #14 from ycombinator/lib-golang-folder-validation

Golang library for validation: folder structure validation

README.md

@@ -11,7 +11,7 @@ This repository contains:
 
 Please use this repository to discuss any changes to the specification, either my making issues or PRs to the specification.
 
-# Specification Format 
+# Specification Format
 
 An Elastic Package specification describes:
 1. the folder structure of packages and expected files within these folders; and

code/go/Makefile

@@ -21,4 +21,4 @@ check-spec:
 
 # Runs tests
 test:
-	@go test -v ./...
+	@go test -v ./...

code/go/go.mod

@@ -3,6 +3,9 @@ module github.com/elastic/package-spec/code/go
 go 1.14
 
 require (
+	github.com/Masterminds/semver/v3 v3.1.0
+	github.com/pkg/errors v0.9.1
 	github.com/rakyll/statik v0.1.7
 	github.com/stretchr/testify v1.6.1
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 )

code/go/go.sum

@@ -1,5 +1,9 @@
+github.com/Masterminds/semver/v3 v3.1.0 h1:Y2lUDsFKVRSYGojLJ1yLxSXdMmMYTYls0rCvoqmMUQk=
+github.com/Masterminds/semver/v3 v3.1.0/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=

code/go/internal/spec_test.go

@@ -3,9 +3,10 @@ package internal
 import (
 	"testing"
 
-	_ "github.com/elastic/package-spec/code/go/internal/spec"
 	"github.com/rakyll/statik/fs"
 	"github.com/stretchr/testify/require"
+
+	_ "github.com/elastic/package-spec/code/go/internal/spec"
 )
 
 func TestBundledSpecs(t *testing.T) {

code/go/internal/validator/errors.go

@@ -19,8 +19,8 @@ func (ve ValidationErrors) Error() string {
 		errorWord = "error"
 	}
 	fmt.Fprintf(&message, "found %v validation %v:\n", len(ve), errorWord)
-	for _, err := range ve {
-		fmt.Fprintf(&message, "\t%v\n", err)
+	for idx, err := range ve {
+		fmt.Fprintf(&message, "%4d. %v\n", idx+1, err)
 	}
 
 	return message.String()

code/go/internal/validator/folder.go

@@ -0,0 +1,217 @@
+package validator
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
+)
+
+const itemTypeFile = "file"
+const itemTypeFolder = "folder"
+
+type folderSpec struct {
+	fs       http.FileSystem
+	specPath string
+	commonSpec
+}
+
+type folderItemSpec struct {
+	Description      string `yaml:"description"`
+	ItemType         string `yaml:"type"`
+	ContentMediaType string `yaml:"contentMediaType"`
+	Name             string `yaml:"name"`
+	Pattern          string `yaml:"pattern"`
+	Required         bool   `yaml:"required"`
+	Ref              string `yaml:"$ref"`
+	commonSpec       `yaml:",inline"`
+}
+
+type commonSpec struct {
+	AdditionalContents bool             `yaml:"additionalContents"`
+	Contents           []folderItemSpec `yaml:"contents"`
+}
+
+func newFolderSpec(fs http.FileSystem, specPath string) (*folderSpec, error) {
+	specFile, err := fs.Open(specPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not open folder specification file")
+	}
+	defer specFile.Close()
+
+	data, err := ioutil.ReadAll(specFile)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not read folder specification file")
+	}
+
+	var wrapper struct {
+		Spec commonSpec `yaml:"spec"`
+	}
+	if err := yaml.Unmarshal(data, &wrapper); err != nil {
+		return nil, errors.Wrap(err, "could not parse folder specification file")
+	}
+
+	spec := folderSpec{
+		fs:         fs,
+		specPath:   specPath,
+		commonSpec: wrapper.Spec,
+	}
+
+	return &spec, nil
+}
+
+func (s *folderSpec) validate(folderPath string) ValidationErrors {
+	var errs ValidationErrors
+	files, err := ioutil.ReadDir(folderPath)
+	if err != nil {
+		errs = append(errs, errors.Wrapf(err, "could not read folder [%s]", folderPath))
+		return errs
+	}
+
+	for _, file := range files {
+		fileName := file.Name()
+		itemSpec, err := s.findItemSpec(fileName)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+
+		if itemSpec == nil && s.AdditionalContents {
+			// No spec found for current folder item but we do allow additional contents in folder.
+			continue
+		}
+
+		if itemSpec == nil && !s.AdditionalContents {
+			// No spec found for current folder item and we do not allow additional contents in folder.
+			errs = append(errs, fmt.Errorf("item [%s] is not allowed in folder [%s]", fileName, folderPath))
+			continue
+		}
+
+		if file.IsDir() {
+			if !itemSpec.isSameType(file) {
+				errs = append(errs, fmt.Errorf("[%s] is a folder but is expected to be a file", fileName))
+				continue
+			}
+
+			if itemSpec.Ref == "" && itemSpec.Contents == nil {
+				// No recursive validation needed
+				continue
+			}
+
+			var subFolderSpec *folderSpec
+			if itemSpec.Ref != "" {
+				subFolderSpecPath := path.Join(filepath.Dir(s.specPath), itemSpec.Ref)
+				subFolderSpec, err = newFolderSpec(s.fs, subFolderSpecPath)
+				if err != nil {
+					errs = append(errs, err)
+					continue
+				}
+			} else if itemSpec.Contents != nil {
+				subFolderSpec = &folderSpec{
+					fs:       s.fs,
+					specPath: s.specPath,
+					commonSpec: commonSpec{
+						AdditionalContents: itemSpec.AdditionalContents,
+						Contents:           itemSpec.Contents,
+					},
+				}
+			}
+
+			subFolderPath := path.Join(folderPath, fileName)
+			subErrs := subFolderSpec.validate(subFolderPath)
+			if len(subErrs) > 0 {
+				errs = append(errs, subErrs...)
+			}
+
+		} else {
+			if !itemSpec.isSameType(file) {
+				errs = append(errs, fmt.Errorf("[%s] is a file but is expected to be a folder", fileName))
+				continue
+			}
+			// TODO: more validation for file item
+		}
+	}
+
+	// validate that required items in spec are all accounted for
+	for _, itemSpec := range s.Contents {
+		if !itemSpec.Required {
+			continue
+		}
+
+		fileFound, err := itemSpec.matchingFileExists(files)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+
+		if !fileFound {
+			var err error
+			if itemSpec.Name != "" {
+				err = fmt.Errorf("expecting to find [%s] %s in folder [%s]", itemSpec.Name, itemSpec.ItemType, folderPath)
+			} else if itemSpec.Pattern != "" {
+				err = fmt.Errorf("expecting to find %s matching pattern [%s] in folder [%s]", itemSpec.ItemType, itemSpec.Pattern, folderPath)
+			}
+			errs = append(errs, err)
+		}
+	}
+	return errs
+}
+
+func (s *folderSpec) findItemSpec(folderItemName string) (*folderItemSpec, error) {
+	for _, itemSpec := range s.Contents {
+		if itemSpec.Name != "" && itemSpec.Name == folderItemName {
+			return &itemSpec, nil
+		}
+		if itemSpec.Pattern != "" {
+			isMatch, err := regexp.MatchString(itemSpec.Pattern, folderItemName)
+			if err != nil {
+				return nil, errors.Wrap(err, "invalid folder item spec pattern")
+			}
+			if isMatch {
+				return &itemSpec, nil
+			}
+		}
+	}
+
+	// No item spec found
+	return nil, nil
+}
+
+func (s *folderItemSpec) matchingFileExists(files []os.FileInfo) (bool, error) {
+	if s.Name != "" {
+		for _, file := range files {
+			if file.Name() == s.Name {
+				return s.isSameType(file), nil
+			}
+		}
+	} else if s.Pattern != "" {
+		for _, file := range files {
+			isMatch, err := regexp.MatchString(s.Pattern, file.Name())
+			if err != nil {
+				return false, errors.Wrap(err, "invalid folder item spec pattern")
+			}
+			if isMatch {
+				return s.isSameType(file), nil
+			}
+		}
+	}
+
+	return false, nil
+}
+
+func (s *folderItemSpec) isSameType(file os.FileInfo) bool {
+	switch s.ItemType {
+	case itemTypeFile:
+		return !file.IsDir()
+	case itemTypeFolder:
+		return file.IsDir()
+	}
+
+	return false
+}

code/go/internal/validator/package.go

@@ -0,0 +1,61 @@
+package validator
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"github.com/Masterminds/semver/v3"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
+)
+
+// Package represents an Elastic Package Registry package
+type Package struct {
+	SpecVersion *semver.Version
+	RootPath    string
+}
+
+// NewPackage creates a new Package from a path to the package's root folder
+func NewPackage(pkgRootPath string) (*Package, error) {
+	info, err := os.Stat(pkgRootPath)
+	if os.IsNotExist(err) {
+		return nil, errors.Wrapf(err, "no package found at path [%v]", pkgRootPath)
+	}
+
+	if !info.IsDir() {
+		return nil, fmt.Errorf("no package folder found at path [%v]", pkgRootPath)
+	}
+
+	pkgManifestPath := path.Join(pkgRootPath, "manifest.yml")
+	info, err = os.Stat(pkgManifestPath)
+	if os.IsNotExist(err) {
+		return nil, errors.Wrapf(err, "no package manifest file found at path [%v]", pkgManifestPath)
+	}
+
+	data, err := ioutil.ReadFile(pkgManifestPath)
+	if err != nil {
+		return nil, fmt.Errorf("could not read package manifest file [%v]", pkgManifestPath)
+	}
+
+	var manifest struct {
+		SpecVersion string `yaml:"format_version"`
+	}
+	if err := yaml.Unmarshal(data, &manifest); err != nil {
+		return nil, errors.Wrapf(err, "could not parse package manifest file [%v]", pkgManifestPath)
+	}
+
+	specVersion, err := semver.NewVersion(manifest.SpecVersion)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not read specification version from package manifest file [%v]", pkgManifestPath)
+	}
+
+	// Instantiate Package object and return it
+	p := Package{
+		specVersion,
+		pkgRootPath,
+	}
+
+	return &p, nil
+}

code/go/internal/validator/package_test.go

@@ -0,0 +1,43 @@
+package validator
+
+import (
+	"path"
+	"testing"
+
+	"github.com/Masterminds/semver/v3"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewPackage(t *testing.T) {
+	tests := map[string]struct {
+		expectedErrContains string
+		expectedSpecVersion *semver.Version
+	}{
+		"good": {
+			expectedSpecVersion: semver.MustParse("1.0.4"),
+		},
+		"non_existent": {
+			expectedErrContains: "no package found at",
+		},
+		"no_manifest": {
+			expectedErrContains: "no package manifest file found at",
+		},
+		"no_spec_version": {
+			expectedErrContains: "could not read specification version",
+		},
+	}
+
+	for pkgName, test := range tests {
+		pkgRootPath := path.Join("test", "packages", pkgName)
+		pkg, err := NewPackage(pkgRootPath)
+		if test.expectedErrContains == "" {
+			require.NoError(t, err)
+			require.Equal(t, test.expectedSpecVersion, pkg.SpecVersion)
+			require.Equal(t, pkgRootPath, pkg.RootPath)
+		} else {
+			require.Error(t, err)
+			require.Contains(t, err.Error(), test.expectedErrContains)
+			require.Nil(t, pkg)
+		}
+	}
+}