From 59b81eb627e6cff267c255aeaabd70c61a6246a8 Mon Sep 17 00:00:00 2001 From: ruflin Date: Wed, 10 Jun 2020 08:41:51 +0200 Subject: [PATCH 1/2] Add validation for dataset type The valid dataset types at the moment are logs, metrics and events. events will be removed in the future but at the moment it is still used in the endpoint package. As soon as this is removed, the code here will be cleaned up. With recent changes to the code, the datasets were not validated anymore during build or running the registry. This is now changed by adding validate for datasets every time a package is created. --- CHANGELOG.md | 1 + util/dataset.go | 16 ++++++++++++++++ util/package.go | 24 +++++++++++++++++++++++- util/packages.go | 1 + 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d4f9a73a..f46dfc389 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Generate index.json file. [#470](https://github.com/elastic/package-registry/pull/470) * Stream archived package content. [#472](https://github.com/elastic/package-registry/pull/472) * Generate package index.json files. [#479](https://github.com/elastic/package-registry/pull/479) +* Add validation for dataset type. [#](https://github.com/elastic/package-registry/pull/) ### Deprecated diff --git a/util/dataset.go b/util/dataset.go index 738cad708..962e2a2e8 100644 --- a/util/dataset.go +++ b/util/dataset.go @@ -24,6 +24,13 @@ const ( DirIngestPipeline = "ingest-pipeline" ) +var ValidTypes = map[string]interface{}{ + "logs": nil, + "metrics": nil, + // TODO: Remove as soon as endpoint package does not use it anymore + "events": nil, +} + type DataSet struct { ID string `config:"id" json:"id,omitempty" yaml:"id,omitempty"` Title string `config:"title" json:"title" validate:"required"` @@ -140,6 +147,10 @@ func (d *DataSet) Validate() error { return fmt.Errorf("dataset name is not allowed to contain `-`: %s", d.ID) } + if !d.ValidType() { + return fmt.Errorf("type is not valid: %s", d.Type) + } + if d.IngestPipeline == "" { // Check that no ingest pipeline exists in the directory except default for _, path := range paths { @@ -191,6 +202,11 @@ func (d *DataSet) Validate() error { return nil } +func (d *DataSet) ValidType() bool { + _, exists := ValidTypes[d.Type] + return exists +} + func validateIngestPipelineFile(pipelinePath string) error { f, err := ioutil.ReadFile(pipelinePath) if err != nil { diff --git a/util/package.go b/util/package.go index ad8057aa9..6affc3ecf 100644 --- a/util/package.go +++ b/util/package.go @@ -317,7 +317,7 @@ func (p *Package) Validate() error { } } - return nil + return p.ValidateDatasets() } // GetDatasetPaths returns a list with the dataset paths inside this package @@ -396,6 +396,28 @@ func (p *Package) LoadDataSets(packagePath string) error { return nil } +// ValidateDatasets loads all datasets and with it validates them +func (p *Package) ValidateDatasets() error { + + datasetPaths, err := p.GetDatasetPaths() + if err != nil { + return err + } + + datasetsBasePath := filepath.Join(p.BasePath, "dataset") + + for _, datasetPath := range datasetPaths { + + datasetBasePath := filepath.Join(datasetsBasePath, datasetPath) + + _, err := NewDataset(datasetBasePath, p) + if err != nil { + return err + } + } + return nil +} + func (p *Package) GetPath() string { return p.Name + "/" + p.Version } diff --git a/util/packages.go b/util/packages.go index 2f9777631..8b0b8a33d 100644 --- a/util/packages.go +++ b/util/packages.go @@ -32,6 +32,7 @@ func GetPackages(packagesBasePath string) ([]Package, error) { if err != nil { return nil, err } + packageList = append(packageList, *p) } From 5de300e16b9466839d35735dff6a1eeea9902fa7 Mon Sep 17 00:00:00 2001 From: ruflin Date: Wed, 10 Jun 2020 09:53:24 +0200 Subject: [PATCH 2/2] apply pr review feedback --- CHANGELOG.md | 2 +- util/dataset.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f46dfc389..1ce57eabb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Generate index.json file. [#470](https://github.com/elastic/package-registry/pull/470) * Stream archived package content. [#472](https://github.com/elastic/package-registry/pull/472) * Generate package index.json files. [#479](https://github.com/elastic/package-registry/pull/479) -* Add validation for dataset type. [#](https://github.com/elastic/package-registry/pull/) +* Add validation for dataset type. [#501](https://github.com/elastic/package-registry/pull/501) ### Deprecated diff --git a/util/dataset.go b/util/dataset.go index 962e2a2e8..4a8d26b7a 100644 --- a/util/dataset.go +++ b/util/dataset.go @@ -24,11 +24,11 @@ const ( DirIngestPipeline = "ingest-pipeline" ) -var ValidTypes = map[string]interface{}{ - "logs": nil, - "metrics": nil, +var validTypes = map[string]string{ + "logs": "Logs", + "metrics": "Metrics", // TODO: Remove as soon as endpoint package does not use it anymore - "events": nil, + "events": "Events", } type DataSet struct { @@ -147,7 +147,7 @@ func (d *DataSet) Validate() error { return fmt.Errorf("dataset name is not allowed to contain `-`: %s", d.ID) } - if !d.ValidType() { + if !d.validType() { return fmt.Errorf("type is not valid: %s", d.Type) } @@ -202,8 +202,8 @@ func (d *DataSet) Validate() error { return nil } -func (d *DataSet) ValidType() bool { - _, exists := ValidTypes[d.Type] +func (d *DataSet) validType() bool { + _, exists := validTypes[d.Type] return exists }