diff --git a/go.mod b/go.mod index f41e26bc42..def530b972 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/elastic/package-storage go 1.12 require ( - github.com/elastic/package-registry v0.4.1-0.20200618213757-98d7184cfe5b + github.com/elastic/package-registry v0.4.1-0.20200630074455-f91d80e4786f github.com/magefile/mage v1.9.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.4.0 diff --git a/go.sum b/go.sum index d8ed9aff54..097c805f88 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/elastic/package-registry v0.4.1-0.20200618094131-9b38a6660e60 h1:Piz/ github.com/elastic/package-registry v0.4.1-0.20200618094131-9b38a6660e60/go.mod h1:oQx3Tg9ynuC6APd0o0OHud9kyPX6S6IzdJp/R4Hj1HY= github.com/elastic/package-registry v0.4.1-0.20200618213757-98d7184cfe5b h1:NKaNWxpZjr2V750h2Q6q52W4ZGGHdX0TOcNaFrhnk3w= github.com/elastic/package-registry v0.4.1-0.20200618213757-98d7184cfe5b/go.mod h1:oQx3Tg9ynuC6APd0o0OHud9kyPX6S6IzdJp/R4Hj1HY= +github.com/elastic/package-registry v0.4.1-0.20200630074455-f91d80e4786f h1:49Xi4IqaCYpD5v4orj+Ssv1Y1502Fn2Nx4S2qTAVng8= +github.com/elastic/package-registry v0.4.1-0.20200630074455-f91d80e4786f/go.mod h1:ERTTIxAsQOCVZJDqR4LJbDDAtxV+pz4wdPPrKheiAUc= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= @@ -22,6 +24,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE= +github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/packages/aws/0.0.3/manifest.yml b/packages/aws/0.0.3/manifest.yml index cbf66a99dc..0a8aee68ca 100644 --- a/packages/aws/0.0.3/manifest.yml +++ b/packages/aws/0.0.3/manifest.yml @@ -125,7 +125,7 @@ icons: title: logo aws size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: aws title: AWS logs and metrics description: Collect logs and metrics from AWS instances diff --git a/packages/aws/0.1.0/manifest.yml b/packages/aws/0.1.0/manifest.yml index 9273012622..8018176b34 100644 --- a/packages/aws/0.1.0/manifest.yml +++ b/packages/aws/0.1.0/manifest.yml @@ -125,7 +125,7 @@ icons: title: logo aws size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: aws title: AWS logs and metrics description: Collect logs and metrics from AWS instances diff --git a/packages/aws/0.1.1/manifest.yml b/packages/aws/0.1.1/manifest.yml index 4546e19ae9..08abc6a4a6 100644 --- a/packages/aws/0.1.1/manifest.yml +++ b/packages/aws/0.1.1/manifest.yml @@ -125,7 +125,7 @@ icons: title: logo aws size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: aws title: AWS logs and metrics description: Collect logs and metrics from AWS instances diff --git a/packages/cisco/0.1.0/manifest.yml b/packages/cisco/0.1.0/manifest.yml index 1c1415c3a4..f97a7c4a50 100644 --- a/packages/cisco/0.1.0/manifest.yml +++ b/packages/cisco/0.1.0/manifest.yml @@ -24,7 +24,7 @@ icons: title: cisco size: 216x216 type: image/svg+xml -datasources: +config_templates: - name: cisco title: Cisco logs description: Collect logs from Cisco instances diff --git a/packages/cisco/0.1.1/manifest.yml b/packages/cisco/0.1.1/manifest.yml index 44f41739a6..c29acdb8b6 100644 --- a/packages/cisco/0.1.1/manifest.yml +++ b/packages/cisco/0.1.1/manifest.yml @@ -24,7 +24,7 @@ icons: title: cisco size: 216x216 type: image/svg+xml -datasources: +config_templates: - name: cisco title: Cisco logs description: Collect logs from Cisco instances diff --git a/packages/cisco/0.1.2/manifest.yml b/packages/cisco/0.1.2/manifest.yml index bde126f073..a362d7f3ab 100644 --- a/packages/cisco/0.1.2/manifest.yml +++ b/packages/cisco/0.1.2/manifest.yml @@ -24,7 +24,7 @@ icons: title: cisco size: 216x216 type: image/svg+xml -datasources: +config_templates: - name: cisco title: Cisco logs description: Collect logs from Cisco instances diff --git a/packages/endpoint/0.1.0/manifest.yml b/packages/endpoint/0.1.0/manifest.yml index c9e33e5775..9ce23607fc 100644 --- a/packages/endpoint/0.1.0/manifest.yml +++ b/packages/endpoint/0.1.0/manifest.yml @@ -13,7 +13,7 @@ license: basic # The endpoint package cannot be removed removable: false -datasources: +config_templates: - name: endpoint title: Endpoint data source description: Interact with the endpoint. diff --git a/packages/endpoint/0.2.0/manifest.yml b/packages/endpoint/0.2.0/manifest.yml index 861e173627..bdd0fb2bc4 100644 --- a/packages/endpoint/0.2.0/manifest.yml +++ b/packages/endpoint/0.2.0/manifest.yml @@ -13,7 +13,7 @@ license: basic # The endpoint package cannot be removed removable: false -datasources: +config_templates: - name: endpoint title: Endpoint data source description: Interact with the endpoint. diff --git a/packages/endpoint/0.3.0/manifest.yml b/packages/endpoint/0.3.0/manifest.yml index 1c08633485..6e107f4bf4 100644 --- a/packages/endpoint/0.3.0/manifest.yml +++ b/packages/endpoint/0.3.0/manifest.yml @@ -13,7 +13,7 @@ license: basic # The endpoint package cannot be removed removable: false -datasources: +config_templates: - name: endpoint title: Endpoint data source description: Interact with the endpoint. diff --git a/packages/endpoint/0.4.0/manifest.yml b/packages/endpoint/0.4.0/manifest.yml index f4ed6d15af..178ab6c026 100644 --- a/packages/endpoint/0.4.0/manifest.yml +++ b/packages/endpoint/0.4.0/manifest.yml @@ -13,7 +13,7 @@ license: basic # The endpoint package cannot be removed removable: false -datasources: +config_templates: - name: endpoint title: Endpoint data source description: Interact with the endpoint. diff --git a/packages/endpoint/0.5.0/manifest.yml b/packages/endpoint/0.5.0/manifest.yml index a9c9733641..8aa3144e08 100644 --- a/packages/endpoint/0.5.0/manifest.yml +++ b/packages/endpoint/0.5.0/manifest.yml @@ -13,7 +13,7 @@ license: basic # The endpoint package cannot be removed removable: false -datasources: +config_templates: - name: endpoint title: Endpoint data source description: Interact with the endpoint. diff --git a/packages/endpoint/0.6.0/manifest.yml b/packages/endpoint/0.6.0/manifest.yml index 0bd877b950..2f9058dbcf 100644 --- a/packages/endpoint/0.6.0/manifest.yml +++ b/packages/endpoint/0.6.0/manifest.yml @@ -11,7 +11,7 @@ release: beta type: solution license: basic -datasources: +config_templates: - name: endpoint title: Endpoint data source description: Interact with the endpoint. diff --git a/packages/endpoint/0.7.0/manifest.yml b/packages/endpoint/0.7.0/manifest.yml index 962a44878b..595dc4c3ba 100644 --- a/packages/endpoint/0.7.0/manifest.yml +++ b/packages/endpoint/0.7.0/manifest.yml @@ -11,7 +11,7 @@ release: beta type: solution license: basic -datasources: +config_templates: - name: endpoint title: Endpoint data source description: Interact with the endpoint. diff --git a/packages/kafka/0.1.0/manifest.yml b/packages/kafka/0.1.0/manifest.yml index a42d2e35a8..a7b1d4d501 100644 --- a/packages/kafka/0.1.0/manifest.yml +++ b/packages/kafka/0.1.0/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo kafka size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: kafka title: Kafka logs and metrics description: Collect logs and metrics from Kafka brokers diff --git a/packages/kafka/0.1.1/manifest.yml b/packages/kafka/0.1.1/manifest.yml index bbfcf0c1d0..afbafbf9ba 100644 --- a/packages/kafka/0.1.1/manifest.yml +++ b/packages/kafka/0.1.1/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo kafka size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: kafka title: Kafka logs and metrics description: Collect logs and metrics from Kafka brokers diff --git a/packages/log/0.1.0/manifest.yml b/packages/log/0.1.0/manifest.yml index 3cd9764fe4..4d648dc568 100644 --- a/packages/log/0.1.0/manifest.yml +++ b/packages/log/0.1.0/manifest.yml @@ -10,7 +10,7 @@ release: ga license: basic -datasources: +config_templates: - name: logs title: Custom logs description: Collect your custom log files. diff --git a/packages/mysql/0.1.0/manifest.yml b/packages/mysql/0.1.0/manifest.yml index 920a2b4adf..9bb45ebdfa 100644 --- a/packages/mysql/0.1.0/manifest.yml +++ b/packages/mysql/0.1.0/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo mysql size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: mysql title: MySQL logs and metrics description: Collect logs and metrics from MySQL instances diff --git a/packages/mysql/0.1.1/manifest.yml b/packages/mysql/0.1.1/manifest.yml index 732e73555b..88869cd050 100644 --- a/packages/mysql/0.1.1/manifest.yml +++ b/packages/mysql/0.1.1/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo mysql size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: mysql title: MySQL logs and metrics description: Collect logs and metrics from MySQL instances diff --git a/packages/mysql/0.1.2/manifest.yml b/packages/mysql/0.1.2/manifest.yml index 6b4f8252dc..39d1b70b2b 100644 --- a/packages/mysql/0.1.2/manifest.yml +++ b/packages/mysql/0.1.2/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo mysql size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: mysql title: MySQL logs and metrics description: Collect logs and metrics from MySQL instances diff --git a/packages/mysql/0.1.3/manifest.yml b/packages/mysql/0.1.3/manifest.yml index 36d542ef35..30ddab7291 100644 --- a/packages/mysql/0.1.3/manifest.yml +++ b/packages/mysql/0.1.3/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo mysql size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: mysql title: MySQL logs and metrics description: Collect logs and metrics from MySQL instances diff --git a/packages/netflow/0.1.0/manifest.yml b/packages/netflow/0.1.0/manifest.yml index 03a6f768d7..9d397289f5 100644 --- a/packages/netflow/0.1.0/manifest.yml +++ b/packages/netflow/0.1.0/manifest.yml @@ -14,7 +14,7 @@ requirement: versions: '>=7.3.0 <8.0.0' elasticsearch: versions: '>7.0.1' -datasources: +config_templates: - name: netflow title: NetFlow logs description: Collect Netflow logs from networks via UDP diff --git a/packages/netflow/0.1.1/manifest.yml b/packages/netflow/0.1.1/manifest.yml index 5e883b16b6..176a706d0d 100644 --- a/packages/netflow/0.1.1/manifest.yml +++ b/packages/netflow/0.1.1/manifest.yml @@ -14,7 +14,7 @@ requirement: versions: '>=7.3.0 <8.0.0' elasticsearch: versions: '>7.0.1' -datasources: +config_templates: - name: netflow title: NetFlow logs description: Collect Netflow logs from networks via UDP diff --git a/packages/nginx/0.1.0/manifest.yml b/packages/nginx/0.1.0/manifest.yml index 4f7353e960..a026d8635c 100644 --- a/packages/nginx/0.1.0/manifest.yml +++ b/packages/nginx/0.1.0/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo nginx size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: nginx title: Nginx logs and metrics description: Collect logs and metrics from Nginx instances diff --git a/packages/nginx/0.1.1/manifest.yml b/packages/nginx/0.1.1/manifest.yml index d615878ab6..a5b8b5a8d1 100644 --- a/packages/nginx/0.1.1/manifest.yml +++ b/packages/nginx/0.1.1/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo nginx size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: nginx title: Nginx logs and metrics description: Collect logs and metrics from Nginx instances diff --git a/packages/nginx/0.1.2/manifest.yml b/packages/nginx/0.1.2/manifest.yml index c43ca07a56..6ad5c7216b 100644 --- a/packages/nginx/0.1.2/manifest.yml +++ b/packages/nginx/0.1.2/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo nginx size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: nginx title: Nginx logs and metrics description: Collect logs and metrics from Nginx instances diff --git a/packages/nginx/0.1.3/manifest.yml b/packages/nginx/0.1.3/manifest.yml index 6fdde95e68..dfa78414c0 100644 --- a/packages/nginx/0.1.3/manifest.yml +++ b/packages/nginx/0.1.3/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo nginx size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: nginx title: Nginx logs and metrics description: Collect logs and metrics from Nginx instances diff --git a/packages/redis/0.1.0/manifest.yml b/packages/redis/0.1.0/manifest.yml index 022cf2e60b..c57b8e5fba 100644 --- a/packages/redis/0.1.0/manifest.yml +++ b/packages/redis/0.1.0/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo redis size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: redis title: Redis logs and metrics description: Collect logs and metrics from Redis instances diff --git a/packages/redis/0.1.1/manifest.yml b/packages/redis/0.1.1/manifest.yml index 6e595e0535..76f798204d 100644 --- a/packages/redis/0.1.1/manifest.yml +++ b/packages/redis/0.1.1/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo redis size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: redis title: Redis logs and metrics description: Collect logs and metrics from Redis instances diff --git a/packages/redis/0.1.2/manifest.yml b/packages/redis/0.1.2/manifest.yml index 8672dc5adb..a15841c49a 100644 --- a/packages/redis/0.1.2/manifest.yml +++ b/packages/redis/0.1.2/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo redis size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: redis title: Redis logs and metrics description: Collect logs and metrics from Redis instances diff --git a/packages/redis/0.1.3/manifest.yml b/packages/redis/0.1.3/manifest.yml index 5c9848f96b..bf48d6c03a 100644 --- a/packages/redis/0.1.3/manifest.yml +++ b/packages/redis/0.1.3/manifest.yml @@ -29,7 +29,7 @@ icons: title: logo redis size: 32x32 type: image/svg+xml -datasources: +config_templates: - name: redis title: Redis logs and metrics description: Collect logs and metrics from Redis instances diff --git a/packages/system/0.1.0/manifest.yml b/packages/system/0.1.0/manifest.yml index 4073974439..4f76ee7e9b 100644 --- a/packages/system/0.1.0/manifest.yml +++ b/packages/system/0.1.0/manifest.yml @@ -28,7 +28,7 @@ icons: title: system size: 1000x1000 type: image/svg+xml -datasources: +config_templates: - name: system title: System logs and metrics description: Collect logs and metrics from System instances diff --git a/packages/system/0.2.0/manifest.yml b/packages/system/0.2.0/manifest.yml index 83d1048b76..6f97bab7e1 100644 --- a/packages/system/0.2.0/manifest.yml +++ b/packages/system/0.2.0/manifest.yml @@ -28,7 +28,7 @@ icons: title: system size: 1000x1000 type: image/svg+xml -datasources: +config_templates: - name: system title: System logs and metrics description: Collect logs and metrics from System instances diff --git a/packages/system/0.3.0/manifest.yml b/packages/system/0.3.0/manifest.yml index d9e66957b0..56d1f6105a 100644 --- a/packages/system/0.3.0/manifest.yml +++ b/packages/system/0.3.0/manifest.yml @@ -28,7 +28,7 @@ icons: title: system size: 1000x1000 type: image/svg+xml -datasources: +config_templates: - name: system title: System logs and metrics description: Collect logs and metrics from System instances diff --git a/vendor/github.com/elastic/package-registry/util/dataset.go b/vendor/github.com/elastic/package-registry/util/dataset.go index 0e8200cd0f..2fe87f1924 100644 --- a/vendor/github.com/elastic/package-registry/util/dataset.go +++ b/vendor/github.com/elastic/package-registry/util/dataset.go @@ -31,11 +31,13 @@ var validTypes = map[string]string{ "events": "Events", } -type DataSet struct { - ID string `config:"id" json:"id,omitempty" yaml:"id,omitempty"` +type Dataset struct { + // Name and type of the dataset. This is linked to dataset.name and dataset.type fields. + Type string `config:"type" json:"type" validate:"required"` + Name string `config:"name" json:"name,omitempty" yaml:"name,omitempty"` + Title string `config:"title" json:"title" validate:"required"` Release string `config:"release" json:"release"` - Type string `config:"type" json:"type" validate:"required"` IngestPipeline string `config:"ingest_pipeline,omitempty" config:"ingest_pipeline" json:"ingest_pipeline,omitempty" yaml:"ingest_pipeline,omitempty"` Streams []Stream `config:"streams" json:"streams,omitempty" yaml:"streams,omitempty" ` Package string `json:"package,omitempty" yaml:"package,omitempty"` @@ -60,11 +62,10 @@ type Stream struct { Vars []Variable `config:"vars" json:"vars,omitempty" yaml:"vars,omitempty"` Dataset string `config:"dataset" json:"dataset,omitempty" yaml:"dataset,omitempty"` // TODO: This might cause issues when consuming the json as the key contains . (had been an issue in the past if I remember correctly) - TemplatePath string `config:"template_path" json:"template_path,omitempty" yaml:"template_path,omitempty"` - TemplateContent string `json:"template,omitempty" yaml:"template,omitempty"` // This is always generated in the json output - Title string `config:"title" json:"title,omitempty" yaml:"title,omitempty"` - Description string `config:"description" json:"description,omitempty" yaml:"description,omitempty"` - Enabled *bool `config:"enabled" json:"enabled,omitempty" yaml:"enabled,omitempty"` + TemplatePath string `config:"template_path" json:"template_path,omitempty" yaml:"template_path,omitempty"` + Title string `config:"title" json:"title,omitempty" yaml:"title,omitempty"` + Description string `config:"description" json:"description,omitempty" yaml:"description,omitempty"` + Enabled *bool `config:"enabled" json:"enabled,omitempty" yaml:"enabled,omitempty"` } type Variable struct { @@ -76,12 +77,6 @@ type Variable struct { Required bool `config:"required" json:"required" yaml:"required"` ShowUser bool `config:"show_user" json:"show_user" yaml:"show_user"` Default interface{} `config:"default" json:"default,omitempty" yaml:"default,omitempty"` - Os *Os `config:"os" json:"os,omitempty" yaml:"os,omitempty"` -} - -type Os struct { - Darwin interface{} `config:"darwin" json:"darwin,omitempty" yaml:"darwin,omitempty"` - Windows interface{} `config:"windows" json:"windows,omitempty" yaml:"windows,omitempty"` } type fieldEntry struct { @@ -89,7 +84,7 @@ type fieldEntry struct { aType string } -func NewDataset(basePath string, p *Package) (*DataSet, error) { +func NewDataset(basePath string, p *Package) (*Dataset, error) { // Check if manifest exists manifestPath := filepath.Join(basePath, "manifest.yml") _, err := os.Stat(manifestPath) @@ -103,7 +98,7 @@ func NewDataset(basePath string, p *Package) (*DataSet, error) { if err != nil { return nil, errors.Wrapf(err, "error creating new manifest config %s", manifestPath) } - var d = &DataSet{ + var d = &Dataset{ Package: p.Name, // This is the name of the directory of the dataset Path: datasetPath, @@ -117,8 +112,8 @@ func NewDataset(basePath string, p *Package) (*DataSet, error) { } // if id is not set, {package}.{datasetPath} is the default - if d.ID == "" { - d.ID = p.Name + "." + datasetPath + if d.Name == "" { + d.Name = p.Name + "." + datasetPath } if d.Release == "" { @@ -130,6 +125,10 @@ func NewDataset(basePath string, p *Package) (*DataSet, error) { for i, _ := range d.Streams { if d.Streams[i].Enabled == nil { d.Streams[i].Enabled = &trueValue + // TODO: validate that the template path actually exists + if d.Streams[i].TemplatePath == "" { + d.Streams[i].TemplatePath = "stream.yml.hbs" + } } } @@ -139,15 +138,15 @@ func NewDataset(basePath string, p *Package) (*DataSet, error) { return d, nil } -func (d *DataSet) Validate() error { +func (d *Dataset) Validate() error { pipelineDir := filepath.Join(d.BasePath, "elasticsearch", DirIngestPipeline) paths, err := filepath.Glob(filepath.Join(pipelineDir, "*")) if err != nil { return err } - if strings.Contains(d.ID, "-") { - return fmt.Errorf("dataset name is not allowed to contain `-`: %s", d.ID) + if strings.Contains(d.Name, "-") { + return fmt.Errorf("dataset name is not allowed to contain `-`: %s", d.Name) } if !d.validType() { @@ -165,7 +164,7 @@ func (d *DataSet) Validate() error { } if d.IngestPipeline == "" && len(paths) > 0 { - return fmt.Errorf("unused pipelines in the package (dataSetID: %s): %s", d.ID, strings.Join(paths, ",")) + return fmt.Errorf("unused pipelines in the package (dataSetID: %s): %s", d.Name, strings.Join(paths, ",")) } // In case an ingest pipeline is set, check if it is around @@ -210,7 +209,7 @@ func (d *DataSet) Validate() error { return nil } -func (d *DataSet) validType() bool { +func (d *Dataset) validType() bool { _, exists := validTypes[d.Type] return exists } @@ -240,7 +239,7 @@ func validateIngestPipelineFile(pipelinePath string) error { } // validateRequiredFields method loads fields from all files and checks if required fields are present. -func (d *DataSet) validateRequiredFields() error { +func (d *Dataset) validateRequiredFields() error { fieldsDirPath := filepath.Join(d.BasePath, "fields") // Collect fields from all files diff --git a/vendor/github.com/elastic/package-registry/util/helper.go b/vendor/github.com/elastic/package-registry/util/helper.go index be20fa5929..b4540f038b 100644 --- a/vendor/github.com/elastic/package-registry/util/helper.go +++ b/vendor/github.com/elastic/package-registry/util/helper.go @@ -11,6 +11,7 @@ const ( // Default release if no release is configured DefaultRelease = ReleaseExperimental + DefaultLicense = "basic" ) var ReleaseTypes = map[string]interface{}{ diff --git a/vendor/github.com/elastic/package-registry/util/package.go b/vendor/github.com/elastic/package-registry/util/package.go index 1ef80d8ce0..f3ef8cb983 100644 --- a/vendor/github.com/elastic/package-registry/util/package.go +++ b/vendor/github.com/elastic/package-registry/util/package.go @@ -6,7 +6,6 @@ package util import ( "fmt" - "io/ioutil" "os" "path" "path/filepath" @@ -22,40 +21,71 @@ import ( const defaultType = "integration" var CategoryTitles = map[string]string{ - "logs": "Logs", - "metrics": "Metrics", - "security": "Security", + "aws": "AWS", + "azure": "Azure", + "cloud": "Cloud", + "config_management": "Config management", + "containers": "Containers", + "crm": "CRM", + "custom": "Custom", + "datastore": "Datastore", + "elastic_stack": "Elastic Stack", + "google_loud": "Google Cloud", + "kubernetes": "Kubernetes", + "languages": "Languages", + "message_queue": "Message Queue", + "monitoring": "Monitoring", + "network": "Network", + "notification": "Notification", + "os_system": "OS & System", + "productivity": "Productivity", + "security": "Security", + "support": "Support", + "ticketing": "Ticketing", + "version_control": "Version Control", + "web": "Web", + + // Old categories, to be removed + "logs": "Logs", + "metrics": "Metrics", + //"security": "Security", } type Package struct { + BasePackage `config:",inline" json:",inline" yaml:",inline"` FormatVersion string `config:"format_version" json:"format_version" yaml:"format_version"` - Name string `config:"name" json:"name"` - Title *string `config:"title,omitempty" json:"title,omitempty" yaml:"title,omitempty"` - Version string `config:"version" json:"version"` - Readme *string `config:"readme,omitempty" json:"readme,omitempty" yaml:"readme,omitempty"` - License string `config:"license,omitempty" json:"license,omitempty" yaml:"license,omitempty"` - versionSemVer *semver.Version - Description string `config:"description" json:"description"` - Type string `config:"type" json:"type"` - Categories []string `config:"categories" json:"categories"` - Release string `config:"release,omitempty" json:"release,omitempty"` - Removable bool `config:"removable" json:"removable"` - Requirement Requirement `config:"requirement" json:"requirement"` - Screenshots []Image `config:"screenshots,omitempty" json:"screenshots,omitempty" yaml:"screenshots,omitempty"` - Icons []Image `config:"icons,omitempty" json:"icons,omitempty" yaml:"icons,omitempty"` - Assets []string `config:"assets,omitempty" json:"assets,omitempty" yaml:"assets,omitempty"` - Internal bool `config:"internal,omitempty" json:"internal,omitempty" yaml:"internal,omitempty"` - DataSets []*DataSet `config:"datasets,omitempty" json:"datasets,omitempty" yaml:"datasets,omitempty"` - Datasources []Datasource `config:"datasources,omitempty" json:"datasources,omitempty" yaml:"datasources,omitempty"` - Download string `json:"download" yaml:"download,omitempty"` - Path string `json:"path" yaml:"path,omitempty"` + Readme *string `config:"readme,omitempty" json:"readme,omitempty" yaml:"readme,omitempty"` + License string `config:"license,omitempty" json:"license,omitempty" yaml:"license,omitempty"` + versionSemVer *semver.Version + Categories []string `config:"categories" json:"categories"` + Release string `config:"release,omitempty" json:"release,omitempty"` + Requirement Requirement `config:"requirement" json:"requirement"` + Screenshots []Image `config:"screenshots,omitempty" json:"screenshots,omitempty" yaml:"screenshots,omitempty"` + Assets []string `config:"assets,omitempty" json:"assets,omitempty" yaml:"assets,omitempty"` + ConfigTemplates []ConfigTemplate `config:"config_templates,omitempty" json:"config_templates,omitempty" yaml:"config_templates,omitempty"` + Datasets []*Dataset `config:"datasets,omitempty" json:"datasets,omitempty" yaml:"datasets,omitempty"` + Owner *Owner `config:"owner,omitempty" json:"owner,omitempty" yaml:"owner,omitempty"` // Local path to the package dir BasePath string `json:"-" yaml:"-"` } -type Datasource struct { +// BasePackage is used for the output of the package info in the /search endpoint +type BasePackage struct { + Name string `config:"name" json:"name"` + Title *string `config:"title,omitempty" json:"title,omitempty" yaml:"title,omitempty"` + Version string `config:"version" json:"version"` + Description string `config:"description" json:"description"` + Type string `config:"type" json:"type"` + Download string `json:"download" yaml:"download,omitempty"` + Downloads []Download `config:"downloads,omitempty" json:"downloads,omitempty" yaml:"downloads,omitempty"` + Path string `json:"path" yaml:"path,omitempty"` + Icons []Image `config:"icons,omitempty" json:"icons,omitempty" yaml:"icons,omitempty"` + Internal bool `config:"internal,omitempty" json:"internal,omitempty" yaml:"internal,omitempty"` +} + +type ConfigTemplate struct { Name string `config:"name" json:"name" validate:"required"` Title string `config:"title" json:"title" validate:"required"` Description string `config:"description" json:"description" validate:"required"` @@ -77,6 +107,10 @@ type Version struct { Max string `config:"max,omitempty" json:"max,omitempty"` } +type Owner struct { + Github string `config:"github,omitempty" json:"github,omitempty"` +} + type Image struct { Src string `config:"src" json:"src" validate:"required"` Title string `config:"title" json:"title,omitempty"` @@ -88,6 +122,22 @@ func (i Image) getPath(p *Package) string { return path.Join("/package", p.Name, p.Version, i.Src) } +type Download struct { + Path string `config:"path" json:"path" validate:"required"` + Type string `config:"type" json:"type" validate:"required"` +} + +func NewDownload(p Package, t string) Download { + return Download{ + Path: getDownloadPath(p, t), + Type: t, + } +} + +func getDownloadPath(p Package, t string) string { + return path.Join("/epr", p.Name, p.Name+"-"+p.Version+".tar.gz") +} + // NewPackage creates a new package instances based on the given base path. // The path passed goes to the root of the package where the manifest.yml is. func NewPackage(basePath string) (*Package, error) { @@ -98,8 +148,7 @@ func NewPackage(basePath string) (*Package, error) { } var p = &Package{ - BasePath: basePath, - Removable: true, + BasePath: basePath, } err = manifest.Unpack(p) if err != nil { @@ -108,9 +157,9 @@ func NewPackage(basePath string) (*Package, error) { // Default for the multiple flags is true. trueValue := true - for i, _ := range p.Datasources { - if p.Datasources[i].Multiple == nil { - p.Datasources[i].Multiple = &trueValue + for i, _ := range p.ConfigTemplates { + if p.ConfigTemplates[i].Multiple == nil { + p.ConfigTemplates[i].Multiple = &trueValue } } if p.Type == "" { @@ -119,7 +168,7 @@ func NewPackage(basePath string) (*Package, error) { // If not license is set, basic is assumed if p.License == "" { - p.License = "basic" + p.License = DefaultLicense } if p.Icons != nil { @@ -134,6 +183,8 @@ func NewPackage(basePath string) (*Package, error) { } } + p.Downloads = []Download{NewDownload(*p, "tar")} + if p.Requirement.Kibana.Versions != "" { p.Requirement.Kibana.semVerRange, err = semver.NewConstraint(p.Requirement.Kibana.Versions) if err != nil { @@ -149,11 +200,6 @@ func NewPackage(basePath string) (*Package, error) { return nil, fmt.Errorf("invalid release: %s", p.Release) } - p.versionSemVer, err = semver.StrictNewVersion(p.Version) - if err != nil { - return nil, err - } - readmePath := filepath.Join(p.BasePath, "docs", "README.md") // Check if readme readme, err := os.Stat(readmePath) @@ -177,21 +223,21 @@ func NewPackage(basePath string) (*Package, error) { } func NewPackageWithResources(path string) (*Package, error) { - aPackage, err := NewPackage(path) + p, err := NewPackage(path) if err != nil { return nil, errors.Wrapf(err, "building package from path '%s' failed", path) } - err = aPackage.LoadAssets(aPackage.GetPath()) + err = p.LoadAssets() if err != nil { return nil, errors.Wrapf(err, "loading package assets failed (path '%s')", path) } - err = aPackage.LoadDataSets() + err = p.LoadDataSets() if err != nil { return nil, errors.Wrapf(err, "loading package datasets failed (path '%s')", path) } - return aPackage, nil + return p, nil } func (p *Package) HasCategory(category string) bool { @@ -219,13 +265,13 @@ func (p *Package) HasKibanaVersion(version *semver.Version) bool { return true } -func (p *Package) IsNewer(pp Package) bool { - return p.versionSemVer.GreaterThan(pp.versionSemVer) +func (p *Package) IsNewerOrEqual(pp Package) bool { + return !p.versionSemVer.LessThan(pp.versionSemVer) } // LoadAssets (re)loads all the assets of the package // Based on the time when this is called, it might be that not all assets for a package exist yet, so it is reset every time. -func (p *Package) LoadAssets(packagePath string) (err error) { +func (p *Package) LoadAssets() (err error) { // Reset Assets p.Assets = nil @@ -255,7 +301,7 @@ func (p *Package) LoadAssets(packagePath string) (err error) { // Strip away the basePath from the local system a = a[len(p.BasePath)+1:] - a = path.Join("/package", packagePath, a) + a = path.Join("/package", p.GetPath(), a) p.Assets = append(p.Assets, a) } return nil @@ -276,6 +322,8 @@ func collectAssets(pattern string) ([]string, error) { return nil, nil } +// Validate is called during Unpack of the manifest. +// The validation here is only related to the fields directly specified in the manifest itself. func (p *Package) Validate() error { if p.FormatVersion == "" { return fmt.Errorf("no format_version set: %v", p) @@ -312,6 +360,11 @@ func (p *Package) Validate() error { } } + p.versionSemVer, err = semver.StrictNewVersion(p.Version) + if err != nil { + return errors.Wrap(err, "invalid package version") + } + err = p.validateVersionConsistency() if err != nil { return errors.Wrap(err, "version in manifest file is not consistent with path") @@ -329,15 +382,13 @@ func (p *Package) validateVersionConsistency() error { baseDir := filepath.Base(p.BasePath) versionDir, err := semver.NewVersion(baseDir) if err != nil { + // TODO: There should be a flag passed to the registry to accept these kind of packages + // as otherwise these could hide some errors in the structure of the package-storage return nil // package content is not rooted in version directory } if !versionPackage.Equal(versionDir) { - manifestVer := "unknown" - if p.versionSemVer != nil { - manifestVer = p.versionSemVer.String() - } - return fmt.Errorf("inconsistent versions (path: %s, manifest: %s)", versionDir.String(), manifestVer) + return fmt.Errorf("inconsistent versions (path: %s, manifest: %s)", versionDir.String(), p.versionSemVer.String()) } return nil } @@ -387,32 +438,9 @@ func (p *Package) LoadDataSets() error { return err } - // Iterate through all datasources and inputs to find the matching streams and add them to the output. - for dK, datasource := range p.Datasources { - for iK, _ := range datasource.Inputs { - for _, stream := range d.Streams { - if stream.Input == p.Datasources[dK].Inputs[iK].Type { - if stream.TemplatePath == "" { - stream.TemplatePath = "stream.yml.hbs" - } - stream.Dataset = d.ID - streamTemplate := filepath.Join(datasetBasePath, "agent", "stream", stream.TemplatePath) - - streamTemplateData, err := ioutil.ReadFile(streamTemplate) - if err != nil { - return err - } - - stream.TemplateContent = string(streamTemplateData) - - // Add template to stream - p.Datasources[dK].Inputs[iK].Streams = append(p.Datasources[dK].Inputs[iK].Streams, stream) - } - } - } - } + // TODO: Validate that each input specified in a stream also is defined in the package - p.DataSets = append(p.DataSets, d) + p.Datasets = append(p.Datasets, d) } return nil diff --git a/vendor/github.com/elastic/package-registry/util/package_watcher.go b/vendor/github.com/elastic/package-registry/util/package_watcher.go new file mode 100644 index 0000000000..634532f0d4 --- /dev/null +++ b/vendor/github.com/elastic/package-registry/util/package_watcher.go @@ -0,0 +1,78 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package util + +import ( + "log" + "os" + "time" + + "github.com/pkg/errors" + "github.com/radovskyb/watcher" +) + +const watcherPollingPeriod = 2 * time.Second + +var ( + w *watcher.Watcher +) + +func MustUsePackageWatcher(packagePaths []string) { + log.Println("Use package watcher") + + var err error + packageList, err = getPackagesFromFilesystem(packagePaths) + if err != nil { + log.Println(errors.Wrap(err, "watcher error: reading packages failed")) + } + + w = watcher.New() + w.SetMaxEvents(1) + + for _, p := range packagePaths { + err = w.AddRecursive(p) + if err != nil && !os.IsNotExist(err) { + log.Fatal(errors.Wrapf(err, "watching directory failed (path: %s)", p)) + } + } + + go w.Start(watcherPollingPeriod) + + go func() { + for { + select { + case _, ok := <-w.Event: + if !ok { + log.Println("Package watcher is stopped") + return // channel is closed + } + + time.Sleep(watcherPollingPeriod) // reload at the end of watch frame + log.Println("Reloading packages...") + packageList, err = getPackagesFromFilesystem(packagePaths) + if err != nil { + log.Println(errors.Wrap(err, "watcher error: reading packages failed")) + } + case err, ok := <-w.Error: + if !ok { + log.Println("Package watcher is stopped") + return // channel is closed + } + log.Println(errors.Wrap(err, "watcher error")) + } + } + }() +} + +func ClosePackageWatcher() { + if !packageWatcherEnabled() { + return + } + w.Close() +} + +func packageWatcherEnabled() bool { + return w != nil +} diff --git a/vendor/github.com/elastic/package-registry/util/packages.go b/vendor/github.com/elastic/package-registry/util/packages.go index 899a08ea5c..b1f1e88f1f 100644 --- a/vendor/github.com/elastic/package-registry/util/packages.go +++ b/vendor/github.com/elastic/package-registry/util/packages.go @@ -13,63 +13,70 @@ import ( "github.com/pkg/errors" ) -var packageList []Package +var packageList Packages + +type Packages []Package // GetPackages returns a slice with all existing packages. -// The list is stored in memory and on the second request directly -// served from memory. This assumes chnages to packages only happen on restart. +// The list is stored in memory and on the second request directly served from memory. +// This assumes changes to packages only happen on restart (unless development mode is enabled). // Caching the packages request many file reads every time this method is called. -func GetPackages(packagesBasePath string) ([]Package, error) { +func GetPackages(packagesBasePaths []string) (Packages, error) { if packageList != nil { return packageList, nil } - packagePaths, err := getPackagePaths(packagesBasePath) + var err error + packageList, err = getPackagesFromFilesystem(packagesBasePaths) + if err != nil { + return nil, errors.Wrapf(err, "reading packages from filesystem failed") + } + return packageList, nil +} + +func getPackagesFromFilesystem(packagesBasePaths []string) (Packages, error) { + packagePaths, err := getPackagePaths(packagesBasePaths) if err != nil { return nil, err } + var pList Packages for _, path := range packagePaths { p, err := NewPackage(path) if err != nil { return nil, errors.Wrapf(err, "loading package failed (path: %s)", path) } - err = p.Validate() - if err != nil { - return nil, errors.Wrapf(err, "validating package failed (path: %s)", path) - } - packageList = append(packageList, *p) + pList = append(pList, *p) } - - return packageList, nil + return pList, nil } // getPackagePaths returns list of available packages, one for each version. -func getPackagePaths(packagesPath string) ([]string, error) { - log.Printf("List packages in %s", packagesPath) - +func getPackagePaths(allPaths []string) ([]string, error) { var foundPaths []string - return foundPaths, filepath.Walk(packagesPath, func(path string, info os.FileInfo, err error) error { - relativePath, err := filepath.Rel(packagesPath, path) - if err != nil { - return err - } + for _, packagesPath := range allPaths { + log.Printf("Packages in %s:", packagesPath) + err := filepath.Walk(packagesPath, func(path string, info os.FileInfo, err error) error { + relativePath, err := filepath.Rel(packagesPath, path) + if err != nil { + return err + } - dirs := strings.Split(relativePath, string(filepath.Separator)) - if len(dirs) < 2 { - return nil // need to go to the package version level - } + dirs := strings.Split(relativePath, string(filepath.Separator)) + if len(dirs) < 2 { + return nil // need to go to the package version level + } - p, err := os.Stat(path) + if info.IsDir() { + log.Printf("%-20s\t%10s\t%s", dirs[0], dirs[1], path) + foundPaths = append(foundPaths, path) + } + return filepath.SkipDir // don't need to go deeper + }) if err != nil { - return err - } - - if p.IsDir() { - log.Printf("%-20s\t%10s\t%s", dirs[0], dirs[1], path) - foundPaths = append(foundPaths, path) + return nil, errors.Wrapf(err, "listing packages failed (path: %s)", packagesPath) } - return filepath.SkipDir // don't need to go deeper - }) + } + return foundPaths, nil } diff --git a/vendor/github.com/radovskyb/watcher/.travis.yml b/vendor/github.com/radovskyb/watcher/.travis.yml new file mode 100644 index 0000000000..3f478f10f3 --- /dev/null +++ b/vendor/github.com/radovskyb/watcher/.travis.yml @@ -0,0 +1,4 @@ +language: go +go: + - 1.7 + - tip \ No newline at end of file diff --git a/vendor/github.com/radovskyb/watcher/LICENSE b/vendor/github.com/radovskyb/watcher/LICENSE new file mode 100644 index 0000000000..92ef0e9115 --- /dev/null +++ b/vendor/github.com/radovskyb/watcher/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2016, Benjamin Radovsky. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of watcher nor the names of its contributors may be used to + endorse or promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/radovskyb/watcher/README.md b/vendor/github.com/radovskyb/watcher/README.md new file mode 100644 index 0000000000..1c822e20ad --- /dev/null +++ b/vendor/github.com/radovskyb/watcher/README.md @@ -0,0 +1,181 @@ +# watcher + +[![Build Status](https://travis-ci.org/radovskyb/watcher.svg?branch=master)](https://travis-ci.org/radovskyb/watcher) + +`watcher` is a Go package for watching for files or directory changes (recursively or non recursively) without using filesystem events, which allows it to work cross platform consistently. + +`watcher` watches for changes and notifies over channels either anytime an event or an error has occurred. + +Events contain the `os.FileInfo` of the file or directory that the event is based on and the type of event and file or directory path. + +[Installation](#installation) +[Features](#features) +[Example](#example) +[Contributing](#contributing) +[Watcher Command](#command) + +# Update +- Event.OldPath has been added [Aug 17, 2019] +- Added new file filter hooks (Including a built in regexp filtering hook) [Dec 12, 2018] +- Event.Path for Rename and Move events is now returned in the format of `fromPath -> toPath` + +#### Chmod event is not supported under windows. + +# Installation + +```shell +go get -u github.com/radovskyb/watcher/... +``` + +# Features + +- Customizable polling interval. +- Filter Events. +- Watch folders recursively or non-recursively. +- Choose to ignore hidden files. +- Choose to ignore specified files and folders. +- Notifies the `os.FileInfo` of the file that the event is based on. e.g `Name`, `ModTime`, `IsDir`, etc. +- Notifies the full path of the file that the event is based on or the old and new paths if the event was a `Rename` or `Move` event. +- Limit amount of events that can be received per watching cycle. +- List the files being watched. +- Trigger custom events. + +# Todo + +- Write more tests. +- Write benchmarks. + +# Example + +```go +package main + +import ( + "fmt" + "log" + "time" + + "github.com/radovskyb/watcher" +) + +func main() { + w := watcher.New() + + // SetMaxEvents to 1 to allow at most 1 event's to be received + // on the Event channel per watching cycle. + // + // If SetMaxEvents is not set, the default is to send all events. + w.SetMaxEvents(1) + + // Only notify rename and move events. + w.FilterOps(watcher.Rename, watcher.Move) + + // Only files that match the regular expression during file listings + // will be watched. + r := regexp.MustCompile("^abc$") + w.AddFilterHook(watcher.RegexFilterHook(r, false)) + + go func() { + for { + select { + case event := <-w.Event: + fmt.Println(event) // Print the event's info. + case err := <-w.Error: + log.Fatalln(err) + case <-w.Closed: + return + } + } + }() + + // Watch this folder for changes. + if err := w.Add("."); err != nil { + log.Fatalln(err) + } + + // Watch test_folder recursively for changes. + if err := w.AddRecursive("../test_folder"); err != nil { + log.Fatalln(err) + } + + // Print a list of all of the files and folders currently + // being watched and their paths. + for path, f := range w.WatchedFiles() { + fmt.Printf("%s: %s\n", path, f.Name()) + } + + fmt.Println() + + // Trigger 2 events after watcher started. + go func() { + w.Wait() + w.TriggerEvent(watcher.Create, nil) + w.TriggerEvent(watcher.Remove, nil) + }() + + // Start the watching process - it'll check for changes every 100ms. + if err := w.Start(time.Millisecond * 100); err != nil { + log.Fatalln(err) + } +} +``` + +# Contributing +If you would ike to contribute, simply submit a pull request. + +# Command + +`watcher` comes with a simple command which is installed when using the `go get` command from above. + +# Usage + +``` +Usage of watcher: + -cmd string + command to run when an event occurs + -dotfiles + watch dot files (default true) + -ignore string + comma separated list of paths to ignore + -interval string + watcher poll interval (default "100ms") + -keepalive + keep alive when a cmd returns code != 0 + -list + list watched files on start + -pipe + pipe event's info to command's stdin + -recursive + watch folders recursively (default true) + -startcmd + run the command when watcher starts +``` + +All of the flags are optional and watcher can also be called by itself: +```shell +watcher +``` +(watches the current directory recursively for changes and notifies any events that occur.) + +A more elaborate example using the `watcher` command: +```shell +watcher -dotfiles=false -recursive=false -cmd="./myscript" main.go ../ +``` +In this example, `watcher` will ignore dot files and folders and won't watch any of the specified folders recursively. It will also run the script `./myscript` anytime an event occurs while watching `main.go` or any files or folders in the previous directory (`../`). + +Using the `pipe` and `cmd` flags together will send the event's info to the command's stdin when changes are detected. + +First create a file called `script.py` with the following contents: +```python +import sys + +for line in sys.stdin: + print (line + " - python") +``` + +Next, start watcher with the `pipe` and `cmd` flags enabled: +```shell +watcher -cmd="python script.py" -pipe=true +``` + +Now when changes are detected, the event's info will be output from the running python script. diff --git a/vendor/github.com/radovskyb/watcher/ishidden.go b/vendor/github.com/radovskyb/watcher/ishidden.go new file mode 100644 index 0000000000..0f242e8aec --- /dev/null +++ b/vendor/github.com/radovskyb/watcher/ishidden.go @@ -0,0 +1,12 @@ +// +build !windows + +package watcher + +import ( + "path/filepath" + "strings" +) + +func isHiddenFile(path string) (bool, error) { + return strings.HasPrefix(filepath.Base(path), "."), nil +} diff --git a/vendor/github.com/radovskyb/watcher/ishidden_windows.go b/vendor/github.com/radovskyb/watcher/ishidden_windows.go new file mode 100644 index 0000000000..306c6b7111 --- /dev/null +++ b/vendor/github.com/radovskyb/watcher/ishidden_windows.go @@ -0,0 +1,21 @@ +// +build windows + +package watcher + +import ( + "syscall" +) + +func isHiddenFile(path string) (bool, error) { + pointer, err := syscall.UTF16PtrFromString(path) + if err != nil { + return false, err + } + + attributes, err := syscall.GetFileAttributes(pointer) + if err != nil { + return false, err + } + + return attributes&syscall.FILE_ATTRIBUTE_HIDDEN != 0, nil +} diff --git a/vendor/github.com/radovskyb/watcher/samefile.go b/vendor/github.com/radovskyb/watcher/samefile.go new file mode 100644 index 0000000000..5968bb6fe2 --- /dev/null +++ b/vendor/github.com/radovskyb/watcher/samefile.go @@ -0,0 +1,9 @@ +// +build !windows + +package watcher + +import "os" + +func sameFile(fi1, fi2 os.FileInfo) bool { + return os.SameFile(fi1, fi2) +} diff --git a/vendor/github.com/radovskyb/watcher/samefile_windows.go b/vendor/github.com/radovskyb/watcher/samefile_windows.go new file mode 100644 index 0000000000..7785a129b7 --- /dev/null +++ b/vendor/github.com/radovskyb/watcher/samefile_windows.go @@ -0,0 +1,12 @@ +// +build windows + +package watcher + +import "os" + +func sameFile(fi1, fi2 os.FileInfo) bool { + return fi1.ModTime() == fi2.ModTime() && + fi1.Size() == fi2.Size() && + fi1.Mode() == fi2.Mode() && + fi1.IsDir() == fi2.IsDir() +} diff --git a/vendor/github.com/radovskyb/watcher/watcher.go b/vendor/github.com/radovskyb/watcher/watcher.go new file mode 100644 index 0000000000..4da4dfe3b9 --- /dev/null +++ b/vendor/github.com/radovskyb/watcher/watcher.go @@ -0,0 +1,715 @@ +package watcher + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" +) + +var ( + // ErrDurationTooShort occurs when calling the watcher's Start + // method with a duration that's less than 1 nanosecond. + ErrDurationTooShort = errors.New("error: duration is less than 1ns") + + // ErrWatcherRunning occurs when trying to call the watcher's + // Start method and the polling cycle is still already running + // from previously calling Start and not yet calling Close. + ErrWatcherRunning = errors.New("error: watcher is already running") + + // ErrWatchedFileDeleted is an error that occurs when a file or folder that was + // being watched has been deleted. + ErrWatchedFileDeleted = errors.New("error: watched file or folder deleted") + + // ErrSkip is less of an error, but more of a way for path hooks to skip a file or + // directory. + ErrSkip = errors.New("error: skipping file") +) + +// An Op is a type that is used to describe what type +// of event has occurred during the watching process. +type Op uint32 + +// Ops +const ( + Create Op = iota + Write + Remove + Rename + Chmod + Move +) + +var ops = map[Op]string{ + Create: "CREATE", + Write: "WRITE", + Remove: "REMOVE", + Rename: "RENAME", + Chmod: "CHMOD", + Move: "MOVE", +} + +// String prints the string version of the Op consts +func (e Op) String() string { + if op, found := ops[e]; found { + return op + } + return "???" +} + +// An Event describes an event that is received when files or directory +// changes occur. It includes the os.FileInfo of the changed file or +// directory and the type of event that's occurred and the full path of the file. +type Event struct { + Op + Path string + OldPath string + os.FileInfo +} + +// String returns a string depending on what type of event occurred and the +// file name associated with the event. +func (e Event) String() string { + if e.FileInfo == nil { + return "???" + } + + pathType := "FILE" + if e.IsDir() { + pathType = "DIRECTORY" + } + return fmt.Sprintf("%s %q %s [%s]", pathType, e.Name(), e.Op, e.Path) +} + +// FilterFileHookFunc is a function that is called to filter files during listings. +// If a file is ok to be listed, nil is returned otherwise ErrSkip is returned. +type FilterFileHookFunc func(info os.FileInfo, fullPath string) error + +// RegexFilterHook is a function that accepts or rejects a file +// for listing based on whether it's filename or full path matches +// a regular expression. +func RegexFilterHook(r *regexp.Regexp, useFullPath bool) FilterFileHookFunc { + return func(info os.FileInfo, fullPath string) error { + str := info.Name() + + if useFullPath { + str = fullPath + } + + // Match + if r.MatchString(str) { + return nil + } + + // No match. + return ErrSkip + } +} + +// Watcher describes a process that watches files for changes. +type Watcher struct { + Event chan Event + Error chan error + Closed chan struct{} + close chan struct{} + wg *sync.WaitGroup + + // mu protects the following. + mu *sync.Mutex + ffh []FilterFileHookFunc + running bool + names map[string]bool // bool for recursive or not. + files map[string]os.FileInfo // map of files. + ignored map[string]struct{} // ignored files or directories. + ops map[Op]struct{} // Op filtering. + ignoreHidden bool // ignore hidden files or not. + maxEvents int // max sent events per cycle +} + +// New creates a new Watcher. +func New() *Watcher { + // Set up the WaitGroup for w.Wait(). + var wg sync.WaitGroup + wg.Add(1) + + return &Watcher{ + Event: make(chan Event), + Error: make(chan error), + Closed: make(chan struct{}), + close: make(chan struct{}), + mu: new(sync.Mutex), + wg: &wg, + files: make(map[string]os.FileInfo), + ignored: make(map[string]struct{}), + names: make(map[string]bool), + } +} + +// SetMaxEvents controls the maximum amount of events that are sent on +// the Event channel per watching cycle. If max events is less than 1, there is +// no limit, which is the default. +func (w *Watcher) SetMaxEvents(delta int) { + w.mu.Lock() + w.maxEvents = delta + w.mu.Unlock() +} + +// AddFilterHook +func (w *Watcher) AddFilterHook(f FilterFileHookFunc) { + w.mu.Lock() + w.ffh = append(w.ffh, f) + w.mu.Unlock() +} + +// IgnoreHiddenFiles sets the watcher to ignore any file or directory +// that starts with a dot. +func (w *Watcher) IgnoreHiddenFiles(ignore bool) { + w.mu.Lock() + w.ignoreHidden = ignore + w.mu.Unlock() +} + +// FilterOps filters which event op types should be returned +// when an event occurs. +func (w *Watcher) FilterOps(ops ...Op) { + w.mu.Lock() + w.ops = make(map[Op]struct{}) + for _, op := range ops { + w.ops[op] = struct{}{} + } + w.mu.Unlock() +} + +// Add adds either a single file or directory to the file list. +func (w *Watcher) Add(name string) (err error) { + w.mu.Lock() + defer w.mu.Unlock() + + name, err = filepath.Abs(name) + if err != nil { + return err + } + + // If name is on the ignored list or if hidden files are + // ignored and name is a hidden file or directory, simply return. + _, ignored := w.ignored[name] + + isHidden, err := isHiddenFile(name) + if err != nil { + return err + } + + if ignored || (w.ignoreHidden && isHidden) { + return nil + } + + // Add the directory's contents to the files list. + fileList, err := w.list(name) + if err != nil { + return err + } + for k, v := range fileList { + w.files[k] = v + } + + // Add the name to the names list. + w.names[name] = false + + return nil +} + +func (w *Watcher) list(name string) (map[string]os.FileInfo, error) { + fileList := make(map[string]os.FileInfo) + + // Make sure name exists. + stat, err := os.Stat(name) + if err != nil { + return nil, err + } + + fileList[name] = stat + + // If it's not a directory, just return. + if !stat.IsDir() { + return fileList, nil + } + + // It's a directory. + fInfoList, err := ioutil.ReadDir(name) + if err != nil { + return nil, err + } + // Add all of the files in the directory to the file list as long + // as they aren't on the ignored list or are hidden files if ignoreHidden + // is set to true. +outer: + for _, fInfo := range fInfoList { + path := filepath.Join(name, fInfo.Name()) + _, ignored := w.ignored[path] + + isHidden, err := isHiddenFile(path) + if err != nil { + return nil, err + } + + if ignored || (w.ignoreHidden && isHidden) { + continue + } + + for _, f := range w.ffh { + err := f(fInfo, path) + if err == ErrSkip { + continue outer + } + if err != nil { + return nil, err + } + } + + fileList[path] = fInfo + } + return fileList, nil +} + +// AddRecursive adds either a single file or directory recursively to the file list. +func (w *Watcher) AddRecursive(name string) (err error) { + w.mu.Lock() + defer w.mu.Unlock() + + name, err = filepath.Abs(name) + if err != nil { + return err + } + + fileList, err := w.listRecursive(name) + if err != nil { + return err + } + for k, v := range fileList { + w.files[k] = v + } + + // Add the name to the names list. + w.names[name] = true + + return nil +} + +func (w *Watcher) listRecursive(name string) (map[string]os.FileInfo, error) { + fileList := make(map[string]os.FileInfo) + + return fileList, filepath.Walk(name, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + for _, f := range w.ffh { + err := f(info, path) + if err == ErrSkip { + return nil + } + if err != nil { + return err + } + } + + // If path is ignored and it's a directory, skip the directory. If it's + // ignored and it's a single file, skip the file. + _, ignored := w.ignored[path] + + isHidden, err := isHiddenFile(path) + if err != nil { + return err + } + + if ignored || (w.ignoreHidden && isHidden) { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + // Add the path and it's info to the file list. + fileList[path] = info + return nil + }) +} + +// Remove removes either a single file or directory from the file's list. +func (w *Watcher) Remove(name string) (err error) { + w.mu.Lock() + defer w.mu.Unlock() + + name, err = filepath.Abs(name) + if err != nil { + return err + } + + // Remove the name from w's names list. + delete(w.names, name) + + // If name is a single file, remove it and return. + info, found := w.files[name] + if !found { + return nil // Doesn't exist, just return. + } + if !info.IsDir() { + delete(w.files, name) + return nil + } + + // Delete the actual directory from w.files + delete(w.files, name) + + // If it's a directory, delete all of it's contents from w.files. + for path := range w.files { + if filepath.Dir(path) == name { + delete(w.files, path) + } + } + return nil +} + +// RemoveRecursive removes either a single file or a directory recursively from +// the file's list. +func (w *Watcher) RemoveRecursive(name string) (err error) { + w.mu.Lock() + defer w.mu.Unlock() + + name, err = filepath.Abs(name) + if err != nil { + return err + } + + // Remove the name from w's names list. + delete(w.names, name) + + // If name is a single file, remove it and return. + info, found := w.files[name] + if !found { + return nil // Doesn't exist, just return. + } + if !info.IsDir() { + delete(w.files, name) + return nil + } + + // If it's a directory, delete all of it's contents recursively + // from w.files. + for path := range w.files { + if strings.HasPrefix(path, name) { + delete(w.files, path) + } + } + return nil +} + +// Ignore adds paths that should be ignored. +// +// For files that are already added, Ignore removes them. +func (w *Watcher) Ignore(paths ...string) (err error) { + for _, path := range paths { + path, err = filepath.Abs(path) + if err != nil { + return err + } + // Remove any of the paths that were already added. + if err := w.RemoveRecursive(path); err != nil { + return err + } + w.mu.Lock() + w.ignored[path] = struct{}{} + w.mu.Unlock() + } + return nil +} + +// WatchedFiles returns a map of files added to a Watcher. +func (w *Watcher) WatchedFiles() map[string]os.FileInfo { + w.mu.Lock() + defer w.mu.Unlock() + + files := make(map[string]os.FileInfo) + for k, v := range w.files { + files[k] = v + } + + return files +} + +// fileInfo is an implementation of os.FileInfo that can be used +// as a mocked os.FileInfo when triggering an event when the specified +// os.FileInfo is nil. +type fileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time + sys interface{} + dir bool +} + +func (fs *fileInfo) IsDir() bool { + return fs.dir +} +func (fs *fileInfo) ModTime() time.Time { + return fs.modTime +} +func (fs *fileInfo) Mode() os.FileMode { + return fs.mode +} +func (fs *fileInfo) Name() string { + return fs.name +} +func (fs *fileInfo) Size() int64 { + return fs.size +} +func (fs *fileInfo) Sys() interface{} { + return fs.sys +} + +// TriggerEvent is a method that can be used to trigger an event, separate to +// the file watching process. +func (w *Watcher) TriggerEvent(eventType Op, file os.FileInfo) { + w.Wait() + if file == nil { + file = &fileInfo{name: "triggered event", modTime: time.Now()} + } + w.Event <- Event{Op: eventType, Path: "-", FileInfo: file} +} + +func (w *Watcher) retrieveFileList() map[string]os.FileInfo { + w.mu.Lock() + defer w.mu.Unlock() + + fileList := make(map[string]os.FileInfo) + + var list map[string]os.FileInfo + var err error + + for name, recursive := range w.names { + if recursive { + list, err = w.listRecursive(name) + if err != nil { + if os.IsNotExist(err) { + w.mu.Unlock() + if name == err.(*os.PathError).Path { + w.Error <- ErrWatchedFileDeleted + w.RemoveRecursive(name) + } + w.mu.Lock() + } else { + w.Error <- err + } + } + } else { + list, err = w.list(name) + if err != nil { + if os.IsNotExist(err) { + w.mu.Unlock() + if name == err.(*os.PathError).Path { + w.Error <- ErrWatchedFileDeleted + w.Remove(name) + } + w.mu.Lock() + } else { + w.Error <- err + } + } + } + // Add the file's to the file list. + for k, v := range list { + fileList[k] = v + } + } + + return fileList +} + +// Start begins the polling cycle which repeats every specified +// duration until Close is called. +func (w *Watcher) Start(d time.Duration) error { + // Return an error if d is less than 1 nanosecond. + if d < time.Nanosecond { + return ErrDurationTooShort + } + + // Make sure the Watcher is not already running. + w.mu.Lock() + if w.running { + w.mu.Unlock() + return ErrWatcherRunning + } + w.running = true + w.mu.Unlock() + + // Unblock w.Wait(). + w.wg.Done() + + for { + // done lets the inner polling cycle loop know when the + // current cycle's method has finished executing. + done := make(chan struct{}) + + // Any events that are found are first piped to evt before + // being sent to the main Event channel. + evt := make(chan Event) + + // Retrieve the file list for all watched file's and dirs. + fileList := w.retrieveFileList() + + // cancel can be used to cancel the current event polling function. + cancel := make(chan struct{}) + + // Look for events. + go func() { + w.pollEvents(fileList, evt, cancel) + done <- struct{}{} + }() + + // numEvents holds the number of events for the current cycle. + numEvents := 0 + + inner: + for { + select { + case <-w.close: + close(cancel) + close(w.Closed) + return nil + case event := <-evt: + if len(w.ops) > 0 { // Filter Ops. + _, found := w.ops[event.Op] + if !found { + continue + } + } + numEvents++ + if w.maxEvents > 0 && numEvents > w.maxEvents { + close(cancel) + break inner + } + w.Event <- event + case <-done: // Current cycle is finished. + break inner + } + } + + // Update the file's list. + w.mu.Lock() + w.files = fileList + w.mu.Unlock() + + // Sleep and then continue to the next loop iteration. + time.Sleep(d) + } +} + +func (w *Watcher) pollEvents(files map[string]os.FileInfo, evt chan Event, + cancel chan struct{}) { + w.mu.Lock() + defer w.mu.Unlock() + + // Store create and remove events for use to check for rename events. + creates := make(map[string]os.FileInfo) + removes := make(map[string]os.FileInfo) + + // Check for removed files. + for path, info := range w.files { + if _, found := files[path]; !found { + removes[path] = info + } + } + + // Check for created files, writes and chmods. + for path, info := range files { + oldInfo, found := w.files[path] + if !found { + // A file was created. + creates[path] = info + continue + } + if oldInfo.ModTime() != info.ModTime() { + select { + case <-cancel: + return + case evt <- Event{Write, path, path, info}: + } + } + if oldInfo.Mode() != info.Mode() { + select { + case <-cancel: + return + case evt <- Event{Chmod, path, path, info}: + } + } + } + + // Check for renames and moves. + for path1, info1 := range removes { + for path2, info2 := range creates { + if sameFile(info1, info2) { + e := Event{ + Op: Move, + Path: path2, + OldPath: path1, + FileInfo: info1, + } + // If they are from the same directory, it's a rename + // instead of a move event. + if filepath.Dir(path1) == filepath.Dir(path2) { + e.Op = Rename + } + + delete(removes, path1) + delete(creates, path2) + + select { + case <-cancel: + return + case evt <- e: + } + } + } + } + + // Send all the remaining create and remove events. + for path, info := range creates { + select { + case <-cancel: + return + case evt <- Event{Create, path, "", info}: + } + } + for path, info := range removes { + select { + case <-cancel: + return + case evt <- Event{Remove, path, path, info}: + } + } +} + +// Wait blocks until the watcher is started. +func (w *Watcher) Wait() { + w.wg.Wait() +} + +// Close stops a Watcher and unlocks its mutex, then sends a close signal. +func (w *Watcher) Close() { + w.mu.Lock() + if !w.running { + w.mu.Unlock() + return + } + w.running = false + w.files = make(map[string]os.FileInfo) + w.names = make(map[string]bool) + w.mu.Unlock() + // Send a close signal to the Start method. + w.close <- struct{}{} +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 2f144d5a7a..59d5d58f48 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -11,7 +11,7 @@ github.com/davecgh/go-spew/spew github.com/elastic/go-ucfg github.com/elastic/go-ucfg/parse github.com/elastic/go-ucfg/yaml -# github.com/elastic/package-registry v0.4.1-0.20200618213757-98d7184cfe5b +# github.com/elastic/package-registry v0.4.1-0.20200630074455-f91d80e4786f github.com/elastic/package-registry/util # github.com/magefile/mage v1.9.0 github.com/magefile/mage/mg @@ -20,6 +20,8 @@ github.com/magefile/mage/sh github.com/pkg/errors # github.com/pmezard/go-difflib v1.0.0 github.com/pmezard/go-difflib/difflib +# github.com/radovskyb/watcher v1.0.7 +github.com/radovskyb/watcher # github.com/stretchr/testify v1.4.0 github.com/stretchr/testify/assert # gopkg.in/yaml.v2 v2.3.0