diff --git a/dev/generator/main.go b/dev/generator/main.go index ac9825601..269307b02 100644 --- a/dev/generator/main.go +++ b/dev/generator/main.go @@ -13,10 +13,9 @@ import ( "os" "path/filepath" - "gopkg.in/yaml.v2" - "github.com/magefile/mage/sh" "github.com/pkg/errors" + "gopkg.in/yaml.v2" "github.com/elastic/package-registry/util" ) @@ -26,27 +25,7 @@ var ( copy bool ) -const ( - packageDirName = "package" - streamFields = ` -- name: stream.type - type: constant_keyword - description: > - Stream type -- name: stream.dataset - type: constant_keyword - description: > - Stream dataset. -- name: stream.namespace - type: constant_keyword - description: > - Stream namespace. -- name: "@timestamp" - type: date - description: > - Event timestamp. -` -) +const packageDirName = "package" type fieldEntry struct { name string @@ -178,16 +157,6 @@ func buildPackage(packagesBasePath string, p util.Package) error { return err } - // Add base-fields.yml if missing to all dataset with the basic stream fields and @timestamp - for _, dataset := range datasets { - datasetPath := filepath.Join(p.BasePath, "dataset", dataset) - - err = createBaseFieldsFile(datasetPath) - if err != nil { - return errors.Wrapf(err, "creating file with basic fields failed (datasetPath: %s)", datasetPath) - } - } - // Validate if basic stream fields and @timestamp are present for _, dataset := range datasets { datasetPath := filepath.Join(p.BasePath, "dataset", dataset) @@ -239,24 +208,6 @@ func buildPackage(packagesBasePath string, p util.Package) error { return nil } -// createBaseFieldsFile method creates the base-fields.yml file if missing. -func createBaseFieldsFile(datasetPath string) error { - fieldsDirPath := filepath.Join(datasetPath, "fields") - baseFieldsPath := filepath.Join(fieldsDirPath, "base-fields.yml") - _, err := os.Stat(baseFieldsPath) - if err == nil { - return nil - } else if !os.IsNotExist(err) { - return errors.Wrapf(err, "stat file failed (fieldsPath: %s)", baseFieldsPath) - } - - err = os.MkdirAll(fieldsDirPath, 0755) - if err != nil { - return errors.Wrapf(err, "creating directories failed (fieldsDirPath: %s)", fieldsDirPath) - } - return ioutil.WriteFile(baseFieldsPath, []byte(streamFields), 0644) -} - // validateRequiredFields method loads fields from all files and checks if required fields are present. func validateRequiredFields(datasetPath string) error { fieldsDirPath := filepath.Join(datasetPath, "fields") diff --git a/dev/packages/example/log/0.9.0/dataset/log/fields/base-fields.yml b/dev/packages/example/log/0.9.0/dataset/log/fields/base-fields.yml new file mode 100644 index 000000000..1fb7d7b3b --- /dev/null +++ b/dev/packages/example/log/0.9.0/dataset/log/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: stream.type + type: constant_keyword + description: > + Stream type +- name: stream.dataset + type: constant_keyword + description: > + Stream dataset. +- name: stream.namespace + type: constant_keyword + description: > + Stream namespace. +- name: "@timestamp" + type: date + description: > + Event timestamp. diff --git a/dev/packages/example/reference/1.0.0/dataset/reference/fields/base-fields.yml b/dev/packages/example/reference/1.0.0/dataset/reference/fields/base-fields.yml new file mode 100644 index 000000000..1fb7d7b3b --- /dev/null +++ b/dev/packages/example/reference/1.0.0/dataset/reference/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: stream.type + type: constant_keyword + description: > + Stream type +- name: stream.dataset + type: constant_keyword + description: > + Stream dataset. +- name: stream.namespace + type: constant_keyword + description: > + Stream namespace. +- name: "@timestamp" + type: date + description: > + Event timestamp. diff --git a/dev/packages/example/yamlpipeline/1.0.0/dataset/log/fields/base-fields.yml b/dev/packages/example/yamlpipeline/1.0.0/dataset/log/fields/base-fields.yml new file mode 100644 index 000000000..1fb7d7b3b --- /dev/null +++ b/dev/packages/example/yamlpipeline/1.0.0/dataset/log/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: stream.type + type: constant_keyword + description: > + Stream type +- name: stream.dataset + type: constant_keyword + description: > + Stream dataset. +- name: stream.namespace + type: constant_keyword + description: > + Stream namespace. +- name: "@timestamp" + type: date + description: > + Event timestamp. diff --git a/testdata/package/datasources/1.0.0/dataset/examplelog1/fields/base-fields.yml b/testdata/package/datasources/1.0.0/dataset/examplelog1/fields/base-fields.yml new file mode 100644 index 000000000..1fb7d7b3b --- /dev/null +++ b/testdata/package/datasources/1.0.0/dataset/examplelog1/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: stream.type + type: constant_keyword + description: > + Stream type +- name: stream.dataset + type: constant_keyword + description: > + Stream dataset. +- name: stream.namespace + type: constant_keyword + description: > + Stream namespace. +- name: "@timestamp" + type: date + description: > + Event timestamp. diff --git a/testdata/package/datasources/1.0.0/dataset/examplelog2/fields/base-fields.yml b/testdata/package/datasources/1.0.0/dataset/examplelog2/fields/base-fields.yml new file mode 100644 index 000000000..1fb7d7b3b --- /dev/null +++ b/testdata/package/datasources/1.0.0/dataset/examplelog2/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: stream.type + type: constant_keyword + description: > + Stream type +- name: stream.dataset + type: constant_keyword + description: > + Stream dataset. +- name: stream.namespace + type: constant_keyword + description: > + Stream namespace. +- name: "@timestamp" + type: date + description: > + Event timestamp. diff --git a/testdata/package/datasources/1.0.0/dataset/examplemetric/fields/base-fields.yml b/testdata/package/datasources/1.0.0/dataset/examplemetric/fields/base-fields.yml new file mode 100644 index 000000000..1fb7d7b3b --- /dev/null +++ b/testdata/package/datasources/1.0.0/dataset/examplemetric/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: stream.type + type: constant_keyword + description: > + Stream type +- name: stream.dataset + type: constant_keyword + description: > + Stream dataset. +- name: stream.namespace + type: constant_keyword + description: > + Stream namespace. +- name: "@timestamp" + type: date + description: > + Event timestamp. diff --git a/testdata/package/default-pipeline/0.0.2/dataset/foo/fields/base-fields.yml b/testdata/package/default-pipeline/0.0.2/dataset/foo/fields/base-fields.yml new file mode 100644 index 000000000..1fb7d7b3b --- /dev/null +++ b/testdata/package/default-pipeline/0.0.2/dataset/foo/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: stream.type + type: constant_keyword + description: > + Stream type +- name: stream.dataset + type: constant_keyword + description: > + Stream dataset. +- name: stream.namespace + type: constant_keyword + description: > + Stream namespace. +- name: "@timestamp" + type: date + description: > + Event timestamp. diff --git a/testdata/package/example/1.0.0/dataset/foo/fields/base-fields.yml b/testdata/package/example/1.0.0/dataset/foo/fields/base-fields.yml new file mode 100644 index 000000000..1fb7d7b3b --- /dev/null +++ b/testdata/package/example/1.0.0/dataset/foo/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: stream.type + type: constant_keyword + description: > + Stream type +- name: stream.dataset + type: constant_keyword + description: > + Stream dataset. +- name: stream.namespace + type: constant_keyword + description: > + Stream namespace. +- name: "@timestamp" + type: date + description: > + Event timestamp. diff --git a/testdata/package/multiple-false/0.0.1/dataset/foo/fields/base-fields.yml b/testdata/package/multiple-false/0.0.1/dataset/foo/fields/base-fields.yml new file mode 100644 index 000000000..1fb7d7b3b --- /dev/null +++ b/testdata/package/multiple-false/0.0.1/dataset/foo/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: stream.type + type: constant_keyword + description: > + Stream type +- name: stream.dataset + type: constant_keyword + description: > + Stream dataset. +- name: stream.namespace + type: constant_keyword + description: > + Stream namespace. +- name: "@timestamp" + type: date + description: > + Event timestamp. diff --git a/testdata/package/no_stream_configs/1.0.0/dataset/log/fields/base-fields.yml b/testdata/package/no_stream_configs/1.0.0/dataset/log/fields/base-fields.yml new file mode 100644 index 000000000..1fb7d7b3b --- /dev/null +++ b/testdata/package/no_stream_configs/1.0.0/dataset/log/fields/base-fields.yml @@ -0,0 +1,16 @@ +- name: stream.type + type: constant_keyword + description: > + Stream type +- name: stream.dataset + type: constant_keyword + description: > + Stream dataset. +- name: stream.namespace + type: constant_keyword + description: > + Stream namespace. +- name: "@timestamp" + type: date + description: > + Event timestamp. diff --git a/testdata/public/package/datasources/1.0.0/dataset/examplelog1/fields/base-fields.yml b/testdata/public/package/datasources/1.0.0/dataset/examplelog1/fields/base-fields.yml index 9b6147211..1fb7d7b3b 100644 --- a/testdata/public/package/datasources/1.0.0/dataset/examplelog1/fields/base-fields.yml +++ b/testdata/public/package/datasources/1.0.0/dataset/examplelog1/fields/base-fields.yml @@ -1,4 +1,3 @@ - - name: stream.type type: constant_keyword description: > diff --git a/testdata/public/package/datasources/1.0.0/dataset/examplelog2/fields/base-fields.yml b/testdata/public/package/datasources/1.0.0/dataset/examplelog2/fields/base-fields.yml index 9b6147211..1fb7d7b3b 100644 --- a/testdata/public/package/datasources/1.0.0/dataset/examplelog2/fields/base-fields.yml +++ b/testdata/public/package/datasources/1.0.0/dataset/examplelog2/fields/base-fields.yml @@ -1,4 +1,3 @@ - - name: stream.type type: constant_keyword description: > diff --git a/testdata/public/package/datasources/1.0.0/dataset/examplemetric/fields/base-fields.yml b/testdata/public/package/datasources/1.0.0/dataset/examplemetric/fields/base-fields.yml index 9b6147211..1fb7d7b3b 100644 --- a/testdata/public/package/datasources/1.0.0/dataset/examplemetric/fields/base-fields.yml +++ b/testdata/public/package/datasources/1.0.0/dataset/examplemetric/fields/base-fields.yml @@ -1,4 +1,3 @@ - - name: stream.type type: constant_keyword description: > diff --git a/testdata/public/package/default-pipeline/0.0.2/dataset/foo/fields/base-fields.yml b/testdata/public/package/default-pipeline/0.0.2/dataset/foo/fields/base-fields.yml index 9b6147211..1fb7d7b3b 100644 --- a/testdata/public/package/default-pipeline/0.0.2/dataset/foo/fields/base-fields.yml +++ b/testdata/public/package/default-pipeline/0.0.2/dataset/foo/fields/base-fields.yml @@ -1,4 +1,3 @@ - - name: stream.type type: constant_keyword description: > diff --git a/testdata/public/package/example/1.0.0/dataset/foo/fields/base-fields.yml b/testdata/public/package/example/1.0.0/dataset/foo/fields/base-fields.yml index 9b6147211..1fb7d7b3b 100644 --- a/testdata/public/package/example/1.0.0/dataset/foo/fields/base-fields.yml +++ b/testdata/public/package/example/1.0.0/dataset/foo/fields/base-fields.yml @@ -1,4 +1,3 @@ - - name: stream.type type: constant_keyword description: > diff --git a/testdata/public/package/multiple-false/0.0.1/dataset/foo/fields/base-fields.yml b/testdata/public/package/multiple-false/0.0.1/dataset/foo/fields/base-fields.yml index 9b6147211..1fb7d7b3b 100644 --- a/testdata/public/package/multiple-false/0.0.1/dataset/foo/fields/base-fields.yml +++ b/testdata/public/package/multiple-false/0.0.1/dataset/foo/fields/base-fields.yml @@ -1,4 +1,3 @@ - - name: stream.type type: constant_keyword description: > diff --git a/testdata/public/package/no_stream_configs/1.0.0/dataset/log/fields/base-fields.yml b/testdata/public/package/no_stream_configs/1.0.0/dataset/log/fields/base-fields.yml index 9b6147211..1fb7d7b3b 100644 --- a/testdata/public/package/no_stream_configs/1.0.0/dataset/log/fields/base-fields.yml +++ b/testdata/public/package/no_stream_configs/1.0.0/dataset/log/fields/base-fields.yml @@ -1,4 +1,3 @@ - - name: stream.type type: constant_keyword description: >