Disallow unknown properties on fields definitions

code/go/pkg/validator/validator_test.go

@@ -77,6 +77,14 @@ func TestValidateFile(t *testing.T) {
 				"field \"example.agent.call_duration\" of type histogram can't be a dimension, allowed types for dimensions: constant_keyword, keyword, long, integer, short, byte, double, float, half_float, scaled_float, unsigned_long, ip",
 			},
 		},
+		"bad_field_typos": {
+			"data_stream/example/fields/fields.yml",
+			[]string{
+				"field 0: Additional property descriiption is not allowed",
+				"field 0: Additional property dimensions is not allowed",
+				"field 1: Additional property default_fields is not allowed",
+			},
+		},
 	}
 
 	for pkgName, test := range tests {

test/packages/bad_field_typos/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/0 # FIXME Replace with the real PR link

test/packages/bad_field_typos/data_stream/example/agent/stream/stream.yml.hbs

@@ -0,0 +1,6 @@
+metricsets: ["sample_metricset"]
+hosts:
+{{#each hosts}}
+  - {{this}}
+{{/each}}
+period: {{period}}

test/packages/bad_field_typos/data_stream/example/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/bad_field_typos/data_stream/example/fields/ecs.yml

@@ -0,0 +1,3 @@
+- name: container.id
+  external: ecs
+  dimension: true

test/packages/bad_field_typos/data_stream/example/fields/fields.yml

@@ -0,0 +1,11 @@
+- name: example.agent.id
+  descriiption: Agent ID
+  type: keyword
+  dimensions: true
+- name: example.agent.call_count
+  type: long
+  metric_type: counter
+  default_fields: false
+- name: example.agent.current_count
+  type: long
+  metric_type: gauge

test/packages/bad_field_typos/data_stream/example/manifest.yml

@@ -0,0 +1,17 @@
+title: "Example data stream for time series"
+type: metrics
+streams:
+  - input: sample/metrics
+    title: Sample metrics
+    description: Collect sample metrics
+    vars:
+      - name: period
+        type: text
+        title: Period
+        default: 10s
+
+elasticsearch:
+  index_template:
+    settings:
+      # Defaults to 16
+      index.mapping.dimension_fields.limit: 32

test/packages/bad_field_typos/docs/README.md

@@ -0,0 +1,5 @@
+# Package with time series
+
+This is a new integration created using the [elastic-package](https://github.com/elastic/elastic-package) tool.
+
+Consider using the README template file `_dev/build/docs/README.md`to generate a list of exported fields or include a sample event.

test/packages/bad_field_typos/manifest.yml

@@ -0,0 +1,33 @@
+format_version: 1.2.0
+name: time_series
+title: "Package with time series"
+version: 0.0.1
+license: basic
+description: "This is a package with time series data"
+type: integration
+categories:
+  - custom
+  - monitoring
+release: beta
+conditions:
+  kibana.version: "^7.16.0"
+screenshots:
+  - src: /img/sample-screenshot.png
+    title: Sample screenshot
+    size: 600x600
+    type: image/png
+icons:
+  - src: /img/sample-logo.svg
+    title: Sample logo
+    size: 32x32
+    type: image/svg+xml
+policy_templates:
+  - name: sample
+    title: Sample logs
+    description: Collect sample logs
+    inputs:
+      - type: logfile
+        title: Collect sample logs from instances
+        description: Collecting sample logs
+owner:
+  github: elastic/integrations-developer-experience

test/packages/good/data_stream/foo/fields/some_fields.yml

@@ -1,5 +1,6 @@
 - name: source
   title: Source
+  footnote: "Some detailed information about these fields"
   group: 2
   type: group
   fields:
@@ -26,9 +27,16 @@
   type: text
   description: A field with a pattern defined
   pattern: '^[a-zA-Z]$'
+  example: foobartext
 - name: aaa
   type: integer
   metric_type: gauge
 - name: vehicle_type
   type: constant_keyword
   value: truck
+- name: magic_number
+  type: integer
+  # TODO: Fix this: example: 42
+- name: location
+  type: alias
+  path: source.geo.location

test/packages/good/data_stream/k8s_data_stream/fields/base-fields.yml

@@ -21,4 +21,24 @@
       level: core
       type: keyword
       description: Region name.
-      ignore_above: 1024
+      ignore_above: 1024
+- name: host
+  title: Host
+  group: 2
+  type: group
+  fields:
+    - name: os.name
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      multi_fields:
+        - name: text
+          type: text
+          norms: false
+          default_field: false
+    - name: labels
+      level: extended
+      type: object
+      object_type: keyword
+      object_type_mapping_type: "*"
+      description: Image labels.

test/packages/good/data_stream/skipped_tests/fields/some_fields.yml

@@ -25,4 +25,4 @@
 - name: foobar
   type: text
   description: A field with a pattern defined
-  pattern: '^[a-zA-Z]$'
+  pattern: '^[a-zA-Z]$'

test/packages/time_series/data_stream/example/fields/fields.yml

@@ -7,3 +7,8 @@
 - name: example.agent.current_count
   type: long
   metric_type: gauge
+- name: example.agent.received.bytes
+  type: long
+  metric_type: gauge
+  unit: byte
+  format: bytes

versions/1/changelog.yml

@@ -10,6 +10,9 @@
   - description: Add kibana/csp-rule-template asset
     type: enhancement
     link: https://github.com/elastic/package-spec/pull/276
+  - description: Disallow unknown properties on fields definitions
+    type: breaking-change
+    link: https://github.com/elastic/package-spec/pull/281
 - version: 1.4.1
   changes:
   - description: ML model file name now matches the id of the model.

versions/1/data_stream/fields/fields.spec.yml

@@ -6,7 +6,7 @@ spec:
   type: array
   items:
     type: object
-    additionalProperties: true
+    additionalProperties: false
     properties:
       name:
         description: Name of field
@@ -51,12 +51,65 @@ spec:
         - wildcard
         - version
         - unsigned_long
+      object_type:
+        description: Datatype of elements in object field
+        type: string
+        enum:
+        - histogram
+        - text
+        - keyword
+        - long
+        - integer
+        - short
+        - byte
+        - double
+        - float
+        - half_float
+        - scaled_float
+        - date
+        - date_nanos
+        - boolean
+        - binary
+        - integer_range
+        - float_range
+        - long_range
+        - double_range
+        - date_range
+        - ip_range
+        - geo_point
+        - ip
+        - array
+        - flattened
+        - wildcard
+        - version
+        - unsigned_long
+      object_type_mapping_type:
+        description: Types in the source document that should be mapped to the type defined in object_type, it can be "*" to indicate all types.
+        type: string
+        examples:
+          - "double"
+          - "*"
+      group:
+        description: Group the field belongs to.
+        type: integer
+      level:
+        description: ECS level the field belongs to.
+        type: string
+        examples:
+          - core
       description:
-        description: Short description of field
+        description: Short description of field.
+        type: string
+      footnote:
+        description: Additional detains on the field.
         type: string
       value:
         description: The value to associate with a constant_keyword field.
         type: string
+      default_field:
+        description: Set to false if the field should not be installed as a default field in Kibana index patterns.
+        type: boolean
+        default: true
       metric_type:
         description: Metric type
         type: string
@@ -76,20 +129,43 @@ spec:
         - ms
         - micros
         - nanos
+      format:
+        description: Formatting used by Kibana to display this field
+        type: string
+        enum:
+          - percent
+          - bytes
+          - duration
+      path:
+        description: Target of alias field.
+        type: string # TODO: Add sematic validation to check that the referenced field exists.
       dimension:
         description: Declare a field as dimension of time series
         type: boolean
         default: false
+      example:
+        description: Examples for this field.
+        type: string
       pattern:
         description: Regular expression pattern of the field value
         type: string
         examples:
           - '^[a-zA-Z]$'
+      ignore_above:
+        description: Index only the first number of characters indicated by this option.
+        type: integer
       external:
         description: External source reference
         type: string
         enum:
           - ecs
+      norms:
+        description: Set to false to disable normalization factors used for scoring calculation.
+        type: boolean
+        default: true
+      multi_fields:
+        description: Additional mappings for this field.
+        $ref: "#"
       fields:
         description: Sub-fields, when type is group
         $ref: "#"     # JSON-schema syntax for pointing to the root of the schema