From 2ca9758ec03b5d893247359853d05560c1494a83 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 2 Oct 2023 16:51:29 +0200
Subject: [PATCH] Require object_type for object fields (#628)

A field of type object without object_type probably assumes that the type should be keyword,
but Fleet is producing invalid mappings (see #624). So better to disallow these mappings to
remove ambiguities.
---
 code/go/pkg/validator/validator_test.go       |  1 +
 spec/changelog.yml                            |  3 +++
 .../data_stream/fields/fields.spec.yml        | 19 ++++++++++++++++++-
 .../data_stream/foo/fields/fields.yml         |  4 +++-
 test/packages/bad_fields/manifest.yml         |  9 ++++++---
 .../metadata_united/fields/fields.yml         |  1 +
 6 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go
index a1507b896..1607f08e7 100644
--- a/code/go/pkg/validator/validator_test.go
+++ b/code/go/pkg/validator/validator_test.go
@@ -136,6 +136,7 @@ func TestValidateFile(t *testing.T) {
 			[]string{
 				`field 0.type: 0.type must be one of the following: "aggregate_metric_double", "alias", "histogram", "constant_keyword", "text", "match_only_text", "keyword", "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "date", "date_nanos", "boolean", "binary", "integer_range", "float_range", "long_range", "double_range", "date_range", "ip_range", "group", "geo_point", "object", "ip", "nested", "flattened", "wildcard", "version", "unsigned_long"`,
 				`field "my_custom_date" of type keyword can't set date_format. date_format is allowed for date field type only`,
+				`field 2: object_type is required`,
 			},
 		},
 		"deploy_custom_agent_invalid_property": {
diff --git a/spec/changelog.yml b/spec/changelog.yml
index 2a571fb09..57b8b7c65 100644
--- a/spec/changelog.yml
+++ b/spec/changelog.yml
@@ -28,6 +28,9 @@
   - description: Make exception for legacy input controls
     type: enhancement
     link: https://github.com/elastic/package-spec/pull/619
+  - description: object_type is mandatory when object type is used
+    type: bugfix
+    link: https://github.com/elastic/package-spec/pull/628
 - version: 2.13.1-next
   changes:
   - description: Prepare for next version
diff --git a/spec/integration/data_stream/fields/fields.spec.yml b/spec/integration/data_stream/fields/fields.spec.yml
index 3c3a2e02e..385fa3796 100644
--- a/spec/integration/data_stream/fields/fields.spec.yml
+++ b/spec/integration/data_stream/fields/fields.spec.yml
@@ -46,11 +46,14 @@ spec:
         description: >
           Name of field. Names containing dots are automatically split into
           sub-fields.
+          Names with wildcards generate dynamic mappings.
         type: string
         pattern: '^[\-*_\/@A-Za-z0-9]+(\.[\-*_\/@A-Za-z0-9]+)*$'
 
       type:
-        description: Datatype of field
+        description: >
+          Datatype of field. If the type is set to object, a dynamic mapping is created. In this case, if the name doesn't
+          contain any wildcard, the wildcard is added as the last segment of the path.
         type: string
         enum:
         - aggregate_metric_double
@@ -521,11 +524,25 @@ spec:
           required:
             - type
             - object_type
+      - if:
+          properties:
+            type:
+              const: object
+          required:
+            - type
+        then:
+          required:
+            - object_type
+
     required:
     - name
 
 # JSON patches for newer versions should be placed on top
 versions:
+  - before: 3.0.0
+    patch:
+      - op: remove
+        path: "/items/allOf/6" # removing requirement of object_type when type is object
   - before: 2.10.0
     patch:
       - op: remove
diff --git a/test/packages/bad_fields/data_stream/foo/fields/fields.yml b/test/packages/bad_fields/data_stream/foo/fields/fields.yml
index 375b25c18..c4351bec7 100644
--- a/test/packages/bad_fields/data_stream/foo/fields/fields.yml
+++ b/test/packages/bad_fields/data_stream/foo/fields/fields.yml
@@ -2,4 +2,6 @@
   type: array
 - name: my_custom_date
   type: keyword
-  date_format: yyyy-MM-dd
\ No newline at end of file
+  date_format: yyyy-MM-dd
+- name: object_without_object_type.*
+  type: object
diff --git a/test/packages/bad_fields/manifest.yml b/test/packages/bad_fields/manifest.yml
index b7cea1738..030c2e1a3 100644
--- a/test/packages/bad_fields/manifest.yml
+++ b/test/packages/bad_fields/manifest.yml
@@ -1,4 +1,4 @@
-format_version: 2.7.0
+format_version: 3.0.0
 name: bad_fields
 title: "Bad Fields"
 version: 0.0.1
@@ -9,8 +9,10 @@ type: integration
 categories:
   - custom
 conditions:
-  kibana.version: "^8.4.1"
-  elastic.subscription: "basic"
+  kibana:
+    version: "^8.4.1"
+  elastic:
+    subscription: "basic"
 screenshots:
   - src: /img/sample-screenshot.png
     title: Sample screenshot
@@ -30,4 +32,5 @@ policy_templates:
         title: Collect sample logs from instances
         description: Collecting sample logs
 owner:
+  type: elastic
   github: elastic/ecosystem
diff --git a/test/packages/good_v3/elasticsearch/transform/metadata_united/fields/fields.yml b/test/packages/good_v3/elasticsearch/transform/metadata_united/fields/fields.yml
index df1ad9e80..c54d2ff36 100644
--- a/test/packages/good_v3/elasticsearch/transform/metadata_united/fields/fields.yml
+++ b/test/packages/good_v3/elasticsearch/transform/metadata_united/fields/fields.yml
@@ -356,4 +356,5 @@
           type: date
         - name: user_provided_metadata
           type: object
+          object_type: keyword
           enabled: false