iter

martijnvg · Oct 20, 2024 · 4a01d14 · 4a01d14
1 parent e77db0b
commit 4a01d14
Show file tree

Hide file tree

Showing 4 changed files with 107 additions and 54 deletions.
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java
@@ -117,8 +117,8 @@ private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, Sourc
         }
 
         // Assumes synthetic source only
-        private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String additionalField) {
-            this(fetcher, lookup, NEEDS_SOURCE_AND_IGNORED_SOURCE.merge(new StoredFieldsSpec(true, false, Set.of(additionalField))));
+        private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, Set<String> requiredStoredFields) {
+            this(fetcher, lookup, NEEDS_SOURCE_AND_IGNORED_SOURCE.merge(new StoredFieldsSpec(true, false, requiredStoredFields)));
         }
 
         @Override
@@ -208,8 +208,8 @@ public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, Sou
             super(fetcher, lookup, sourceMode);
         }
 
-        public BytesRefsBlockLoader(SourceValueFetcher fetcher, LeafIteratorLookup lookup, String originalFieldName) {
-            super(fetcher, lookup, originalFieldName);
+        public BytesRefsBlockLoader(SourceValueFetcher fetcher, LeafIteratorLookup lookup, Set<String> requiredStoredFields) {
+            super(fetcher, lookup, requiredStoredFields);
         }
 
         @Override

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -83,6 +83,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.IntPredicate;
 
 import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
@@ -1024,12 +1025,20 @@ protected String delegatingTo() {
                 return null;
             }
             SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
+            // TODO push this logic to keyword field mapper (which is the delegate) (this should fix the
             var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
             if (sourceMode == SourceFieldMapper.Mode.SYNTHETIC && syntheticSourceDelegate().ignoreAbove() != Integer.MAX_VALUE) {
                 // Can't just read the original field, because only when ignore_above threshold is exceeded,
                 // then this stored fields gets added. In other cases this field does not exist. But we do need to include the field name:
                 String originalName = syntheticSourceDelegate().name() + "._original";
-                return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext), originalName);
+                Set<String> requiredStoredFields;
+                if (syntheticSourceDelegate.isStored()) {
+                    // in case field is stored and ignore_above is configured:
+                    requiredStoredFields = Set.of(syntheticSourceDelegate().name(), originalName);
+                } else {
+                    requiredStoredFields = Set.of(originalName);
+                }
+                return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext), requiredStoredFields);
             } else {
                 return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext), sourceMode);
             }

diff --git a/...rc/yamlRestTest/resources/rest-api-spec/test/50_esql_synthetic_source_disabled_fields.yml b/...rc/yamlRestTest/resources/rest-api-spec/test/50_esql_synthetic_source_disabled_fields.yml
@@ -305,7 +305,7 @@ teardown:
   - match: {values.5.0: null}
 
 ---
-"message field keyword multi-field with ignore_above":
+"message field with keyword multi-field with ignore_above":
   - do:
       indices.create:
         index: my-index2
@@ -319,25 +319,6 @@ teardown:
                 type: date
               host.name:
                 type: keyword
-              agent_id:
-                type: keyword
-                doc_values: false
-                store: false
-              process_id:
-                type: integer
-                doc_values: false
-                store: false
-              http_method:
-                type: keyword
-                doc_values: false
-                store: false
-              is_https:
-                type: boolean
-                doc_values: false
-                store: false
-              location:
-                type: geo_point
-                doc_values: false
                 store: false
               message:
                 type: text
@@ -353,17 +334,17 @@ teardown:
         refresh: true
         body:
           - { "index": { } }
-          - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "No, I am your father." }
+          - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "message": "No, I am your father." }
           - { "index": { } }
-          - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "Do. Or do not. There is no try." }
+          - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "message": "Do. Or do not. There is no try." }
           - { "index": { } }
-          - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "May the force be with you." }
+          - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "message": "May the force be with you." }
           - { "index": { } }
-          - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "is_https": true, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "I find your lack of faith disturbing." }
+          - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "message": "I find your lack of faith disturbing." }
           - { "index": { } }
-          - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "Wars not make one great." }
+          - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "message": "Wars not make one great." }
           - { "index": { } }
-          - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "That's no moon. It's a space station." }
+          - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "message": "That's no moon. It's a space station." }
 
   - do:
       esql.query:
@@ -372,32 +353,18 @@ teardown:
 
   - match: {columns.0.name: "@timestamp"}
   - match: {columns.0.type: "date"}
-  - match: {columns.1.name: "agent_id"}
+  - match: {columns.1.name: "host.name"}
   - match: {columns.1.type: "keyword"}
-  - match: {columns.2.name: "host.name"}
-  - match: {columns.2.type: "keyword"}
-  - match: {columns.3.name: "http_method" }
-  - match: {columns.3.type: "keyword" }
-  - match: {columns.4.name: "is_https"}
-  - match: {columns.4.type: "boolean"}
-  - match: {columns.5.name: "location"}
-  - match: {columns.5.type: "geo_point"}
-  - match: {columns.6.name: "message"}
-  - match: {columns.6.type: "text"}
-  - match: {columns.7.name: "message.raw"}
-  - match: {columns.7.type: "keyword"}
-  - match: {columns.8.name: "process_id"}
-  - match: {columns.8.type: "integer"}
+  - match: {columns.2.name: "message"}
+  - match: {columns.2.type: "text"}
+  - match: {columns.3.name: "message.raw"}
+  - match: {columns.3.type: "keyword"}
 
   - match: {values.0.0: "2024-02-12T10:31:00.000Z"}
-  - match: {values.0.1: "yoda"}
-  - match: {values.0.2: "bar"}
-  - match: {values.0.3: "PUT"}
-  - match: {values.0.4: false}
-  - match: {values.0.5: "POINT (-74.006 40.7128)"}
-  - match: {values.0.6: "Do. Or do not. There is no try."}
-#  - match: {values.0.7: "Do. Or do not. There is no try."}
-  - match: {values.0.8: 102}
+  - match: {values.0.1: "bar"}
+  - match: {values.0.2: "Do. Or do not. There is no try."}
+  # TODO: this should be fixed:
+#  - match: {values.0.3: "Do. Or do not. There is no try."}
 
   - do:
       esql.query:

diff --git a/.../plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/51_esql_synthetic_source.yml b/.../plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/51_esql_synthetic_source.yml
@@ -175,3 +175,80 @@ teardown:
   - match: {values.3.0: "No, I am your father."}
   - match: {values.4.0: "May the force be with you."}
   - match: {values.5.0: "That's no moon. It's a space station."}
+
+---
+"message field with stored keyword multi-field with ignore_above":
+  - do:
+      indices.create:
+        index: my-index2
+        body:
+          settings:
+            index:
+              mode: logsdb
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              host.name:
+                type: keyword
+                store: false
+              message:
+                type: text
+                store: false
+                fields:
+                  raw:
+                    type: keyword
+                    store: true
+
+  - do:
+      bulk:
+        index: my-index2
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "message": "No, I am your father." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "message": "Do. Or do not. There is no try." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "message": "May the force be with you." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "message": "I find your lack of faith disturbing." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "message": "Wars not make one great." }
+          - { "index": { } }
+          - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "message": "That's no moon. It's a space station." }
+
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index2 | SORT host.name, @timestamp | LIMIT 1'
+
+  - match: {columns.0.name: "@timestamp"}
+  - match: {columns.0.type: "date"}
+  - match: {columns.1.name: "host.name"}
+  - match: {columns.1.type: "keyword"}
+  - match: {columns.2.name: "message"}
+  - match: {columns.2.type: "text"}
+  - match: {columns.3.name: "message.raw"}
+  - match: {columns.3.type: "keyword"}
+
+  - match: {values.0.0: "2024-02-12T10:31:00.000Z"}
+  - match: {values.0.1: "bar"}
+  - match: {values.0.2: "Do. Or do not. There is no try."}
+  - match: {values.0.3: "Do. Or do not. There is no try."}
+
+  - do:
+      esql.query:
+        body:
+          query: 'FROM my-index2 | SORT host.name, @timestamp | KEEP message | LIMIT 10'
+
+  - match: {columns.0.name: "message"}
+  - match: {columns.0.type: "text"}
+
+  - match: {values.0.0: "Do. Or do not. There is no try."}
+  - match: {values.1.0: "I find your lack of faith disturbing."}
+  - match: {values.2.0: "Wars not make one great."}
+  - match: {values.3.0: "No, I am your father."}
+  - match: {values.4.0: "May the force be with you."}
+  - match: {values.5.0: "That's no moon. It's a space station."}
+