[fix](es catalog) fix issue with select and insert from es catalog co…

…re (#24318) Issue Number: close #24315 The root cause of this issue is that Elasticsearch's long type allows inserting floats and strings. Doris did not handle these cases when doing type conversion. The current strategy is to take the integer before the decimal point if a float or string is found.
apache · Sep 13, 2023 · 11afd32 · 11afd32
1 parent d5b490b
commit 11afd32
Show file tree

Hide file tree

Showing 14 changed files with 70 additions and 36 deletions.
diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp
@@ -347,7 +347,16 @@ template <typename T>
 Status insert_int_value(const rapidjson::Value& col, PrimitiveType type,
                         vectorized::IColumn* col_ptr, bool pure_doc_value, bool nullable) {
     if (col.IsNumber()) {
-        T value = (T)(sizeof(T) < 8 ? col.GetInt() : col.GetInt64());
+        T value;
+        // ES allows inserting float and double in int/long types.
+        // To parse these numbers in Doris, we direct cast them to int types.
+        if (col.IsDouble()) {
+            value = static_cast<T>(col.GetDouble());
+        } else if (col.IsFloat()) {
+            value = static_cast<T>(col.GetFloat());
+        } else {
+            value = (T)(sizeof(T) < 8 ? col.GetInt() : col.GetInt64());
+        }
         col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&value)), 0);
         return Status::OK();
     }
@@ -363,8 +372,14 @@ Status insert_int_value(const rapidjson::Value& col, PrimitiveType type,
     RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type);
 
     StringParser::ParseResult result;
-    const std::string& val = col.GetString();
-    size_t len = col.GetStringLength();
+    std::string val = col.GetString();
+    // ES allows inserting numbers and characters containing decimals in numeric types.
+    // To parse these numbers in Doris, we remove the decimals here.
+    size_t pos = val.find(".");
+    if (pos != std::string::npos) {
+        val = val.substr(0, pos);
+    }
+    size_t len = val.length();
     T v = StringParser::string_to_int<T>(val.c_str(), len, &result);
     RETURN_ERROR_IF_PARSING_FAILED(result, col, type);
 

diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1.json
@@ -7,6 +7,7 @@
   "test6": 1659931810000,
   "test7": 1659931810000,
   "test8": "2022-08-08T12:10:10Z",
+  "test9": 12345,
   "c_bool": [true, false, true, true],
   "c_byte": [1, -2, -3, 4],
   "c_short": [128, 129, -129, -130],
@@ -26,4 +27,4 @@
     {"name": "Andy", "age": 18},
     {"name": "Tim", "age": 28}
   ]
-}
+}
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1_es6.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1_es6.json
@@ -3,6 +3,7 @@
   "test2": "text#1",
   "test3": 3.14,
   "test4": "2022-08-08",
+  "test5": 12345,
   "c_bool": [true, false, true, true],
   "c_byte": [1, -2, -3, 4],
   "c_short": [128, 129, -129, -130],
@@ -22,4 +23,4 @@
     {"name": "Andy", "age": 18},
     {"name": "Tim", "age": 28}
   ]
-}
+}
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json
@@ -7,6 +7,7 @@
   "test6": 1660018210000,
   "test7": "2022-08-09 12:10:10",
   "test8": 1660018210000,
+  "test9": 2222.2,
   "c_bool": [true, false, true, true],
   "c_byte": [1, -2, -3, 4],
   "c_short": [128, 129, -129, -130],
@@ -26,4 +27,4 @@
     {"name": "Andy", "age": 18},
     {"name": "Tim", "age": 28}
   ]
-}
+}
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2_es6.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2_es6.json
@@ -3,6 +3,7 @@
   "test2": "text2",
   "test3": 4,
   "test4": "2022-08-08",
+  "test5": 2222.2,
   "c_bool": [true, false, true, true],
   "c_byte": [1, -2, -3, 4],
   "c_short": [128, 129, -129, -130],
@@ -22,4 +23,4 @@
     {"name": "Andy", "age": 18},
     {"name": "Tim", "age": 28}
   ]
-}
+}
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json
@@ -7,6 +7,7 @@
   "test6": 1660104610000,
   "test7": 1660104610000,
   "test8": "2022-08-10T12:10:10",
+  "test9": 3333.22,
   "c_bool": [true, false, true, true],
   "c_byte": [1, -2, -3, 4],
   "c_short": [128, 129, -129, -130],
@@ -26,4 +27,4 @@
     {"name": "Andy", "age": 18},
     {"name": "Tim", "age": 28}
   ]
-}
+}
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3_es6.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3_es6.json
@@ -3,6 +3,7 @@
   "test2": "text3_4*5",
   "test3": 5.0,
   "test4": "2022-08-08",
+  "test5": "3333.22",
   "c_bool": [true, false, true, true],
   "c_byte": [1, -2, -3, 4],
   "c_short": [128, 129, -129, -130],
@@ -22,4 +23,4 @@
     {"name": "Andy", "age": 18},
     {"name": "Tim", "age": 28}
   ]
-}
+}
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4.json
@@ -7,6 +7,7 @@
   "test6": 1660191010000,
   "test7": "2022-08-11 12:10:10",
   "test8": "2022-08-11T12:10:10+09:00",
+  "test9": "4444.22",
   "c_bool": [true, false, true, true],
   "c_byte": [1, -2, -3, 4],
   "c_short": [128, 129, -129, -130],
@@ -26,4 +27,4 @@
     {"name": "Andy", "age": 18},
     {"name": "Tim", "age": 28}
   ]
-}
+}
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json
@@ -24,6 +24,9 @@
         "test4": {
           "type": "date"
         },
+        "test5": {
+          "type": "long"
+        },
         "c_bool": {
           "type": "boolean"
         },
@@ -85,4 +88,4 @@
       }
     }
   }
-}
+}
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test2.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test2.json
@@ -27,6 +27,9 @@
         "test4": {
           "type": "date"
         },
+        "test5": {
+          "type": "long"
+        },
         "c_bool": {
           "type": "boolean"
         },
@@ -88,4 +91,4 @@
       }
     }
   }
-}
+}
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json
@@ -38,6 +38,9 @@
       "test8": {
         "type": "date"
       },
+      "test9": {
+        "type": "long"
+      },
       "c_bool": {
         "type": "boolean"
       },
@@ -98,4 +101,4 @@
       }
     }
   }
-}
+}
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test2.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test2.json
@@ -41,6 +41,9 @@
       "test8": {
         "type": "date"
       },
+      "test9": {
+        "type": "long"
+      },
       "c_bool": {
         "type": "boolean"
       },
@@ -101,4 +104,4 @@
       }
     }
   }
-}
+}