Refactor ES access with new es client (#4900)

* refact with new es client * add FTIndexUtilsTest * remove useless code * enbale limit and timeout * fix ut * address some comments
vesoft-inc · Dec 8, 2022 · b8e6101 · b8e6101
1 parent 55212e2
commit b8e6101
Show file tree

Hide file tree

Showing 53 changed files with 886 additions and 1,816 deletions.
diff --git a/src/common/expression/TextSearchExpression.h b/src/common/expression/TextSearchExpression.h
@@ -85,8 +85,8 @@ class TextSearchArgument final {
   std::string val_;
   std::string op_;
   int32_t fuzziness_{-2};
-  int32_t limit_{100};
-  int32_t timeout_{200};
+  int32_t limit_{-1};
+  int32_t timeout_{-1};
 };
 
 class TextSearchExpression : public Expression {

diff --git a/src/common/expression/test/CMakeLists.txt b/src/common/expression/test/CMakeLists.txt
@@ -33,7 +33,7 @@ set(expression_test_common_libs
     $<TARGET_OBJECTS:file_based_cluster_id_man_obj>
     $<TARGET_OBJECTS:process_obj>
     $<TARGET_OBJECTS:time_utils_obj>
-    $<TARGET_OBJECTS:ft_es_graph_adapter_obj>
+    $<TARGET_OBJECTS:es_adapter_obj>
     $<TARGET_OBJECTS:ws_common_obj>
     $<TARGET_OBJECTS:version_obj>
     $<TARGET_OBJECTS:graph_session_obj>
@@ -47,6 +47,7 @@ set(expression_test_common_libs
     $<TARGET_OBJECTS:ssl_obj>
     $<TARGET_OBJECTS:memory_obj>
     $<TARGET_OBJECTS:gc_obj>
+    $<TARGET_OBJECTS:http_client_obj>
 )
 
 
@@ -95,6 +96,7 @@ nebula_add_test(
         gtest
         ${THRIFT_LIBRARIES}
         ${PROXYGEN_LIBRARIES}
+        curl
 )
 
 nebula_add_executable(
@@ -110,6 +112,7 @@ nebula_add_executable(
         boost_regex
         ${THRIFT_LIBRARIES}
         ${PROXYGEN_LIBRARIES}
+        curl
 )
 
 nebula_add_executable(

diff --git a/src/common/http/HttpClient.cpp b/src/common/http/HttpClient.cpp
@@ -31,29 +31,61 @@ HttpResponse HttpClient::get(const std::string& url) {
 }
 
 HttpResponse HttpClient::get(const std::string& url, const std::vector<std::string>& header) {
-  return sendRequest(url, "GET", header, "");
+  return sendRequest(url, "GET", header, "", "", "");
+}
+
+HttpResponse HttpClient::get(const std::string& url,
+                             const std::vector<std::string>& header,
+                             const std::string& username,
+                             const std::string& password) {
+  return sendRequest(url, "GET", header, "", username, password);
 }
 
 HttpResponse HttpClient::post(const std::string& url,
                               const std::vector<std::string>& header,
                               const std::string& body) {
-  return sendRequest(url, "POST", header, body);
+  return sendRequest(url, "POST", header, body, "", "");
+}
+
+HttpResponse HttpClient::post(const std::string& url,
+                              const std::vector<std::string>& header,
+                              const std::string& body,
+                              const std::string& username,
+                              const std::string& password) {
+  return sendRequest(url, "POST", header, body, username, password);
 }
 
 HttpResponse HttpClient::delete_(const std::string& url, const std::vector<std::string>& header) {
-  return sendRequest(url, "DELETE", header, "");
+  return sendRequest(url, "DELETE", header, "", "", "");
+}
+
+HttpResponse HttpClient::delete_(const std::string& url,
+                                 const std::vector<std::string>& header,
+                                 const std::string& username,
+                                 const std::string& password) {
+  return sendRequest(url, "DELETE", header, "", username, password);
 }
 
 HttpResponse HttpClient::put(const std::string& url,
                              const std::vector<std::string>& header,
                              const std::string& body) {
-  return sendRequest(url, "PUT", header, body);
+  return sendRequest(url, "PUT", header, body, "", "");
+}
+
+HttpResponse HttpClient::put(const std::string& url,
+                             const std::vector<std::string>& header,
+                             const std::string& body,
+                             const std::string& username,
+                             const std::string& password) {
+  return sendRequest(url, "PUT", header, body, username, password);
 }
 
 HttpResponse HttpClient::sendRequest(const std::string& url,
                                      const std::string& method,
                                      const std::vector<std::string>& header,
-                                     const std::string& body) {
+                                     const std::string& body,
+                                     const std::string& username,
+                                     const std::string& password) {
   CurlHandle::instance();
   HttpResponse resp;
   CURL* curl = curl_easy_init();
@@ -67,6 +99,7 @@ HttpResponse HttpClient::sendRequest(const std::string& url,
   setRespHeader(curl, resp.header);
   setRespBody(curl, resp.body);
   setTimeout(curl);
+  setAuth(curl, username, password);
   resp.curlCode = curl_easy_perform(curl);
   if (resp.curlCode != 0) {
     resp.curlMessage = std::string(curl_easy_strerror(resp.curlCode));
@@ -112,6 +145,15 @@ void HttpClient::setTimeout(CURL* curl) {
   curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3);
 }
 
+void HttpClient::setAuth(CURL* curl, const std::string& username, const std::string& password) {
+  if (!username.empty()) {
+    curl_easy_setopt(curl, CURLOPT_USERNAME, username.c_str());
+    if (!password.empty()) {
+      curl_easy_setopt(curl, CURLOPT_PASSWORD, password.c_str());
+    }
+  }
+}
+
 size_t HttpClient::onWriteData(void* ptr, size_t size, size_t nmemb, void* stream) {
   if (ptr == nullptr || size == 0) {
     return 0;

diff --git a/src/common/http/HttpClient.h b/src/common/http/HttpClient.h
@@ -33,33 +33,63 @@ struct HttpResponse {
 class HttpClient {
  public:
   static HttpClient& instance();
+
   virtual ~HttpClient() = default;
+
   virtual HttpResponse get(const std::string& url);
+
   virtual HttpResponse get(const std::string& url, const std::vector<std::string>& headers);
 
+  virtual HttpResponse get(const std::string& url,
+                           const std::vector<std::string>& headers,
+                           const std::string& username,
+                           const std::string& password);
+
   virtual HttpResponse post(const std::string& url,
                             const std::vector<std::string>& headers,
                             const std::string& body);
+
+  virtual HttpResponse post(const std::string& url,
+                            const std::vector<std::string>& headers,
+                            const std::string& body,
+                            const std::string& username,
+                            const std::string& password);
+
   virtual HttpResponse delete_(const std::string& url, const std::vector<std::string>& headers);
+
+  virtual HttpResponse delete_(const std::string& url,
+                               const std::vector<std::string>& headers,
+                               const std::string& username,
+                               const std::string& password);
+
   virtual HttpResponse put(const std::string& url,
                            const std::vector<std::string>& headers,
                            const std::string& body);
 
+  virtual HttpResponse put(const std::string& url,
+                           const std::vector<std::string>& headers,
+                           const std::string& body,
+                           const std::string& username,
+                           const std::string& password);
+
  protected:
   HttpClient() = default;
 
  private:
   static HttpResponse sendRequest(const std::string& url,
                                   const std::string& method,
                                   const std::vector<std::string>& headers,
-                                  const std::string& body);
+                                  const std::string& body,
+                                  const std::string& username,
+                                  const std::string& password);
   static void setUrl(CURL* curl, const std::string& url);
   static void setMethod(CURL* curl, const std::string& method);
   static curl_slist* setHeaders(CURL* curl, const std::vector<std::string>& headers);
 
   static void setRespBody(CURL* curl, const std::string& body);
   static void setRespHeader(CURL* curl, const std::string& header);
   static void setTimeout(CURL* curl);
+  static void setAuth(CURL* curl, const std::string& username, const std::string& password);
   static size_t onWriteData(void* ptr, size_t size, size_t nmemb, void* stream);
 };
 

diff --git a/src/common/plugin/fulltext/CMakeLists.txt b/src/common/plugin/fulltext/CMakeLists.txt
@@ -2,16 +2,6 @@
 #
 # This source code is licensed under Apache 2.0 License.
 
-nebula_add_library(
-    ft_es_graph_adapter_obj OBJECT
-    elasticsearch/ESGraphAdapter.cpp
-)
-
-nebula_add_library(
-    ft_es_storage_adapter_obj OBJECT
-    elasticsearch/ESStorageAdapter.cpp
-)
-
 nebula_add_library(
     es_adapter_obj OBJECT
     elasticsearch/ESAdapter.cpp

diff --git a/src/common/plugin/fulltext/FTGraphAdapter.h b/src/common/plugin/fulltext/FTGraphAdapter.h
diff --git a/src/common/plugin/fulltext/FTStorageAdapter.h b/src/common/plugin/fulltext/FTStorageAdapter.h