Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor ES access with new es client #4900

Merged
merged 11 commits into from
Dec 8, 2022
Merged

Refactor ES access with new es client #4900

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8613c25
refact with new es client
cangfengzhs Dec 1, 2022
f0699e3
add FTIndexUtilsTest
cangfengzhs Dec 3, 2022
bc43304
remove useless code
cangfengzhs Dec 3, 2022
72a4428
Merge branch 'master' into refactor-es
cangfengzhs Dec 5, 2022
56bcbb9
enbale limit and timeout
cangfengzhs Dec 5, 2022
819dd98
fix ut
cangfengzhs Dec 5, 2022
ab1b16d
Merge branch 'master' into refactor-es
cangfengzhs Dec 6, 2022
bfcaecb
address some comments
cangfengzhs Dec 8, 2022
bb1bf97
Merge branch 'master' into refactor-es
cangfengzhs Dec 8, 2022
0f6320f
Merge branch 'master' into refactor-es
cangfengzhs Dec 8, 2022
1cc474f
Merge branch 'master' into refactor-es
cangfengzhs Dec 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/common/expression/TextSearchExpression.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,8 @@ class TextSearchArgument final {
std::string val_;
std::string op_;
int32_t fuzziness_{-2};
int32_t limit_{100};
int32_t timeout_{200};
int32_t limit_{-1};
int32_t timeout_{-1};
};

class TextSearchExpression : public Expression {
Expand Down
5 changes: 4 additions & 1 deletion src/common/expression/test/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ set(expression_test_common_libs
$<TARGET_OBJECTS:file_based_cluster_id_man_obj>
$<TARGET_OBJECTS:process_obj>
$<TARGET_OBJECTS:time_utils_obj>
$<TARGET_OBJECTS:ft_es_graph_adapter_obj>
$<TARGET_OBJECTS:es_adapter_obj>
$<TARGET_OBJECTS:ws_common_obj>
$<TARGET_OBJECTS:version_obj>
$<TARGET_OBJECTS:graph_session_obj>
Expand All @@ -47,6 +47,7 @@ set(expression_test_common_libs
$<TARGET_OBJECTS:ssl_obj>
$<TARGET_OBJECTS:memory_obj>
$<TARGET_OBJECTS:gc_obj>
$<TARGET_OBJECTS:http_client_obj>
)


Expand Down Expand Up @@ -95,6 +96,7 @@ nebula_add_test(
gtest
${THRIFT_LIBRARIES}
${PROXYGEN_LIBRARIES}
curl
)

nebula_add_executable(
Expand All @@ -110,6 +112,7 @@ nebula_add_executable(
boost_regex
${THRIFT_LIBRARIES}
${PROXYGEN_LIBRARIES}
curl
)

nebula_add_executable(
Expand Down
52 changes: 47 additions & 5 deletions src/common/http/HttpClient.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,29 +31,61 @@ HttpResponse HttpClient::get(const std::string& url) {
}

HttpResponse HttpClient::get(const std::string& url, const std::vector<std::string>& header) {
return sendRequest(url, "GET", header, "");
return sendRequest(url, "GET", header, "", "", "");
}

HttpResponse HttpClient::get(const std::string& url,
const std::vector<std::string>& header,
const std::string& username,
const std::string& password) {
return sendRequest(url, "GET", header, "", username, password);
}

HttpResponse HttpClient::post(const std::string& url,
const std::vector<std::string>& header,
const std::string& body) {
return sendRequest(url, "POST", header, body);
return sendRequest(url, "POST", header, body, "", "");
}

HttpResponse HttpClient::post(const std::string& url,
const std::vector<std::string>& header,
const std::string& body,
const std::string& username,
const std::string& password) {
return sendRequest(url, "POST", header, body, username, password);
}

HttpResponse HttpClient::delete_(const std::string& url, const std::vector<std::string>& header) {
return sendRequest(url, "DELETE", header, "");
return sendRequest(url, "DELETE", header, "", "", "");
}

HttpResponse HttpClient::delete_(const std::string& url,
const std::vector<std::string>& header,
const std::string& username,
const std::string& password) {
return sendRequest(url, "DELETE", header, "", username, password);
}

HttpResponse HttpClient::put(const std::string& url,
const std::vector<std::string>& header,
const std::string& body) {
return sendRequest(url, "PUT", header, body);
return sendRequest(url, "PUT", header, body, "", "");
}

HttpResponse HttpClient::put(const std::string& url,
const std::vector<std::string>& header,
const std::string& body,
const std::string& username,
const std::string& password) {
return sendRequest(url, "PUT", header, body, username, password);
}

HttpResponse HttpClient::sendRequest(const std::string& url,
const std::string& method,
const std::vector<std::string>& header,
const std::string& body) {
const std::string& body,
const std::string& username,
const std::string& password) {
CurlHandle::instance();
HttpResponse resp;
CURL* curl = curl_easy_init();
Expand All @@ -67,6 +99,7 @@ HttpResponse HttpClient::sendRequest(const std::string& url,
setRespHeader(curl, resp.header);
setRespBody(curl, resp.body);
setTimeout(curl);
setAuth(curl, username, password);
resp.curlCode = curl_easy_perform(curl);
if (resp.curlCode != 0) {
resp.curlMessage = std::string(curl_easy_strerror(resp.curlCode));
Expand Down Expand Up @@ -112,6 +145,15 @@ void HttpClient::setTimeout(CURL* curl) {
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3);
}

void HttpClient::setAuth(CURL* curl, const std::string& username, const std::string& password) {
if (!username.empty()) {
curl_easy_setopt(curl, CURLOPT_USERNAME, username.c_str());
if (!password.empty()) {
curl_easy_setopt(curl, CURLOPT_PASSWORD, password.c_str());
}
}
}

size_t HttpClient::onWriteData(void* ptr, size_t size, size_t nmemb, void* stream) {
if (ptr == nullptr || size == 0) {
return 0;
Expand Down
32 changes: 31 additions & 1 deletion src/common/http/HttpClient.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,33 +33,63 @@ struct HttpResponse {
class HttpClient {
public:
static HttpClient& instance();

virtual ~HttpClient() = default;

virtual HttpResponse get(const std::string& url);

virtual HttpResponse get(const std::string& url, const std::vector<std::string>& headers);

virtual HttpResponse get(const std::string& url,
const std::vector<std::string>& headers,
const std::string& username,
const std::string& password);

virtual HttpResponse post(const std::string& url,
const std::vector<std::string>& headers,
const std::string& body);

virtual HttpResponse post(const std::string& url,
const std::vector<std::string>& headers,
const std::string& body,
const std::string& username,
const std::string& password);

virtual HttpResponse delete_(const std::string& url, const std::vector<std::string>& headers);

virtual HttpResponse delete_(const std::string& url,
const std::vector<std::string>& headers,
const std::string& username,
const std::string& password);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not store the username and password in HttpClient to avoid to pass them in each function?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the beginning, httpclient was designed to be stateless, and it only has some static functions. Later, in order to facilitate the use of gmock in the test, we changed it to non static, but the httpclient is still stateless.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In addition, username, password and url are bound, but httpclient and url are not bound. Therefore, it is not reasonable to put username and password in httpclient. Maybe you have other better suggestions?


virtual HttpResponse put(const std::string& url,
const std::vector<std::string>& headers,
const std::string& body);

virtual HttpResponse put(const std::string& url,
const std::vector<std::string>& headers,
const std::string& body,
const std::string& username,
const std::string& password);

protected:
HttpClient() = default;

private:
static HttpResponse sendRequest(const std::string& url,
const std::string& method,
const std::vector<std::string>& headers,
const std::string& body);
const std::string& body,
const std::string& username,
const std::string& password);
static void setUrl(CURL* curl, const std::string& url);
static void setMethod(CURL* curl, const std::string& method);
static curl_slist* setHeaders(CURL* curl, const std::vector<std::string>& headers);

static void setRespBody(CURL* curl, const std::string& body);
static void setRespHeader(CURL* curl, const std::string& header);
static void setTimeout(CURL* curl);
static void setAuth(CURL* curl, const std::string& username, const std::string& password);
static size_t onWriteData(void* ptr, size_t size, size_t nmemb, void* stream);
};

Expand Down
10 changes: 0 additions & 10 deletions src/common/plugin/fulltext/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,6 @@
#
# This source code is licensed under Apache 2.0 License.

nebula_add_library(
ft_es_graph_adapter_obj OBJECT
elasticsearch/ESGraphAdapter.cpp
)

nebula_add_library(
ft_es_storage_adapter_obj OBJECT
elasticsearch/ESStorageAdapter.cpp
)

nebula_add_library(
es_adapter_obj OBJECT
elasticsearch/ESAdapter.cpp
Expand Down
58 changes: 0 additions & 58 deletions src/common/plugin/fulltext/FTGraphAdapter.h
View file
Open in desktop

This file was deleted.

32 changes: 0 additions & 32 deletions src/common/plugin/fulltext/FTStorageAdapter.h
View file
Open in desktop

This file was deleted.

Loading