From a7e352efe21d8b47d93b68d0d3de608a28bac6df Mon Sep 17 00:00:00 2001 From: affonsoBQ <67347924+affonsoBQ@users.noreply.github.com> Date: Wed, 22 Jun 2022 18:14:28 -0700 Subject: [PATCH] [AD-758] - Performance investigation SELECT * TABLE and SELECT * VIRTUAL-TABLE (#84) This fixes some issues. Warning console messages while querying. The performance framework was not cleaning up bind_fetch vector. Split between bind and fetch time in the performance framework added default batch fetch size. --- src/odbc/src/connection.cpp | 4 +- src/odbc/src/query/data_query.cpp | 4 +- .../include/performance_test_runner.h | 2 + .../src/performance_test_runner.cpp | 45 ++++++++++++++++--- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/src/odbc/src/connection.cpp b/src/odbc/src/connection.cpp index d1970b16c..8f60dab59 100644 --- a/src/odbc/src/connection.cpp +++ b/src/odbc/src/connection.cpp @@ -660,9 +660,11 @@ std::string Connection::FormatMongoCppConnectionString( mongoConnectionString.append("@" + host); mongoConnectionString.append(":" + port); mongoConnectionString.append("/admin"); + mongoConnectionString.append("?authMechanism=SCRAM-SHA-1"); if (config_.IsTls()) { - mongoConnectionString.append("?tlsAllowInvalidHostnames=true"); + mongoConnectionString.append("&tlsAllowInvalidHostnames=true"); } + // tls configuration is handled using tls_options in connectionCPP // TODO handle the other DSN configuration // https://bitquill.atlassian.net/browse/AD-599 diff --git a/src/odbc/src/query/data_query.cpp b/src/odbc/src/query/data_query.cpp index 35d210dd1..2fd220f74 100644 --- a/src/odbc/src/query/data_query.cpp +++ b/src/odbc/src/query/data_query.cpp @@ -297,7 +297,9 @@ SqlResult::Type DataQuery::MakeRequestFetch() { for (auto const& stage : aggregateOperations) { pipeline.append_stage(bsoncxx::from_json(stage)); } - mongocxx::cursor cursor = collection.aggregate(pipeline); + auto options = mongocxx::options::aggregate{}; + options.batch_size(config.GetDefaultFetchSize()); + mongocxx::cursor cursor = collection.aggregate(pipeline, options); this->cursor_.reset(new DocumentDbCursor(cursor, columnMetadata, paths)); diff --git a/src/tests/performance/include/performance_test_runner.h b/src/tests/performance/include/performance_test_runner.h index 86d7aafe1..beed32217 100644 --- a/src/tests/performance/include/performance_test_runner.h +++ b/src/tests/performance/include/performance_test_runner.h @@ -91,6 +91,8 @@ struct TestCase { std::vector< long long > time_ms; // time for exec->bind->fetch combined std::vector< long long > time_exec_ms; std::vector< long long > time_bind_fetch_ms; + std::vector< long long > time_bind_ms; + std::vector< long long > time_fetch_ms; StatisticalInfo stat_info; StatisticalInfo stat_info_exec; StatisticalInfo stat_info_bind_fetch; diff --git a/src/tests/performance/src/performance_test_runner.cpp b/src/tests/performance/src/performance_test_runner.cpp index 9e024f1bd..f7d2c9190 100644 --- a/src/tests/performance/src/performance_test_runner.cpp +++ b/src/tests/performance/src/performance_test_runner.cpp @@ -406,6 +406,8 @@ void performance::PerformanceTestRunner::RecordExecBindFetch( SQLROWSETSIZE row_count; long long time_exec_ms; long long time_bind_fetch_ms; + long long time_bind_ms; + long long time_fetch_ms; std::vector< Col > cols; // Query @@ -450,7 +452,7 @@ void performance::PerformanceTestRunner::RecordExecBindFetch( } // Bind and fetch and record time - auto time_bind_fetch_start = std::chrono::steady_clock::now(); + auto time_bind_start = std::chrono::steady_clock::now(); for (size_t i = 0; i < static_cast< size_t >(total_columns); i++) { ret = SQLBindCol(*hstmt, static_cast< SQLUSMALLINT >(i + 1), SQL_C_CHAR, static_cast< SQLPOINTER >(&cols[i].data_dat[0]), @@ -462,18 +464,28 @@ void performance::PerformanceTestRunner::RecordExecBindFetch( return; // continue to next test case } } + auto time_bind_end = std::chrono::steady_clock::now(); + auto time_fetch_start = std::chrono::steady_clock::now(); while (SQLFetch(*hstmt) == SQL_SUCCESS) { row_count++; } + auto time_fetch_end = std::chrono::steady_clock::now(); - auto time_bind_fetch_end = std::chrono::steady_clock::now(); + // Store bind and fetch time + time_bind_ms = std::chrono::duration_cast< std::chrono::milliseconds >( + time_bind_end - time_bind_start) + .count(); + test_case.time_bind_ms.push_back(time_bind_ms); + + // Store bind and fetch time + time_fetch_ms = std::chrono::duration_cast< std::chrono::milliseconds >( + time_fetch_end - time_fetch_start) + .count(); + test_case.time_fetch_ms.push_back(time_fetch_ms); // Store bind and fetch time - time_bind_fetch_ms = - std::chrono::duration_cast< std::chrono::milliseconds >( - time_bind_fetch_end - time_bind_fetch_start) - .count(); + time_bind_fetch_ms = time_bind_ms + time_fetch_ms; test_case.time_bind_fetch_ms.push_back(time_bind_fetch_ms); test_case.time_ms.push_back(time_exec_ms + time_bind_fetch_ms); @@ -646,6 +658,22 @@ void performance::PerformanceTestRunner::ReportTime(const TestCase& test_case) { } std::cout << "\n\n"; + std::cout << "SQLBindCol time dump: "; + for (size_t i = 0; i < test_case.time_bind_ms.size(); i++) { + std::cout << test_case.time_bind_ms[i] << " ms"; + if (i != (test_case.time_bind_ms.size() - 1)) + std::cout << ", "; + } + std::cout << "\n\n"; + std::cout << "SQLFetch time dump: "; + for (size_t i = 0; i < test_case.time_fetch_ms.size(); i++) { + std::cout << test_case.time_fetch_ms[i] << " ms"; + if (i != (test_case.time_fetch_ms.size() - 1)) + std::cout << ", "; + } + + std::cout << "\n\n"; + } else if (_output_mode == 2) { std::cout << sync_min << test_case.stat_info_bind_fetch.min << " ms" << std::endl; @@ -981,6 +1009,11 @@ void performance::PerformanceTestRunner::RunPerformanceTestPlan() { test_case.test_name.clear(); test_case.time_ms.clear(); + test_case.time_exec_ms.clear(); + test_case.time_bind_fetch_ms.clear(); + test_case.time_bind_ms.clear(); + test_case.time_fetch_ms.clear(); + // Deallocate Statement Handle if (SQL_NULL_HSTMT != _hstmt) { CloseCursor(&_hstmt, true, true);