Add fetch_size for pagination support

src/IntegrationTests/CMakeLists.txt

@@ -24,6 +24,7 @@ set(INFO_ITEST "${CMAKE_CURRENT_SOURCE_DIR}/ITODBCInfo")
 set(RESULTS_ITEST "${CMAKE_CURRENT_SOURCE_DIR}/ITODBCResults")
 set(TABLEAU_QUERIES_ITEST "${CMAKE_CURRENT_SOURCE_DIR}/ITODBCTableauQueries")
 set(AWS_AUTH_ITEST "${CMAKE_CURRENT_SOURCE_DIR}/ITODBCAwsAuth")
+set(PAGINATION_ITEST "${CMAKE_CURRENT_SOURCE_DIR}/ITODBCPagination")
 
 # Projects to build
 add_subdirectory(${HELPER_ITEST})
@@ -34,4 +35,5 @@ add_subdirectory(${EXECUTION_ITEST})
 add_subdirectory(${INFO_ITEST})
 add_subdirectory(${RESULTS_ITEST})
 add_subdirectory(${TABLEAU_QUERIES_ITEST})
-add_subdirectory(${AWS_AUTH_ITEST})
+add_subdirectory(${AWS_AUTH_ITEST})
+add_subdirectory(${PAGINATION_ITEST})

src/IntegrationTests/ITODBCHelper/it_odbc_helper.h

@@ -47,7 +47,8 @@ std::vector< std::pair< std::wstring, std::wstring > > conn_str_pair = {
     {L"hostnameVerification", L"0"},
     {L"logLevel", L"0"},
     {L"logOutput", L"C:\\"},
-    {L"responseTimeout", L"10"}};
+    {L"responseTimeout", L"10"},
+    {L"fetchSize", L"0"}};
 
 std::wstring conn_string = []() {
     std::wstring temp;

src/IntegrationTests/ITODBCPagination/CMakeLists.txt

@@ -0,0 +1,30 @@
+#
+#   Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License").
+#   You may not use this file except in compliance with the License.
+#   A copy of the License is located at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#   or in the "license" file accompanying this file. This file is distributed
+#   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#   express or implied. See the License for the specific language governing
+#   permissions and limitations under the License.
+#
+
+project(itodbc_pagination)
+
+# Source, headers, and include dirs
+set(SOURCE_FILES test_odbc_pagination.cpp)
+include_directories(	${UT_HELPER} 
+			${IT_HELPER} 
+			${ODFEODBC_SRC} 
+			${VLD_SRC}	)
+
+# Generate executable
+add_executable(itodbc_pagination ${SOURCE_FILES})
+
+# Library dependencies
+target_link_libraries(itodbc_pagination odfesqlodbc itodbc_helper ut_helper gtest_main)
+target_compile_definitions(itodbc_pagination PUBLIC _UNICODE UNICODE)

src/IntegrationTests/ITODBCPagination/packages.config

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="Microsoft.googletest.v140.windesktop.msvcstl.static.rt-dyn" version="1.8.1" targetFramework="native" />
+</packages>

src/IntegrationTests/ITODBCPagination/pch.cpp

@@ -0,0 +1,21 @@
+/*
+ *   Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License").
+ *   You may not use this file except in compliance with the License.
+ *   A copy of the License is located at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file. This file is distributed
+ *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *   express or implied. See the License for the specific language governing
+ *   permissions and limitations under the License.
+ */
+
+//
+// pch.cpp
+// Include the standard header and generate the precompiled header.
+//
+
+#include "pch.h"

src/IntegrationTests/ITODBCPagination/pch.h

@@ -0,0 +1,23 @@
+/*
+ *   Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License").
+ *   You may not use this file except in compliance with the License.
+ *   A copy of the License is located at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file. This file is distributed
+ *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *   express or implied. See the License for the specific language governing
+ *   permissions and limitations under the License.
+ */
+
+//
+// pch.h
+// Header for standard system include files.
+//
+
+#pragma once
+
+#include "gtest/gtest.h"

src/IntegrationTests/ITODBCPagination/test_odbc_pagination.cpp

@@ -0,0 +1,152 @@
+/*
+ * Copyright <2019> Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+// clang-format off
+#include "pch.h"
+#include "unit_test_helper.h"
+#include "it_odbc_helper.h"
+// clang-format on
+
+#define BIND_SIZE 255
+#define SINGLE_ROW 1
+typedef struct Col {
+    SQLLEN data_len;
+    SQLCHAR data_dat[BIND_SIZE];
+} Col;
+
+class TestPagination : public testing::Test {
+   public:
+    TestPagination() {
+    }
+
+    void SetUp() {
+        AllocConnection(&m_env, &m_conn, true, true);
+    }
+
+    void TearDown() {
+        if (SQL_NULL_HDBC != m_conn) {
+            SQLFreeHandle(SQL_HANDLE_DBC, m_conn);
+            SQLFreeHandle(SQL_HANDLE_ENV, m_env);
+        }
+    }
+
+    int GetTotalRowsAfterQueryExecution() {
+        SQLAllocHandle(SQL_HANDLE_STMT, m_conn, &m_hstmt);
+        SQLRETURN ret = SQLExecDirect(m_hstmt, (SQLTCHAR*)m_query.c_str(), SQL_NTS);
+        EXPECT_EQ(SQL_SUCCESS, ret);
+
+        // Get column count
+        SQLSMALLINT total_columns = -1;
+        SQLNumResultCols(m_hstmt, &total_columns);
+        std::vector< std::vector< Col > > cols(total_columns);
+        for (size_t i = 0; i < cols.size(); i++) {
+            cols[i].resize(SINGLE_ROW);
+        }
+
+        // Bind and fetch
+        for (size_t i = 0; i < cols.size(); i++) {
+            ret = SQLBindCol(m_hstmt, (SQLUSMALLINT)i + 1, SQL_C_CHAR,
+                             (SQLPOINTER)&cols[i][0].data_dat[i], 255,
+                             &cols[i][0].data_len);
+        }
+
+        // Get total number of rows
+        int row_count = 0;
+        while (SQLFetch(m_hstmt) == SQL_SUCCESS) {
+            row_count++;
+        }
+        return row_count;
+    }
+
+    ~TestPagination() {
+        // cleanup any pending stuff, but no exceptions allowed
+    }
+
+    SQLHENV m_env = SQL_NULL_HENV;
+    SQLHDBC m_conn = SQL_NULL_HDBC;
+    SQLHSTMT m_hstmt = SQL_NULL_HSTMT;
+    SQLTCHAR m_out_conn_string[1024];
+    SQLSMALLINT m_out_conn_string_length;
+    std::wstring m_query =
+        L"SELECT Origin FROM kibana_sample_data_flights";
+};
+
+TEST_F(TestPagination, Fetch15Rows) {
+    int total_rows = 15;
+    std::wstring fetch_size_15_conn_string =
+        use_ssl ? L"Driver={Elasticsearch ODBC};"
+                  L"host=https://localhost;port=9200;"
+                  L"user=admin;password=admin;auth=BASIC;useSSL="
+                  L"1;hostnameVerification=0;logLevel=0;logOutput=C:\\;"
+                  L"responseTimeout=1;fetchSize=15"
+                : L"Driver={Elasticsearch ODBC};"
+                  L"host=localhost;port=9200;"
+                  L"user=admin;password=admin;auth=BASIC;useSSL="
+                  L"0;hostnameVerification=0;logLevel=0;logOutput=C:\\;"
+                  L"responseTimeout=1;fetchSize=15";
+    ASSERT_EQ(SQL_SUCCESS,
+              SQLDriverConnect(
+                  m_conn, NULL, (SQLTCHAR*)fetch_size_15_conn_string.c_str(),
+                  SQL_NTS, m_out_conn_string, IT_SIZEOF(m_out_conn_string),
+                  &m_out_conn_string_length, SQL_DRIVER_PROMPT));
+    EXPECT_EQ(total_rows, GetTotalRowsAfterQueryExecution());
+}
+
+TEST_F(TestPagination, NoFetchSize) {
+    // Default size when pagination is disabled i.e. fetch size is 0.
+    int total_rows = 200;
+    std::wstring fetch_size_15_conn_string =
+        use_ssl ? L"Driver={Elasticsearch ODBC};"
+                  L"host=https://localhost;port=9200;"
+                  L"user=admin;password=admin;auth=BASIC;useSSL="
+                  L"1;hostnameVerification=0;logLevel=0;logOutput=C:\\;"
+                  L"responseTimeout=1;"
+                : L"Driver={Elasticsearch ODBC};"
+                  L"host=localhost;port=9200;"
+                  L"user=admin;password=admin;auth=BASIC;useSSL="
+                  L"0;hostnameVerification=0;logLevel=0;logOutput=C:\\;"
+                  L"responseTimeout=1;";
+    ASSERT_EQ(SQL_SUCCESS,
+              SQLDriverConnect(
+                  m_conn, NULL, (SQLTCHAR*)fetch_size_15_conn_string.c_str(),
+                  SQL_NTS, m_out_conn_string, IT_SIZEOF(m_out_conn_string),
+                  &m_out_conn_string_length, SQL_DRIVER_PROMPT));
+    EXPECT_EQ(total_rows, GetTotalRowsAfterQueryExecution());
+}
+
+int main(int argc, char** argv) {
+#ifdef __APPLE__
+    // Enable malloc logging for detecting memory leaks.
+    system("export MallocStackLogging=1");
+#endif
+    testing::internal::CaptureStdout();
+    ::testing::InitGoogleTest(&argc, argv);
+
+    int failures = RUN_ALL_TESTS();
+
+    std::string output = testing::internal::GetCapturedStdout();
+    std::cout << output << std::endl;
+    std::cout << (failures ? "Not all tests passed." : "All tests passed")
+              << std::endl;
+    WriteFileIfSpecified(argv, argv + argc, "-fout", output);
+
+#ifdef __APPLE__
+    // Disable malloc logging and report memory leaks
+    system("unset MallocStackLogging");
+    system("leaks itodbc_pagination > leaks_itodbc_pagination");
+#endif
+    return failures;
+}

src/PerformanceTests/PTODBCInfo/performance_odbc_info.cpp

@@ -83,10 +83,10 @@ void GetVersionInfoString(std::string& version_info) {
     ASSERT_TRUE(es_comm.ConnectDBStart());
 
     // Issue request
-    std::string endpoint, content_type, query;
+    std::string endpoint, content_type, query, fetch_size;
     std::shared_ptr< Aws::Http::HttpResponse > response = nullptr;
     es_comm.IssueRequest(endpoint, Aws::Http::HttpMethod::HTTP_GET,
-                         content_type, query, response);
+                         content_type, query, response, fetch_size);
 
     // Convert response to string
     ASSERT_TRUE(response != nullptr);

src/UnitTests/UTConn/test_conn.cpp

@@ -33,14 +33,14 @@ const std::string invalid_port = "920";
 const std::string invalid_user = "amin";
 const std::string invalid_pw = "amin";
 const std::string invalid_region = "bad-region";
-runtime_options valid_opt_val = {{valid_host, valid_port, "1"},
+runtime_options valid_opt_val = {{valid_host, valid_port, "1", "0"},
                                  {"BASIC", valid_user, valid_pw, valid_region},
                                  {use_ssl, false, "", "", "", ""}};
 runtime_options invalid_opt_val = {
-    {invalid_host, invalid_port, "1"},
+    {invalid_host, invalid_port, "1", "0"},
     {"BASIC", invalid_user, invalid_pw, valid_region},
     {use_ssl, false, "", "", "", ""}};
-runtime_options missing_opt_val = {{"", "", "1"},
+runtime_options missing_opt_val = {{"", "", "1", "0"},
                                    {"BASIC", "", invalid_pw, valid_region},
                                    {use_ssl, false, "", "", "", ""}};
 

src/UnitTests/UTConn/test_query_execution.cpp

@@ -35,11 +35,11 @@ const std::string some_columns_flights_query =
 const std::string invalid_query = "SELECT";
 const int EXECUTION_SUCCESS = 0;
 const int EXECUTION_ERROR = -1;
-
+const std::string fetch_size = "0";
 const int all_columns_flights_count = 25;
 const int some_columns_flights_count = 2;
 runtime_options valid_conn_opt_val = {
-    {valid_host, valid_port, "1"},
+    {valid_host, valid_port, "1", "0"},
     {"BASIC", valid_user, valid_pw, valid_region},
     {use_ssl, false, "", "", "", ""}};
 
@@ -48,18 +48,19 @@ TEST(TestESExecDirect, ValidQuery) {
     ASSERT_TRUE(conn.ConnectionOptions(valid_conn_opt_val, false, 0, 0));
     ASSERT_TRUE(conn.ConnectDBStart());
     EXPECT_EQ(EXECUTION_SUCCESS,
-              ESExecDirect(&conn, some_columns_flights_query.c_str()));
+        ESExecDirect(&conn, some_columns_flights_query.c_str(), fetch_size.c_str()));
 }
 
 TEST(TestESExecDirect, MissingQuery) {
     ESCommunication conn;
     ASSERT_TRUE(conn.ConnectionOptions(valid_conn_opt_val, false, 0, 0));
     ASSERT_TRUE(conn.ConnectDBStart());
-    EXPECT_EQ(EXECUTION_ERROR, ESExecDirect(&conn, NULL));
+    EXPECT_EQ(EXECUTION_ERROR, ESExecDirect(&conn, NULL, fetch_size.c_str()));
 }
 
 TEST(TestESExecDirect, MissingConnection) {
-    EXPECT_EQ(EXECUTION_ERROR, ESExecDirect(NULL, query.c_str()));
+    EXPECT_EQ(EXECUTION_ERROR,
+              ESExecDirect(NULL, query.c_str(), fetch_size.c_str()));
 }
 
 // Conn::ExecDirect
@@ -69,7 +70,7 @@ TEST(TestConnExecDirect, ValidQueryAllColumns) {
     ASSERT_TRUE(conn.ConnectionOptions(valid_conn_opt_val, false, 0, 0));
     ASSERT_TRUE(conn.ConnectDBStart());
 
-    conn.ExecDirect(all_columns_flights_query.c_str());
+    conn.ExecDirect(all_columns_flights_query.c_str(), fetch_size.c_str());
     ESResult* result = conn.PopResult();
     EXPECT_EQ("SELECT", result->command_type);
     EXPECT_FALSE(result->result_json.empty());
@@ -82,7 +83,7 @@ TEST(TestConnExecDirect, ValidQuerySomeColumns) {
     ASSERT_TRUE(conn.ConnectionOptions(valid_conn_opt_val, false, 0, 0));
     ASSERT_TRUE(conn.ConnectDBStart());
 
-    conn.ExecDirect(some_columns_flights_query.c_str());
+    conn.ExecDirect(some_columns_flights_query.c_str(), fetch_size.c_str());
     ESResult* result = conn.PopResult();
     EXPECT_EQ("SELECT", result->command_type);
     EXPECT_FALSE(result->result_json.empty());
@@ -95,7 +96,7 @@ TEST(TestConnExecDirect, InvalidQuery) {
     ASSERT_TRUE(conn.ConnectionOptions(valid_conn_opt_val, false, 0, 0));
     ASSERT_TRUE(conn.ConnectDBStart());
 
-    conn.ExecDirect(invalid_query.c_str());
+    conn.ExecDirect(invalid_query.c_str(), fetch_size.c_str());
     ESResult* result = conn.PopResult();
     EXPECT_EQ(NULL, (void*)result);
 }
@@ -116,8 +117,8 @@ TEST(TestConnPopResult, PopTwoQueryResults) {
     ASSERT_TRUE(conn.ConnectionOptions(valid_conn_opt_val, false, 0, 0));
     ASSERT_TRUE(conn.ConnectDBStart());
 
-    conn.ExecDirect(some_columns_flights_query.c_str());
-    conn.ExecDirect(all_columns_flights_query.c_str());
+    conn.ExecDirect(some_columns_flights_query.c_str(), fetch_size.c_str());
+    conn.ExecDirect(all_columns_flights_query.c_str(), fetch_size.c_str());
 
     // Pop some_columns
     ESResult* result = conn.PopResult();