From a97e2aa9ca645c85c5dcf8ade66b3c0d2f28656a Mon Sep 17 00:00:00 2001 From: Rupal Mahajan Date: Thu, 9 Jul 2020 15:17:16 -0700 Subject: [PATCH] adding filter for columns in SQLColumns call --- .../ITODBCCatalog/test_odbc_catalog.cpp | 6 +- sql-odbc/src/odfesqlodbc/connection.c | 12 +++- sql-odbc/src/odfesqlodbc/es_communication.cpp | 61 +++++++++++++++++++ sql-odbc/src/odfesqlodbc/es_communication.h | 1 + sql-odbc/src/odfesqlodbc/es_helper.cpp | 6 ++ sql-odbc/src/odfesqlodbc/es_helper.h | 2 + sql-odbc/src/odfesqlodbc/es_info.cpp | 42 ++++++++++--- 7 files changed, 117 insertions(+), 13 deletions(-) diff --git a/sql-odbc/src/IntegrationTests/ITODBCCatalog/test_odbc_catalog.cpp b/sql-odbc/src/IntegrationTests/ITODBCCatalog/test_odbc_catalog.cpp index 217bc42d9e..9210a3c056 100644 --- a/sql-odbc/src/IntegrationTests/ITODBCCatalog/test_odbc_catalog.cpp +++ b/sql-odbc/src/IntegrationTests/ITODBCCatalog/test_odbc_catalog.cpp @@ -71,7 +71,7 @@ typedef struct bind_info { // Column test constants and macro const std::vector< std::string > flights_column_name = { - "FlightNum", "Origin", "OriginLocation", "DestLocation", + "FlightNum", "Origin", "FlightDelay", "DistanceMiles", "FlightTimeMin", "OriginWeather", "dayOfWeek", "AvgTicketPrice", "Carrier", "FlightDelayMin", "OriginRegion", "DestAirportID", "FlightDelayType", "timestamp", @@ -79,7 +79,7 @@ const std::vector< std::string > flights_column_name = { "OriginCityName", "DestWeather", "OriginCountry", "DestCountry", "DestRegion", "DestCityName", "OriginAirportID"}; const std::vector< std::string > flights_data_type = { - "keyword", "keyword", "geo_point", "geo_point", "boolean", "float", + "keyword", "keyword", "boolean", "float", "float", "keyword", "integer", "float", "keyword", "integer", "keyword", "keyword", "keyword", "date", "keyword", "keyword", "boolean", "float", "keyword", "keyword", "keyword", "keyword", @@ -404,7 +404,7 @@ TEST_F(TestSQLColumns, FlightsValidation) { } column_idx++; } - EXPECT_EQ(column_idx, static_cast< size_t >(27)); + EXPECT_EQ(column_idx, static_cast< size_t >(25)); } // We expect an empty result set for PrimaryKeys and ForeignKeys diff --git a/sql-odbc/src/odfesqlodbc/connection.c b/sql-odbc/src/odfesqlodbc/connection.c index afb8cdc1d5..f42c0786b0 100644 --- a/sql-odbc/src/odfesqlodbc/connection.c +++ b/sql-odbc/src/odfesqlodbc/connection.c @@ -655,7 +655,17 @@ void CC_log_error(const char *func, const char *desc, } const char *CurrCat(const ConnectionClass *conn) { - return conn->cluster_name; + // Keeping this NULL since it's required for getting list of tables in + // Microsoft Excel with ODBC Connection mode. This causes error in Data + // Connection wizard mode but data connection wizard sends query with + // catalog.table_name which fails. So setting this to NULL will enable more + // functionality + UNUSED(conn); + return NULL; + + // Change this to following return value when found a solution which works + // with ODBC and Data Connection wizard in Microsoft Excel return + // conn->cluster_name; } const char *CurrCatString(const ConnectionClass *conn) { diff --git a/sql-odbc/src/odfesqlodbc/es_communication.cpp b/sql-odbc/src/odfesqlodbc/es_communication.cpp index cf25b070a7..24ad4600c1 100644 --- a/sql-odbc/src/odfesqlodbc/es_communication.cpp +++ b/sql-odbc/src/odfesqlodbc/es_communication.cpp @@ -407,6 +407,67 @@ bool ESCommunication::EstablishConnection() { return false; } +std::vector< std::string > ESCommunication::GetColumnsWithSelectQuery( + const std::string table_name) { + std::vector< std::string > list_of_column; + if (table_name.empty()) { + m_error_message = "Query is NULL"; + LogMsg(ES_ERROR, m_error_message.c_str()); + return list_of_column; + } + + // Prepare query + std::string query = "SELECT * FROM " + table_name + " LIMIT 0"; + std::string msg = "Attempting to execute a query \"" + query + "\""; + LogMsg(ES_DEBUG, msg.c_str()); + + // Issue request + std::shared_ptr< Aws::Http::HttpResponse > response = + IssueRequest(SQL_ENDPOINT_FORMAT_JDBC, Aws::Http::HttpMethod::HTTP_POST, + ctype, query); + + // Validate response + if (response == nullptr) { + m_error_message = + "Failed to receive response from query. " + "Received NULL response."; + LogMsg(ES_ERROR, m_error_message.c_str()); + return list_of_column; + } + + // Convert body from Aws IOStream to string + std::unique_ptr< ESResult > result = std::make_unique< ESResult >(); + AwsHttpResponseToString(response, result->result_json); + + // If response was not valid, set error + if (response->GetResponseCode() != Aws::Http::HttpResponseCode::OK) { + m_error_message = + "Http response code was not OK. Code received: " + + std::to_string(static_cast< long >(response->GetResponseCode())) + + "."; + if (response->HasClientError()) + m_error_message += + " Client error: '" + response->GetClientErrorMessage() + "'."; + if (!result->result_json.empty()) { + m_error_message += + " Response error: '" + result->result_json + "'."; + } + LogMsg(ES_ERROR, m_error_message.c_str()); + return list_of_column; + } + + GetJsonSchema(*result); + + rabbit::array schema_array = result->es_result_doc["schema"]; + for (rabbit::array::iterator it = schema_array.begin(); + it != schema_array.end(); ++it) { + std::string column_name = it->at("name").as_string(); + list_of_column.push_back(column_name); + } + + return list_of_column; +} + int ESCommunication::ExecDirect(const char* query, const char* fetch_size_) { if (!query) { m_error_message = "Query is NULL"; diff --git a/sql-odbc/src/odfesqlodbc/es_communication.h b/sql-odbc/src/odfesqlodbc/es_communication.h index 38ca25b1de..f1ed8f33d4 100644 --- a/sql-odbc/src/odfesqlodbc/es_communication.h +++ b/sql-odbc/src/odfesqlodbc/es_communication.h @@ -74,6 +74,7 @@ class ESCommunication { std::string& output); void SendCloseCursorRequest(const std::string& cursor); void StopResultRetrieval(); + std::vector< std::string > GetColumnsWithSelectQuery(const std::string table_name); private: void InitializeConnection(); diff --git a/sql-odbc/src/odfesqlodbc/es_helper.cpp b/sql-odbc/src/odfesqlodbc/es_helper.cpp index cf243137ce..a540bfc4fb 100644 --- a/sql-odbc/src/odfesqlodbc/es_helper.cpp +++ b/sql-odbc/src/odfesqlodbc/es_helper.cpp @@ -114,6 +114,12 @@ void ESStopRetrieval(void* es_conn) { static_cast< ESCommunication* >(es_conn)->StopResultRetrieval(); } +std::vector< std::string > ESGetColumnsWithSelectQuery( + void* es_conn, const std::string table_name) { + return static_cast< ESCommunication* >(es_conn)->GetColumnsWithSelectQuery( + table_name); +} + // This class provides a cross platform way of entering critical sections class CriticalSectionHelper { public: diff --git a/sql-odbc/src/odfesqlodbc/es_helper.h b/sql-odbc/src/odfesqlodbc/es_helper.h index 2328b07d7a..61d26fb7e3 100644 --- a/sql-odbc/src/odfesqlodbc/es_helper.h +++ b/sql-odbc/src/odfesqlodbc/es_helper.h @@ -30,6 +30,8 @@ void* ESConnectDBParams(runtime_options& rt_opts, int expand_dbname, std::string GetServerVersion(void* es_conn); std::string GetClusterName(void* es_conn); std::string GetErrorMsg(void* es_conn); +std::vector< std::string > ESGetColumnsWithSelectQuery( + void* es_conn, const std::string table_name); // C Interface extern "C" { diff --git a/sql-odbc/src/odfesqlodbc/es_info.cpp b/sql-odbc/src/odfesqlodbc/es_info.cpp index 4c037f1ba9..639b965368 100644 --- a/sql-odbc/src/odfesqlodbc/es_info.cpp +++ b/sql-odbc/src/odfesqlodbc/es_info.cpp @@ -306,8 +306,8 @@ void GetCatalogData(const std::string &query, StatementClass *stmt, StatementClass *sub_stmt, const TableResultSet res_type, std::string &table_type, void (*populate_binds)(bind_vector &), - void (*setup_qres_info)(QResultClass *, - EnvironmentClass *)); + void (*setup_qres_info)(QResultClass *, EnvironmentClass *), + std::vector< std::string > *list_of_columns = NULL); // Common function declarations void ConvertToString(std::string &out, bool &valid, const SQLCHAR *sql_char, @@ -394,7 +394,8 @@ void AssignTableBindTemplates(bind_vector &tabs); void SetupTableQResInfo(QResultClass *res, EnvironmentClass *env); void SetTableTuples(QResultClass *res, const TableResultSet res_type, const bind_vector &bind_tbl, std::string &table_type, - StatementClass *stmt, StatementClass *tbl_stmt); + StatementClass *stmt, StatementClass *tbl_stmt, + std::vector< std::string > *list_of_columns = NULL); // Table specific function declarations void split(const std::string &input, const std::string &delim, @@ -457,7 +458,8 @@ void SetupTableQResInfo(QResultClass *res, EnvironmentClass *env) { void SetTableTuples(QResultClass *res, const TableResultSet res_type, const bind_vector &bind_tbl, std::string &table_type, - StatementClass *stmt, StatementClass *tbl_stmt) { + StatementClass *stmt, StatementClass *tbl_stmt, + std::vector< std::string > *list_of_columns) { auto CheckResult = [&](const auto &res) { if (res != SQL_NO_DATA_FOUND) { SC_full_error_copy(stmt, tbl_stmt, FALSE); @@ -476,12 +478,24 @@ void SetTableTuples(QResultClass *res, const TableResultSet res_type, // General case if (res_type == TableResultSet::All) { RETCODE result = SQL_NO_DATA_FOUND; + int ordinal_position = 0; while (SQL_SUCCEEDED(result = ESAPI_Fetch(tbl_stmt))) { if (bind_tbl[TABLES_TABLE_TYPE]->AsString() == "BASE TABLE") { std::string table("TABLE"); bind_tbl[TABLES_TABLE_TYPE]->UpdateData(&table, table.size()); } - AssignData(res, bind_tbl); + if (list_of_columns != NULL && !list_of_columns->empty()) { + if (std::find(list_of_columns->begin(), list_of_columns->end(), + bind_tbl[COLUMNS_COLUMN_NAME]->AsString()) + != list_of_columns->end()) { + ordinal_position++; + bind_tbl[COLUMNS_ORDINAL_POSITION]->UpdateData( + &ordinal_position, 0); + AssignData(res, bind_tbl); + } + } else { + AssignData(res, bind_tbl); + } } CheckResult(result); } else if (res_type == TableResultSet::TableLookUp) { @@ -639,8 +653,8 @@ void GetCatalogData(const std::string &query, StatementClass *stmt, StatementClass *sub_stmt, const TableResultSet res_type, std::string &table_type, void (*populate_binds)(bind_vector &), - void (*setup_qres_info)(QResultClass *, - EnvironmentClass *)) { + void (*setup_qres_info)(QResultClass *, EnvironmentClass *), + std::vector< std::string > *list_of_columns) { // Execute query ExecuteQuery(SC_get_conn(stmt), reinterpret_cast< HSTMT * >(&sub_stmt), query); @@ -656,7 +670,8 @@ void GetCatalogData(const std::string &query, StatementClass *stmt, // Setup QResultClass (*setup_qres_info)( res, static_cast< EnvironmentClass * >(CC_get_env(SC_get_conn(stmt)))); - SetTableTuples(res, res_type, binds, table_type, stmt, sub_stmt); + SetTableTuples(res, res_type, binds, table_type, stmt, sub_stmt, + list_of_columns); CleanUp(stmt, sub_stmt, SQL_SUCCESS); } @@ -770,13 +785,22 @@ ESAPI_Columns(HSTMT hstmt, const SQLCHAR *catalog_name_sql, std::string query; GenerateColumnQuery(query, table_name, column_name, table_valid, column_valid, flag); + + // Get list of columns with SELECT * query since columns doesn't match with DESCRIBE & SELECT * query + std::vector< std::string > list_of_columns; + if (table_valid) { + ConnectionClass *conn = SC_get_conn(stmt); + list_of_columns = + ESGetColumnsWithSelectQuery(conn->esconn, table_name); + } // TODO #324 (SQL Plugin)- evaluate catalog & schema support // Execute query std::string table_type = ""; GetCatalogData(query, stmt, col_stmt, TableResultSet::All, table_type, - AssignColumnBindTemplates, SetupColumnQResInfo); + AssignColumnBindTemplates, SetupColumnQResInfo, + &list_of_columns); return SQL_SUCCESS; } catch (std::bad_alloc &e) { std::string error_msg = std::string("Bad allocation exception: '")