Remove columns from result which are not returned by SELECT * query but returned by DESCRIBE query

sql-odbc/src/IntegrationTests/ITODBCCatalog/test_odbc_catalog.cpp

@@ -71,15 +71,15 @@ typedef struct bind_info {
 
 // Column test constants and macro
 const std::vector< std::string > flights_column_name = {
-    "FlightNum",      "Origin",         "OriginLocation",  "DestLocation",
+    "FlightNum",      "Origin",
     "FlightDelay",    "DistanceMiles",  "FlightTimeMin",   "OriginWeather",
     "dayOfWeek",      "AvgTicketPrice", "Carrier",         "FlightDelayMin",
     "OriginRegion",   "DestAirportID",  "FlightDelayType", "timestamp",
     "Dest",           "FlightTimeHour", "Cancelled",       "DistanceKilometers",
     "OriginCityName", "DestWeather",    "OriginCountry",   "DestCountry",
     "DestRegion",     "DestCityName",   "OriginAirportID"};
 const std::vector< std::string > flights_data_type = {
-    "keyword", "keyword", "geo_point", "geo_point", "boolean", "float",
+    "keyword", "keyword", "boolean", "float",
     "float",   "keyword", "integer",   "float",     "keyword", "integer",
     "keyword", "keyword", "keyword",   "date",      "keyword", "keyword",
     "boolean", "float",   "keyword",   "keyword",   "keyword", "keyword",
@@ -404,7 +404,7 @@ TEST_F(TestSQLColumns, FlightsValidation) {
         }
         column_idx++;
     }
-    EXPECT_EQ(column_idx, static_cast< size_t >(27));
+    EXPECT_EQ(column_idx, static_cast< size_t >(25));
 }
 
 // We expect an empty result set for PrimaryKeys and ForeignKeys

sql-odbc/src/odfesqlodbc/connection.c

@@ -655,7 +655,17 @@ void CC_log_error(const char *func, const char *desc,
 }
 
 const char *CurrCat(const ConnectionClass *conn) {
-    return conn->cluster_name;
+    // Keeping this NULL since it's required for getting list of tables in
+    // Microsoft Excel with ODBC Connection mode. This causes error in Data
+    // Connection wizard mode but data connection wizard sends query with
+    // catalog.table_name which fails. So setting this to NULL will enable more
+    // functionality
+    UNUSED(conn);
+    return NULL;
+
+    // Change this to following return value when found a solution which works
+    // with ODBC and Data Connection wizard in Microsoft Excel return
+    // conn->cluster_name;
 }
 
 const char *CurrCatString(const ConnectionClass *conn) {

sql-odbc/src/odfesqlodbc/es_communication.cpp

@@ -407,6 +407,67 @@ bool ESCommunication::EstablishConnection() {
     return false;
 }
 
+std::vector< std::string > ESCommunication::GetColumnsWithSelectQuery(
+    const std::string table_name) {
+    std::vector< std::string > list_of_column;
+    if (table_name.empty()) {
+        m_error_message = "Query is NULL";
+        LogMsg(ES_ERROR, m_error_message.c_str());
+        return list_of_column;
+    }
+
+    // Prepare query
+    std::string query = "SELECT * FROM " + table_name + " LIMIT 0";
+    std::string msg = "Attempting to execute a query \"" + query + "\"";
+    LogMsg(ES_DEBUG, msg.c_str());
+
+    // Issue request
+    std::shared_ptr< Aws::Http::HttpResponse > response =
+        IssueRequest(SQL_ENDPOINT_FORMAT_JDBC, Aws::Http::HttpMethod::HTTP_POST,
+                     ctype, query);
+
+    // Validate response
+    if (response == nullptr) {
+        m_error_message =
+            "Failed to receive response from query. "
+            "Received NULL response.";
+        LogMsg(ES_ERROR, m_error_message.c_str());
+        return list_of_column;
+    }
+
+    // Convert body from Aws IOStream to string
+    std::unique_ptr< ESResult > result = std::make_unique< ESResult >();
+    AwsHttpResponseToString(response, result->result_json);
+
+    // If response was not valid, set error
+    if (response->GetResponseCode() != Aws::Http::HttpResponseCode::OK) {
+        m_error_message =
+            "Http response code was not OK. Code received: "
+            + std::to_string(static_cast< long >(response->GetResponseCode()))
+            + ".";
+        if (response->HasClientError())
+            m_error_message +=
+                " Client error: '" + response->GetClientErrorMessage() + "'.";
+        if (!result->result_json.empty()) {
+            m_error_message +=
+                " Response error: '" + result->result_json + "'.";
+        }
+        LogMsg(ES_ERROR, m_error_message.c_str());
+        return list_of_column;
+    }
+
+    GetJsonSchema(*result);
+
+    rabbit::array schema_array = result->es_result_doc["schema"];
+    for (rabbit::array::iterator it = schema_array.begin();
+         it != schema_array.end(); ++it) {
+        std::string column_name = it->at("name").as_string();
+        list_of_column.push_back(column_name);
+    }
+
+    return list_of_column;
+}
+
 int ESCommunication::ExecDirect(const char* query, const char* fetch_size_) {
     if (!query) {
         m_error_message = "Query is NULL";

sql-odbc/src/odfesqlodbc/es_communication.h

@@ -74,6 +74,7 @@ class ESCommunication {
         std::string& output);
     void SendCloseCursorRequest(const std::string& cursor);
     void StopResultRetrieval();
+    std::vector< std::string > GetColumnsWithSelectQuery(const std::string table_name);
 
    private:
     void InitializeConnection();

sql-odbc/src/odfesqlodbc/es_helper.cpp

@@ -114,6 +114,12 @@ void ESStopRetrieval(void* es_conn) {
      static_cast< ESCommunication* >(es_conn)->StopResultRetrieval();
 }
 
+std::vector< std::string > ESGetColumnsWithSelectQuery(
+    void* es_conn, const std::string table_name) {
+    return static_cast< ESCommunication* >(es_conn)->GetColumnsWithSelectQuery(
+        table_name);
+}
+
 // This class provides a cross platform way of entering critical sections
 class CriticalSectionHelper {
    public:

sql-odbc/src/odfesqlodbc/es_helper.h

@@ -30,6 +30,8 @@ void* ESConnectDBParams(runtime_options& rt_opts, int expand_dbname,
 std::string GetServerVersion(void* es_conn);
 std::string GetClusterName(void* es_conn);
 std::string GetErrorMsg(void* es_conn);
+std::vector< std::string > ESGetColumnsWithSelectQuery(
+    void* es_conn, const std::string table_name);
 
 // C Interface
 extern "C" {

sql-odbc/src/odfesqlodbc/es_info.cpp

@@ -306,8 +306,8 @@ void GetCatalogData(const std::string &query, StatementClass *stmt,
                     StatementClass *sub_stmt, const TableResultSet res_type,
                     std::string &table_type,
                     void (*populate_binds)(bind_vector &),
-                    void (*setup_qres_info)(QResultClass *,
-                                            EnvironmentClass *));
+                    void (*setup_qres_info)(QResultClass *, EnvironmentClass *),
+                    std::vector< std::string > *list_of_columns = NULL);
 
 // Common function declarations
 void ConvertToString(std::string &out, bool &valid, const SQLCHAR *sql_char,
@@ -394,7 +394,8 @@ void AssignTableBindTemplates(bind_vector &tabs);
 void SetupTableQResInfo(QResultClass *res, EnvironmentClass *env);
 void SetTableTuples(QResultClass *res, const TableResultSet res_type,
                     const bind_vector &bind_tbl, std::string &table_type,
-                    StatementClass *stmt, StatementClass *tbl_stmt);
+                    StatementClass *stmt, StatementClass *tbl_stmt,
+                    std::vector< std::string > *list_of_columns = NULL);
 
 // Table specific function declarations
 void split(const std::string &input, const std::string &delim,
@@ -457,7 +458,8 @@ void SetupTableQResInfo(QResultClass *res, EnvironmentClass *env) {
 
 void SetTableTuples(QResultClass *res, const TableResultSet res_type,
                     const bind_vector &bind_tbl, std::string &table_type,
-                    StatementClass *stmt, StatementClass *tbl_stmt) {
+                    StatementClass *stmt, StatementClass *tbl_stmt,
+                    std::vector< std::string > *list_of_columns) {
     auto CheckResult = [&](const auto &res) {
         if (res != SQL_NO_DATA_FOUND) {
             SC_full_error_copy(stmt, tbl_stmt, FALSE);
@@ -476,12 +478,24 @@ void SetTableTuples(QResultClass *res, const TableResultSet res_type,
     // General case
     if (res_type == TableResultSet::All) {
         RETCODE result = SQL_NO_DATA_FOUND;
+        int ordinal_position = 0;
         while (SQL_SUCCEEDED(result = ESAPI_Fetch(tbl_stmt))) {
             if (bind_tbl[TABLES_TABLE_TYPE]->AsString() == "BASE TABLE") {
                 std::string table("TABLE");
                 bind_tbl[TABLES_TABLE_TYPE]->UpdateData(&table, table.size());
             }
-            AssignData(res, bind_tbl);
+            if (list_of_columns != NULL && !list_of_columns->empty()) {
+                if (std::find(list_of_columns->begin(), list_of_columns->end(),
+                              bind_tbl[COLUMNS_COLUMN_NAME]->AsString())
+                    != list_of_columns->end()) {
+                    ordinal_position++;
+                    bind_tbl[COLUMNS_ORDINAL_POSITION]->UpdateData(
+                        &ordinal_position, 0);
+                    AssignData(res, bind_tbl);
+                }
+            } else {
+                AssignData(res, bind_tbl);
+            }
         }
         CheckResult(result);
     } else if (res_type == TableResultSet::TableLookUp) {
@@ -639,8 +653,8 @@ void GetCatalogData(const std::string &query, StatementClass *stmt,
                     StatementClass *sub_stmt, const TableResultSet res_type,
                     std::string &table_type,
                     void (*populate_binds)(bind_vector &),
-                    void (*setup_qres_info)(QResultClass *,
-                                            EnvironmentClass *)) {
+                    void (*setup_qres_info)(QResultClass *, EnvironmentClass *),
+                    std::vector< std::string > *list_of_columns) {
     // Execute query
     ExecuteQuery(SC_get_conn(stmt), reinterpret_cast< HSTMT * >(&sub_stmt),
                  query);
@@ -656,7 +670,8 @@ void GetCatalogData(const std::string &query, StatementClass *stmt,
     // Setup QResultClass
     (*setup_qres_info)(
         res, static_cast< EnvironmentClass * >(CC_get_env(SC_get_conn(stmt))));
-    SetTableTuples(res, res_type, binds, table_type, stmt, sub_stmt);
+    SetTableTuples(res, res_type, binds, table_type, stmt, sub_stmt,
+                   list_of_columns);
 
     CleanUp(stmt, sub_stmt, SQL_SUCCESS);
 }
@@ -770,13 +785,22 @@ ESAPI_Columns(HSTMT hstmt, const SQLCHAR *catalog_name_sql,
         std::string query;
         GenerateColumnQuery(query, table_name, column_name, table_valid,
                             column_valid, flag);
+
+        // Get list of columns with SELECT * query since columns doesn't match with DESCRIBE & SELECT * query
+        std::vector< std::string > list_of_columns;
+        if (table_valid) {
+            ConnectionClass *conn = SC_get_conn(stmt);
+            list_of_columns =
+                ESGetColumnsWithSelectQuery(conn->esconn, table_name);
+        }
 
         // TODO #324 (SQL Plugin)- evaluate catalog & schema support
 
         // Execute query
         std::string table_type = "";
         GetCatalogData(query, stmt, col_stmt, TableResultSet::All, table_type,
-                       AssignColumnBindTemplates, SetupColumnQResInfo);
+                       AssignColumnBindTemplates, SetupColumnQResInfo,
+                       &list_of_columns);
         return SQL_SUCCESS;
     } catch (std::bad_alloc &e) {
         std::string error_msg = std::string("Bad allocation exception: '")