support "SELECT *" queries

ad-freiburg · Jan 8, 2024 · 8238c10 · 8238c10
1 parent 74f5d34
commit 8238c10
Show file tree

Hide file tree

Showing 3 changed files with 94 additions and 14 deletions.
diff --git a/src/qlever-petrimaps/Misc.cpp b/src/qlever-petrimaps/Misc.cpp
@@ -13,6 +13,64 @@
 
 using petrimaps::RequestReader;
 
+// _____________________________________________________________________________
+std::vector<std::string> RequestReader::requestColumns(const std::string& query) {
+  CURLcode res;
+  char errbuf[CURL_ERROR_SIZE];
+
+  std::string resString;
+
+  if (_curl) {
+    auto url = queryUrl(query) + "&action=tsv_export";
+    curl_easy_setopt(_curl, CURLOPT_URL, url.c_str());
+    curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, RequestReader::writeStringCb);
+    curl_easy_setopt(_curl, CURLOPT_WRITEDATA, &resString);
+    curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYPEER, false);
+    curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYHOST, false);
+    curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, 0);
+
+    // accept any compression supported
+    curl_easy_setopt(_curl, CURLOPT_ACCEPT_ENCODING, "");
+    curl_easy_setopt(_curl, CURLOPT_ERRORBUFFER, errbuf);
+    res = curl_easy_perform(_curl);
+
+    long httpCode = 0;
+    curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &httpCode);
+
+    if (httpCode != 200) {
+      std::stringstream ss;
+      ss << "QLever backend returned status code " << httpCode;
+      ss << "\n";
+      ss << _raw;
+      throw std::runtime_error(ss.str());
+    }
+
+    if (exceptionPtr) std::rethrow_exception(exceptionPtr);
+
+  } else {
+    std::stringstream ss;
+    ss << "[REQUESTREADER] Failed to perform curl request.\n";
+    throw std::runtime_error(ss.str());
+  }
+
+  if (res != CURLE_OK) {
+    std::stringstream ss;
+    ss << "QLever backend request failed: ";
+    size_t len = strlen(errbuf);
+    if (len > 0) {
+      LOG(ERROR) << "[REQUESTREADER] " << errbuf;
+      ss << errbuf;
+    } else {
+      LOG(ERROR) << "[REQUESTREADER] " << curl_easy_strerror(res);
+      ss << curl_easy_strerror(res);
+    }
+
+    throw std::runtime_error(ss.str());
+  }
+
+  return util::split(util::trim(resString), '\t');
+}
+
 // _____________________________________________________________________________
 void RequestReader::requestIds(const std::string& query) {
   CURLcode res;
@@ -154,6 +212,13 @@ std::string RequestReader::queryUrl(const std::string& query) const {
   return _backendUrl + "/?send=18446744073709551615" + "&query=" + esc;
 }
 
+// _____________________________________________________________________________
+size_t RequestReader::writeStringCb(void* contents, size_t size, size_t nmemb,
+                              void* userp) {
+  ((std::string*)userp)->append((char*)contents, size * nmemb);
+  return size * nmemb;
+}
+
 // _____________________________________________________________________________
 size_t RequestReader::writeCb(void* contents, size_t size, size_t nmemb,
                               void* userp) {
@@ -195,7 +260,7 @@ void RequestReader::parseIds(const char* c, size_t size) {
     _curByte = (_curByte + 1) % 8;
 
     if (_curByte == 0) {
-      ids.push_back({_curId.val, ids.size()});
+      _ids.push_back({_curId.val, _ids.size()});
     }
   }
 }

diff --git a/src/qlever-petrimaps/Misc.h b/src/qlever-petrimaps/Misc.h
@@ -95,13 +95,16 @@ struct RequestReader {
     if (_curl) curl_easy_cleanup(_curl);
   }
 
+  std::vector<std::string> requestColumns(const std::string& query);
   void requestIds(const std::string& qurl);
   void requestRows(const std::string& qurl);
   void requestRows(const std::string& query,
                    size_t (*writeCb)(void*, size_t, size_t, void*), void* ptr);
   void parse(const char*, size_t size);
   void parseIds(const char*, size_t size);
 
+  static size_t writeStringCb(void* contents, size_t size, size_t nmemb,
+                              void* userp);
   static size_t writeCb(void* contents, size_t size, size_t nmemb, void* userp);
   static size_t writeCbIds(void* contents, size_t size, size_t nmemb,
                            void* userp);
@@ -124,7 +127,7 @@ struct RequestReader {
   uint8_t _curByte = 0;
   ID _curId;
   size_t _received = 0;
-  std::vector<IdMapping> ids;
+  std::vector<IdMapping> _ids;
   size_t _maxMemory;
   std::exception_ptr exceptionPtr;
 };

diff --git a/src/qlever-petrimaps/server/Requestor.cpp b/src/qlever-petrimaps/server/Requestor.cpp
@@ -49,20 +49,20 @@ void Requestor::request(const std::string& qry) {
   LOG(INFO) << "[REQUESTOR] Requesting IDs for query " << qry;
   reader.requestIds(prepQuery(qry));
 
-  LOG(INFO) << "[REQUESTOR] Done, have " << reader.ids.size()
+  LOG(INFO) << "[REQUESTOR] Done, have " << reader._ids.size()
             << " ids in total.";
 
   // join with geoms from GeomCache
 
   // sort by qlever id
   LOG(INFO) << "[REQUESTOR] Sorting results by qlever ID...";
-  std::sort(reader.ids.begin(), reader.ids.end());
+  std::sort(reader._ids.begin(), reader._ids.end());
   LOG(INFO) << "[REQUESTOR] ... done";
 
   LOG(INFO) << "[REQUESTOR] Retrieving geoms from cache...";
 
   // (geom id, result row)
-  const auto& ret = _cache->getRelObjects(reader.ids);
+  const auto& ret = _cache->getRelObjects(reader._ids);
   _objects = ret.first;
   _numObjects = ret.second;
   LOG(INFO) << "[REQUESTOR] ... done, got " << _objects.size() << " objects.";
@@ -361,30 +361,42 @@ void Requestor::requestRows(
 
 // _____________________________________________________________________________
 std::string Requestor::prepQuery(std::string query) const {
-  // only use last column
-  std::regex expr("select[^{]*(\\?[A-Z0-9_\\-+]*)+[^{]*\\s*\\{",
+  std::cout << "BEFORE: \n" << query << std::endl;
+  std::regex expr("select[^{]*(\\*|[\\?$][A-Z0-9_\\-+]*)+[^{]*\\s*\\{",
                   std::regex_constants::icase);
 
-  // only remove columns the first (=outer) SELECT statement
-  query = std::regex_replace(query, expr, "SELECT $1 WHERE {$&",
-                             std::regex_constants::format_first_only) + "}";
+  std::string var;
+
+  std::smatch m;
+  std::regex_search(query, m, expr);
+
+  if (m.size() == 2) var = m[1].str();
 
-  if (util::toLower(query).find("limit") == std::string::npos) {
-    query += " LIMIT 18446744073709551615";
+  if (var == "*") {
+    // if we have a wildcard variable (*), we request the list of variables
+    // from the backend by sending a LIMIT 0 requests.
+    RequestReader reader(_cache->getBackendURL(), _maxMemory);
+    auto cols = reader.requestColumns(query + " LIMIT 0");
+    if (cols.size() > 0) var = cols.back();
   }
 
+  query = std::regex_replace(query, expr, "SELECT " + var + " WHERE {$&",
+                             std::regex_constants::format_first_only) + "}";
+
+  query += " LIMIT 18446744073709551615";
+
   return query;
 }
 
 // _____________________________________________________________________________
 std::string Requestor::prepQueryRow(std::string query, uint64_t row) const {
   // replace first select
-  std::regex expr("select[^{]*\\?[A-Z0-9_\\-+]*+[^{]*\\s*\\{",
+  std::regex expr("select[^{]*(\\*|[\\?$][A-Z0-9_\\-+]*)+[^{]*\\s*\\{",
                   std::regex_constants::icase);
 
   query = std::regex_replace(query, expr, "SELECT * {$&",
                              std::regex_constants::format_first_only) + "}";
-  query += "OFFSET " + std::to_string(row) + " LIMIT 1";
+  query += " OFFSET " + std::to_string(row) + " LIMIT 1";
   return query;
 }