diff --git a/src/qlever-petrimaps/Misc.cpp b/src/qlever-petrimaps/Misc.cpp index 7e3378c..10f61f4 100644 --- a/src/qlever-petrimaps/Misc.cpp +++ b/src/qlever-petrimaps/Misc.cpp @@ -13,6 +13,64 @@ using petrimaps::RequestReader; +// _____________________________________________________________________________ +std::vector RequestReader::requestColumns(const std::string& query) { + CURLcode res; + char errbuf[CURL_ERROR_SIZE]; + + std::string resString; + + if (_curl) { + auto url = queryUrl(query) + "&action=tsv_export"; + curl_easy_setopt(_curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, RequestReader::writeStringCb); + curl_easy_setopt(_curl, CURLOPT_WRITEDATA, &resString); + curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYPEER, false); + curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYHOST, false); + curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, 0); + + // accept any compression supported + curl_easy_setopt(_curl, CURLOPT_ACCEPT_ENCODING, ""); + curl_easy_setopt(_curl, CURLOPT_ERRORBUFFER, errbuf); + res = curl_easy_perform(_curl); + + long httpCode = 0; + curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &httpCode); + + if (httpCode != 200) { + std::stringstream ss; + ss << "QLever backend returned status code " << httpCode; + ss << "\n"; + ss << _raw; + throw std::runtime_error(ss.str()); + } + + if (exceptionPtr) std::rethrow_exception(exceptionPtr); + + } else { + std::stringstream ss; + ss << "[REQUESTREADER] Failed to perform curl request.\n"; + throw std::runtime_error(ss.str()); + } + + if (res != CURLE_OK) { + std::stringstream ss; + ss << "QLever backend request failed: "; + size_t len = strlen(errbuf); + if (len > 0) { + LOG(ERROR) << "[REQUESTREADER] " << errbuf; + ss << errbuf; + } else { + LOG(ERROR) << "[REQUESTREADER] " << curl_easy_strerror(res); + ss << curl_easy_strerror(res); + } + + throw std::runtime_error(ss.str()); + } + + return util::split(util::trim(resString), '\t'); +} + // _____________________________________________________________________________ void RequestReader::requestIds(const std::string& query) { CURLcode res; @@ -154,6 +212,13 @@ std::string RequestReader::queryUrl(const std::string& query) const { return _backendUrl + "/?send=18446744073709551615" + "&query=" + esc; } +// _____________________________________________________________________________ +size_t RequestReader::writeStringCb(void* contents, size_t size, size_t nmemb, + void* userp) { + ((std::string*)userp)->append((char*)contents, size * nmemb); + return size * nmemb; +} + // _____________________________________________________________________________ size_t RequestReader::writeCb(void* contents, size_t size, size_t nmemb, void* userp) { @@ -195,7 +260,7 @@ void RequestReader::parseIds(const char* c, size_t size) { _curByte = (_curByte + 1) % 8; if (_curByte == 0) { - ids.push_back({_curId.val, ids.size()}); + _ids.push_back({_curId.val, _ids.size()}); } } } diff --git a/src/qlever-petrimaps/Misc.h b/src/qlever-petrimaps/Misc.h index eee20c7..e6f1e7a 100644 --- a/src/qlever-petrimaps/Misc.h +++ b/src/qlever-petrimaps/Misc.h @@ -95,6 +95,7 @@ struct RequestReader { if (_curl) curl_easy_cleanup(_curl); } + std::vector requestColumns(const std::string& query); void requestIds(const std::string& qurl); void requestRows(const std::string& qurl); void requestRows(const std::string& query, @@ -102,6 +103,8 @@ struct RequestReader { void parse(const char*, size_t size); void parseIds(const char*, size_t size); + static size_t writeStringCb(void* contents, size_t size, size_t nmemb, + void* userp); static size_t writeCb(void* contents, size_t size, size_t nmemb, void* userp); static size_t writeCbIds(void* contents, size_t size, size_t nmemb, void* userp); @@ -124,7 +127,7 @@ struct RequestReader { uint8_t _curByte = 0; ID _curId; size_t _received = 0; - std::vector ids; + std::vector _ids; size_t _maxMemory; std::exception_ptr exceptionPtr; }; diff --git a/src/qlever-petrimaps/server/Requestor.cpp b/src/qlever-petrimaps/server/Requestor.cpp index 6f55f19..de04c5a 100644 --- a/src/qlever-petrimaps/server/Requestor.cpp +++ b/src/qlever-petrimaps/server/Requestor.cpp @@ -49,20 +49,20 @@ void Requestor::request(const std::string& qry) { LOG(INFO) << "[REQUESTOR] Requesting IDs for query " << qry; reader.requestIds(prepQuery(qry)); - LOG(INFO) << "[REQUESTOR] Done, have " << reader.ids.size() + LOG(INFO) << "[REQUESTOR] Done, have " << reader._ids.size() << " ids in total."; // join with geoms from GeomCache // sort by qlever id LOG(INFO) << "[REQUESTOR] Sorting results by qlever ID..."; - std::sort(reader.ids.begin(), reader.ids.end()); + std::sort(reader._ids.begin(), reader._ids.end()); LOG(INFO) << "[REQUESTOR] ... done"; LOG(INFO) << "[REQUESTOR] Retrieving geoms from cache..."; // (geom id, result row) - const auto& ret = _cache->getRelObjects(reader.ids); + const auto& ret = _cache->getRelObjects(reader._ids); _objects = ret.first; _numObjects = ret.second; LOG(INFO) << "[REQUESTOR] ... done, got " << _objects.size() << " objects."; @@ -361,30 +361,42 @@ void Requestor::requestRows( // _____________________________________________________________________________ std::string Requestor::prepQuery(std::string query) const { - // only use last column - std::regex expr("select[^{]*(\\?[A-Z0-9_\\-+]*)+[^{]*\\s*\\{", + std::cout << "BEFORE: \n" << query << std::endl; + std::regex expr("select[^{]*(\\*|[\\?$][A-Z0-9_\\-+]*)+[^{]*\\s*\\{", std::regex_constants::icase); - // only remove columns the first (=outer) SELECT statement - query = std::regex_replace(query, expr, "SELECT $1 WHERE {$&", - std::regex_constants::format_first_only) + "}"; + std::string var; + + std::smatch m; + std::regex_search(query, m, expr); + + if (m.size() == 2) var = m[1].str(); - if (util::toLower(query).find("limit") == std::string::npos) { - query += " LIMIT 18446744073709551615"; + if (var == "*") { + // if we have a wildcard variable (*), we request the list of variables + // from the backend by sending a LIMIT 0 requests. + RequestReader reader(_cache->getBackendURL(), _maxMemory); + auto cols = reader.requestColumns(query + " LIMIT 0"); + if (cols.size() > 0) var = cols.back(); } + query = std::regex_replace(query, expr, "SELECT " + var + " WHERE {$&", + std::regex_constants::format_first_only) + "}"; + + query += " LIMIT 18446744073709551615"; + return query; } // _____________________________________________________________________________ std::string Requestor::prepQueryRow(std::string query, uint64_t row) const { // replace first select - std::regex expr("select[^{]*\\?[A-Z0-9_\\-+]*+[^{]*\\s*\\{", + std::regex expr("select[^{]*(\\*|[\\?$][A-Z0-9_\\-+]*)+[^{]*\\s*\\{", std::regex_constants::icase); query = std::regex_replace(query, expr, "SELECT * {$&", std::regex_constants::format_first_only) + "}"; - query += "OFFSET " + std::to_string(row) + " LIMIT 1"; + query += " OFFSET " + std::to_string(row) + " LIMIT 1"; return query; }