From 0e1394a33561f38f4f8f22f721a300378eed25fb Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Fri, 15 Nov 2024 14:38:20 +0100 Subject: [PATCH 01/19] Move `checkParameter` from `Server` to `ad_utility::url_parser` --- src/engine/Server.cpp | 32 +++++++------------------------- src/engine/Server.h | 12 ------------ src/util/http/UrlParser.cpp | 22 ++++++++++++++++++++++ src/util/http/UrlParser.h | 7 +++++++ test/ServerTest.cpp | 23 ----------------------- test/UrlParserTest.cpp | 23 +++++++++++++++++++++++ 6 files changed, 59 insertions(+), 60 deletions(-) diff --git a/src/engine/Server.cpp b/src/engine/Server.cpp index 87ca3897fc..ffe4aafe52 100644 --- a/src/engine/Server.cpp +++ b/src/engine/Server.cpp @@ -346,8 +346,8 @@ Awaitable Server::process( // We always want to call `Server::checkParameter` with the same first // parameter. - auto checkParameter = - std::bind_front(&Server::checkParameter, std::cref(parameters)); + auto checkParameter = std::bind_front(&ad_utility::url_parser::checkParameter, + std::cref(parameters)); // Check the access token. If an access token is provided and the check fails, // throw an exception and do not process any part of the query (even if the @@ -516,9 +516,11 @@ Awaitable Server::process( std::pair Server::determineResultPinning( const ad_utility::url_parser::ParamValueMap& params) { const bool pinSubtrees = - checkParameter(params, "pinsubtrees", "true").has_value(); + ad_utility::url_parser::checkParameter(params, "pinsubtrees", "true") + .has_value(); const bool pinResult = - checkParameter(params, "pinresult", "true").has_value(); + ad_utility::url_parser::checkParameter(params, "pinresult", "true") + .has_value(); return {pinSubtrees, pinResult}; } // ____________________________________________________________________________ @@ -719,6 +721,7 @@ class NoSupportedMediatypeError : public std::runtime_error { MediaType Server::determineMediaType( const ad_utility::url_parser::ParamValueMap& params, const ad_utility::httpUtils::HttpRequest auto& request) { + using namespace ad_utility::url_parser; // The following code block determines the media type to be used for the // result. The media type is either determined by the "Accept:" header of // the request or by the URL parameter "action=..." (for TSV and CSV export, @@ -1001,24 +1004,3 @@ bool Server::checkAccessToken( return true; } } - -// _____________________________________________________________________________ -std::optional Server::checkParameter( - const ad_utility::url_parser::ParamValueMap& parameters, - std::string_view key, std::optional value) { - auto param = - ad_utility::url_parser::getParameterCheckAtMostOnce(parameters, key); - if (!param.has_value()) { - return std::nullopt; - } - std::string parameterValue = param.value(); - - // If value is given, but not equal to param value, return std::nullopt. If - // no value is given, set it to param value. - if (value == std::nullopt) { - value = parameterValue; - } else if (value != parameterValue) { - return std::nullopt; - } - return value; -} diff --git a/src/engine/Server.h b/src/engine/Server.h index e7e514adf1..ecbea2ac9a 100644 --- a/src/engine/Server.h +++ b/src/engine/Server.h @@ -236,18 +236,6 @@ class Server { /// HTTP error response. bool checkAccessToken(std::optional accessToken) const; - /// Checks if a URL parameter exists in the request, and it matches the - /// expected `value`. If yes, return the value, otherwise return - /// `std::nullopt`. If `value` is `std::nullopt`, only check if the key - /// exists. We need this because we have parameters like "cmd=stats", where a - /// fixed combination of the key and value determines the kind of action, as - /// well as parameters like "index-decription=...", where the key determines - /// the kind of action. If the key is not found, always return `std::nullopt`. - static std::optional checkParameter( - const ad_utility::url_parser::ParamValueMap& parameters, - std::string_view key, std::optional value); - FRIEND_TEST(ServerTest, checkParameter); - /// Check if user-provided timeout is authorized with a valid access-token or /// lower than the server default. Return an empty optional and send a 403 /// Forbidden HTTP response if the change is not allowed. Return the new diff --git a/src/util/http/UrlParser.cpp b/src/util/http/UrlParser.cpp index 3149f97aec..1670fa937d 100644 --- a/src/util/http/UrlParser.cpp +++ b/src/util/http/UrlParser.cpp @@ -8,6 +8,7 @@ using namespace ad_utility::url_parser; +// _____________________________________________________________________________ std::optional ad_utility::url_parser::getParameterCheckAtMostOnce( const ParamValueMap& map, string_view key) { if (!map.contains(key)) { @@ -21,6 +22,27 @@ std::optional ad_utility::url_parser::getParameterCheckAtMostOnce( } return value.front(); } + +// _____________________________________________________________________________ +std::optional ad_utility::url_parser::checkParameter( + const ParamValueMap& parameters, std::string_view key, + std::optional value) { + const auto param = getParameterCheckAtMostOnce(parameters, key); + if (!param.has_value()) { + return std::nullopt; + } + std::string parameterValue = param.value(); + + // If no value is given, return the parameter's value. If value is given, but + // not equal to the parameter's value, return `std::nullopt`. + if (value == std::nullopt) { + value = parameterValue; + } else if (value != parameterValue) { + return std::nullopt; + } + return value; +} + // _____________________________________________________________________________ ParsedUrl ad_utility::url_parser::parseRequestTarget(std::string_view target) { auto urlResult = boost::urls::parse_origin_form(target); diff --git a/src/util/http/UrlParser.h b/src/util/http/UrlParser.h index bc1374504f..54dc34472e 100644 --- a/src/util/http/UrlParser.h +++ b/src/util/http/UrlParser.h @@ -29,6 +29,13 @@ using ParamValueMap = ad_utility::HashMap>; std::optional getParameterCheckAtMostOnce(const ParamValueMap& map, string_view key); +// Checks if a parameter exists, and it matches the +// expected `value`. If yes, return the value, otherwise return +// `std::nullopt`. +std::optional checkParameter(const ParamValueMap& parameters, + std::string_view key, + std::optional value); + // A parsed URL. // - `path_` is the URL path // - `parameters_` is a map of the HTTP Query parameters diff --git a/test/ServerTest.cpp b/test/ServerTest.cpp index 5ed589fc42..766eef7668 100644 --- a/test/ServerTest.cpp +++ b/test/ServerTest.cpp @@ -142,29 +142,6 @@ TEST(ServerTest, parseHttpRequest) { ParsedRequestIs("/", {}, Update{"DELETE * WHERE {}"})); } -TEST(ServerTest, checkParameter) { - const ParamValueMap exampleParams = {{"foo", {"bar"}}, - {"baz", {"qux", "quux"}}}; - - EXPECT_THAT(Server::checkParameter(exampleParams, "doesNotExist", ""), - testing::Eq(std::nullopt)); - EXPECT_THAT(Server::checkParameter(exampleParams, "foo", "baz"), - testing::Eq(std::nullopt)); - EXPECT_THAT(Server::checkParameter(exampleParams, "foo", "bar"), - testing::Optional(testing::StrEq("bar"))); - AD_EXPECT_THROW_WITH_MESSAGE( - Server::checkParameter(exampleParams, "baz", "qux"), - testing::StrEq("Parameter \"baz\" must be given exactly once. Is: 2")); - EXPECT_THAT(Server::checkParameter(exampleParams, "foo", std::nullopt), - testing::Optional(testing::StrEq("bar"))); - AD_EXPECT_THROW_WITH_MESSAGE( - Server::checkParameter(exampleParams, "baz", std::nullopt), - testing::StrEq("Parameter \"baz\" must be given exactly once. Is: 2")); - AD_EXPECT_THROW_WITH_MESSAGE( - Server::checkParameter(exampleParams, "baz", std::nullopt), - testing::StrEq("Parameter \"baz\" must be given exactly once. Is: 2")); -} - TEST(ServerTest, determineResultPinning) { EXPECT_THAT(Server::determineResultPinning( {{"pinsubtrees", {"true"}}, {"pinresult", {"true"}}}), diff --git a/test/UrlParserTest.cpp b/test/UrlParserTest.cpp index eb6aeebc4d..d0b09ab030 100644 --- a/test/UrlParserTest.cpp +++ b/test/UrlParserTest.cpp @@ -128,3 +128,26 @@ TEST(UrlParserTest, parseDatasetClauses) { {iri(""), true}, {iri(""), true}})); } + +TEST(UrlParserTest, checkParameter) { + const url_parser::ParamValueMap exampleParams = {{"foo", {"bar"}}, + {"baz", {"qux", "quux"}}}; + + EXPECT_THAT(url_parser::checkParameter(exampleParams, "doesNotExist", ""), + ::testing::Eq(std::nullopt)); + EXPECT_THAT(url_parser::checkParameter(exampleParams, "foo", "baz"), + ::testing::Eq(std::nullopt)); + EXPECT_THAT(url_parser::checkParameter(exampleParams, "foo", "bar"), + ::testing::Optional(::testing::StrEq("bar"))); + AD_EXPECT_THROW_WITH_MESSAGE( + url_parser::checkParameter(exampleParams, "baz", "qux"), + ::testing::StrEq("Parameter \"baz\" must be given exactly once. Is: 2")); + EXPECT_THAT(url_parser::checkParameter(exampleParams, "foo", std::nullopt), + ::testing::Optional(::testing::StrEq("bar"))); + AD_EXPECT_THROW_WITH_MESSAGE( + url_parser::checkParameter(exampleParams, "baz", std::nullopt), + ::testing::StrEq("Parameter \"baz\" must be given exactly once. Is: 2")); + AD_EXPECT_THROW_WITH_MESSAGE( + url_parser::checkParameter(exampleParams, "baz", std::nullopt), + ::testing::StrEq("Parameter \"baz\" must be given exactly once. Is: 2")); +} From f840757e750019464e5f48d0ef69fda2b1251061 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Wed, 20 Nov 2024 12:57:59 +0100 Subject: [PATCH 02/19] Extract helpers for creating http requests --- test/ServerTest.cpp | 21 ++------------------- test/util/HttpRequestHelpers.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 19 deletions(-) create mode 100644 test/util/HttpRequestHelpers.h diff --git a/test/ServerTest.cpp b/test/ServerTest.cpp index 766eef7668..e1bd132957 100644 --- a/test/ServerTest.cpp +++ b/test/ServerTest.cpp @@ -9,6 +9,7 @@ #include #include "util/GTestHelpers.h" +#include "util/HttpRequestHelpers.h" #include "util/http/HttpUtils.h" #include "util/http/UrlParser.h" @@ -30,24 +31,6 @@ auto ParsedRequestIs = [](const std::string& path, } // namespace TEST(ServerTest, parseHttpRequest) { - namespace http = boost::beast::http; - - auto MakeBasicRequest = [](http::verb method, const std::string& target) { - // version 11 stands for HTTP/1.1 - return http::request{method, target, 11}; - }; - auto MakeGetRequest = [&MakeBasicRequest](const std::string& target) { - return MakeBasicRequest(http::verb::get, target); - }; - auto MakePostRequest = [&MakeBasicRequest](const std::string& target, - const std::string& contentType, - const std::string& body) { - auto req = MakeBasicRequest(http::verb::post, target); - req.set(http::field::content_type, contentType); - req.body() = body; - req.prepare_payload(); - return req; - }; auto parse = [](const ad_utility::httpUtils::HttpRequest auto& request) { return Server::parseHttpRequest(request); }; @@ -120,7 +103,7 @@ TEST(ServerTest, parseHttpRequest) { parse(MakePostRequest("/?send=100", QUERY, "SELECT * WHERE {}")), ParsedRequestIs("/", {{"send", {"100"}}}, Query{"SELECT * WHERE {}"})); AD_EXPECT_THROW_WITH_MESSAGE( - parse(MakeBasicRequest(http::verb::patch, "/")), + parse(MakeRequest(http::verb::patch, "/")), testing::StrEq( "Request method \"PATCH\" not supported (has to be GET or POST)")); AD_EXPECT_THROW_WITH_MESSAGE( diff --git a/test/util/HttpRequestHelpers.h b/test/util/HttpRequestHelpers.h new file mode 100644 index 0000000000..4b19baa359 --- /dev/null +++ b/test/util/HttpRequestHelpers.h @@ -0,0 +1,34 @@ +// Copyright 2024, University of Freiburg +// Chair of Algorithms and Data Structures +// Authors: Julian Mundhahs + +#pragma once + +namespace http = boost::beast::http; + +using Headers = ad_utility::HashMap; +inline auto MakeRequest(const http::verb method = http::verb::get, + const std::string& target = "/", + const Headers& headers = {}, + const std::optional& body = std::nullopt) { + // version 11 stands for HTTP/1.1 + auto req = http::request{method, target, 11}; + for (const auto& [key, value] : headers) { + req.set(key, value); + } + if (body.has_value()) { + req.body() = body.value(); + req.prepare_payload(); + } + return req; +} + +inline auto MakeGetRequest(const std::string& target) { + return MakeRequest(http::verb::get, target); +} +inline auto MakePostRequest(const std::string& target, + const std::string& contentType, + const std::string& body) { + return MakeRequest(http::verb::post, target, + {{http::field::content_type, contentType}}, body); +} From a1a96e69e54b3e03f365fd9397707a751f9bb038 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Thu, 21 Nov 2024 14:36:36 +0100 Subject: [PATCH 03/19] change HttpRequestHelpers.h --- test/util/HttpRequestHelpers.h | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test/util/HttpRequestHelpers.h b/test/util/HttpRequestHelpers.h index 4b19baa359..6634832ac6 100644 --- a/test/util/HttpRequestHelpers.h +++ b/test/util/HttpRequestHelpers.h @@ -4,13 +4,17 @@ #pragma once -namespace http = boost::beast::http; +#include "util/HashMap.h" +#include "util/http/beast.h" -using Headers = ad_utility::HashMap; -inline auto MakeRequest(const http::verb method = http::verb::get, - const std::string& target = "/", - const Headers& headers = {}, - const std::optional& body = std::nullopt) { +namespace ad_utility::testing { + +namespace http = beast::http; + +inline auto MakeRequest( + const http::verb method = http::verb::get, const std::string& target = "/", + const ad_utility::HashMap& headers = {}, + const std::optional& body = std::nullopt) { // version 11 stands for HTTP/1.1 auto req = http::request{method, target, 11}; for (const auto& [key, value] : headers) { @@ -26,9 +30,12 @@ inline auto MakeRequest(const http::verb method = http::verb::get, inline auto MakeGetRequest(const std::string& target) { return MakeRequest(http::verb::get, target); } + inline auto MakePostRequest(const std::string& target, const std::string& contentType, const std::string& body) { return MakeRequest(http::verb::post, target, {{http::field::content_type, contentType}}, body); } + +} // namespace ad_utility::testing From a7cb7ba19c9dfffe63e6333b7eed00bb2e5efc77 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Thu, 21 Nov 2024 14:38:21 +0100 Subject: [PATCH 04/19] begin implementation --- src/engine/CMakeLists.txt | 3 +- src/engine/GraphStoreProtocol.cpp | 106 ++++++++++++++++++++++++++++++ src/engine/GraphStoreProtocol.h | 28 ++++++++ test/CMakeLists.txt | 2 + test/GraphStoreProtocolTest.cpp | 39 +++++++++++ 5 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 src/engine/GraphStoreProtocol.cpp create mode 100644 src/engine/GraphStoreProtocol.h create mode 100644 test/GraphStoreProtocolTest.cpp diff --git a/src/engine/CMakeLists.txt b/src/engine/CMakeLists.txt index cbfb3344c3..d14486e961 100644 --- a/src/engine/CMakeLists.txt +++ b/src/engine/CMakeLists.txt @@ -13,5 +13,6 @@ add_library(engine VariableToColumnMap.cpp ExportQueryExecutionTrees.cpp CartesianProductJoin.cpp TextIndexScanForWord.cpp TextIndexScanForEntity.cpp TextLimit.cpp LazyGroupBy.cpp GroupByHashMapOptimization.cpp SpatialJoin.cpp - CountConnectedSubgraphs.cpp SpatialJoinAlgorithms.cpp PathSearch.cpp ExecuteUpdate.cpp) + CountConnectedSubgraphs.cpp SpatialJoinAlgorithms.cpp PathSearch.cpp ExecuteUpdate.cpp + GraphStoreProtocol.cpp) qlever_target_link_libraries(engine util index parser sparqlExpressions http SortPerformanceEstimator Boost::iostreams s2) diff --git a/src/engine/GraphStoreProtocol.cpp b/src/engine/GraphStoreProtocol.cpp new file mode 100644 index 0000000000..a0fa6d9fe0 --- /dev/null +++ b/src/engine/GraphStoreProtocol.cpp @@ -0,0 +1,106 @@ +// Copyright 2024, University of Freiburg +// Chair of Algorithms and Data Structures +// Authors: Julian Mundhahs + +#include "engine/GraphStoreProtocol.h" + +#include + +#include "parser/RdfParser.h" + +// ____________________________________________________________________________ +GraphOrDefault GraphStoreProtocol::extractTargetGraph( + const ad_utility::url_parser::ParamValueMap& params) { + // Extract the graph to be acted upon using `Indirect Graph + // Identification`. + const auto graphIri = + ad_utility::url_parser::checkParameter(params, "graph", std::nullopt); + const auto isDefault = + ad_utility::url_parser::checkParameter(params, "default", ""); + if (!(graphIri || isDefault)) { + throw std::runtime_error("No graph IRI specified in the request."); + } + if (graphIri && isDefault) { + throw std::runtime_error( + "Only one of `default` and `graph` may be used for graph " + "identification."); + } + if (graphIri) { + return GraphRef::fromIriref(graphIri.value()); + } else { + AD_CORRECTNESS_CHECK(isDefault); + return DEFAULT{}; + } +} + +// ____________________________________________________________________________ +ParsedQuery GraphStoreProtocol::transformGraphStoreProtocol( + const ad_utility::httpUtils::HttpRequest auto& rawRequest, + const ad_utility::url_parser::ParsedRequest& request) { + // TODO: only use the raw request. It makes no sense to use `ParsedRequest` + // here - the interface just doesn't fit. + GraphOrDefault graph = extractTargetGraph(request.parameters_); + auto visitGet = [&graph]() -> ParsedQuery { + ParsedQuery res; + res._clause = parsedQuery::ConstructClause( + {{Variable("?s"), Variable("?p"), Variable("?o")}}); + res._rootGraphPattern = {}; + parsedQuery::GraphPattern selectSPO; + selectSPO._graphPatterns.emplace_back(parsedQuery::BasicGraphPattern{ + {SparqlTriple(Variable("?s"), "?p", Variable("?o"))}}); + // TODO: extract this wrapping into a lambda + if (std::holds_alternative(graph)) { + parsedQuery::GroupGraphPattern selectSPOWithGraph{ + std::move(selectSPO), + std::get(graph)}; + res._rootGraphPattern._graphPatterns.emplace_back( + std::move(selectSPOWithGraph)); + } else { + AD_CORRECTNESS_CHECK(std::holds_alternative(graph)); + res._rootGraphPattern = std::move(selectSPO); + } + return res; + }; + + using namespace boost::beast::http; + using Re2Parser = RdfStringParser>; + using NQuadRe2Parser = RdfStringParser>; + auto visitPost = [&graph, &rawRequest]() -> ParsedQuery { + auto contentType = rawRequest.at(field::content_type); + auto type = ad_utility::getMediaTypeFromAcceptHeader(contentType); + ParsedQuery res; + updateClause::GraphUpdate up{{}, {}}; + if (std::holds_alternative(graph)) { + up.with_ = std::get(graph); + } + res._clause = parsedQuery::UpdateClause{up}; + return res; + }; + + auto method = rawRequest.method(); + if (method == verb::get) { + return visitGet(); + } else if (method == verb::put) { + throw std::runtime_error( + "PUT in the SPARQL Graph Store HTTP Protocol is not yet implemented in " + "QLever."); + } else if (method == verb::delete_) { + throw std::runtime_error( + "DELETE in the SPARQL Graph Store HTTP Protocol is not yet implemented " + "in QLever."); + } else if (method == verb::post) { + return visitPost(); + } else if (method == verb::head) { + throw std::runtime_error( + "HEAD in the SPARQL Graph Store HTTP Protocol is not yet implemented " + "in QLever."); + } else if (method == verb::patch) { + throw std::runtime_error( + "PATCH in the SPARQL Graph Store HTTP Protocol is not yet implemented " + "in QLever."); + } else { + throw std::runtime_error( + absl::StrCat("Unsupported HTTP method \"", "", + "\" for the SPARQL Graph Store HTTP Protocol.")); + } +} diff --git a/src/engine/GraphStoreProtocol.h b/src/engine/GraphStoreProtocol.h new file mode 100644 index 0000000000..9792cae36b --- /dev/null +++ b/src/engine/GraphStoreProtocol.h @@ -0,0 +1,28 @@ +// Copyright 2024, University of Freiburg +// Chair of Algorithms and Data Structures +// Authors: Julian Mundhahs + +#pragma once + +#include + +#include "parser/ParsedQuery.h" +#include "util/http/HttpUtils.h" +#include "util/http/UrlParser.h" + +class GraphStoreProtocol { + public: + // Every Graph Store Protocol requests has equivalent SPARQL Query or Update. + // Transform the Graph Store Protocol request into it's equivalent Query or + // Update. + static ParsedQuery transformGraphStoreProtocol( + const ad_utility::httpUtils::HttpRequest auto& rawRequest, + const ad_utility::url_parser::ParsedRequest& request); + + private: + // Extract the graph to be acted upon using `Indirect Graph + // Identification`. + static GraphOrDefault extractTargetGraph( + const ad_utility::url_parser::ParamValueMap& params); + FRIEND_TEST(GraphStoreProtocolTest, extractTargetGraph); +}; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e540b448eb..44e8b619fc 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -420,3 +420,5 @@ addLinkAndDiscoverTest(UrlParserTest) addLinkAndDiscoverTest(ServerTest engine) addLinkAndDiscoverTest(ExecuteUpdateTest engine) + +addLinkAndDiscoverTest(GraphStoreProtocolTest engine) diff --git a/test/GraphStoreProtocolTest.cpp b/test/GraphStoreProtocolTest.cpp new file mode 100644 index 0000000000..1bbe7aa9d0 --- /dev/null +++ b/test/GraphStoreProtocolTest.cpp @@ -0,0 +1,39 @@ +// Copyright 2024, University of Freiburg +// Chair of Algorithms and Data Structures +// Authors: Julian Mundhahs + +#include +#include + +#include "./util/GTestHelpers.h" +#include "./util/HttpRequestHelpers.h" +#include "engine/GraphStoreProtocol.h" + +TEST(GraphStoreProtocolTest, extractTargetGraph) { + EXPECT_THAT(GraphStoreProtocol::extractTargetGraph({{"default", {""}}}), + DEFAULT{}); + EXPECT_THAT(GraphStoreProtocol::extractTargetGraph({{"graph", {""}}}), + TripleComponent::Iri::fromIriref("")); + AD_EXPECT_THROW_WITH_MESSAGE( + GraphStoreProtocol::extractTargetGraph({}), + testing::HasSubstr("No graph IRI specified in the request.")); + AD_EXPECT_THROW_WITH_MESSAGE( + GraphStoreProtocol::extractTargetGraph({{"unrelated", {"a", "b"}}}), + testing::HasSubstr("No graph IRI specified in the request.")); + AD_EXPECT_THROW_WITH_MESSAGE( + GraphStoreProtocol::extractTargetGraph( + {{"default", {""}}, {"graph", {""}}}), + testing::HasSubstr("Only one of `default` and `graph` may be used for " + "graph identification.")); +} + +TEST(GraphStoreProtocolTest, transformGraphStoreProtocol) { + ad_utility::httpUtils::HttpRequest auto f = + ad_utility::testing::MakeGetRequest("/?default"); + ad_utility::url_parser::ParsedRequest ff = + ad_utility::url_parser::ParsedRequest{}; + GraphStoreProtocol::transformGraphStoreProtocol(f, ff); + EXPECT_THAT(GraphStoreProtocol::transformGraphStoreProtocol( + f, ad_utility::url_parser::ParsedRequest{}), + testing::_); +} From e863950a26c56750b954cbdb9e5bd9c1c9cb5179 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Thu, 21 Nov 2024 14:40:24 +0100 Subject: [PATCH 05/19] revert wrong ide suggestion --- test/util/HttpRequestHelpers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/util/HttpRequestHelpers.h b/test/util/HttpRequestHelpers.h index 6634832ac6..b05d070a3b 100644 --- a/test/util/HttpRequestHelpers.h +++ b/test/util/HttpRequestHelpers.h @@ -9,7 +9,7 @@ namespace ad_utility::testing { -namespace http = beast::http; +namespace http = boost::beast::http; inline auto MakeRequest( const http::verb method = http::verb::get, const std::string& target = "/", From 6c9cc6557a26d336524297a8e772d69a6a78a0ff Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Sat, 7 Dec 2024 14:53:38 +0100 Subject: [PATCH 06/19] continue work --- src/engine/GraphStoreProtocol.cpp | 84 ++------------------- src/engine/GraphStoreProtocol.h | 117 +++++++++++++++++++++++++++++- src/parser/TripleComponent.h | 4 + src/util/http/MediaTypes.cpp | 7 +- src/util/http/MediaTypes.h | 2 + test/GraphStoreProtocolTest.cpp | 77 ++++++++++++++++++-- 6 files changed, 201 insertions(+), 90 deletions(-) diff --git a/src/engine/GraphStoreProtocol.cpp b/src/engine/GraphStoreProtocol.cpp index a0fa6d9fe0..d0300436ad 100644 --- a/src/engine/GraphStoreProtocol.cpp +++ b/src/engine/GraphStoreProtocol.cpp @@ -6,8 +6,6 @@ #include -#include "parser/RdfParser.h" - // ____________________________________________________________________________ GraphOrDefault GraphStoreProtocol::extractTargetGraph( const ad_utility::url_parser::ParamValueMap& params) { @@ -16,91 +14,19 @@ GraphOrDefault GraphStoreProtocol::extractTargetGraph( const auto graphIri = ad_utility::url_parser::checkParameter(params, "graph", std::nullopt); const auto isDefault = - ad_utility::url_parser::checkParameter(params, "default", ""); - if (!(graphIri || isDefault)) { + ad_utility::url_parser::checkParameter(params, "default", "").has_value(); + if (!(graphIri.has_value() || isDefault)) { throw std::runtime_error("No graph IRI specified in the request."); } - if (graphIri && isDefault) { + if (graphIri.has_value() && isDefault) { throw std::runtime_error( "Only one of `default` and `graph` may be used for graph " "identification."); } - if (graphIri) { - return GraphRef::fromIriref(graphIri.value()); + if (graphIri.has_value()) { + return GraphRef::fromIrirefWithoutBrackets(graphIri.value()); } else { AD_CORRECTNESS_CHECK(isDefault); return DEFAULT{}; } } - -// ____________________________________________________________________________ -ParsedQuery GraphStoreProtocol::transformGraphStoreProtocol( - const ad_utility::httpUtils::HttpRequest auto& rawRequest, - const ad_utility::url_parser::ParsedRequest& request) { - // TODO: only use the raw request. It makes no sense to use `ParsedRequest` - // here - the interface just doesn't fit. - GraphOrDefault graph = extractTargetGraph(request.parameters_); - auto visitGet = [&graph]() -> ParsedQuery { - ParsedQuery res; - res._clause = parsedQuery::ConstructClause( - {{Variable("?s"), Variable("?p"), Variable("?o")}}); - res._rootGraphPattern = {}; - parsedQuery::GraphPattern selectSPO; - selectSPO._graphPatterns.emplace_back(parsedQuery::BasicGraphPattern{ - {SparqlTriple(Variable("?s"), "?p", Variable("?o"))}}); - // TODO: extract this wrapping into a lambda - if (std::holds_alternative(graph)) { - parsedQuery::GroupGraphPattern selectSPOWithGraph{ - std::move(selectSPO), - std::get(graph)}; - res._rootGraphPattern._graphPatterns.emplace_back( - std::move(selectSPOWithGraph)); - } else { - AD_CORRECTNESS_CHECK(std::holds_alternative(graph)); - res._rootGraphPattern = std::move(selectSPO); - } - return res; - }; - - using namespace boost::beast::http; - using Re2Parser = RdfStringParser>; - using NQuadRe2Parser = RdfStringParser>; - auto visitPost = [&graph, &rawRequest]() -> ParsedQuery { - auto contentType = rawRequest.at(field::content_type); - auto type = ad_utility::getMediaTypeFromAcceptHeader(contentType); - ParsedQuery res; - updateClause::GraphUpdate up{{}, {}}; - if (std::holds_alternative(graph)) { - up.with_ = std::get(graph); - } - res._clause = parsedQuery::UpdateClause{up}; - return res; - }; - - auto method = rawRequest.method(); - if (method == verb::get) { - return visitGet(); - } else if (method == verb::put) { - throw std::runtime_error( - "PUT in the SPARQL Graph Store HTTP Protocol is not yet implemented in " - "QLever."); - } else if (method == verb::delete_) { - throw std::runtime_error( - "DELETE in the SPARQL Graph Store HTTP Protocol is not yet implemented " - "in QLever."); - } else if (method == verb::post) { - return visitPost(); - } else if (method == verb::head) { - throw std::runtime_error( - "HEAD in the SPARQL Graph Store HTTP Protocol is not yet implemented " - "in QLever."); - } else if (method == verb::patch) { - throw std::runtime_error( - "PATCH in the SPARQL Graph Store HTTP Protocol is not yet implemented " - "in QLever."); - } else { - throw std::runtime_error( - absl::StrCat("Unsupported HTTP method \"", "", - "\" for the SPARQL Graph Store HTTP Protocol.")); - } -} diff --git a/src/engine/GraphStoreProtocol.h b/src/engine/GraphStoreProtocol.h index 9792cae36b..f8b8022247 100644 --- a/src/engine/GraphStoreProtocol.h +++ b/src/engine/GraphStoreProtocol.h @@ -7,6 +7,7 @@ #include #include "parser/ParsedQuery.h" +#include "parser/RdfParser.h" #include "util/http/HttpUtils.h" #include "util/http/UrlParser.h" @@ -16,8 +17,120 @@ class GraphStoreProtocol { // Transform the Graph Store Protocol request into it's equivalent Query or // Update. static ParsedQuery transformGraphStoreProtocol( - const ad_utility::httpUtils::HttpRequest auto& rawRequest, - const ad_utility::url_parser::ParsedRequest& request); + const ad_utility::httpUtils::HttpRequest auto& rawRequest) { + ad_utility::url_parser::ParsedUrl parsedUrl = + ad_utility::url_parser::parseRequestTarget(rawRequest.target()); + GraphOrDefault graph = extractTargetGraph(parsedUrl.parameters_); + auto visitGet = [&graph]() -> ParsedQuery { + ParsedQuery res; + res._clause = parsedQuery::ConstructClause( + {{Variable("?s"), Variable("?p"), Variable("?o")}}); + res._rootGraphPattern = {}; + parsedQuery::GraphPattern selectSPO; + selectSPO._graphPatterns.emplace_back(parsedQuery::BasicGraphPattern{ + {SparqlTriple(Variable("?s"), "?p", Variable("?o"))}}); + // TODO: extract this wrapping into a lambda + if (std::holds_alternative(graph)) { + parsedQuery::GroupGraphPattern selectSPOWithGraph{ + std::move(selectSPO), + std::get(graph)}; + res._rootGraphPattern._graphPatterns.emplace_back( + std::move(selectSPOWithGraph)); + } else { + AD_CORRECTNESS_CHECK(std::holds_alternative(graph)); + res._rootGraphPattern = std::move(selectSPO); + } + return res; + }; + + using namespace boost::beast::http; + using Re2Parser = RdfStringParser>; + using NQuadRe2Parser = RdfStringParser>; + auto visitPost = [&graph, &rawRequest]() -> ParsedQuery { + auto contentType = rawRequest.at(field::content_type); + auto type = ad_utility::getMediaTypeFromAcceptHeader(contentType); + std::vector triples; + switch (type.value()) { + case ad_utility::MediaType::turtle: + case ad_utility::MediaType::ntriples: { + Re2Parser parser = Re2Parser(); + parser.setInputStream(rawRequest.body()); + triples = parser.parseAndReturnAllTriples(); + break; + } + case ad_utility::MediaType::nquads: { + NQuadRe2Parser parser = NQuadRe2Parser(); + parser.setInputStream(rawRequest.body()); + triples = parser.parseAndReturnAllTriples(); + break; + } + default: { + throw std::runtime_error( + absl::StrCat("Mediatype \"", ad_utility::toString(type.value()), + "\" is not supported for SPARQL Graph Store HTTP " + "Protocol in QLever.")); + break; + } + } + ParsedQuery res; + auto transformTurtleTriple = + [&graph](const TurtleTriple& triple) -> SparqlTripleSimpleWithGraph { + auto triplesGraph = + [&triple]() -> std::variant { + if (triple.graphIri_.isIri()) { + return Iri(triple.graphIri_.getIri().toStringRepresentation()); + } else if (triple.graphIri_.isVariable()) { + return triple.graphIri_.getVariable(); + } else { + AD_CORRECTNESS_CHECK(triple.graphIri_.isId()); + AD_CORRECTNESS_CHECK(triple.graphIri_.getId() == + qlever::specialIds().at(DEFAULT_GRAPH_IRI)); + return std::monostate{}; + } + }(); + if (std::holds_alternative(graph)) { + triplesGraph = + Iri(std::get(graph).toStringRepresentation()); + } + return SparqlTripleSimpleWithGraph(triple.subject_, triple.predicate_, + triple.object_, triplesGraph); + }; + updateClause::GraphUpdate up{ + ad_utility::transform(triples, transformTurtleTriple), {}}; + if (std::holds_alternative(graph)) { + up.with_ = std::get(graph); + } + res._clause = parsedQuery::UpdateClause{up}; + return res; + }; + + auto method = rawRequest.method(); + if (method == verb::get) { + return visitGet(); + } else if (method == verb::put) { + throw std::runtime_error( + "PUT in the SPARQL Graph Store HTTP Protocol is not yet implemented " + "in QLever."); + } else if (method == verb::delete_) { + throw std::runtime_error( + "DELETE in the SPARQL Graph Store HTTP Protocol is not yet " + "implemented in QLever."); + } else if (method == verb::post) { + return visitPost(); + } else if (method == verb::head) { + throw std::runtime_error( + "HEAD in the SPARQL Graph Store HTTP Protocol is not yet implemented " + "in QLever."); + } else if (method == verb::patch) { + throw std::runtime_error( + "PATCH in the SPARQL Graph Store HTTP Protocol is not yet " + "implemented in QLever."); + } else { + throw std::runtime_error( + absl::StrCat("Unsupported HTTP method \"", "", + "\" for the SPARQL Graph Store HTTP Protocol.")); + } + } private: // Extract the graph to be acted upon using `Indirect Graph diff --git a/src/parser/TripleComponent.h b/src/parser/TripleComponent.h index fb874fc3c1..9041c3b637 100644 --- a/src/parser/TripleComponent.h +++ b/src/parser/TripleComponent.h @@ -175,6 +175,10 @@ class TripleComponent { } [[nodiscard]] Variable& getVariable() { return std::get(_variant); } + bool isId() const { return std::holds_alternative(_variant); } + [[nodiscard]] const Id& getId() const { return std::get(_variant); } + [[nodiscard]] Id& getId() { return std::get(_variant); } + /// Convert to an RDF literal. `std::strings` will be emitted directly, /// `int64_t` is converted to a `xsd:integer` literal, and a `double` is /// converted to a `xsd:double`. diff --git a/src/util/http/MediaTypes.cpp b/src/util/http/MediaTypes.cpp index 096cb3a3c1..1e85acbfde 100644 --- a/src/util/http/MediaTypes.cpp +++ b/src/util/http/MediaTypes.cpp @@ -17,8 +17,9 @@ using enum MediaType; // The first media type in this list is the default, if no other type is // specified in the request. It's "application/sparql-results+json", as // required by the SPARQL standard. -constexpr std::array SUPPORTED_MEDIA_TYPES{ - sparqlJson, sparqlXml, qleverJson, tsv, csv, turtle, octetStream}; +constexpr std::array SUPPORTED_MEDIA_TYPES{sparqlJson, sparqlXml, qleverJson, + tsv, csv, turtle, + ntriples, nquads, octetStream}; // _____________________________________________________________ const ad_utility::HashMap& getAllMediaTypes() { @@ -40,6 +41,8 @@ const ad_utility::HashMap& getAllMediaTypes() { add(sparqlXml, "application", "sparql-results+xml", {}); add(qleverJson, "application", "qlever-results+json", {}); add(turtle, "text", "turtle", {".ttl"}); + add(ntriples, "application", "n-triples", {".nt"}); + add(nquads, "application", "n-quads", {".nq"}); add(octetStream, "application", "octet-stream", {}); return t; }(); diff --git a/src/util/http/MediaTypes.h b/src/util/http/MediaTypes.h index 0b2634b8ce..7f4607a8d9 100644 --- a/src/util/http/MediaTypes.h +++ b/src/util/http/MediaTypes.h @@ -28,6 +28,8 @@ enum class MediaType { tsv, csv, turtle, + ntriples, + nquads, octetStream }; diff --git a/test/GraphStoreProtocolTest.cpp b/test/GraphStoreProtocolTest.cpp index 1bbe7aa9d0..cec0e3b3b3 100644 --- a/test/GraphStoreProtocolTest.cpp +++ b/test/GraphStoreProtocolTest.cpp @@ -4,9 +4,11 @@ #include #include +#include #include "./util/GTestHelpers.h" #include "./util/HttpRequestHelpers.h" +#include "SparqlAntlrParserTestHelpers.h" #include "engine/GraphStoreProtocol.h" TEST(GraphStoreProtocolTest, extractTargetGraph) { @@ -28,12 +30,73 @@ TEST(GraphStoreProtocolTest, extractTargetGraph) { } TEST(GraphStoreProtocolTest, transformGraphStoreProtocol) { - ad_utility::httpUtils::HttpRequest auto f = - ad_utility::testing::MakeGetRequest("/?default"); - ad_utility::url_parser::ParsedRequest ff = - ad_utility::url_parser::ParsedRequest{}; - GraphStoreProtocol::transformGraphStoreProtocol(f, ff); + using Var = Variable; + using TC = TripleComponent; + auto Iri = [](std::string_view stringWithBrackets) { + return TripleComponent::Iri::fromIriref(stringWithBrackets); + }; + namespace m = matchers; EXPECT_THAT(GraphStoreProtocol::transformGraphStoreProtocol( - f, ad_utility::url_parser::ParsedRequest{}), - testing::_); + ad_utility::testing::MakeGetRequest("/?default")), + m::ConstructQuery({{Var{"?s"}, Var{"?p"}, Var{"?o"}}}, + m::GraphPattern(matchers::Triples({SparqlTriple( + TC(Var{"?s"}), "?p", TC(Var{"?o"}))})))); + EXPECT_THAT( + GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakeGetRequest("/?graph=foo")), + m::ConstructQuery({{Var{"?s"}, Var{"?p"}, Var{"?o"}}}, + m::GraphPattern(m::GroupGraphPatternWithGraph( + {}, TC::Iri::fromIriref(""), + matchers::Triples({SparqlTriple( + TC(Var{"?s"}), "?p", TC(Var{"?o"}))}))))); + EXPECT_THAT(GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakePostRequest( + "/?default", "text/turtle", " .")), + m::UpdateClause(m::GraphUpdate({}, + {{Iri(""), Iri(""), + Iri(""), std::monostate{}}}, + std::nullopt), + m::GraphPattern())); + EXPECT_THAT(GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakePostRequest( + "/?default", "application/n-triples", " .")), + m::UpdateClause(m::GraphUpdate({}, + {{Iri(""), Iri(""), + Iri(""), std::monostate{}}}, + std::nullopt), + m::GraphPattern())); + EXPECT_THAT(GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakePostRequest( + "/?default", "application/n-quads", " .")), + m::UpdateClause( + m::GraphUpdate( + {}, {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, + std::nullopt), + m::GraphPattern())); + AD_EXPECT_THROW_WITH_MESSAGE( + GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakePostRequest( + "/?default", "application/unknown", "fantasy")), + testing::HasSubstr("Not a single media type known to this parser was " + "detected in \"application/unknown\".")); + AD_EXPECT_THROW_WITH_MESSAGE( + GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakeRequest(boost::beast::http::verb::put, + "/?default")), + testing::HasSubstr("PUT in the SPARQL Graph Store HTTP Protocol")); + AD_EXPECT_THROW_WITH_MESSAGE( + GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakeRequest(boost::beast::http::verb::delete_, + "/?default")), + testing::HasSubstr("DELETE in the SPARQL Graph Store HTTP Protocol")); + AD_EXPECT_THROW_WITH_MESSAGE( + GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakeRequest(boost::beast::http::verb::head, + "/?default")), + testing::HasSubstr("HEAD in the SPARQL Graph Store HTTP Protocol")); + AD_EXPECT_THROW_WITH_MESSAGE( + GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakeRequest(boost::beast::http::verb::patch, + "/?default")), + testing::HasSubstr("PATCH in the SPARQL Graph Store HTTP Protocol")); } From eaa76d4c21dd116a939654c717dc3238287cbc25 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Sun, 8 Dec 2024 18:03:04 +0100 Subject: [PATCH 07/19] correct behaviour if a graph is specified for a post --- src/engine/GraphStoreProtocol.h | 3 --- test/GraphStoreProtocolTest.cpp | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/engine/GraphStoreProtocol.h b/src/engine/GraphStoreProtocol.h index f8b8022247..093b818ffa 100644 --- a/src/engine/GraphStoreProtocol.h +++ b/src/engine/GraphStoreProtocol.h @@ -97,9 +97,6 @@ class GraphStoreProtocol { }; updateClause::GraphUpdate up{ ad_utility::transform(triples, transformTurtleTriple), {}}; - if (std::holds_alternative(graph)) { - up.with_ = std::get(graph); - } res._clause = parsedQuery::UpdateClause{up}; return res; }; diff --git a/test/GraphStoreProtocolTest.cpp b/test/GraphStoreProtocolTest.cpp index cec0e3b3b3..94bb477d53 100644 --- a/test/GraphStoreProtocolTest.cpp +++ b/test/GraphStoreProtocolTest.cpp @@ -65,6 +65,15 @@ TEST(GraphStoreProtocolTest, transformGraphStoreProtocol) { Iri(""), std::monostate{}}}, std::nullopt), m::GraphPattern())); + EXPECT_THAT( + GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakePostRequest( + "/?graph=bar", "application/n-triples", " .")), + m::UpdateClause( + m::GraphUpdate({}, + {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, + std::nullopt), + m::GraphPattern())); EXPECT_THAT(GraphStoreProtocol::transformGraphStoreProtocol( ad_utility::testing::MakePostRequest( "/?default", "application/n-quads", " .")), @@ -73,6 +82,15 @@ TEST(GraphStoreProtocolTest, transformGraphStoreProtocol) { {}, {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, std::nullopt), m::GraphPattern())); + EXPECT_THAT( + GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakePostRequest( + "/?graph=baz", "application/n-quads", " .")), + m::UpdateClause( + m::GraphUpdate({}, + {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, + std::nullopt), + m::GraphPattern())); AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::transformGraphStoreProtocol( ad_utility::testing::MakePostRequest( From 8deab381538c01fcc16995bb7eecfa7dc0c56f9d Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Sun, 8 Dec 2024 18:16:39 +0100 Subject: [PATCH 08/19] more concrete types --- src/engine/GraphStoreProtocol.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engine/GraphStoreProtocol.cpp b/src/engine/GraphStoreProtocol.cpp index d0300436ad..4b58407e74 100644 --- a/src/engine/GraphStoreProtocol.cpp +++ b/src/engine/GraphStoreProtocol.cpp @@ -11,9 +11,9 @@ GraphOrDefault GraphStoreProtocol::extractTargetGraph( const ad_utility::url_parser::ParamValueMap& params) { // Extract the graph to be acted upon using `Indirect Graph // Identification`. - const auto graphIri = + const std::optional graphIri = ad_utility::url_parser::checkParameter(params, "graph", std::nullopt); - const auto isDefault = + const bool isDefault = ad_utility::url_parser::checkParameter(params, "default", "").has_value(); if (!(graphIri.has_value() || isDefault)) { throw std::runtime_error("No graph IRI specified in the request."); From 63aa1da213f7e50bc9725a9729f7f79a1b9a5232 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Sun, 8 Dec 2024 18:34:40 +0100 Subject: [PATCH 09/19] extract method handling into methods --- src/engine/GraphStoreProtocol.h | 164 ++++++++++++++++---------------- 1 file changed, 84 insertions(+), 80 deletions(-) diff --git a/src/engine/GraphStoreProtocol.h b/src/engine/GraphStoreProtocol.h index 093b818ffa..a598484cb5 100644 --- a/src/engine/GraphStoreProtocol.h +++ b/src/engine/GraphStoreProtocol.h @@ -12,6 +12,88 @@ #include "util/http/UrlParser.h" class GraphStoreProtocol { + private: + static ParsedQuery transformPost( + const ad_utility::httpUtils::HttpRequest auto& rawRequest, + const GraphOrDefault& graph) { + using namespace boost::beast::http; + using Re2Parser = RdfStringParser>; + using NQuadRe2Parser = RdfStringParser>; + auto contentType = ad_utility::getMediaTypeFromAcceptHeader( + rawRequest.at(field::content_type)); + std::vector triples; + switch (contentType.value()) { + case ad_utility::MediaType::turtle: + case ad_utility::MediaType::ntriples: { + Re2Parser parser = Re2Parser(); + parser.setInputStream(rawRequest.body()); + triples = parser.parseAndReturnAllTriples(); + break; + } + case ad_utility::MediaType::nquads: { + NQuadRe2Parser parser = NQuadRe2Parser(); + parser.setInputStream(rawRequest.body()); + triples = parser.parseAndReturnAllTriples(); + break; + } + default: { + throw std::runtime_error(absl::StrCat( + "Mediatype \"", ad_utility::toString(contentType.value()), + "\" is not supported for SPARQL Graph Store HTTP " + "Protocol in QLever.")); + break; + } + } + ParsedQuery res; + auto transformTurtleTriple = + [&graph](const TurtleTriple& triple) -> SparqlTripleSimpleWithGraph { + auto triplesGraph = + [&triple]() -> std::variant { + if (triple.graphIri_.isIri()) { + return Iri(triple.graphIri_.getIri().toStringRepresentation()); + } else if (triple.graphIri_.isVariable()) { + return triple.graphIri_.getVariable(); + } else { + AD_CORRECTNESS_CHECK(triple.graphIri_.isId()); + AD_CORRECTNESS_CHECK(triple.graphIri_.getId() == + qlever::specialIds().at(DEFAULT_GRAPH_IRI)); + return std::monostate{}; + } + }(); + if (std::holds_alternative(graph)) { + triplesGraph = Iri(std::get(graph).toStringRepresentation()); + } + return SparqlTripleSimpleWithGraph(triple.subject_, triple.predicate_, + triple.object_, triplesGraph); + }; + updateClause::GraphUpdate up{ + ad_utility::transform(triples, transformTurtleTriple), {}}; + res._clause = parsedQuery::UpdateClause{up}; + return res; + } + + static ParsedQuery transformGet(const GraphOrDefault& graph) { + ParsedQuery res; + res._clause = parsedQuery::ConstructClause( + {{Variable("?s"), Variable("?p"), Variable("?o")}}); + res._rootGraphPattern = {}; + parsedQuery::GraphPattern selectSPO; + selectSPO._graphPatterns.emplace_back(parsedQuery::BasicGraphPattern{ + {SparqlTriple(Variable("?s"), "?p", Variable("?o"))}}); + // TODO: extract this wrapping into a lambda + if (std::holds_alternative(graph)) { + parsedQuery::GroupGraphPattern selectSPOWithGraph{ + std::move(selectSPO), + std::get(graph)}; + res._rootGraphPattern._graphPatterns.emplace_back( + std::move(selectSPOWithGraph)); + } else { + AD_CORRECTNESS_CHECK(std::holds_alternative(graph)); + res._rootGraphPattern = std::move(selectSPO); + } + return res; + } + public: // Every Graph Store Protocol requests has equivalent SPARQL Query or Update. // Transform the Graph Store Protocol request into it's equivalent Query or @@ -21,89 +103,11 @@ class GraphStoreProtocol { ad_utility::url_parser::ParsedUrl parsedUrl = ad_utility::url_parser::parseRequestTarget(rawRequest.target()); GraphOrDefault graph = extractTargetGraph(parsedUrl.parameters_); - auto visitGet = [&graph]() -> ParsedQuery { - ParsedQuery res; - res._clause = parsedQuery::ConstructClause( - {{Variable("?s"), Variable("?p"), Variable("?o")}}); - res._rootGraphPattern = {}; - parsedQuery::GraphPattern selectSPO; - selectSPO._graphPatterns.emplace_back(parsedQuery::BasicGraphPattern{ - {SparqlTriple(Variable("?s"), "?p", Variable("?o"))}}); - // TODO: extract this wrapping into a lambda - if (std::holds_alternative(graph)) { - parsedQuery::GroupGraphPattern selectSPOWithGraph{ - std::move(selectSPO), - std::get(graph)}; - res._rootGraphPattern._graphPatterns.emplace_back( - std::move(selectSPOWithGraph)); - } else { - AD_CORRECTNESS_CHECK(std::holds_alternative(graph)); - res._rootGraphPattern = std::move(selectSPO); - } - return res; - }; using namespace boost::beast::http; - using Re2Parser = RdfStringParser>; - using NQuadRe2Parser = RdfStringParser>; - auto visitPost = [&graph, &rawRequest]() -> ParsedQuery { - auto contentType = rawRequest.at(field::content_type); - auto type = ad_utility::getMediaTypeFromAcceptHeader(contentType); - std::vector triples; - switch (type.value()) { - case ad_utility::MediaType::turtle: - case ad_utility::MediaType::ntriples: { - Re2Parser parser = Re2Parser(); - parser.setInputStream(rawRequest.body()); - triples = parser.parseAndReturnAllTriples(); - break; - } - case ad_utility::MediaType::nquads: { - NQuadRe2Parser parser = NQuadRe2Parser(); - parser.setInputStream(rawRequest.body()); - triples = parser.parseAndReturnAllTriples(); - break; - } - default: { - throw std::runtime_error( - absl::StrCat("Mediatype \"", ad_utility::toString(type.value()), - "\" is not supported for SPARQL Graph Store HTTP " - "Protocol in QLever.")); - break; - } - } - ParsedQuery res; - auto transformTurtleTriple = - [&graph](const TurtleTriple& triple) -> SparqlTripleSimpleWithGraph { - auto triplesGraph = - [&triple]() -> std::variant { - if (triple.graphIri_.isIri()) { - return Iri(triple.graphIri_.getIri().toStringRepresentation()); - } else if (triple.graphIri_.isVariable()) { - return triple.graphIri_.getVariable(); - } else { - AD_CORRECTNESS_CHECK(triple.graphIri_.isId()); - AD_CORRECTNESS_CHECK(triple.graphIri_.getId() == - qlever::specialIds().at(DEFAULT_GRAPH_IRI)); - return std::monostate{}; - } - }(); - if (std::holds_alternative(graph)) { - triplesGraph = - Iri(std::get(graph).toStringRepresentation()); - } - return SparqlTripleSimpleWithGraph(triple.subject_, triple.predicate_, - triple.object_, triplesGraph); - }; - updateClause::GraphUpdate up{ - ad_utility::transform(triples, transformTurtleTriple), {}}; - res._clause = parsedQuery::UpdateClause{up}; - return res; - }; - auto method = rawRequest.method(); if (method == verb::get) { - return visitGet(); + return transformGet(graph); } else if (method == verb::put) { throw std::runtime_error( "PUT in the SPARQL Graph Store HTTP Protocol is not yet implemented " @@ -113,7 +117,7 @@ class GraphStoreProtocol { "DELETE in the SPARQL Graph Store HTTP Protocol is not yet " "implemented in QLever."); } else if (method == verb::post) { - return visitPost(); + return transformPost(rawRequest, graph); } else if (method == verb::head) { throw std::runtime_error( "HEAD in the SPARQL Graph Store HTTP Protocol is not yet implemented " From ff6d023475d78b7a618bbabb059e3d4132bc8124 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Sun, 8 Dec 2024 18:57:21 +0100 Subject: [PATCH 10/19] split code and tests into smaller parts --- src/engine/GraphStoreProtocol.h | 2 + test/GraphStoreProtocolTest.cpp | 136 ++++++++++++++++++++------------ 2 files changed, 86 insertions(+), 52 deletions(-) diff --git a/src/engine/GraphStoreProtocol.h b/src/engine/GraphStoreProtocol.h index a598484cb5..67bdf073fa 100644 --- a/src/engine/GraphStoreProtocol.h +++ b/src/engine/GraphStoreProtocol.h @@ -71,6 +71,7 @@ class GraphStoreProtocol { res._clause = parsedQuery::UpdateClause{up}; return res; } + FRIEND_TEST(GraphStoreProtocolTest, transformPost); static ParsedQuery transformGet(const GraphOrDefault& graph) { ParsedQuery res; @@ -93,6 +94,7 @@ class GraphStoreProtocol { } return res; } + FRIEND_TEST(GraphStoreProtocolTest, transformGet); public: // Every Graph Store Protocol requests has equivalent SPARQL Query or Update. diff --git a/test/GraphStoreProtocolTest.cpp b/test/GraphStoreProtocolTest.cpp index 94bb477d53..c84baced80 100644 --- a/test/GraphStoreProtocolTest.cpp +++ b/test/GraphStoreProtocolTest.cpp @@ -14,7 +14,7 @@ TEST(GraphStoreProtocolTest, extractTargetGraph) { EXPECT_THAT(GraphStoreProtocol::extractTargetGraph({{"default", {""}}}), DEFAULT{}); - EXPECT_THAT(GraphStoreProtocol::extractTargetGraph({{"graph", {""}}}), + EXPECT_THAT(GraphStoreProtocol::extractTargetGraph({{"graph", {"foo"}}}), TripleComponent::Iri::fromIriref("")); AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::extractTargetGraph({}), @@ -24,79 +24,111 @@ TEST(GraphStoreProtocolTest, extractTargetGraph) { testing::HasSubstr("No graph IRI specified in the request.")); AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::extractTargetGraph( - {{"default", {""}}, {"graph", {""}}}), + {{"default", {""}}, {"graph", {"foo"}}}), testing::HasSubstr("Only one of `default` and `graph` may be used for " "graph identification.")); } -TEST(GraphStoreProtocolTest, transformGraphStoreProtocol) { - using Var = Variable; - using TC = TripleComponent; +TEST(GraphStoreProtocolTest, transformPost) { auto Iri = [](std::string_view stringWithBrackets) { return TripleComponent::Iri::fromIriref(stringWithBrackets); }; namespace m = matchers; - EXPECT_THAT(GraphStoreProtocol::transformGraphStoreProtocol( - ad_utility::testing::MakeGetRequest("/?default")), - m::ConstructQuery({{Var{"?s"}, Var{"?p"}, Var{"?o"}}}, - m::GraphPattern(matchers::Triples({SparqlTriple( - TC(Var{"?s"}), "?p", TC(Var{"?o"}))})))); - EXPECT_THAT( - GraphStoreProtocol::transformGraphStoreProtocol( - ad_utility::testing::MakeGetRequest("/?graph=foo")), - m::ConstructQuery({{Var{"?s"}, Var{"?p"}, Var{"?o"}}}, - m::GraphPattern(m::GroupGraphPatternWithGraph( - {}, TC::Iri::fromIriref(""), - matchers::Triples({SparqlTriple( - TC(Var{"?s"}), "?p", TC(Var{"?o"}))}))))); - EXPECT_THAT(GraphStoreProtocol::transformGraphStoreProtocol( - ad_utility::testing::MakePostRequest( - "/?default", "text/turtle", " .")), - m::UpdateClause(m::GraphUpdate({}, - {{Iri(""), Iri(""), - Iri(""), std::monostate{}}}, - std::nullopt), - m::GraphPattern())); - EXPECT_THAT(GraphStoreProtocol::transformGraphStoreProtocol( - ad_utility::testing::MakePostRequest( - "/?default", "application/n-triples", " .")), - m::UpdateClause(m::GraphUpdate({}, - {{Iri(""), Iri(""), - Iri(""), std::monostate{}}}, - std::nullopt), - m::GraphPattern())); - EXPECT_THAT( - GraphStoreProtocol::transformGraphStoreProtocol( - ad_utility::testing::MakePostRequest( - "/?graph=bar", "application/n-triples", " .")), + auto expectTransformPost = + [](const ad_utility::httpUtils::HttpRequest auto& request, + const testing::Matcher& matcher, + ad_utility::source_location l = + ad_utility::source_location::current()) { + auto trace = generateLocationTrace(l); + const ad_utility::url_parser::ParsedUrl parsedUrl = + ad_utility::url_parser::parseRequestTarget(request.target()); + const GraphOrDefault graph = + GraphStoreProtocol::extractTargetGraph(parsedUrl.parameters_); + EXPECT_THAT(GraphStoreProtocol::transformPost(request, graph), matcher); + }; + + expectTransformPost( + ad_utility::testing::MakePostRequest("/?default", "text/turtle", + " ."), + m::UpdateClause( + m::GraphUpdate( + {}, {{Iri(""), Iri(""), Iri(""), std::monostate{}}}, + std::nullopt), + m::GraphPattern())); + expectTransformPost( + ad_utility::testing::MakePostRequest("/?default", "application/n-triples", + " ."), + m::UpdateClause( + m::GraphUpdate( + {}, {{Iri(""), Iri(""), Iri(""), std::monostate{}}}, + std::nullopt), + m::GraphPattern())); + expectTransformPost( + ad_utility::testing::MakePostRequest( + "/?graph=bar", "application/n-triples", " ."), m::UpdateClause( m::GraphUpdate({}, {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, std::nullopt), m::GraphPattern())); - EXPECT_THAT(GraphStoreProtocol::transformGraphStoreProtocol( - ad_utility::testing::MakePostRequest( - "/?default", "application/n-quads", " .")), - m::UpdateClause( - m::GraphUpdate( - {}, {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, - std::nullopt), - m::GraphPattern())); - EXPECT_THAT( - GraphStoreProtocol::transformGraphStoreProtocol( - ad_utility::testing::MakePostRequest( - "/?graph=baz", "application/n-quads", " .")), + expectTransformPost( + ad_utility::testing::MakePostRequest("/?default", "application/n-quads", + " ."), + m::UpdateClause( + m::GraphUpdate({}, + {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, + std::nullopt), + m::GraphPattern())); + expectTransformPost( + ad_utility::testing::MakePostRequest("/?graph=baz", "application/n-quads", + " ."), m::UpdateClause( m::GraphUpdate({}, {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, std::nullopt), m::GraphPattern())); AD_EXPECT_THROW_WITH_MESSAGE( - GraphStoreProtocol::transformGraphStoreProtocol( + GraphStoreProtocol::transformPost( ad_utility::testing::MakePostRequest( - "/?default", "application/unknown", "fantasy")), + "/?default", "application/unknown", "fantasy"), + DEFAULT{}), testing::HasSubstr("Not a single media type known to this parser was " "detected in \"application/unknown\".")); +} + +TEST(GraphStoreProtocolTest, transformGet) { + using Var = Variable; + using TC = TripleComponent; + namespace m = matchers; + auto expectTransformGet = + [](const ad_utility::httpUtils::HttpRequest auto& request, + const testing::Matcher& matcher, + ad_utility::source_location l = + ad_utility::source_location::current()) { + auto trace = generateLocationTrace(l); + const ad_utility::url_parser::ParsedUrl parsedUrl = + ad_utility::url_parser::parseRequestTarget(request.target()); + const GraphOrDefault graph = + GraphStoreProtocol::extractTargetGraph(parsedUrl.parameters_); + EXPECT_THAT(GraphStoreProtocol::transformGet(graph), matcher); + }; + expectTransformGet( + ad_utility::testing::MakeGetRequest("/?default"), + m::ConstructQuery({{Var{"?s"}, Var{"?p"}, Var{"?o"}}}, + m::GraphPattern(matchers::Triples({SparqlTriple( + TC(Var{"?s"}), "?p", TC(Var{"?o"}))})))); + expectTransformGet( + ad_utility::testing::MakeGetRequest("/?graph=foo"), + m::ConstructQuery({{Var{"?s"}, Var{"?p"}, Var{"?o"}}}, + m::GraphPattern(m::GroupGraphPatternWithGraph( + {}, TC::Iri::fromIriref(""), + matchers::Triples({SparqlTriple( + TC(Var{"?s"}), "?p", TC(Var{"?o"}))}))))); +} + +TEST(GraphStoreProtocolTest, transformGraphStoreProtocol) { + namespace m = matchers; + // TODO: re-add some simple tests for POST and GET AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::transformGraphStoreProtocol( ad_utility::testing::MakeRequest(boost::beast::http::verb::put, From a2a95c9421af67b269f0802194d38227f5d55cd0 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Sun, 8 Dec 2024 19:02:05 +0100 Subject: [PATCH 11/19] global namespace alia --- test/GraphStoreProtocolTest.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/GraphStoreProtocolTest.cpp b/test/GraphStoreProtocolTest.cpp index c84baced80..e6f816a014 100644 --- a/test/GraphStoreProtocolTest.cpp +++ b/test/GraphStoreProtocolTest.cpp @@ -11,6 +11,8 @@ #include "SparqlAntlrParserTestHelpers.h" #include "engine/GraphStoreProtocol.h" +namespace m = matchers; + TEST(GraphStoreProtocolTest, extractTargetGraph) { EXPECT_THAT(GraphStoreProtocol::extractTargetGraph({{"default", {""}}}), DEFAULT{}); @@ -33,7 +35,6 @@ TEST(GraphStoreProtocolTest, transformPost) { auto Iri = [](std::string_view stringWithBrackets) { return TripleComponent::Iri::fromIriref(stringWithBrackets); }; - namespace m = matchers; auto expectTransformPost = [](const ad_utility::httpUtils::HttpRequest auto& request, const testing::Matcher& matcher, @@ -99,7 +100,6 @@ TEST(GraphStoreProtocolTest, transformPost) { TEST(GraphStoreProtocolTest, transformGet) { using Var = Variable; using TC = TripleComponent; - namespace m = matchers; auto expectTransformGet = [](const ad_utility::httpUtils::HttpRequest auto& request, const testing::Matcher& matcher, @@ -127,7 +127,6 @@ TEST(GraphStoreProtocolTest, transformGet) { } TEST(GraphStoreProtocolTest, transformGraphStoreProtocol) { - namespace m = matchers; // TODO: re-add some simple tests for POST and GET AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::transformGraphStoreProtocol( From 2fd96778020664bb73b42ebeb248d4bc2963c2f1 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Sun, 8 Dec 2024 19:09:55 +0100 Subject: [PATCH 12/19] add some comments in tests --- test/GraphStoreProtocolTest.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/GraphStoreProtocolTest.cpp b/test/GraphStoreProtocolTest.cpp index e6f816a014..2de34332da 100644 --- a/test/GraphStoreProtocolTest.cpp +++ b/test/GraphStoreProtocolTest.cpp @@ -14,16 +14,21 @@ namespace m = matchers; TEST(GraphStoreProtocolTest, extractTargetGraph) { + // Equivalent to `/?default` EXPECT_THAT(GraphStoreProtocol::extractTargetGraph({{"default", {""}}}), DEFAULT{}); + // Equivalent to `/?graph=foo` EXPECT_THAT(GraphStoreProtocol::extractTargetGraph({{"graph", {"foo"}}}), TripleComponent::Iri::fromIriref("")); + // Equivalent to `/` or `/?` AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::extractTargetGraph({}), testing::HasSubstr("No graph IRI specified in the request.")); + // Equivalent to `/?unrelated=a&unrelated=b` AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::extractTargetGraph({{"unrelated", {"a", "b"}}}), testing::HasSubstr("No graph IRI specified in the request.")); + // Equivalent to `/?default&graph=foo` AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::extractTargetGraph( {{"default", {""}}, {"graph", {"foo"}}}), From c700734a22b03392619e3885a00aebbc050b68e2 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Mon, 9 Dec 2024 14:31:01 +0100 Subject: [PATCH 13/19] add missing using --- test/ServerTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/ServerTest.cpp b/test/ServerTest.cpp index 5a65824036..cd6b579e05 100644 --- a/test/ServerTest.cpp +++ b/test/ServerTest.cpp @@ -15,6 +15,7 @@ using namespace ad_utility::url_parser; using namespace ad_utility::url_parser::sparqlOperation; +using namespace ad_utility::testing; namespace { auto ParsedRequestIs = [](const std::string& path, From c2ec8cfc0a56a12c2f8bee26a78c4281a48df62b Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Mon, 9 Dec 2024 19:49:53 +0100 Subject: [PATCH 14/19] Resolve sonarcloud issues --- src/engine/GraphStoreProtocol.h | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/engine/GraphStoreProtocol.h b/src/engine/GraphStoreProtocol.h index 67bdf073fa..9c799c60b6 100644 --- a/src/engine/GraphStoreProtocol.h +++ b/src/engine/GraphStoreProtocol.h @@ -25,13 +25,13 @@ class GraphStoreProtocol { switch (contentType.value()) { case ad_utility::MediaType::turtle: case ad_utility::MediaType::ntriples: { - Re2Parser parser = Re2Parser(); + auto parser = Re2Parser(); parser.setInputStream(rawRequest.body()); triples = parser.parseAndReturnAllTriples(); break; } case ad_utility::MediaType::nquads: { - NQuadRe2Parser parser = NQuadRe2Parser(); + auto parser = NQuadRe2Parser(); parser.setInputStream(rawRequest.body()); triples = parser.parseAndReturnAllTriples(); break; @@ -45,8 +45,7 @@ class GraphStoreProtocol { } } ParsedQuery res; - auto transformTurtleTriple = - [&graph](const TurtleTriple& triple) -> SparqlTripleSimpleWithGraph { + auto transformTurtleTriple = [&graph](const TurtleTriple& triple) { auto triplesGraph = [&triple]() -> std::variant { if (triple.graphIri_.isIri()) { @@ -106,25 +105,25 @@ class GraphStoreProtocol { ad_utility::url_parser::parseRequestTarget(rawRequest.target()); GraphOrDefault graph = extractTargetGraph(parsedUrl.parameters_); - using namespace boost::beast::http; + using enum boost::beast::http::verb; auto method = rawRequest.method(); - if (method == verb::get) { + if (method == get) { return transformGet(graph); - } else if (method == verb::put) { + } else if (method == put) { throw std::runtime_error( "PUT in the SPARQL Graph Store HTTP Protocol is not yet implemented " "in QLever."); - } else if (method == verb::delete_) { + } else if (method == delete_) { throw std::runtime_error( "DELETE in the SPARQL Graph Store HTTP Protocol is not yet " "implemented in QLever."); - } else if (method == verb::post) { + } else if (method == post) { return transformPost(rawRequest, graph); - } else if (method == verb::head) { + } else if (method == head) { throw std::runtime_error( "HEAD in the SPARQL Graph Store HTTP Protocol is not yet implemented " "in QLever."); - } else if (method == verb::patch) { + } else if (method == patch) { throw std::runtime_error( "PATCH in the SPARQL Graph Store HTTP Protocol is not yet " "implemented in QLever."); From 5bafc3d4e3e609f99a26a93c00fb32863a3fdba6 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Tue, 10 Dec 2024 14:11:34 +0100 Subject: [PATCH 15/19] don't allow n-quads --- src/engine/GraphStoreProtocol.h | 36 +++++++++++---------------------- src/util/http/MediaTypes.cpp | 6 ++---- src/util/http/MediaTypes.h | 1 - test/GraphStoreProtocolTest.cpp | 23 +++++++-------------- 4 files changed, 21 insertions(+), 45 deletions(-) diff --git a/src/engine/GraphStoreProtocol.h b/src/engine/GraphStoreProtocol.h index 9c799c60b6..da9642cb88 100644 --- a/src/engine/GraphStoreProtocol.h +++ b/src/engine/GraphStoreProtocol.h @@ -18,9 +18,12 @@ class GraphStoreProtocol { const GraphOrDefault& graph) { using namespace boost::beast::http; using Re2Parser = RdfStringParser>; - using NQuadRe2Parser = RdfStringParser>; - auto contentType = ad_utility::getMediaTypeFromAcceptHeader( - rawRequest.at(field::content_type)); + const std::string contentTypeString = rawRequest.at(field::content_type); + if (contentTypeString.empty()) { + // No ContentType specified; we don't try to guess -> 400 Bad Request + } + const auto contentType = + ad_utility::getMediaTypeFromAcceptHeader(contentTypeString); std::vector triples; switch (contentType.value()) { case ad_utility::MediaType::turtle: @@ -30,13 +33,8 @@ class GraphStoreProtocol { triples = parser.parseAndReturnAllTriples(); break; } - case ad_utility::MediaType::nquads: { - auto parser = NQuadRe2Parser(); - parser.setInputStream(rawRequest.body()); - triples = parser.parseAndReturnAllTriples(); - break; - } default: { + // Unsupported media type -> 415 Unsupported Media Type throw std::runtime_error(absl::StrCat( "Mediatype \"", ad_utility::toString(contentType.value()), "\" is not supported for SPARQL Graph Store HTTP " @@ -46,24 +44,15 @@ class GraphStoreProtocol { } ParsedQuery res; auto transformTurtleTriple = [&graph](const TurtleTriple& triple) { - auto triplesGraph = - [&triple]() -> std::variant { - if (triple.graphIri_.isIri()) { - return Iri(triple.graphIri_.getIri().toStringRepresentation()); - } else if (triple.graphIri_.isVariable()) { - return triple.graphIri_.getVariable(); - } else { - AD_CORRECTNESS_CHECK(triple.graphIri_.isId()); - AD_CORRECTNESS_CHECK(triple.graphIri_.getId() == + AD_CORRECTNESS_CHECK(triple.graphIri_.isId() && + triple.graphIri_.getId() == qlever::specialIds().at(DEFAULT_GRAPH_IRI)); - return std::monostate{}; - } - }(); + SparqlTripleSimpleWithGraph::Graph g{std::monostate{}}; if (std::holds_alternative(graph)) { - triplesGraph = Iri(std::get(graph).toStringRepresentation()); + g = Iri(std::get(graph).toStringRepresentation()); } return SparqlTripleSimpleWithGraph(triple.subject_, triple.predicate_, - triple.object_, triplesGraph); + triple.object_, g); }; updateClause::GraphUpdate up{ ad_utility::transform(triples, transformTurtleTriple), {}}; @@ -80,7 +69,6 @@ class GraphStoreProtocol { parsedQuery::GraphPattern selectSPO; selectSPO._graphPatterns.emplace_back(parsedQuery::BasicGraphPattern{ {SparqlTriple(Variable("?s"), "?p", Variable("?o"))}}); - // TODO: extract this wrapping into a lambda if (std::holds_alternative(graph)) { parsedQuery::GroupGraphPattern selectSPOWithGraph{ std::move(selectSPO), diff --git a/src/util/http/MediaTypes.cpp b/src/util/http/MediaTypes.cpp index 1e85acbfde..ed1f7b4958 100644 --- a/src/util/http/MediaTypes.cpp +++ b/src/util/http/MediaTypes.cpp @@ -17,9 +17,8 @@ using enum MediaType; // The first media type in this list is the default, if no other type is // specified in the request. It's "application/sparql-results+json", as // required by the SPARQL standard. -constexpr std::array SUPPORTED_MEDIA_TYPES{sparqlJson, sparqlXml, qleverJson, - tsv, csv, turtle, - ntriples, nquads, octetStream}; +constexpr std::array SUPPORTED_MEDIA_TYPES{ + sparqlJson, sparqlXml, qleverJson, tsv, csv, turtle, ntriples, octetStream}; // _____________________________________________________________ const ad_utility::HashMap& getAllMediaTypes() { @@ -42,7 +41,6 @@ const ad_utility::HashMap& getAllMediaTypes() { add(qleverJson, "application", "qlever-results+json", {}); add(turtle, "text", "turtle", {".ttl"}); add(ntriples, "application", "n-triples", {".nt"}); - add(nquads, "application", "n-quads", {".nq"}); add(octetStream, "application", "octet-stream", {}); return t; }(); diff --git a/src/util/http/MediaTypes.h b/src/util/http/MediaTypes.h index 7f4607a8d9..e03efd6842 100644 --- a/src/util/http/MediaTypes.h +++ b/src/util/http/MediaTypes.h @@ -29,7 +29,6 @@ enum class MediaType { csv, turtle, ntriples, - nquads, octetStream }; diff --git a/test/GraphStoreProtocolTest.cpp b/test/GraphStoreProtocolTest.cpp index 2de34332da..aa214fe460 100644 --- a/test/GraphStoreProtocolTest.cpp +++ b/test/GraphStoreProtocolTest.cpp @@ -77,22 +77,13 @@ TEST(GraphStoreProtocolTest, transformPost) { {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, std::nullopt), m::GraphPattern())); - expectTransformPost( - ad_utility::testing::MakePostRequest("/?default", "application/n-quads", - " ."), - m::UpdateClause( - m::GraphUpdate({}, - {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, - std::nullopt), - m::GraphPattern())); - expectTransformPost( - ad_utility::testing::MakePostRequest("/?graph=baz", "application/n-quads", - " ."), - m::UpdateClause( - m::GraphUpdate({}, - {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, - std::nullopt), - m::GraphPattern())); + AD_EXPECT_THROW_WITH_MESSAGE( + GraphStoreProtocol::transformPost( + ad_utility::testing::MakePostRequest( + "/?default", "application/n-quads", " ."), + DEFAULT{}), + testing::HasSubstr("Not a single media type known to this parser was " + "detected in \"application/n-quads\".")); AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::transformPost( ad_utility::testing::MakePostRequest( From 4dd0ab7fa65bde5c1a5bf583cf0255ff4222bf74 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Tue, 10 Dec 2024 15:11:24 +0100 Subject: [PATCH 16/19] add missing tests --- src/engine/GraphStoreProtocol.h | 3 ++- test/GraphStoreProtocolTest.cpp | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/engine/GraphStoreProtocol.h b/src/engine/GraphStoreProtocol.h index da9642cb88..c5bafa3f4b 100644 --- a/src/engine/GraphStoreProtocol.h +++ b/src/engine/GraphStoreProtocol.h @@ -117,7 +117,8 @@ class GraphStoreProtocol { "implemented in QLever."); } else { throw std::runtime_error( - absl::StrCat("Unsupported HTTP method \"", "", + absl::StrCat("Unsupported HTTP method \"", + std::string_view{rawRequest.method_string()}, "\" for the SPARQL Graph Store HTTP Protocol.")); } } diff --git a/test/GraphStoreProtocolTest.cpp b/test/GraphStoreProtocolTest.cpp index aa214fe460..6029d8cc97 100644 --- a/test/GraphStoreProtocolTest.cpp +++ b/test/GraphStoreProtocolTest.cpp @@ -77,6 +77,14 @@ TEST(GraphStoreProtocolTest, transformPost) { {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, std::nullopt), m::GraphPattern())); + AD_EXPECT_THROW_WITH_MESSAGE( + GraphStoreProtocol::transformPost( + ad_utility::testing::MakePostRequest( + "/?default", "application/sparql-results+json", "{}"), + DEFAULT{}), + testing::HasSubstr( + "Mediatype \"application/sparql-results+json\" is not supported for " + "SPARQL Graph Store HTTP Protocol in QLever.")); AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::transformPost( ad_utility::testing::MakePostRequest( @@ -144,4 +152,9 @@ TEST(GraphStoreProtocolTest, transformGraphStoreProtocol) { ad_utility::testing::MakeRequest(boost::beast::http::verb::patch, "/?default")), testing::HasSubstr("PATCH in the SPARQL Graph Store HTTP Protocol")); + AD_EXPECT_THROW_WITH_MESSAGE( + GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakeRequest(boost::beast::http::verb::connect, + "/?default")), + testing::HasSubstr("Unsupported HTTP method \"CONNECT\"")); } From 750dd6f21f23ef0b460c44afe70f12688af38718 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Tue, 10 Dec 2024 15:20:36 +0100 Subject: [PATCH 17/19] add TripleComponent unit tests --- test/TripleComponentTest.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/TripleComponentTest.cpp b/test/TripleComponentTest.cpp index 2cc5c74cb3..2c67b16823 100644 --- a/test/TripleComponentTest.cpp +++ b/test/TripleComponentTest.cpp @@ -69,6 +69,24 @@ TEST(TripleComponent, setAndGetVariable) { ASSERT_EQ(tc.getVariable(), Variable{"?x"}); } +TEST(TripleComponent, setAndGetId) { + Id id = Id::makeFromVocabIndex(VocabIndex::make(1)); + TripleComponent tc{id}; + ASSERT_TRUE(tc.isId()); + ASSERT_FALSE(tc.isVariable()); + ASSERT_FALSE(tc.isString()); + ASSERT_FALSE(tc.isDouble()); + ASSERT_FALSE(tc.isInt()); + ASSERT_FALSE(tc.isBool()); + ASSERT_FALSE(tc.isIri()); + ASSERT_FALSE(tc.isLiteral()); + ASSERT_FALSE(tc.isUndef()); + ASSERT_EQ(tc, id); + ASSERT_EQ(tc.getId(), id); + const TripleComponent tcConst = std::move(tc); + ASSERT_EQ(tcConst.getId(), id); +} + TEST(TripleComponent, assignmentOperator) { TripleComponent object; object = -12.435; From ec8f076367b2b4f08d1cc14a20228a812fab8a99 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Wed, 11 Dec 2024 14:00:19 +0100 Subject: [PATCH 18/19] improve test coverage --- src/engine/GraphStoreProtocol.h | 8 +++++--- test/GraphStoreProtocolTest.cpp | 32 +++++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/engine/GraphStoreProtocol.h b/src/engine/GraphStoreProtocol.h index c5bafa3f4b..44e6338d8f 100644 --- a/src/engine/GraphStoreProtocol.h +++ b/src/engine/GraphStoreProtocol.h @@ -18,9 +18,12 @@ class GraphStoreProtocol { const GraphOrDefault& graph) { using namespace boost::beast::http; using Re2Parser = RdfStringParser>; - const std::string contentTypeString = rawRequest.at(field::content_type); + std::string contentTypeString; + if (rawRequest.find(field::content_type) != rawRequest.end()) { + contentTypeString = rawRequest.at(field::content_type); + } if (contentTypeString.empty()) { - // No ContentType specified; we don't try to guess -> 400 Bad Request + // ContentType not set or empty; we don't try to guess -> 400 Bad Request } const auto contentType = ad_utility::getMediaTypeFromAcceptHeader(contentTypeString); @@ -39,7 +42,6 @@ class GraphStoreProtocol { "Mediatype \"", ad_utility::toString(contentType.value()), "\" is not supported for SPARQL Graph Store HTTP " "Protocol in QLever.")); - break; } } ParsedQuery res; diff --git a/test/GraphStoreProtocolTest.cpp b/test/GraphStoreProtocolTest.cpp index 6029d8cc97..d942239ef8 100644 --- a/test/GraphStoreProtocolTest.cpp +++ b/test/GraphStoreProtocolTest.cpp @@ -79,12 +79,20 @@ TEST(GraphStoreProtocolTest, transformPost) { m::GraphPattern())); AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::transformPost( - ad_utility::testing::MakePostRequest( - "/?default", "application/sparql-results+json", "{}"), + ad_utility::testing::MakeRequest(boost::beast::http::verb::post, + "/?default", {}, " "), DEFAULT{}), testing::HasSubstr( "Mediatype \"application/sparql-results+json\" is not supported for " "SPARQL Graph Store HTTP Protocol in QLever.")); + AD_EXPECT_THROW_WITH_MESSAGE( + GraphStoreProtocol::transformPost( + ad_utility::testing::MakePostRequest( + "/?default", "application/sparql-results+xml", ""), + DEFAULT{}), + testing::HasSubstr( + "Mediatype \"application/sparql-results+xml\" is not supported for " + "SPARQL Graph Store HTTP Protocol in QLever.")); AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::transformPost( ad_utility::testing::MakePostRequest( @@ -131,7 +139,25 @@ TEST(GraphStoreProtocolTest, transformGet) { } TEST(GraphStoreProtocolTest, transformGraphStoreProtocol) { - // TODO: re-add some simple tests for POST and GET + using Var = Variable; + using TC = TripleComponent; + auto Iri = [](std::string_view stringWithBrackets) { + return TripleComponent::Iri::fromIriref(stringWithBrackets); + }; + EXPECT_THAT(GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakeGetRequest("/?default")), + m::ConstructQuery({{Var{"?s"}, Var{"?p"}, Var{"?o"}}}, + m::GraphPattern(matchers::Triples({SparqlTriple( + TC(Var{"?s"}), "?p", TC(Var{"?o"}))})))); + EXPECT_THAT( + GraphStoreProtocol::transformGraphStoreProtocol( + ad_utility::testing::MakePostRequest( + "/?default", "application/n-triples", " .")), + m::UpdateClause(m::GraphUpdate({}, + {{Iri(""), Iri(""), Iri(""), + std::monostate{}}}, + std::nullopt), + m::GraphPattern())); AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::transformGraphStoreProtocol( ad_utility::testing::MakeRequest(boost::beast::http::verb::put, From d45b2da2dd9bdc007d112b07f1979e6c45a48d11 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Wed, 11 Dec 2024 14:08:40 +0100 Subject: [PATCH 19/19] reduce duplication of helpers in tests --- test/GraphStoreProtocolTest.cpp | 49 +++++++++++++++------------------ 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/test/GraphStoreProtocolTest.cpp b/test/GraphStoreProtocolTest.cpp index d942239ef8..7b22706c8c 100644 --- a/test/GraphStoreProtocolTest.cpp +++ b/test/GraphStoreProtocolTest.cpp @@ -8,10 +8,15 @@ #include "./util/GTestHelpers.h" #include "./util/HttpRequestHelpers.h" +#include "./util/TripleComponentTestHelpers.h" #include "SparqlAntlrParserTestHelpers.h" #include "engine/GraphStoreProtocol.h" namespace m = matchers; +namespace t = ad_utility::testing; + +using Var = Variable; +using TC = TripleComponent; TEST(GraphStoreProtocolTest, extractTargetGraph) { // Equivalent to `/?default` @@ -19,7 +24,7 @@ TEST(GraphStoreProtocolTest, extractTargetGraph) { DEFAULT{}); // Equivalent to `/?graph=foo` EXPECT_THAT(GraphStoreProtocol::extractTargetGraph({{"graph", {"foo"}}}), - TripleComponent::Iri::fromIriref("")); + t::iri("")); // Equivalent to `/` or `/?` AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::extractTargetGraph({}), @@ -37,9 +42,6 @@ TEST(GraphStoreProtocolTest, extractTargetGraph) { } TEST(GraphStoreProtocolTest, transformPost) { - auto Iri = [](std::string_view stringWithBrackets) { - return TripleComponent::Iri::fromIriref(stringWithBrackets); - }; auto expectTransformPost = [](const ad_utility::httpUtils::HttpRequest auto& request, const testing::Matcher& matcher, @@ -56,26 +58,26 @@ TEST(GraphStoreProtocolTest, transformPost) { expectTransformPost( ad_utility::testing::MakePostRequest("/?default", "text/turtle", " ."), - m::UpdateClause( - m::GraphUpdate( - {}, {{Iri(""), Iri(""), Iri(""), std::monostate{}}}, - std::nullopt), - m::GraphPattern())); + m::UpdateClause(m::GraphUpdate({}, + {{t::iri(""), t::iri(""), + t::iri(""), std::monostate{}}}, + std::nullopt), + m::GraphPattern())); expectTransformPost( ad_utility::testing::MakePostRequest("/?default", "application/n-triples", " ."), - m::UpdateClause( - m::GraphUpdate( - {}, {{Iri(""), Iri(""), Iri(""), std::monostate{}}}, - std::nullopt), - m::GraphPattern())); + m::UpdateClause(m::GraphUpdate({}, + {{t::iri(""), t::iri(""), + t::iri(""), std::monostate{}}}, + std::nullopt), + m::GraphPattern())); expectTransformPost( ad_utility::testing::MakePostRequest( "/?graph=bar", "application/n-triples", " ."), m::UpdateClause( - m::GraphUpdate({}, - {{Iri(""), Iri(""), Iri(""), ::Iri("")}}, - std::nullopt), + m::GraphUpdate( + {}, {{t::iri(""), t::iri(""), t::iri(""), Iri("")}}, + std::nullopt), m::GraphPattern())); AD_EXPECT_THROW_WITH_MESSAGE( GraphStoreProtocol::transformPost( @@ -110,8 +112,6 @@ TEST(GraphStoreProtocolTest, transformPost) { } TEST(GraphStoreProtocolTest, transformGet) { - using Var = Variable; - using TC = TripleComponent; auto expectTransformGet = [](const ad_utility::httpUtils::HttpRequest auto& request, const testing::Matcher& matcher, @@ -133,17 +133,12 @@ TEST(GraphStoreProtocolTest, transformGet) { ad_utility::testing::MakeGetRequest("/?graph=foo"), m::ConstructQuery({{Var{"?s"}, Var{"?p"}, Var{"?o"}}}, m::GraphPattern(m::GroupGraphPatternWithGraph( - {}, TC::Iri::fromIriref(""), + {}, t::iri(""), matchers::Triples({SparqlTriple( TC(Var{"?s"}), "?p", TC(Var{"?o"}))}))))); } TEST(GraphStoreProtocolTest, transformGraphStoreProtocol) { - using Var = Variable; - using TC = TripleComponent; - auto Iri = [](std::string_view stringWithBrackets) { - return TripleComponent::Iri::fromIriref(stringWithBrackets); - }; EXPECT_THAT(GraphStoreProtocol::transformGraphStoreProtocol( ad_utility::testing::MakeGetRequest("/?default")), m::ConstructQuery({{Var{"?s"}, Var{"?p"}, Var{"?o"}}}, @@ -154,8 +149,8 @@ TEST(GraphStoreProtocolTest, transformGraphStoreProtocol) { ad_utility::testing::MakePostRequest( "/?default", "application/n-triples", " .")), m::UpdateClause(m::GraphUpdate({}, - {{Iri(""), Iri(""), Iri(""), - std::monostate{}}}, + {{t::iri(""), t::iri(""), + t::iri(""), std::monostate{}}}, std::nullopt), m::GraphPattern())); AD_EXPECT_THROW_WITH_MESSAGE(