Add a VALUES clause to the query of a SERVICE clause to simplify the execution

src/engine/QueryPlanner.cpp

@@ -1641,6 +1641,13 @@
     return {makeSubtreePlan<OptionalJoin>(_qec, a._qet, b._qet)};
   }
 
+  // Check if one of the two Operations is a SERVICE. If so, we can try
+  // to simplify the Service Query using the result of the other operation.
+  if (auto opt = createJoinWithService(a, b, jcs)) {
+    candidates.push_back(std::move(opt.value()));
+    return candidates;
+  }
+
   if (jcs.size() >= 2) {
     // If there are two or more join columns and we are not using the
     // TwoColumnJoin (the if part before this comment), use a multiColumnJoin.
@@ -1769,6 +1776,35 @@
   return plan;
 }
 
+// _____________________________________________________________________
+auto QueryPlanner::createJoinWithService(
+    SubtreePlan a, SubtreePlan b,
+    const std::vector<std::array<ColumnIndex, 2>>& jcs)
+    -> std::optional<SubtreePlan> {
+  auto aRootOp = std::dynamic_pointer_cast<Service>(a._qet->getRootOperation());
+  auto bRootOp = std::dynamic_pointer_cast<Service>(b._qet->getRootOperation());
+
+  // Exactly one of the two Operations can be a service.
+  if (static_cast<bool>(aRootOp) == static_cast<bool>(bRootOp)) {
+    return std::nullopt;
+  }
+
+  auto service = aRootOp ? aRootOp : bRootOp;
+  auto sibling = bRootOp ? a : b;
+
+  service->setSiblingTree(sibling._qet);
+
+  const auto& qec = service->getExecutionContext();
+
+  SubtreePlan plan =
+      jcs.size() == 1
+          ? makeSubtreePlan<Join>(qec, a._qet, b._qet, jcs[0][0], jcs[0][1])
+          : makeSubtreePlan<MultiColumnJoin>(qec, a._qet, b._qet);
+  mergeSubtreePlanIds(plan, a, b);
+
+  return plan;
+}
+
 // _____________________________________________________________________
 void QueryPlanner::QueryGraph::setupGraph(
     const std::vector<SubtreePlan>& leafOperations) {

src/engine/QueryPlanner.h

@@ -332,6 +332,10 @@ class QueryPlanner {
       SubtreePlan a, SubtreePlan b,
       const std::vector<std::array<ColumnIndex, 2>>& jcs);
 
+  [[nodiscard]] static std::optional<SubtreePlan> createJoinWithService(
+      SubtreePlan a, SubtreePlan b,
+      const std::vector<std::array<ColumnIndex, 2>>& jcs);
+
   [[nodiscard]] vector<SubtreePlan> getOrderByRow(
       const ParsedQuery& pq,
       const std::vector<std::vector<SubtreePlan>>& dpTab) const;

src/engine/Service.cpp

@@ -5,11 +5,14 @@
 #include "engine/Service.h"
 
 #include <absl/strings/str_cat.h>
+#include <absl/strings/str_join.h>
 #include <absl/strings/str_split.h>
 
 #include "engine/CallFixedSize.h"
+#include "engine/ExportQueryExecutionTrees.h"
 #include "engine/Values.h"
 #include "engine/VariableToColumnMap.h"
+#include "global/RuntimeParameters.h"
 #include "parser/TokenizerCtre.h"
 #include "parser/TurtleParser.h"
 #include "util/Exception.h"
@@ -21,10 +24,12 @@
 // ____________________________________________________________________________
 Service::Service(QueryExecutionContext* qec,
                  parsedQuery::Service parsedServiceClause,
-                 GetTsvFunction getTsvFunction)
+                 GetTsvFunction getTsvFunction,
+                 std::shared_ptr<QueryExecutionTree> siblingTree)
     : Operation{qec},
       parsedServiceClause_{std::move(parsedServiceClause)},
-      getTsvFunction_{std::move(getTsvFunction)} {}
+      getTsvFunction_{std::move(getTsvFunction)},
+      siblingTree_{std::move(siblingTree)} {}
 
 // ____________________________________________________________________________
 std::string Service::getCacheKeyImpl() const {
@@ -92,6 +97,14 @@
   serviceIriString.remove_suffix(1);
   ad_utility::httpUtils::Url serviceUrl{serviceIriString};
 
+  // Try to simplify the Service Query using it's sibling Operation.
+  if (auto valuesClause = getSiblingValuesClause(); valuesClause.has_value()) {
+    auto openBracketPos = parsedServiceClause_.graphPatternAsString_.find('{');
+    parsedServiceClause_.graphPatternAsString_ =
+        "{\n" + valuesClause.value() + '\n' +
+        parsedServiceClause_.graphPatternAsString_.substr(openBracketPos + 1);
+  }
+
   // Construct the query to be sent to the SPARQL endpoint.
   std::string variablesForSelectClause = absl::StrJoin(
       parsedServiceClause_.visibleVariables_, " ", Variable::AbslFormatter);
@@ -159,6 +172,59 @@
   return {std::move(idTable), resultSortedOn(), std::move(localVocab)};
 }
 
+// ____________________________________________________________________________
+std::optional<std::string> Service::getSiblingValuesClause() const {
+  if (siblingTree_ == nullptr) {
+    return std::nullopt;
+  }
+
+  const auto& siblingResult = siblingTree_->getResult();
+  if (siblingResult->size() >
+      RuntimeParameters().get<"service-max-value-rows">()) {
+    return std::nullopt;
+  }
+
+  std::vector<ColumnIndex> commonColumnIndices;
+  const auto& siblingVars = siblingTree_->getVariableColumns();
+  std::string vars = "(";
+  for (const auto& localVar : parsedServiceClause_.visibleVariables_) {
+    auto it = siblingVars.find(localVar);
+    if (it == siblingVars.end()) {
+      continue;
+    }
+    vars += it->first.name() + " ";
+    commonColumnIndices.push_back(it->second.columnIndex_);
+  }
+  vars.back() = ')';
+
+  ad_utility::HashSet<std::string> rowSet;
+
+  std::string values = " { ";
+  for (size_t rowIndex = 0; rowIndex < siblingResult->size(); ++rowIndex) {
+    std::string row = "(";
+    for (size_t i = 0; i < commonColumnIndices.size(); ++i) {
+      const auto& optionalString = ExportQueryExecutionTrees::idToStringAndType(
+          siblingTree_->getRootOperation()->getIndex(),
+          siblingResult->idTable()(rowIndex, commonColumnIndices[i]),
+          siblingResult->localVocab());
+
+      if (optionalString.has_value()) {
+        row += optionalString.value().first + ' ';
+      }
+    }
+    row.back() = ')';
+
+    if (rowSet.contains(row)) {
+      continue;
+    }
+
+    rowSet.insert(row);
+    values += row + " ";
+  }
+
+  return "VALUES " + vars + values + "} . ";
+}
+
 // ____________________________________________________________________________
 template <size_t I>
 void Service::writeTsvResult(cppcoro::generator<std::string_view> tsvResult,

src/engine/Service.h

@@ -46,6 +46,9 @@
   // The function used to obtain the result from the remote endpoint.
   GetTsvFunction getTsvFunction_;
 
+  // The siblingTree, used for SERVICE clause optimization.
+  std::shared_ptr<QueryExecutionTree> siblingTree_;
+
  public:
   // Construct from parsed Service clause.
   //
@@ -54,7 +57,14 @@
   // but in our tests (`ServiceTest`) we use a mock function that does not
   // require a running `HttpServer`.
   Service(QueryExecutionContext* qec, parsedQuery::Service parsedServiceClause,
-          GetTsvFunction getTsvFunction = sendHttpOrHttpsRequest);
+          GetTsvFunction getTsvFunction = sendHttpOrHttpsRequest,
+          std::shared_ptr<QueryExecutionTree> siblingTree = nullptr);
+
+  // Set the siblingTree (subTree that will later be joined with the Result of
+  // the Service Operation), used to reduce the Service Queries Complexity.
+  void setSiblingTree(std::shared_ptr<QueryExecutionTree> siblingTree) {
+    siblingTree_ = siblingTree;
+  }
 
   // Methods inherited from base class `Operation`.
   std::string getDescriptor() const override;
@@ -82,6 +92,9 @@
   // Compute the result using `getTsvFunction_`.
   ResultTable computeResult() override;
 
+  // Get a VALUES clause that contains the values of the siblingTree's result.
+  std::optional<std::string> getSiblingValuesClause() const;
+
   // Write the given TSV result to the given result object. The `I` is the width
   // of the result table.
   //

src/engine/Values.cpp

@@ -131,7 +131,9 @@ void Values::writeValues(IdTable* idTablePtr, LocalVocab* localVocab) {
   for (auto& row : parsedValues_._values) {
     for (size_t colIdx = 0; colIdx < idTable.numColumns(); colIdx++) {
       TripleComponent& tc = row[colIdx];
-      Id id = std::move(tc).toValueId(getIndex().getVocab(), *localVocab);
+      // TODO<joka921> We don't want to move, but also don't want to
+      // unconditionally copy.
+      Id id = TripleComponent{tc}.toValueId(getIndex().getVocab(), *localVocab);
       idTable(rowIdx, colIdx) = id;
       if (id.getDatatype() == Datatype::LocalVocabIndex) {
         ++numLocalVocabPerColumn[colIdx];

src/global/RuntimeParameters.h

@@ -46,7 +46,8 @@ inline auto& RuntimeParameters() {
                 30s}),
         SizeT<"lazy-index-scan-max-size-materialization">{1'000'000},
         Bool<"use-binsearch-transitive-path">{true},
-        Bool<"group-by-hash-map-enabled">{false}};
+        Bool<"group-by-hash-map-enabled">{false},
+        SizeT<"service-max-value-rows">{100}};
   }();
   return params;
 }

test/ServiceTest.cpp

@@ -11,6 +11,7 @@
 #include "parser/GraphPatternOperation.h"
 #include "util/IdTableHelpers.h"
 #include "util/IndexTestHelpers.h"
+#include "util/TripleComponentTestHelpers.h"
 #include "util/http/HttpUtils.h"
 
 // Fixture that sets up a test index and a factory for producing mocks for the
@@ -192,4 +193,33 @@ TEST_F(ServiceTest, computeResult) {
   IdTable expectedIdTable = makeIdTableFromVector(
       {{idX, idY}, {idBla, idBli}, {idBlu, idBla}, {idBli, idBlu}});
   EXPECT_EQ(result->idTable(), expectedIdTable);
+
+  // Check 5: When a siblingTree with variables common to the Service Clause is
+  // passed, the Service Operation shall use the siblings result to reduce
+  // its Query complexity by injecting them as Value Clause
+  auto iri = ad_utility::testing::iri;
+  using TC = TripleComponent;
+  auto siblingTree = std::make_shared<QueryExecutionTree>(
+      testQec,
+      std::make_shared<Values>(
+          testQec,
+          (parsedQuery::SparqlValues){
+              {Variable{"?x"}, Variable{"?y"}, Variable{"?z"}},
+              {{TC(iri("<x>")), TC(iri("<y>")), TC(iri("<z>"))},
+               {TC(iri("<blu>")), TC(iri("<bla>")), TC(iri("<blo>"))}}}));
+
+  auto parsedServiceClause5 = parsedServiceClause;
+  parsedServiceClause5.graphPatternAsString_ = "{ ?x <ble> ?y . }";
+
+  std::string_view expectedSparqlQuery5 =
+      "PREFIX doof: <http://doof.org> SELECT ?x ?y "
+      "WHERE { VALUES (?x ?y) { (<x> <y>) (<blu> <bla>) } . ?x <ble> ?y . }";
+
+  Service serviceOperation5{
+      testQec, parsedServiceClause5,
+      getTsvFunctionFactory(
+          expectedUrl, expectedSparqlQuery5,
+          "?x\t?y\n<x>\t<y>\n<bla>\t<bli>\n<blu>\t<bla>\n<bli>\t<blu>\n"),
+      siblingTree};
+  EXPECT_NO_THROW(serviceOperation5.getResult());
 }