From 84070c9649454fdcf3c82fbd29eca9ebdbb8d088 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 25 Apr 2023 15:22:13 +0200 Subject: [PATCH 001/150] A function to merge a join column with a block. Also several refactorings for the IndexScan class. --- src/engine/GroupBy.cpp | 4 +- src/engine/IndexScan.cpp | 303 ++++------------------- src/engine/IndexScan.h | 109 +++++--- src/index/CompressedRelation.cpp | 55 +++- src/index/CompressedRelation.h | 8 +- src/util/JoinAlgorithms/JoinAlgorithms.h | 25 +- test/QueryPlannerTest.cpp | 58 +++-- test/QueryPlannerTestHelpers.h | 2 +- 8 files changed, 237 insertions(+), 327 deletions(-) diff --git a/src/engine/GroupBy.cpp b/src/engine/GroupBy.cpp index 3bfb605317..8064f8ad07 100644 --- a/src/engine/GroupBy.cpp +++ b/src/engine/GroupBy.cpp @@ -543,7 +543,7 @@ std::optional GroupBy::getPermutationForThreeVariableTriple( } { auto v = variableThatMustBeContained; - if (v != indexScan->getSubject() && v.name() != indexScan->getPredicate() && + if (v != indexScan->getSubject() && v != indexScan->getPredicate() && v != indexScan->getObject()) { return std::nullopt; } @@ -551,7 +551,7 @@ std::optional GroupBy::getPermutationForThreeVariableTriple( if (variableByWhichToSort == indexScan->getSubject()) { return Index::Permutation::SPO; - } else if (variableByWhichToSort.name() == indexScan->getPredicate()) { + } else if (variableByWhichToSort == indexScan->getPredicate()) { return Index::Permutation::POS; } else if (variableByWhichToSort == indexScan->getObject()) { return Index::Permutation::OSP; diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 04a5b1940a..27280e63b1 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -18,7 +18,9 @@ IndexScan::IndexScan(QueryExecutionContext* qec, ScanType type, : Operation(qec), _type(type), _subject(triple._s), - _predicate(triple._p.getIri()), + _predicate(triple._p.getIri().starts_with("?") + ? TripleComponent(Variable{triple._p.getIri()}) + : TripleComponent(triple._p.getIri())), _object(triple._o), _sizeEstimate(std::numeric_limits::max()) { precomputeSizeEstimate(); @@ -84,8 +86,8 @@ string IndexScan::asStringImpl(size_t indent) const { // _____________________________________________________________________________ string IndexScan::getDescriptor() const { - return "IndexScan " + _subject.toString() + " " + _predicate + " " + - _object.toString(); + return "IndexScan " + _subject.toString() + " " + _predicate.toString() + + " " + _object.toString(); } // _____________________________________________________________________________ @@ -116,24 +118,12 @@ size_t IndexScan::getResultWidth() const { // _____________________________________________________________________________ vector IndexScan::resultSortedOn() const { - switch (_type) { - case PSO_BOUND_S: - case POS_BOUND_O: - case SOP_BOUND_O: + switch (getResultWidth()) { + case 1: return {0}; - case PSO_FREE_S: - case POS_FREE_O: - case SPO_FREE_P: - case SOP_FREE_O: - case OSP_FREE_S: - case OPS_FREE_P: + case 2: return {0, 1}; - case FULL_INDEX_SCAN_SPO: - case FULL_INDEX_SCAN_SOP: - case FULL_INDEX_SCAN_PSO: - case FULL_INDEX_SCAN_POS: - case FULL_INDEX_SCAN_OSP: - case FULL_INDEX_SCAN_OPS: + case 3: return {0, 1, 2}; default: AD_FAIL(); @@ -147,71 +137,13 @@ VariableToColumnMap IndexScan::computeVariableToColumnMap() const { auto makeCol = makeAlwaysDefinedColumn; size_t col = 0; - // Helper lambdas that add the respective triple component as the next column. - auto addSubject = [&]() { - if (_subject.isVariable()) { - res[_subject.getVariable()] = makeCol(col); - ++col; - } - }; - // TODO Refactor the `PropertyPath` class s.t. it also has - //`isVariable` and `getVariable`, then those three lambdas can become one. - auto addPredicate = [&]() { - if (_predicate[0] == '?') { - res[Variable{_predicate}] = makeCol(col); + for (const TripleComponent* const ptr : getPermutedTriple()) { + if (ptr->isVariable()) { + res[ptr->getVariable()] = makeCol(col); ++col; } - }; - auto addObject = [&]() { - if (_object.isVariable()) { - res[_object.getVariable()] = makeCol(col); - ++col; - } - }; - - switch (_type) { - case SPO_FREE_P: - case FULL_INDEX_SCAN_SPO: - addSubject(); - addPredicate(); - addObject(); - return res; - case SOP_FREE_O: - case SOP_BOUND_O: - case FULL_INDEX_SCAN_SOP: - addSubject(); - addObject(); - addPredicate(); - return res; - case PSO_BOUND_S: - case PSO_FREE_S: - case FULL_INDEX_SCAN_PSO: - addPredicate(); - addSubject(); - addObject(); - return res; - case POS_BOUND_O: - case POS_FREE_O: - case FULL_INDEX_SCAN_POS: - addPredicate(); - addObject(); - addSubject(); - return res; - case OPS_FREE_P: - case FULL_INDEX_SCAN_OPS: - addObject(); - addPredicate(); - addSubject(); - return res; - case OSP_FREE_S: - case FULL_INDEX_SCAN_OSP: - addObject(); - addSubject(); - addPredicate(); - return res; - default: - AD_FAIL(); } + return res; } // _____________________________________________________________________________ ResultTable IndexScan::computeResult() { @@ -219,87 +151,25 @@ ResultTable IndexScan::computeResult() { IdTable idTable{getExecutionContext()->getAllocator()}; using enum Index::Permutation; - switch (_type) { - case PSO_BOUND_S: - computePSOboundS(&idTable); - break; - case POS_BOUND_O: - computePOSboundO(&idTable); - break; - case PSO_FREE_S: - computePSOfreeS(&idTable); - break; - case POS_FREE_O: - computePOSfreeO(&idTable); - break; - case SOP_BOUND_O: - computeSOPboundO(&idTable); - break; - case SPO_FREE_P: - computeSPOfreeP(&idTable); - break; - case SOP_FREE_O: - computeSOPfreeO(&idTable); - break; - case OSP_FREE_S: - computeOSPfreeS(&idTable); - break; - case OPS_FREE_P: - computeOPSfreeP(&idTable); - break; - case FULL_INDEX_SCAN_SPO: - computeFullScan(&idTable, SPO); - break; - case FULL_INDEX_SCAN_SOP: - computeFullScan(&idTable, SOP); - break; - case FULL_INDEX_SCAN_PSO: - computeFullScan(&idTable, PSO); - break; - case FULL_INDEX_SCAN_POS: - computeFullScan(&idTable, POS); - break; - case FULL_INDEX_SCAN_OSP: - computeFullScan(&idTable, OSP); - break; - case FULL_INDEX_SCAN_OPS: - computeFullScan(&idTable, OPS); - break; + size_t numVariables = scanTypeToNumVariables(_type); + idTable.setNumColumns(numVariables); + const auto& idx = _executionContext->getIndex(); + const auto permutedTriple = getPermutedTriple(); + const Index::Permutation permutation = scanTypeToPermutation(_type); + if (numVariables == 2) { + idx.scan(*permutedTriple[0], &idTable, permutation, _timeoutTimer); + } else if (numVariables == 1) { + idx.scan(*permutedTriple[0], *permutedTriple[1], &idTable, permutation, + _timeoutTimer); + } else { + AD_CORRECTNESS_CHECK(numVariables == 3); + computeFullScan(&idTable, permutation); } LOG(DEBUG) << "IndexScan result computation done.\n"; return {std::move(idTable), resultSortedOn(), LocalVocab{}}; } -// _____________________________________________________________________________ -void IndexScan::computePSOboundS(IdTable* result) const { - result->setNumColumns(1); - const auto& idx = _executionContext->getIndex(); - idx.scan(_predicate, _subject, result, Index::Permutation::PSO, - _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computePSOfreeS(IdTable* result) const { - result->setNumColumns(2); - const auto& idx = _executionContext->getIndex(); - idx.scan(_predicate, result, Index::Permutation::PSO, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computePOSboundO(IdTable* result) const { - result->setNumColumns(1); - const auto& idx = _executionContext->getIndex(); - idx.scan(_predicate, _object, result, Index::Permutation::POS, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computePOSfreeO(IdTable* result) const { - result->setNumColumns(2); - const auto& idx = _executionContext->getIndex(); - idx.scan(_predicate, result, Index::Permutation::POS, _timeoutTimer); -} - // _____________________________________________________________________________ size_t IndexScan::computeSizeEstimate() { if (_executionContext) { @@ -331,23 +201,19 @@ size_t IndexScan::computeSizeEstimate() { getRuntimeInfo().costEstimate_ = sizeEstimate; return sizeEstimate; } + } else if (getResultWidth() == 2) { + const auto& firstKey = *getPermutedTriple()[0]; + return getIndex().getCardinality(firstKey, scanTypeToPermutation(_type)); + } else { + // The triple consists of three variables. + // TODO As soon as all implementations of a full index scan + // (Including the "dummy joins" in Join.cpp) consistently exclude the + // internal triples, this estimate should be changed to only return + // the number of triples in the actual knowledge graph (excluding the + // internal triples). + AD_CORRECTNESS_CHECK(getResultWidth() == 3); + return getIndex().numTriples().normalAndInternal_(); } - // TODO Should be a oneliner - // getIndex().cardinality(getPermutation(), getFirstKey()); - if (_type == SPO_FREE_P || _type == SOP_FREE_O) { - return getIndex().getCardinality(_subject, Index::Permutation::SPO); - } else if (_type == POS_FREE_O || _type == PSO_FREE_S) { - return getIndex().getCardinality(_predicate, Index::Permutation::PSO); - } else if (_type == OPS_FREE_P || _type == OSP_FREE_S) { - return getIndex().getCardinality(_object, Index::Permutation::OSP); - } - // The triple consists of three variables. - // TODO As soon as all implementations of a full index scan - // (Including the "dummy joins" in Join.cpp) consistently exclude the - // internal triples, this estimate should be changed to only return - // the number of triples in the actual knowledge graph (excluding the - // internal triples). - return getIndex().numTriples().normalAndInternal_(); } else { // Only for test cases. The handling of the objects is to make the // strange query planner tests pass. @@ -356,7 +222,9 @@ size_t IndexScan::computeSizeEstimate() { _object.isString() ? _object.getString() : _object.toString(); std::string subjectStr = _subject.isString() ? _subject.getString() : _subject.toString(); - return 1000 + subjectStr.size() + _predicate.size() + objectStr.size(); + std::string predStr = + _predicate.isString() ? _predicate.getString() : _predicate.toString(); + return 1000 + subjectStr.size() + predStr.size() + objectStr.size(); } } @@ -387,95 +255,20 @@ size_t IndexScan::getCostEstimate() { } } -// _____________________________________________________________________________ -void IndexScan::computeSPOfreeP(IdTable* result) const { - result->setNumColumns(2); - const auto& idx = _executionContext->getIndex(); - idx.scan(_subject, result, Index::Permutation::SPO, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computeSOPboundO(IdTable* result) const { - result->setNumColumns(1); - const auto& idx = _executionContext->getIndex(); - idx.scan(_subject, _object, result, Index::Permutation::SOP, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computeSOPfreeO(IdTable* result) const { - result->setNumColumns(2); - const auto& idx = _executionContext->getIndex(); - idx.scan(_subject, result, Index::Permutation::SOP, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computeOPSfreeP(IdTable* result) const { - result->setNumColumns(2); - const auto& idx = _executionContext->getIndex(); - idx.scan(_object, result, Index::Permutation::OPS, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computeOSPfreeS(IdTable* result) const { - result->setNumColumns(2); - const auto& idx = _executionContext->getIndex(); - idx.scan(_object, result, Index::Permutation::OSP, _timeoutTimer); -} - // _____________________________________________________________________________ void IndexScan::determineMultiplicities() { _multiplicity.clear(); + auto permutation = scanTypeToPermutation(_type); if (_executionContext) { + const auto& idx = getIndex(); if (getResultWidth() == 1) { _multiplicity.emplace_back(1); + } else if (getResultWidth() == 2) { + const auto permutedTriple = getPermutedTriple(); + _multiplicity = idx.getMultiplicities(*permutedTriple[0], permutation); } else { - const auto& idx = getIndex(); - switch (_type) { - case PSO_FREE_S: - _multiplicity = - idx.getMultiplicities(_predicate, Index::Permutation::PSO); - break; - case POS_FREE_O: - _multiplicity = - idx.getMultiplicities(_predicate, Index::Permutation::POS); - break; - case SPO_FREE_P: - _multiplicity = - idx.getMultiplicities(_subject, Index::Permutation::SPO); - break; - case SOP_FREE_O: - _multiplicity = - idx.getMultiplicities(_subject, Index::Permutation::SOP); - break; - case OSP_FREE_S: - _multiplicity = - idx.getMultiplicities(_object, Index::Permutation::OSP); - break; - case OPS_FREE_P: - _multiplicity = - idx.getMultiplicities(_object, Index::Permutation::OPS); - break; - case FULL_INDEX_SCAN_SPO: - _multiplicity = idx.getMultiplicities(Index::Permutation::SPO); - break; - case FULL_INDEX_SCAN_SOP: - _multiplicity = idx.getMultiplicities(Index::Permutation::SOP); - break; - case FULL_INDEX_SCAN_PSO: - _multiplicity = idx.getMultiplicities(Index::Permutation::PSO); - break; - case FULL_INDEX_SCAN_POS: - _multiplicity = idx.getMultiplicities(Index::Permutation::POS); - break; - case FULL_INDEX_SCAN_OSP: - _multiplicity = idx.getMultiplicities(Index::Permutation::OSP); - break; - case FULL_INDEX_SCAN_OPS: - _multiplicity = idx.getMultiplicities(Index::Permutation::OPS); - break; - default: - AD_FAIL(); - } + AD_CORRECTNESS_CHECK(getResultWidth() == 3); + _multiplicity = idx.getMultiplicities(permutation); } } else { _multiplicity.emplace_back(1); diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index f23f52126d..b9054ce248 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -31,10 +31,78 @@ class IndexScan : public Operation { FULL_INDEX_SCAN_OPS = 14 }; + static size_t scanTypeToNumVariables(ScanType scanType) { + switch (scanType) { + case PSO_BOUND_S: + case POS_BOUND_O: + case SOP_BOUND_O: + return 1; + case PSO_FREE_S: + case POS_FREE_O: + case SOP_FREE_O: + case SPO_FREE_P: + case OSP_FREE_S: + case OPS_FREE_P: + return 2; + case FULL_INDEX_SCAN_SPO: + case FULL_INDEX_SCAN_SOP: + case FULL_INDEX_SCAN_PSO: + case FULL_INDEX_SCAN_POS: + case FULL_INDEX_SCAN_OSP: + case FULL_INDEX_SCAN_OPS: + return 3; + } + } + + static Index::Permutation scanTypeToPermutation(ScanType scanType) { + switch (scanType) { + case PSO_BOUND_S: + case PSO_FREE_S: + case FULL_INDEX_SCAN_PSO: + return Index::Permutation::PSO; + case POS_FREE_O: + case POS_BOUND_O: + case FULL_INDEX_SCAN_POS: + return Index::Permutation::POS; + case SPO_FREE_P: + case FULL_INDEX_SCAN_SPO: + return Index::Permutation::SPO; + case SOP_FREE_O: + case SOP_BOUND_O: + case FULL_INDEX_SCAN_SOP: + return Index::Permutation::SOP; + case OSP_FREE_S: + case FULL_INDEX_SCAN_OSP: + return Index::Permutation::OSP; + case OPS_FREE_P: + case FULL_INDEX_SCAN_OPS: + return Index::Permutation::OPS; + } + } + + static std::array permutationToKeyOrder( + Index::Permutation permutation) { + using enum Index::Permutation; + switch (permutation) { + case POS: + return {1, 2, 0}; + case PSO: + return {1, 0, 2}; + case SOP: + return {0, 2, 1}; + case SPO: + return {0, 1, 2}; + case OPS: + return {2, 1, 0}; + case OSP: + return {2, 0, 1}; + } + } + private: ScanType _type; TripleComponent _subject; - string _predicate; + TripleComponent _predicate; TripleComponent _object; size_t _sizeEstimate; vector _multiplicity; @@ -50,7 +118,7 @@ class IndexScan : public Operation { virtual ~IndexScan() = default; - const string& getPredicate() const { return _predicate; } + const TripleComponent& getPredicate() const { return _predicate; } const TripleComponent& getSubject() const { return _subject; } const TripleComponent& getObject() const { return _object; } @@ -90,17 +158,7 @@ class IndexScan : public Operation { // Currently only the full scans support a limit clause. [[nodiscard]] bool supportsLimit() const override { - switch (_type) { - case FULL_INDEX_SCAN_SPO: - case FULL_INDEX_SCAN_SOP: - case FULL_INDEX_SCAN_PSO: - case FULL_INDEX_SCAN_POS: - case FULL_INDEX_SCAN_OSP: - case FULL_INDEX_SCAN_OPS: - return true; - default: - return false; - } + return getResultWidth() == 3; } ScanType getType() const { return _type; } @@ -110,24 +168,6 @@ class IndexScan : public Operation { vector getChildren() override { return {}; } - void computePSOboundS(IdTable* result) const; - - void computePSOfreeS(IdTable* result) const; - - void computePOSboundO(IdTable* result) const; - - void computePOSfreeO(IdTable* result) const; - - void computeSPOfreeP(IdTable* result) const; - - void computeSOPboundO(IdTable* result) const; - - void computeSOPfreeO(IdTable* result) const; - - void computeOPSfreeP(IdTable* result) const; - - void computeOSPfreeS(IdTable* result) const; - void computeFullScan(IdTable* result, Index::Permutation permutation) const; size_t computeSizeEstimate(); @@ -140,4 +180,11 @@ class IndexScan : public Operation { getPrecomputedResultFromQueryPlanning() override { return _precomputedResult; } + + std::array getPermutedTriple() const { + using Arr = std::array; + Arr inp{&_subject, &_predicate, &_object}; + auto permutation = permutationToKeyOrder(scanTypeToPermutation(_type)); + return {inp[permutation[0]], inp[permutation[1]], inp[permutation[2]]}; + } }; diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index bdaa15bd72..77ad5be4ac 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -9,6 +9,8 @@ #include "util/CompressionUsingZstd/ZstdWrapper.h" #include "util/ConcurrentCache.h" #include "util/Generator.h" +#include "util/JoinAlgorithms/JoinAlgorithms.h" +#include "util/OverloadCallOperator.h" #include "util/TypeTraits.h" using namespace std::chrono_literals; @@ -16,7 +18,7 @@ using namespace std::chrono_literals; // ____________________________________________________________________________ void CompressedRelationReader::scan( const CompressedRelationMetadata& metadata, - const vector& blockMetadata, + std::span blockMetadata, ad_utility::File& file, IdTable* result, ad_utility::SharedConcurrentTimeoutTimer timer) const { AD_CONTRACT_CHECK(result->numColumns() == NumColumns); @@ -152,6 +154,57 @@ void CompressedRelationReader::scan( } } +// _____________________________________________________________________________ +std::vector CompressedRelationReader::getBlocksForJoin( + std::span joinColum, const CompressedRelationMetadata& metadata, + std::span blockMetadata) { + // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ + struct KeyLhs { + Id col0FirstId_; + Id col0LastId_; + }; + Id col0Id = metadata.col0Id_; + // TODO Use a structured binding. Structured bindings are + // currently not supported by clang when using OpenMP because clang internally + // transforms the `#pragma`s into lambdas, and capturing structured bindings + // is only supported in clang >= 16. + decltype(blockMetadata.begin()) beginBlock, endBlock; + std::tie(beginBlock, endBlock) = std::equal_range( + // TODO For some reason we can't use `std::ranges::equal_range`, + // find out why. Note: possibly it has something to do with the limited + // support of ranges in clang with versions < 16. Revisit this when + // we use clang 16. + blockMetadata.begin(), blockMetadata.end(), KeyLhs{col0Id, col0Id}, + [](const auto& a, const auto& b) { + return a.col0FirstId_ < b.col0FirstId_ && a.col0LastId_ < b.col0LastId_; + }); + + if (endBlock - beginBlock < 2) { + return std::vector(beginBlock, endBlock); + } + + auto idLessThanBlock = [](Id id, const CompressedBlockMetadata& block) { + return id < block.col1FirstId_; + }; + auto blockLessThanId = [](const CompressedBlockMetadata& block, Id id) { + return block.col1LastId_ < id; + }; + + auto lessThan = + ad_utility::OverloadCallOperator{idLessThanBlock, blockLessThanId}; + + std::vector result; + auto addRow = [&result]([[maybe_unused]] auto it1, auto it2) { + result.push_back(*it2); + }; + + auto noop = ad_utility::noop; + [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( + joinColum, std::span(beginBlock, endBlock), + lessThan, addRow, noop, noop); + return result; +} + // _____________________________________________________________________________ void CompressedRelationReader::scan( const CompressedRelationMetadata& metaData, Id col1Id, diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h index 3c6c5df80a..b05a1a66c2 100644 --- a/src/index/CompressedRelation.h +++ b/src/index/CompressedRelation.h @@ -250,10 +250,16 @@ class CompressedRelationReader { * The same `CompressedRelationWriter` (see below). */ void scan(const CompressedRelationMetadata& metadata, - const vector& blockMetadata, + std::span blockMetadata, ad_utility::File& file, IdTable* result, ad_utility::SharedConcurrentTimeoutTimer timer) const; + // Get all the blocks that can contain an Id from the `joinColumn`. + // TODO Include a timeout check. + std::vector getBlocksForJoin( + std::span joinColum, const CompressedRelationMetadata& metadata, + std::span blockMetadata); + /** * @brief For a permutation XYZ, retrieve all Z for given X and Y. * diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index e7923c9c5a..4cea2c7668 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -93,17 +93,17 @@ concept BinaryIteratorFunction = * be exploited to fix the result in a cheaper way than a full sort. */ template < - std::ranges::random_access_range Range, - BinaryRangePredicate LessThan, typename FindSmallerUndefRangesLeft, - typename FindSmallerUndefRangesRight, - UnaryIteratorFunction ElFromFirstNotFoundAction = decltype(noop)> + bool addDuplicatesFromLeft = true, std::ranges::random_access_range Range1, + std::ranges::random_access_range Range2, typename LessThan, + typename FindSmallerUndefRangesLeft, typename FindSmallerUndefRangesRight, + typename ElFromFirstNotFoundAction = decltype(noop)> [[nodiscard]] size_t zipperJoinWithUndef( - const Range& left, const Range& right, const LessThan& lessThan, - const BinaryIteratorFunction auto& compatibleRowAction, + const Range1& left, const Range2& right, const LessThan& lessThan, + const auto& compatibleRowAction, const FindSmallerUndefRangesLeft& findSmallerUndefRangesLeft, const FindSmallerUndefRangesRight& findSmallerUndefRangesRight, ElFromFirstNotFoundAction elFromFirstNotFoundAction = {}) { - using Iterator = std::ranges::iterator_t; + // using Iterator = std::ranges::iterator_t; // If this is not an OPTIONAL join or a MINUS we can apply several // optimizations, so we store this information. @@ -145,8 +145,8 @@ template < // all elements in `left` that are smaller than `*itFromRight` to work // correctly. It would thus be always correct to pass in `left.begin()` and // `left.end()`, but passing in smaller ranges is more efficient. - auto mergeWithUndefLeft = [&](Iterator itFromRight, Iterator leftBegin, - Iterator leftEnd) { + auto mergeWithUndefLeft = [&](auto itFromRight, auto leftBegin, + auto leftEnd) { if constexpr (!isSimilar) { // We need to bind the const& to a variable, else it will be // dangling inside the `findSmallerUndefRangesLeft` generator. @@ -182,8 +182,8 @@ template < // correct place in the result. The condition about containing no UNDEF values // is important, because otherwise `*itFromLeft` may be compatible to a larger // element in `right` that is only discovered later. - auto mergeWithUndefRight = [&](Iterator itFromLeft, Iterator beginRight, - Iterator endRight, bool hasNoMatch) { + auto mergeWithUndefRight = [&](auto itFromLeft, auto beginRight, + auto endRight, bool hasNoMatch) { if constexpr (!isSimilar) { bool compatibleWasFound = false; // We need to bind the const& to a variable, else it will be @@ -258,6 +258,9 @@ template < for (auto innerIt2 = it2; innerIt2 != endSame2; ++innerIt2) { compatibleRowAction(it1, innerIt2); } + if constexpr (!addDuplicatesFromLeft) { + break; + } } it1 = endSame1; it2 = endSame2; diff --git a/test/QueryPlannerTest.cpp b/test/QueryPlannerTest.cpp index 7d5d47a001..601a82ecc3 100644 --- a/test/QueryPlannerTest.cpp +++ b/test/QueryPlannerTest.cpp @@ -1025,7 +1025,7 @@ TEST(QueryPlannerTest, SimpleTripleOneVariable) { h::expect("SELECT * WHERE { ?s

}", h::IndexScan(Var{"?s"}, "

", "", {POS_BOUND_O})); h::expect("SELECT * WHERE { ?p }", - h::IndexScan("", "?p", "", {SOP_BOUND_O})); + h::IndexScan("", Var{"?p"}, "", {SOP_BOUND_O})); h::expect("SELECT * WHERE {

?o }", h::IndexScan("", "

", Var{"?o"}, {PSO_BOUND_S})); } @@ -1046,20 +1046,22 @@ TEST(QueryPlannerTest, SimpleTripleTwoVariables) { h::IndexScan(Var{"?s"}, "

", Var{"?o"}, {PSO_FREE_S})); // Fixed subject. - h::expect("SELECT * WHERE { ?p ?o }", - h::IndexScan("", "?p", Var{"?o"}, {SOP_FREE_O, SPO_FREE_P})); + h::expect( + "SELECT * WHERE { ?p ?o }", + h::IndexScan("", Var{"?p"}, Var{"?o"}, {SOP_FREE_O, SPO_FREE_P})); h::expect("SELECT * WHERE { ?p ?o } INTERNAL SORT BY ?o", - h::IndexScan("", "?p", Var{"?o"}, {SOP_FREE_O})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, {SOP_FREE_O})); h::expect("SELECT * WHERE { ?p ?o } INTERNAL SORT BY ?p", - h::IndexScan("", "?p", Var{"?o"}, {SPO_FREE_P})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, {SPO_FREE_P})); // Fixed object. - h::expect("SELECT * WHERE { ?p ?o }", - h::IndexScan("", "?p", Var{"?o"}, {SOP_FREE_O, SPO_FREE_P})); + h::expect( + "SELECT * WHERE { ?p ?o }", + h::IndexScan("", Var{"?p"}, Var{"?o"}, {SOP_FREE_O, SPO_FREE_P})); h::expect("SELECT * WHERE { ?p ?o } INTERNAL SORT BY ?o", - h::IndexScan("", "?p", Var{"?o"}, {SOP_FREE_O})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, {SOP_FREE_O})); h::expect("SELECT * WHERE { ?p ?o } INTERNAL SORT BY ?p", - h::IndexScan("", "?p", Var{"?o"}, {SPO_FREE_P})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, {SPO_FREE_P})); } TEST(QueryPlannerTest, SimpleTripleThreeVariables) { @@ -1068,33 +1070,39 @@ TEST(QueryPlannerTest, SimpleTripleThreeVariables) { // Fixed predicate. // Don't care about the sorting. h::expect("SELECT * WHERE { ?s ?p ?o }", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_SPO, FULL_INDEX_SCAN_SOP, FULL_INDEX_SCAN_PSO, FULL_INDEX_SCAN_POS, FULL_INDEX_SCAN_OSP, FULL_INDEX_SCAN_OPS})); // Sorted by one variable, two possible permutations remain. h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_SPO, FULL_INDEX_SCAN_SOP})); h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_POS, FULL_INDEX_SCAN_PSO})); h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_OSP, FULL_INDEX_SCAN_OPS})); // Sorted by two variables, this makes the permutation unique. - h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s ?o", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, {FULL_INDEX_SCAN_SOP})); - h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s ?p", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, {FULL_INDEX_SCAN_SPO})); - h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o ?s", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, {FULL_INDEX_SCAN_OSP})); - h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o ?p", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, {FULL_INDEX_SCAN_OPS})); - h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p ?s", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, {FULL_INDEX_SCAN_PSO})); - h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p ?o", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, {FULL_INDEX_SCAN_POS})); + h::expect( + "SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s ?o", + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_SOP})); + h::expect( + "SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s ?p", + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_SPO})); + h::expect( + "SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o ?s", + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_OSP})); + h::expect( + "SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o ?p", + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_OPS})); + h::expect( + "SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p ?s", + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_PSO})); + h::expect( + "SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p ?o", + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_POS})); } diff --git a/test/QueryPlannerTestHelpers.h b/test/QueryPlannerTestHelpers.h index cbb23f699a..f08e4b2805 100644 --- a/test/QueryPlannerTestHelpers.h +++ b/test/QueryPlannerTestHelpers.h @@ -33,7 +33,7 @@ auto RootOperation(auto matcher) -> Matcher { /// Return a matcher that checks that a given `QueryExecutionTree` consists of a /// single `IndexScan` with the given `subject`, `predicate`, and `object`, and /// that the `ScanType` of this `IndexScan` is any of the given `scanTypes`. -auto IndexScan(TripleComponent subject, std::string predicate, +auto IndexScan(TripleComponent subject, TripleComponent predicate, TripleComponent object, const std::vector& scanTypes = {}) -> Matcher { From 06225fbb3ab149a9250c437e8aa6a524771b8b78 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 25 Apr 2023 16:49:55 +0200 Subject: [PATCH 002/150] Continued the refactoring. The `ScanType` is now completely unused inside the `IndexScan` class. --- src/engine/IndexScan.cpp | 122 ++++++++++----------------------- src/engine/IndexScan.h | 25 ++++++- src/engine/Join.cpp | 26 +------ test/QueryPlannerTest.cpp | 84 ++++++++++------------- test/QueryPlannerTestHelpers.h | 9 +-- 5 files changed, 99 insertions(+), 167 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 27280e63b1..63d7c367c2 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -16,7 +16,8 @@ using std::string; IndexScan::IndexScan(QueryExecutionContext* qec, ScanType type, const SparqlTriple& triple) : Operation(qec), - _type(type), + _permutation(scanTypeToPermutation(type)), + _numVariables(scanTypeToNumVariables(type)), _subject(triple._s), _predicate(triple._p.getIri().starts_with("?") ? TripleComponent(Variable{triple._p.getIri()}) @@ -31,55 +32,28 @@ string IndexScan::asStringImpl(size_t indent) const { for (size_t i = 0; i < indent; ++i) { os << ' '; } - switch (_type) { - case PSO_BOUND_S: - os << "SCAN PSO with P = \"" << _predicate << "\", S = \"" << _subject - << "\""; - break; - case POS_BOUND_O: - os << "SCAN POS with P = \"" << _predicate << "\", O = \"" - << _object.toRdfLiteral() << "\""; - break; - case SOP_BOUND_O: - os << "SCAN SOP with S = \"" << _subject << "\", O = \"" - << _object.toRdfLiteral() << "\""; - break; - case PSO_FREE_S: - os << "SCAN PSO with P = \"" << _predicate << "\""; - break; - case POS_FREE_O: - os << "SCAN POS with P = \"" << _predicate << "\""; - break; - case SPO_FREE_P: - os << "SCAN SPO with S = \"" << _subject << "\""; - break; - case SOP_FREE_O: - os << "SCAN SOP with S = \"" << _subject << "\""; - break; - case OPS_FREE_P: - os << "SCAN OPS with O = \"" << _object.toRdfLiteral() << "\""; - break; - case OSP_FREE_S: - os << "SCAN OSP with O = \"" << _object.toRdfLiteral() << "\""; - break; - case FULL_INDEX_SCAN_SPO: - os << "SCAN FOR FULL INDEX SPO (DUMMY OPERATION)"; - break; - case FULL_INDEX_SCAN_SOP: - os << "SCAN FOR FULL INDEX SOP (DUMMY OPERATION)"; - break; - case FULL_INDEX_SCAN_PSO: - os << "SCAN FOR FULL INDEX PSO (DUMMY OPERATION)"; - break; - case FULL_INDEX_SCAN_POS: - os << "SCAN FOR FULL INDEX POS (DUMMY OPERATION)"; - break; - case FULL_INDEX_SCAN_OSP: - os << "SCAN FOR FULL INDEX OSP (DUMMY OPERATION)"; - break; - case FULL_INDEX_SCAN_OPS: - os << "SCAN FOR FULL INDEX OPS (DUMMY OPERATION)"; - break; + + auto permutationString = permutationToString(_permutation); + + if (getResultWidth() == 3) { + AD_CORRECTNESS_CHECK(getResultWidth() == 3); + os << "SCAN FOR FULL INDEX " << permutationToString(_permutation) + << " (DUMMY OPERATION)"; + + } else { + auto firstKeyString = permutationString.at(0); + auto permutedTriple = getPermutedTriple(); + const auto& firstKey = permutedTriple.at(0)->toRdfLiteral(); + if (getResultWidth() == 1) { + auto secondKeyString = permutationString.at(1); + const auto& secondKey = permutedTriple.at(1)->toRdfLiteral(); + os << "SCAN " << permutationString << " with " << firstKeyString + << " = \"" << firstKey << "\", " << secondKeyString << " = \"" + << secondKey << "\""; + } else if (getResultWidth() == 2) { + os << "SCAN " << permutationString << " with " << firstKeyString + << " = \"" << firstKey << "\""; + } } return std::move(os).str(); } @@ -91,30 +65,7 @@ string IndexScan::getDescriptor() const { } // _____________________________________________________________________________ -size_t IndexScan::getResultWidth() const { - switch (_type) { - case PSO_BOUND_S: - case POS_BOUND_O: - case SOP_BOUND_O: - return 1; - case PSO_FREE_S: - case POS_FREE_O: - case SPO_FREE_P: - case SOP_FREE_O: - case OSP_FREE_S: - case OPS_FREE_P: - return 2; - case FULL_INDEX_SCAN_SPO: - case FULL_INDEX_SCAN_SOP: - case FULL_INDEX_SCAN_PSO: - case FULL_INDEX_SCAN_POS: - case FULL_INDEX_SCAN_OSP: - case FULL_INDEX_SCAN_OPS: - return 3; - default: - AD_FAIL(); - } -} +size_t IndexScan::getResultWidth() const { return _numVariables; } // _____________________________________________________________________________ vector IndexScan::resultSortedOn() const { @@ -151,19 +102,17 @@ ResultTable IndexScan::computeResult() { IdTable idTable{getExecutionContext()->getAllocator()}; using enum Index::Permutation; - size_t numVariables = scanTypeToNumVariables(_type); - idTable.setNumColumns(numVariables); + idTable.setNumColumns(_numVariables); const auto& idx = _executionContext->getIndex(); const auto permutedTriple = getPermutedTriple(); - const Index::Permutation permutation = scanTypeToPermutation(_type); - if (numVariables == 2) { - idx.scan(*permutedTriple[0], &idTable, permutation, _timeoutTimer); - } else if (numVariables == 1) { - idx.scan(*permutedTriple[0], *permutedTriple[1], &idTable, permutation, + if (_numVariables == 2) { + idx.scan(*permutedTriple[0], &idTable, _permutation, _timeoutTimer); + } else if (_numVariables == 1) { + idx.scan(*permutedTriple[0], *permutedTriple[1], &idTable, _permutation, _timeoutTimer); } else { - AD_CORRECTNESS_CHECK(numVariables == 3); - computeFullScan(&idTable, permutation); + AD_CORRECTNESS_CHECK(_numVariables == 3); + computeFullScan(&idTable, _permutation); } LOG(DEBUG) << "IndexScan result computation done.\n"; @@ -203,7 +152,7 @@ size_t IndexScan::computeSizeEstimate() { } } else if (getResultWidth() == 2) { const auto& firstKey = *getPermutedTriple()[0]; - return getIndex().getCardinality(firstKey, scanTypeToPermutation(_type)); + return getIndex().getCardinality(firstKey, _permutation); } else { // The triple consists of three variables. // TODO As soon as all implementations of a full index scan @@ -258,17 +207,16 @@ size_t IndexScan::getCostEstimate() { // _____________________________________________________________________________ void IndexScan::determineMultiplicities() { _multiplicity.clear(); - auto permutation = scanTypeToPermutation(_type); if (_executionContext) { const auto& idx = getIndex(); if (getResultWidth() == 1) { _multiplicity.emplace_back(1); } else if (getResultWidth() == 2) { const auto permutedTriple = getPermutedTriple(); - _multiplicity = idx.getMultiplicities(*permutedTriple[0], permutation); + _multiplicity = idx.getMultiplicities(*permutedTriple[0], _permutation); } else { AD_CORRECTNESS_CHECK(getResultWidth() == 3); - _multiplicity = idx.getMultiplicities(permutation); + _multiplicity = idx.getMultiplicities(_permutation); } } else { _multiplicity.emplace_back(1); diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index b9054ce248..b77a481964 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -99,8 +99,27 @@ class IndexScan : public Operation { } } + static std::string_view permutationToString(Index::Permutation permutation) { + using enum Index::Permutation; + switch (permutation) { + case POS: + return "POS"; + case PSO: + return "PSO"; + case SOP: + return "SOP"; + case SPO: + return "SPO"; + case OPS: + return "OPS"; + case OSP: + return "OSP"; + } + } + private: - ScanType _type; + Index::Permutation _permutation; + size_t _numVariables; TripleComponent _subject; TripleComponent _predicate; TripleComponent _object; @@ -161,7 +180,7 @@ class IndexScan : public Operation { return getResultWidth() == 3; } - ScanType getType() const { return _type; } + Index::Permutation permutation() const { return _permutation; } private: ResultTable computeResult() override; @@ -184,7 +203,7 @@ class IndexScan : public Operation { std::array getPermutedTriple() const { using Arr = std::array; Arr inp{&_subject, &_predicate, &_object}; - auto permutation = permutationToKeyOrder(scanTypeToPermutation(_type)); + auto permutation = permutationToKeyOrder(_permutation); return {inp[permutation[0]], inp[permutation[1]], inp[permutation[2]]}; } }; diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp index 3f92d8100a..8dc0fe186a 100644 --- a/src/engine/Join.cpp +++ b/src/engine/Join.cpp @@ -233,30 +233,8 @@ Join::ScanMethodType Join::getScanMethod( [&idx, perm](Id id, IdTable* idTable) { idx.scan(id, idTable, perm); }; }; - using enum Index::Permutation; - switch (scan.getType()) { - case IndexScan::FULL_INDEX_SCAN_SPO: - scanMethod = scanLambda(SPO); - break; - case IndexScan::FULL_INDEX_SCAN_SOP: - scanMethod = scanLambda(SOP); - break; - case IndexScan::FULL_INDEX_SCAN_PSO: - scanMethod = scanLambda(PSO); - break; - case IndexScan::FULL_INDEX_SCAN_POS: - scanMethod = scanLambda(POS); - break; - case IndexScan::FULL_INDEX_SCAN_OSP: - scanMethod = scanLambda(OSP); - break; - case IndexScan::FULL_INDEX_SCAN_OPS: - scanMethod = scanLambda(OPS); - break; - default: - AD_THROW("Found non-dummy scan where one was expected."); - } - return scanMethod; + AD_CORRECTNESS_CHECK(scan.getResultWidth() == 3); + return scanLambda(scan.permutation()); } // _____________________________________________________________________________ diff --git a/test/QueryPlannerTest.cpp b/test/QueryPlannerTest.cpp index 601a82ecc3..1e43f813a2 100644 --- a/test/QueryPlannerTest.cpp +++ b/test/QueryPlannerTest.cpp @@ -1017,92 +1017,78 @@ TEST(QueryPlannerTest, testSimpleOptional) { } TEST(QueryPlannerTest, SimpleTripleOneVariable) { - using enum IndexScan::ScanType; + using enum Index::Permutation; // With only one variable, there are always two permutations that will yield // exactly the same result. The query planner consistently chosses one of // them. h::expect("SELECT * WHERE { ?s

}", - h::IndexScan(Var{"?s"}, "

", "", {POS_BOUND_O})); + h::IndexScan(Var{"?s"}, "

", "", 1, {POS})); h::expect("SELECT * WHERE { ?p }", - h::IndexScan("", Var{"?p"}, "", {SOP_BOUND_O})); + h::IndexScan("", Var{"?p"}, "", 1, {SOP})); h::expect("SELECT * WHERE {

?o }", - h::IndexScan("", "

", Var{"?o"}, {PSO_BOUND_S})); + h::IndexScan("", "

", Var{"?o"}, 1, {PSO})); } TEST(QueryPlannerTest, SimpleTripleTwoVariables) { - using enum IndexScan::ScanType; + using enum Index::Permutation; // Fixed predicate. // Without `Order By`, two orderings are possible, both are fine. - h::expect( - "SELECT * WHERE { ?s

?o }", - h::IndexScan(Var{"?s"}, "

", Var{"?o"}, {POS_FREE_O, PSO_FREE_S})); + h::expect("SELECT * WHERE { ?s

?o }", + h::IndexScan(Var{"?s"}, "

", Var{"?o"}, 2, {POS, PSO})); // Must always be a single index scan, never index scan + sorting. h::expect("SELECT * WHERE { ?s

?o } INTERNAL SORT BY ?o", - h::IndexScan(Var{"?s"}, "

", Var{"?o"}, {POS_FREE_O})); + h::IndexScan(Var{"?s"}, "

", Var{"?o"}, 2, {POS})); h::expect("SELECT * WHERE { ?s

?o } INTERNAL SORT BY ?s", - h::IndexScan(Var{"?s"}, "

", Var{"?o"}, {PSO_FREE_S})); + h::IndexScan(Var{"?s"}, "

", Var{"?o"}, 2, {PSO})); // Fixed subject. - h::expect( - "SELECT * WHERE { ?p ?o }", - h::IndexScan("", Var{"?p"}, Var{"?o"}, {SOP_FREE_O, SPO_FREE_P})); + h::expect("SELECT * WHERE { ?p ?o }", + h::IndexScan("", Var{"?p"}, Var{"?o"}, 2, {SOP, SPO})); h::expect("SELECT * WHERE { ?p ?o } INTERNAL SORT BY ?o", - h::IndexScan("", Var{"?p"}, Var{"?o"}, {SOP_FREE_O})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, 2, {SOP})); h::expect("SELECT * WHERE { ?p ?o } INTERNAL SORT BY ?p", - h::IndexScan("", Var{"?p"}, Var{"?o"}, {SPO_FREE_P})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, 2, {SPO})); // Fixed object. - h::expect( - "SELECT * WHERE { ?p ?o }", - h::IndexScan("", Var{"?p"}, Var{"?o"}, {SOP_FREE_O, SPO_FREE_P})); + h::expect("SELECT * WHERE { ?p ?o }", + h::IndexScan("", Var{"?p"}, Var{"?o"}, 2, {SOP, SPO})); h::expect("SELECT * WHERE { ?p ?o } INTERNAL SORT BY ?o", - h::IndexScan("", Var{"?p"}, Var{"?o"}, {SOP_FREE_O})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, 2, {SOP})); h::expect("SELECT * WHERE { ?p ?o } INTERNAL SORT BY ?p", - h::IndexScan("", Var{"?p"}, Var{"?o"}, {SPO_FREE_P})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, 2, {SPO})); } TEST(QueryPlannerTest, SimpleTripleThreeVariables) { - using enum IndexScan::ScanType; + using enum Index::Permutation; // Fixed predicate. // Don't care about the sorting. h::expect("SELECT * WHERE { ?s ?p ?o }", - h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, - {FULL_INDEX_SCAN_SPO, FULL_INDEX_SCAN_SOP, - FULL_INDEX_SCAN_PSO, FULL_INDEX_SCAN_POS, - FULL_INDEX_SCAN_OSP, FULL_INDEX_SCAN_OPS})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, + {SPO, SOP, PSO, POS, OSP, OPS})); // Sorted by one variable, two possible permutations remain. h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s", - h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, - {FULL_INDEX_SCAN_SPO, FULL_INDEX_SCAN_SOP})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {SPO, SOP})); h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p", - h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, - {FULL_INDEX_SCAN_POS, FULL_INDEX_SCAN_PSO})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {POS, PSO})); h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o", - h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, - {FULL_INDEX_SCAN_OSP, FULL_INDEX_SCAN_OPS})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {OSP, OPS})); // Sorted by two variables, this makes the permutation unique. - h::expect( - "SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s ?o", - h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_SOP})); - h::expect( - "SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s ?p", - h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_SPO})); - h::expect( - "SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o ?s", - h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_OSP})); - h::expect( - "SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o ?p", - h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_OPS})); - h::expect( - "SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p ?s", - h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_PSO})); - h::expect( - "SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p ?o", - h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, {FULL_INDEX_SCAN_POS})); + h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s ?o", + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {SOP})); + h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s ?p", + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {SPO})); + h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o ?s", + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {OSP})); + h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o ?p", + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {OPS})); + h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p ?s", + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {PSO})); + h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p ?o", + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {POS})); } diff --git a/test/QueryPlannerTestHelpers.h b/test/QueryPlannerTestHelpers.h index f08e4b2805..836c69e302 100644 --- a/test/QueryPlannerTestHelpers.h +++ b/test/QueryPlannerTestHelpers.h @@ -34,13 +34,14 @@ auto RootOperation(auto matcher) -> Matcher { /// single `IndexScan` with the given `subject`, `predicate`, and `object`, and /// that the `ScanType` of this `IndexScan` is any of the given `scanTypes`. auto IndexScan(TripleComponent subject, TripleComponent predicate, - TripleComponent object, - const std::vector& scanTypes = {}) + TripleComponent object, size_t numVariables, + const std::vector& scanTypes = {}) -> Matcher { auto typeMatcher = - scanTypes.empty() ? A() : AnyOfArray(scanTypes); + scanTypes.empty() ? A() : AnyOfArray(scanTypes); return RootOperation<::IndexScan>( - AllOf(Property(&IndexScan::getType, typeMatcher), + AllOf(Property(&IndexScan::permutation, typeMatcher), + Property(&IndexScan::getResultWidth, Eq(numVariables)), Property(&IndexScan::getSubject, Eq(subject)), Property(&IndexScan::getPredicate, Eq(predicate)), Property(&IndexScan::getObject, Eq(object)))); From 5e5352b2ca60c7aac6836184c53c2246a99c54f4 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 25 Apr 2023 18:12:45 +0200 Subject: [PATCH 003/150] Implemented a (serial) coroutine for scanning. --- src/engine/IndexScan.cpp | 10 +++ src/index/CompressedRelation.cpp | 107 +++++++++++++++++++++++++++++-- src/index/CompressedRelation.h | 59 ++++++++++------- 3 files changed, 147 insertions(+), 29 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 63d7c367c2..c970ed2688 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -13,6 +13,7 @@ using std::string; +// _____________________________________________________________________________ IndexScan::IndexScan(QueryExecutionContext* qec, ScanType type, const SparqlTriple& triple) : Operation(qec), @@ -25,7 +26,16 @@ IndexScan::IndexScan(QueryExecutionContext* qec, ScanType type, _object(triple._o), _sizeEstimate(std::numeric_limits::max()) { precomputeSizeEstimate(); + + auto permutedTriple = getPermutedTriple(); + for (size_t i = 0; i < 3 - _numVariables; ++i) { + AD_CONTRACT_CHECK(!permutedTriple.at(i)->isVariable()); + } + for (size_t i = 3 - _numVariables; i < permutedTriple.size(); ++i) { + AD_CONTRACT_CHECK(permutedTriple.at(i)->isVariable()); + } } + // _____________________________________________________________________________ string IndexScan::asStringImpl(size_t indent) const { std::ostringstream os; diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index 77ad5be4ac..a505d5af66 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -78,12 +78,12 @@ void CompressedRelationReader::scan( // We have at most one block that is incomplete and thus requires trimming. // Set up a lambda, that reads this block and decompresses it to // the result. - auto readIncompleteBlock = [&](const auto& block) { + auto readIncompleteBlock = [&](const auto& block) mutable { // A block is uniquely identified by its start position in the file. auto cacheKey = block.offsetsAndCompressedSize_.at(0).offsetInFile_; auto uncompressedBuffer = blockCache_ .computeOnce(cacheKey, - [&]() { + [&]() { return readAndDecompressBlock( block, file, std::nullopt); }) @@ -154,6 +154,102 @@ void CompressedRelationReader::scan( } } +// _____________________________________________________________________________ +cppcoro::generator CompressedRelationReader::lazyScan(const CompressedRelationMetadata& metadata, std::span blockMetadata, ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, ad_utility::SharedConcurrentTimeoutTimer timer) { + // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ + struct KeyLhs { + Id col0FirstId_; + Id col0LastId_; + }; + Id col0Id = metadata.col0Id_; + // TODO Use a structured binding. Structured bindings are + // currently not supported by clang when using OpenMP because clang internally + // transforms the `#pragma`s into lambdas, and capturing structured bindings + // is only supported in clang >= 16. + decltype(blockMetadata.begin()) beginBlock, endBlock; + std::tie(beginBlock, endBlock) = std::equal_range( + // TODO For some reason we can't use `std::ranges::equal_range`, + // find out why. Note: possibly it has something to do with the limited + // support of ranges in clang with versions < 16. Revisit this when + // we use clang 16. + blockMetadata.begin(), blockMetadata.end(), KeyLhs{col0Id, col0Id}, + [](const auto& a, const auto& b) { + return a.col0FirstId_ < b.col0FirstId_ && a.col0LastId_ < b.col0LastId_; + }); + + // The first block might contain entries that are not part of our + // actual scan result. + bool firstBlockIsIncomplete = + beginBlock < endBlock && + (beginBlock->col0FirstId_ < col0Id || beginBlock->col0LastId_ > col0Id); + auto lastBlock = endBlock - 1; + + bool lastBlockIsIncomplete = + beginBlock < lastBlock && + (lastBlock->col0FirstId_ < col0Id || lastBlock->col0LastId_ > col0Id); + + // Invariant: A relation spans multiple blocks exclusively or several + // entities are stored completely in the same Block. + AD_CORRECTNESS_CHECK(!firstBlockIsIncomplete || (beginBlock == lastBlock)); + AD_CORRECTNESS_CHECK(!lastBlockIsIncomplete); + if (firstBlockIsIncomplete) { + AD_CORRECTNESS_CHECK(metadata.offsetInBlock_ != + std::numeric_limits::max()); + } + + // We have at most one block that is incomplete and thus requires trimming. + // Set up a lambda, that reads this block and decompresses it to + // the result. + auto readIncompleteBlock = [&](const auto& block) { + // A block is uniquely identified by its start position in the file. + auto cacheKey = block.offsetsAndCompressedSize_.at(0).offsetInFile_; + auto uncompressedBuffer = blockCache_ + .computeOnce(cacheKey, + [&]() { + return readAndDecompressBlock( + block, file, std::nullopt); + }) + ._resultPointer; + + // Extract the part of the block that actually belongs to the relation + auto numElements = metadata.numRows_; + AD_CORRECTNESS_CHECK(uncompressedBuffer->numColumns() == + metadata.numColumns()); + IdTable result(uncompressedBuffer->numColumns(), allocator); + result.resize(numElements); + for (size_t i = 0; i < uncompressedBuffer->numColumns(); ++i) { + const auto& inputCol = uncompressedBuffer->getColumn(i); + auto begin = inputCol.begin() + metadata.offsetInBlock_; + decltype(auto) resultColumn = result.getColumn(i); + std::copy(begin, begin + numElements, resultColumn.begin()); + } + return result; + }; + + // Read the first block if it is incomplete + if (firstBlockIsIncomplete) { + co_yield readIncompleteBlock(*beginBlock); + ++beginBlock; + if (timer) { + timer->wlock()->checkTimeoutAndThrow("IndexScan :"); + } + } + + // Read all the other (complete!) blocks in parallel + for (; beginBlock < endBlock; ++beginBlock) { + const auto& block = *beginBlock; + // Read a block from disk (serially). + + CompressedBlock compressedBuffer = + readCompressedBlockFromFile(block, file, std::nullopt); + co_yield decompressBlock(compressedBuffer, block.numRows_); + // The `decompressLambda` can now run in parallel + if (timer) { + timer->wlock()->checkTimeoutAndThrow(); + }; + } + } + // _____________________________________________________________________________ std::vector CompressedRelationReader::getBlocksForJoin( std::span joinColum, const CompressedRelationMetadata& metadata, @@ -512,8 +608,9 @@ CompressedBlock CompressedRelationReader::readCompressedBlockFromFile( // ____________________________________________________________________________ DecompressedBlock CompressedRelationReader::decompressBlock( - const CompressedBlock& compressedBlock, size_t numRowsToRead) { - DecompressedBlock decompressedBlock{compressedBlock.size()}; + const CompressedBlock& compressedBlock, size_t numRowsToRead) const { + DecompressedBlock decompressedBlock{allocator}; + decompressedBlock.setNumColumns(compressedBlock.size()); decompressedBlock.resize(numRowsToRead); for (size_t i = 0; i < compressedBlock.size(); ++i) { auto col = decompressedBlock.getColumn(i); @@ -551,7 +648,7 @@ void CompressedRelationReader::decompressColumn( // _____________________________________________________________________________ DecompressedBlock CompressedRelationReader::readAndDecompressBlock( const CompressedBlockMetadata& blockMetaData, ad_utility::File& file, - std::optional> columnIndices) { + std::optional> columnIndices) const { CompressedBlock compressedColumns = readCompressedBlockFromFile( blockMetaData, file, std::move(columnIndices)); const auto numRowsToRead = blockMetaData.numRows_; diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h index b05a1a66c2..e02d51d6a2 100644 --- a/src/index/CompressedRelation.h +++ b/src/index/CompressedRelation.h @@ -20,6 +20,7 @@ #include "util/Serializer/Serializer.h" #include "util/Timer.h" #include "util/TypeTraits.h" +#include "util/Generator.h" // Forward declaration of the `IdTable` class. class IdTable; @@ -40,7 +41,7 @@ using SmallRelationsBuffer = columnBasedIdTable::IdTable; // Sometimes we do not read/decompress all the columns of a block, so we have // to use a dynamic `IdTable`. -using DecompressedBlock = columnBasedIdTable::IdTable; +using DecompressedBlock = IdTable; // After compression the columns have different sizes, so we cannot use an // `IdTable`. @@ -233,6 +234,9 @@ class CompressedRelationReader { ad_utility::HeapBasedLRUCache> blockCache_{20ul}; + // TODO This should probably be the allocator of the global qec. + mutable ad_utility::AllocatorWithLimit allocator{ad_utility::makeAllocationMemoryLeftThreadsafeObject(std::numeric_limits::max())}; + public: /** * @brief For a permutation XYZ, retrieve all YZ for a given X. @@ -260,25 +264,32 @@ class CompressedRelationReader { std::span joinColum, const CompressedRelationMetadata& metadata, std::span blockMetadata); - /** - * @brief For a permutation XYZ, retrieve all Z for given X and Y. - * - * @param metaData The metadata of the given X. - * @param col1Id The ID for Y. - * @param blocks The metadata of the on-disk blocks for the given permutation. - * @param file The file in which the permutation is stored. - * @param result The ID table to which we write the result. It must have - * exactly one column. - * @param timer If specified (!= nullptr) a `TimeoutException` will be thrown - * if the timer runs out during the exeuction of this function. - * - * The arguments `metaData`, `blocks`, and `file` must all be obtained from - * The same `CompressedRelationWriter` (see below). - */ - void scan(const CompressedRelationMetadata& metaData, Id col1Id, - const vector& blocks, - ad_utility::File& file, IdTable* result, - ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const; + cppcoro::generator lazyScan(const CompressedRelationMetadata& metadata, +std::span blockMetadata, +ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, +ad_utility::SharedConcurrentTimeoutTimer timer); + + /** + * @brief For a permutation XYZ, retrieve all Z for given X and Y. + * + * @param metaData The metadata of the given X. + * @param col1Id The ID for Y. + * @param blocks The metadata of the on-disk blocks for the given + * permutation. + * @param file The file in which the permutation is stored. + * @param result The ID table to which we write the result. It must have + * exactly one column. + * @param timer If specified (!= nullptr) a `TimeoutException` will be + * thrown if the timer runs out during the exeuction of this function. + * + * The arguments `metaData`, `blocks`, and `file` must all be obtained + * from The same `CompressedRelationWriter` (see below). + */ + void + scan(const CompressedRelationMetadata& metaData, Id col1Id, + const vector& blocks, + ad_utility::File& file, IdTable* result, + ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const; private: // Read the block that is identified by the `blockMetaData` from the `file`. @@ -292,8 +303,8 @@ class CompressedRelationReader { // have after decompression must be passed in via the `numRowsToRead` // argument. It is typically obtained from the corresponding // `CompressedBlockMetaData`. - static DecompressedBlock decompressBlock( - const CompressedBlock& compressedBlock, size_t numRowsToRead); + DecompressedBlock decompressBlock( + const CompressedBlock& compressedBlock, size_t numRowsToRead) const; // Similar to `decompressBlock`, but the block is directly decompressed into // the `table`, starting at the `offsetInTable`-th row. The `table` and the @@ -315,9 +326,9 @@ class CompressedRelationReader { // decompress and return it. // If `columnIndices` is `nullopt`, then all columns of the block are read, // else only the specified columns are read. - static DecompressedBlock readAndDecompressBlock( + DecompressedBlock readAndDecompressBlock( const CompressedBlockMetadata& blockMetaData, ad_utility::File& file, - std::optional> columnIndices); + std::optional> columnIndices) const; }; #endif // QLEVER_COMPRESSEDRELATION_H From 620231abf79bc6c66c636ca9525516a897d7ef00 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Fri, 28 Apr 2023 13:55:29 +0200 Subject: [PATCH 004/150] In the middle of something not yet working. --- src/index/CompressedRelation.cpp | 55 ++++++++++++++++++++++++++++++++ src/index/CompressedRelation.h | 3 ++ 2 files changed, 58 insertions(+) diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index a505d5af66..4006baeccc 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -301,6 +301,61 @@ std::vector CompressedRelationReader::getBlocksForJoin( return result; } + +std::array, 2> CompressedRelationReader::getBlocksForJoin(const CompressedRelationMetadata& md1, const CompressedRelationMetadata& md2, std::span blockMetadata) { + // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ + struct KeyLhs { + Id col0FirstId_; + Id col0LastId_; + }; + Id col0Id = md1.col0Id_; + // TODO Use a structured binding. Structured bindings are + // currently not supported by clang when using OpenMP because clang internally + // transforms the `#pragma`s into lambdas, and capturing structured bindings + // is only supported in clang >= 16. + decltype(blockMetadata.begin()) beginBlock1, endBlock1, beginBlock2, endBlock2; + std::tie(beginBlock1, endBlock1) = std::equal_range( + // TODO For some reason we can't use `std::ranges::equal_range`, + // find out why. Note: possibly it has something to do with the limited + // support of ranges in clang with versions < 16. Revisit this when + // we use clang 16. + blockMetadata.begin(), blockMetadata.end(), KeyLhs{col0Id, col0Id}, + [](const auto& a, const auto& b) { + return a.col0FirstId_ < b.col0FirstId_ && a.col0LastId_ < b.col0LastId_; + }); + col0Id = md2.col0Id_; + std::tie(beginBlock2, endBlock2) = std::equal_range( + // TODO For some reason we can't use `std::ranges::equal_range`, + // find out why. Note: possibly it has something to do with the limited + // support of ranges in clang with versions < 16. Revisit this when + // we use clang 16. + blockMetadata.begin(), blockMetadata.end(), KeyLhs{col0Id, col0Id}, + [](const auto& a, const auto& b) { + return a.col0FirstId_ < b.col0FirstId_ && a.col0LastId_ < b.col0LastId_; + }); + + auto blockLessThanId = [](const CompressedBlockMetadata& block1, const CompressedBlockMetadata& block2) { + if (block1.col0LastId_ < block2.col0FirstId_ || block1.col0FirstId_ > block2.col0LastId_) { + return block1.col0LastId_ < block2.col0LastId_; + } + }; + + auto lessThan = + ad_utility::OverloadCallOperator{idLessThanBlock, blockLessThanId}; + + std::vector result; + auto addRow = [&result]([[maybe_unused]] auto it1, auto it2) { + result.push_back(*it2); + }; + + auto noop = ad_utility::noop; + [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( + joinColum, std::span(beginBlock, endBlock), + lessThan, addRow, noop, noop); + return result; + +} + // _____________________________________________________________________________ void CompressedRelationReader::scan( const CompressedRelationMetadata& metaData, Id col1Id, diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h index e02d51d6a2..bc2c625a59 100644 --- a/src/index/CompressedRelation.h +++ b/src/index/CompressedRelation.h @@ -263,6 +263,9 @@ class CompressedRelationReader { std::vector getBlocksForJoin( std::span joinColum, const CompressedRelationMetadata& metadata, std::span blockMetadata); + std::array, 2> getBlocksForJoin( + const CompressedRelationMetadata& md1, const CompressedRelationMetadata& md2, std::span blockMetadata + ); cppcoro::generator lazyScan(const CompressedRelationMetadata& metadata, std::span blockMetadata, From f2bc0ce833fc9efd9cffdb3dd9db7a48c9c5ddc6 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 4 May 2023 10:18:12 +0200 Subject: [PATCH 005/150] In the middle of everything, first clean up the permutation business. --- src/engine/IndexScan.cpp | 4 ++++ src/engine/IndexScan.h | 2 ++ src/engine/Join.cpp | 7 ++++++- src/engine/Join.h | 2 ++ src/index/Permutations.h | 19 +++++++++++++++++++ 5 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index c970ed2688..a13a078d8a 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -278,3 +278,7 @@ void IndexScan::computeFullScan(IdTable* result, } *result = std::move(table).toDynamic(); } + +std::array, 2> IndexScan::lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2) { + const auto& index = s1.getExecutionContext()->getIndex().getImpl(); +} diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index b77a481964..268653a281 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -153,6 +153,8 @@ class IndexScan : public Operation { // can be read from the Metadata. size_t getExactSize() const { return _sizeEstimate; } + static std::array, 2> lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2); + private: // TODO Make the `getSizeEstimateBeforeLimit()` function `const` for // ALL the `Operations`. diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp index 8dc0fe186a..c6ee1f0bee 100644 --- a/src/engine/Join.cpp +++ b/src/engine/Join.cpp @@ -37,7 +37,7 @@ Join::Join(QueryExecutionContext* qec, std::shared_ptr t1, std::swap(t1, t2); std::swap(t1JoinCol, t2JoinCol); } - if (isFullScanDummy(t1)) { + if (isFullScanDummy(t1) || t1->getType() == QueryExecutionTree::SCAN) { AD_CONTRACT_CHECK(!isFullScanDummy(t2)); std::swap(t1, t2); std::swap(t1JoinCol, t2JoinCol); @@ -607,3 +607,8 @@ void Join::addCombinedRowToIdTable(const ROW_A& rowA, const ROW_B& rowB, (*table)(backIndex, h + rowA.numColumns() - 1) = rowB[h]; } } + +// ______________________________________________________________________________________________________ +ResultTable Join::computeResultForTwoIndexScans() { + AD_CORRECTNESS_CHECK(_left->getType() == QueryExecutionTree::SCAN && _right->getType() == QueryExecutionTree::SCAN); +} diff --git a/src/engine/Join.h b/src/engine/Join.h index 939cedac65..700c72f819 100644 --- a/src/engine/Join.h +++ b/src/engine/Join.h @@ -133,6 +133,8 @@ class Join : public Operation { ResultTable computeResultForJoinWithFullScanDummy(); + ResultTable computeResultForTwoIndexScans(); + using ScanMethodType = std::function; ScanMethodType getScanMethod( diff --git a/src/index/Permutations.h b/src/index/Permutations.h index 9699bedb30..83dd210964 100644 --- a/src/index/Permutations.h +++ b/src/index/Permutations.h @@ -82,6 +82,25 @@ class PermutationImpl { timer); } + struct MetaDataAndBlocks { + const CompressedRelationMetadata& relationMetadata_; + const std::vector& blockMetadata_; + }; + + std::optional getMetadataAndBlocks(Id col0Id) { + if (!_meta.col0IdExists(col0Id)) { + return std::nullopt; + } + return {_meta.getMetaData(col0Id), _meta.blockData()}; + } + + cppcoro::generator lazyScan(Id col0Id, const std::vector& blocks, ad_utility::AllocatorWithLimit allocator, ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) { + if (!_meta.col0IdExists(col0Id)) { + return {}; + } + return _reader.lazyScan(_meta.getMetaData(col0Id), blocks, _file, std::move(allocator), timer); + } + // _______________________________________________________ void setKbName(const string& name) { _meta.setName(name); } From efe72a7c1aeadda1d493cf33072d4f99aa634a71 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 4 May 2023 10:54:17 +0200 Subject: [PATCH 006/150] Changing everything to a single class of MetaData. --- src/index/IndexImpl.cpp | 63 ++++++++++++++++------------------------ src/index/IndexImpl.h | 48 ++++++++++++------------------ src/index/Permutations.h | 19 ++---------- 3 files changed, 46 insertions(+), 84 deletions(-) diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index 3703b85223..59eea5d864 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -171,7 +171,7 @@ void IndexImpl::createFromFile(const string& filename) { auto uniqueSorter = ad_utility::uniqueView(psoSorter.sortedView()); size_t numPredicatesNormal = 0; - createPermutationPair( + createPermutationPair( std::move(uniqueSorter), _PSO, _POS, spoSorter.makePushCallback(), makeNumEntitiesCounter(numPredicatesNormal, 1), countActualTriples); _configurationJson["num-predicates-normal"] = numPredicatesNormal; @@ -192,14 +192,13 @@ void IndexImpl::createFromFile(const string& filename) { patternCreator.processTriple(triple); } }; - createPermutationPair( - spoSorter.sortedView(), _SPO, _SOP, ospSorter.makePushCallback(), - pushTripleToPatterns, numSubjectCounter); + createPermutationPair(spoSorter.sortedView(), _SPO, _SOP, + ospSorter.makePushCallback(), pushTripleToPatterns, + numSubjectCounter); patternCreator.finish(); } else { - createPermutationPair( - spoSorter.sortedView(), _SPO, _SOP, ospSorter.makePushCallback(), - numSubjectCounter); + createPermutationPair(spoSorter.sortedView(), _SPO, _SOP, + ospSorter.makePushCallback(), numSubjectCounter); } spoSorter.clear(); _configurationJson["num-subjects-normal"] = numSubjectsNormal; @@ -208,9 +207,8 @@ void IndexImpl::createFromFile(const string& filename) { // For the last pair of permutations we don't need a next sorter, so we have // no fourth argument. size_t numObjectsNormal = 0; - createPermutationPair( - ospSorter.sortedView(), _OSP, _OPS, - makeNumEntitiesCounter(numObjectsNormal, 2)); + createPermutationPair(ospSorter.sortedView(), _OSP, _OPS, + makeNumEntitiesCounter(numObjectsNormal, 2)); _configurationJson["num-objects-normal"] = numObjectsNormal; _configurationJson["has-all-permutations"] = true; } else { @@ -487,15 +485,13 @@ std::unique_ptr IndexImpl::convertPartialToGlobalIds( } // _____________________________________________________________________________ -template -std::optional> +std::optional> IndexImpl::createPermutationPairImpl(const string& fileName1, const string& fileName2, - SortedTriples&& sortedTriples, size_t c0, - size_t c1, size_t c2, - auto&&... perTripleCallbacks) { - using MetaData = typename MetaDataDispatcher::WriteType; + auto&& sortedTriples, size_t c0, size_t c1, + size_t c2, auto&&... perTripleCallbacks) { + using MetaData = IndexMetaDataMmapDispatcher::WriteType; MetaData metaData1, metaData2; if constexpr (metaData1._isMmapBased) { metaData1.setup(fileName1 + MMAP_FILE_SUFFIX, ad_utility::CreateTag{}); @@ -585,17 +581,12 @@ CompressedRelationMetadata IndexImpl::writeSwitchedRel( } // ________________________________________________________________________ -template -std::optional> -IndexImpl::createPermutations( - auto&& sortedTriples, - const PermutationImpl& - p1, - const PermutationImpl& - p2, - auto&&... perTripleCallbacks) { - auto metaData = createPermutationPairImpl( +std::optional> +IndexImpl::createPermutations(auto&& sortedTriples, const PermutationImpl& p1, + const PermutationImpl& p2, + auto&&... perTripleCallbacks) { + auto metaData = createPermutationPairImpl( _onDiskBase + ".index" + p1._fileSuffix, _onDiskBase + ".index" + p2._fileSuffix, AD_FWD(sortedTriples), p1._keyOrder[0], p1._keyOrder[1], p1._keyOrder[2], @@ -613,16 +604,12 @@ IndexImpl::createPermutations( } // ________________________________________________________________________ -template -void IndexImpl::createPermutationPair( - auto&& sortedTriples, - const PermutationImpl& - p1, - const PermutationImpl& - p2, - auto&&... perTripleCallbacks) { - auto metaData = createPermutations( - AD_FWD(sortedTriples), p1, p2, AD_FWD(perTripleCallbacks)...); +void IndexImpl::createPermutationPair(auto&& sortedTriples, + const PermutationImpl& p1, + const PermutationImpl& p2, + auto&&... perTripleCallbacks) { + auto metaData = createPermutations(AD_FWD(sortedTriples), p1, p2, + AD_FWD(perTripleCallbacks)...); // Set the name of this newly created pair of `IndexMetaData` objects. // NOTE: When `setKbName` was called, it set the name of _PSO._meta, // _PSO._meta, ... which however are not used during index building. diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h index 11f6d8ed8a..b64a74f757 100644 --- a/src/index/IndexImpl.h +++ b/src/index/IndexImpl.h @@ -107,8 +107,7 @@ class IndexImpl { using ReadType = IndexMetaDataHmap; }; - template - using PermutationImpl = Permutation::PermutationImpl; + using PermutationImpl = Permutation::PermutationImpl; using NumNormalAndInternal = Index::NumNormalAndInternal; @@ -169,12 +168,12 @@ class IndexImpl { // TODO: make those private and allow only const access // instantiations for the six permutations used in QLever. // They simplify the creation of permutations in the index class. - Permutation::POS_T _POS{SortByPOS(), "POS", ".pos", {1, 2, 0}}; - Permutation::PSO_T _PSO{SortByPSO(), "PSO", ".pso", {1, 0, 2}}; - Permutation::SOP_T _SOP{SortBySOP(), "SOP", ".sop", {0, 2, 1}}; - Permutation::SPO_T _SPO{SortBySPO(), "SPO", ".spo", {0, 1, 2}}; - Permutation::OPS_T _OPS{SortByOPS(), "OPS", ".ops", {2, 1, 0}}; - Permutation::OSP_T _OSP{SortByOSP(), "OSP", ".osp", {2, 0, 1}}; + PermutationImpl _POS{"POS", ".pos", {1, 2, 0}}; + PermutationImpl _PSO{"PSO", ".pso", {1, 0, 2}}; + PermutationImpl _SOP{"SOP", ".sop", {0, 2, 1}}; + PermutationImpl _SPO{"SPO", ".spo", {0, 1, 2}}; + PermutationImpl _OPS{"OPS", ".ops", {2, 1, 0}}; + PermutationImpl _OSP{"OSP", ".osp", {2, 0, 1}}; public: IndexImpl(); @@ -691,11 +690,10 @@ class IndexImpl { void processWordsForInvertedLists(const string& contextFile, bool addWordsFromLiterals, TextVec& vec); - template - std::optional> + std::optional> createPermutationPairImpl(const string& fileName1, const string& fileName2, - SortedTriples&& sortedTriples, size_t c0, size_t c1, + auto&& sortedTriples, size_t c0, size_t c1, size_t c2, auto&&... perTripleCallbacks); static CompressedRelationMetadata writeSwitchedRel( @@ -712,14 +710,10 @@ class IndexImpl { // the SPO permutation is also needed for patterns (see usage in // IndexImpl::createFromFile function) - template - void createPermutationPair( - auto&& sortedTriples, - const PermutationImpl& - p1, - const PermutationImpl& - p2, - auto&&... perTripleCallbacks); + template <> + void createPermutationPair(auto&& sortedTriples, const PermutationImpl& p1, + const PermutationImpl& p2, + auto&&... perTripleCallbacks); // wrapper for createPermutation that saves a lot of code duplications // Writes the permutation that is specified by argument permutation @@ -730,16 +724,10 @@ class IndexImpl { // Careful: only multiplicities for first column is valid after call, need to // call exchangeMultiplicities as done by createPermutationPair // the optional is std::nullopt if vec and thus the index is empty - template - std::optional> - createPermutations( - auto&& sortedTriples, - const PermutationImpl& - p1, - const PermutationImpl& - p2, - auto&&... perTripleCallbacks); + std::optional> + createPermutations(auto&& sortedTriples, const PermutationImpl& p1, + const PermutationImpl& p2, auto&&... perTripleCallbacks); void createTextIndex(const string& filename, const TextVec& vec); diff --git a/src/index/Permutations.h b/src/index/Permutations.h index 9699bedb30..7c5ced5ad0 100644 --- a/src/index/Permutations.h +++ b/src/index/Permutations.h @@ -19,14 +19,11 @@ using std::string; // Helper class to store static properties of the different permutations to // avoid code duplication. The first template parameter is a search functor for // STXXL. -template class PermutationImpl { public: - using MetaData = MetaDataT; - PermutationImpl(const Comparator& comp, string name, string suffix, - array order) - : _comp(comp), - _readableName(std::move(name)), + using MetaData = IndexMetaDataMmapView; + PermutationImpl(string name, string suffix, array order) + : _readableName(std::move(name)), _fileSuffix(std::move(suffix)), _keyOrder(order) {} @@ -85,8 +82,6 @@ class PermutationImpl { // _______________________________________________________ void setKbName(const string& name) { _meta.setName(name); } - // stxxl comparison functor - const Comparator _comp; // for Log output, e.g. "POS" const std::string _readableName; // e.g. ".pos" @@ -106,12 +101,4 @@ class PermutationImpl { bool _isLoaded = false; }; -// Type aliases for the 6 permutations used by QLever -using POS_T = PermutationImpl; -using PSO_T = PermutationImpl; -using SOP_T = Permutation::PermutationImpl; -using SPO_T = PermutationImpl; -using OPS_T = PermutationImpl; -using OSP_T = PermutationImpl; - } // namespace Permutation From 6ffc83ea280162b03cb091dd263880a10b2333f5 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 4 May 2023 11:06:11 +0200 Subject: [PATCH 007/150] Also delete some additional unused Code. --- src/index/IndexImpl.h | 5 -- src/index/IndexMetaData.h | 3 - src/index/MetaDataHandler.h | 108 ------------------------------------ test/IndexMetaDataTest.cpp | 30 ---------- 4 files changed, 146 deletions(-) diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h index b64a74f757..6c5e4cafcb 100644 --- a/src/index/IndexImpl.h +++ b/src/index/IndexImpl.h @@ -102,11 +102,6 @@ class IndexImpl { using ReadType = IndexMetaDataMmapView; }; - struct IndexMetaDataHmapDispatcher { - using WriteType = IndexMetaDataHmap; - using ReadType = IndexMetaDataHmap; - }; - using PermutationImpl = Permutation::PermutationImpl; using NumNormalAndInternal = Index::NumNormalAndInternal; diff --git a/src/index/IndexMetaData.h b/src/index/IndexMetaData.h index 282fbd9223..4e3ef4b38f 100644 --- a/src/index/IndexMetaData.h +++ b/src/index/IndexMetaData.h @@ -227,9 +227,6 @@ using MetaWrapperMmap = MetaDataWrapperDense>; using MetaWrapperMmapView = MetaDataWrapperDense< ad_utility::MmapVectorView>; -using MetaWrapperHashMap = - MetaDataWrapperHashMap>; -using IndexMetaDataHmap = IndexMetaData; using IndexMetaDataMmap = IndexMetaData; using IndexMetaDataMmapView = IndexMetaData; diff --git a/src/index/MetaDataHandler.h b/src/index/MetaDataHandler.h index 9c6cb784ae..da84f1158a 100644 --- a/src/index/MetaDataHandler.h +++ b/src/index/MetaDataHandler.h @@ -118,111 +118,3 @@ class MetaDataWrapperDense { } M _vec; }; - -// _____________________________________________________________________ -template -class MetaDataWrapperHashMap { - public: - template - struct AddGetIdIterator : public BaseIterator { - using BaseIterator::BaseIterator; - AddGetIdIterator(BaseIterator base) : BaseIterator{base} {} - [[nodiscard]] Id getId() const { return (*this)->second.col0Id_; } - static Id getIdFromElement(const typename BaseIterator::value_type& v) { - return v.second.col0Id_; - } - static auto getNumRowsFromElement( - const typename BaseIterator::value_type& v) { - return v.second.numRows_; - } - }; - using Iterator = AddGetIdIterator; - using ConstIterator = AddGetIdIterator; - - using value_type = typename hashMap::mapped_type; - - // An iterator on the underlying hashMap that iterates over the elements - // in order. This is used for deterministically exporting the underlying - // permutation. - static inline auto getSortedKey = [](const auto& wrapper, - uint64_t i) -> decltype(auto) { - const auto& m = wrapper.getUnderlyingHashMap(); - return *m.find(wrapper.sortedKeys()[i]); - }; - - using ConstOrderedIteratorBase = - ad_utility::IteratorForAccessOperator; - using ConstOrderedIterator = AddGetIdIterator; - - // nothing to do here, since the default constructor of the hashMap does - // everything we want - explicit MetaDataWrapperHashMap() = default; - // nothing to setup, but has to be defined to meet template requirements - void setup(){}; - - // _______________________________________________________________ - size_t size() const { return _map.size(); } - - // __________________________________________________________________ - ConstIterator cbegin() const { return _map.begin(); } - ConstIterator begin() const { return _map.begin(); } - - // _________________________________________________________________________ - ConstOrderedIterator ordered_begin() const { - return ConstOrderedIterator{this, 0}; - } - - // ____________________________________________________________ - ConstIterator cend() const { return _map.end(); } - ConstIterator end() const { return _map.end(); } - - // _________________________________________________________________________ - ConstOrderedIterator ordered_end() const { - AD_CONTRACT_CHECK(size() == _sortedKeys.size()); - return ConstOrderedIterator{this, size()}; - } - - // ____________________________________________________________ - void set(Id id, value_type value) { - _map[id] = std::move(value); - if (!_sortedKeys.empty()) { - AD_CONTRACT_CHECK(id > _sortedKeys.back()); - } - _sortedKeys.push_back(id); - } - - const auto& getUnderlyingHashMap() const { return _map; } - - // __________________________________________________________ - const value_type& getAsserted(Id id) const { - auto it = _map.find(id); - AD_CONTRACT_CHECK(it != _map.end()); - return std::cref(it->second); - } - - // ________________________________________________________ - size_t count(Id id) const { - // can either be 1 or 0 for map-like types - return _map.count(id); - } - - AD_SERIALIZE_FRIEND_FUNCTION(MetaDataWrapperHashMap) { - serializer | arg._map; - if constexpr (ad_utility::serialization::ReadSerializer) { - arg._sortedKeys.clear(); - arg._sortedKeys.reserve(arg.size()); - for (const auto& [key, value] : arg) { - (void)value; // Silence the warning about `value` being unused. - arg._sortedKeys.push_back(key); - } - std::sort(arg._sortedKeys.begin(), arg._sortedKeys.end()); - } - } - - const auto& sortedKeys() const { return _sortedKeys; } - - private: - hashMap _map; - std::vector _sortedKeys; -}; diff --git a/test/IndexMetaDataTest.cpp b/test/IndexMetaDataTest.cpp index f1af1e04e3..72d9fe8a6f 100644 --- a/test/IndexMetaDataTest.cpp +++ b/test/IndexMetaDataTest.cpp @@ -36,36 +36,6 @@ TEST(RelationMetaDataTest, writeReadTest) { ASSERT_EQ(rmdB, rmdB2); } -TEST(IndexMetaDataTest, writeReadTest2Hmap) { - vector bs; - bs.push_back(CompressedBlockMetadata{ - {{12, 34}, {42, 5}}, 5, V(0), V(2), V(13), V(24), V(62)}); - bs.push_back(CompressedBlockMetadata{ - {{16, 34}, {165, 3}}, 5, V(0), V(2), V(13), V(24), V(62)}); - CompressedRelationMetadata rmdF{V(1), 3, 2.0, 42.0, 16}; - CompressedRelationMetadata rmdF2{V(2), 5, 3.0, 43.0, 10}; - IndexMetaDataHmap imd; - imd.add(rmdF); - imd.add(rmdF2); - imd.blockData() = bs; - - const string filename = "_testtmp.imd"; - imd.writeToFile(filename); - - ad_utility::File in("_testtmp.imd", "r"); - IndexMetaDataHmap imd2; - imd2.readFromFile(&in); - remove("_testtmp.rmd"); - - auto rmdFn = imd2.getMetaData(V(1)); - auto rmdFn2 = imd2.getMetaData(V(2)); - - ASSERT_EQ(rmdF, rmdFn); - ASSERT_EQ(rmdF2, rmdFn2); - - ASSERT_EQ(imd2.blockData(), bs); -} - TEST(IndexMetaDataTest, writeReadTest2Mmap) { std::string imdFilename = "_testtmp.imd"; std::string mmapFilename = imdFilename + ".mmap"; From 975c59a1b2c32b11f0d23a13ace83a6ed131a8c5 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 4 May 2023 12:00:31 +0200 Subject: [PATCH 008/150] Completely removed the permutation-templating out of the code. TODO: Many functions can now be moved into the IndexImpl.cpp --- src/engine/GroupBy.cpp | 91 +++++++++++++++++------------------- src/engine/IndexScan.cpp | 10 ++-- src/index/Index.cpp | 30 ++++-------- src/index/IndexImpl.h | 99 ++++++++++++++++------------------------ test/IndexTest.cpp | 56 ++++++++++++----------- 5 files changed, 125 insertions(+), 161 deletions(-) diff --git a/src/engine/GroupBy.cpp b/src/engine/GroupBy.cpp index 3bfb605317..c52360056b 100644 --- a/src/engine/GroupBy.cpp +++ b/src/engine/GroupBy.cpp @@ -473,55 +473,50 @@ bool GroupBy::computeGroupByForFullIndexScan(IdTable* result) { IdTable* idTable) mutable { auto ignoredRanges = getIndex().getImpl().getIgnoredIdRanges(permutation.value()).first; - // The permutations in the `Index` class have different types, so we - // also have to write a generic lambda here that will be passed to - // `IndexImpl::applyToPermutation`. - auto applyForPermutation = [&](const auto& permutation) mutable { - IdTableStatic table = std::move(*idTable).toStatic(); - const auto& metaData = permutation._meta.data(); - // TODO the reserve is too large because of the ignored - // triples. We would need to incorporate the information how many - // added "relations" are in each permutation during index building. - table.reserve(metaData.size()); - for (auto it = metaData.ordered_begin(); it != metaData.ordered_end(); - ++it) { - Id id = decltype(metaData.ordered_begin())::getIdFromElement(*it); - - // Check whether this is an `@en@...` predicate in a `Pxx` - // permutation, a literal in a `Sxx` permutation or some other - // entity that was added only for internal reasons. - if (std::ranges::any_of(ignoredRanges, [&id](const auto& pair) { - return id >= pair.first && id < pair.second; - })) { - continue; - } - Id count = Id::makeFromInt( - decltype(metaData.ordered_begin())::getNumRowsFromElement(*it)); - // TODO The count is actually not accurate at least for the - // `Sxx` and `Oxx` permutations because it contains the triples with - // predicate - // `@en@rdfs:label` etc. The probably easiest way to fix this is to - // exclude these triples from those permutations (they are only - // relevant for queries with a fixed subject), but then we would - // need to make sure, that we don't accidentally break the language - // filters for queries like - // ` @en@rdfs:label ?labels`, for which the best - // query plan potentially goes through the `SPO` relation. - // Alternatively we would have to write an additional number - // `numNonAddedTriples` to the `IndexMetaData` which would further - // increase their size. - // TODO Discuss this with Hannah. - table.emplace_back(); - table(table.size() - 1, 0) = id; - if (numCounts == 1) { - table(table.size() - 1, 1) = count; - } + const auto& permutationImpl = + getExecutionContext()->getIndex().getPimpl().getPermutation( + permutation.value()); + IdTableStatic table = std::move(*idTable).toStatic(); + const auto& metaData = permutationImpl._meta.data(); + // TODO the reserve is too large because of the ignored + // triples. We would need to incorporate the information how many + // added "relations" are in each permutation during index building. + table.reserve(metaData.size()); + for (auto it = metaData.ordered_begin(); it != metaData.ordered_end(); + ++it) { + Id id = decltype(metaData.ordered_begin())::getIdFromElement(*it); + + // Check whether this is an `@en@...` predicate in a `Pxx` + // permutation, a literal in a `Sxx` permutation or some other + // entity that was added only for internal reasons. + if (std::ranges::any_of(ignoredRanges, [&id](const auto& pair) { + return id >= pair.first && id < pair.second; + })) { + continue; } - *idTable = std::move(table).toDynamic(); - }; - - getExecutionContext()->getIndex().getPimpl().applyToPermutation( - permutation.value(), applyForPermutation); + Id count = Id::makeFromInt( + decltype(metaData.ordered_begin())::getNumRowsFromElement(*it)); + // TODO The count is actually not accurate at least for the + // `Sxx` and `Oxx` permutations because it contains the triples with + // predicate + // `@en@rdfs:label` etc. The probably easiest way to fix this is to + // exclude these triples from those permutations (they are only + // relevant for queries with a fixed subject), but then we would + // need to make sure, that we don't accidentally break the language + // filters for queries like + // ` @en@rdfs:label ?labels`, for which the best + // query plan potentially goes through the `SPO` relation. + // Alternatively we would have to write an additional number + // `numNonAddedTriples` to the `IndexMetaData` which would further + // increase their size. + // TODO Discuss this with Hannah. + table.emplace_back(); + table(table.size() - 1, 0) = id; + if (numCounts == 1) { + table(table.size() - 1, 1) = count; + } + } + *idTable = std::move(table).toDynamic(); }; ad_utility::callFixedSize(numCols, doComputationForNumberOfColumns, result); diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 04a5b1940a..e7e1316714 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -511,13 +511,11 @@ void IndexScan::computeFullScan(IdTable* result, result->reserve(resultSize); auto table = std::move(*result).toStatic<3>(); size_t i = 0; + const auto& permutationImpl = + getExecutionContext()->getIndex().getImpl().getPermutation(permutation); auto triplesView = - getExecutionContext()->getIndex().getImpl().applyToPermutation( - permutation, [&, ignoredRanges = ignoredRanges, - isTripleIgnored = isTripleIgnored](const auto& p) { - return TriplesView(p, getExecutionContext()->getAllocator(), - ignoredRanges, isTripleIgnored); - }); + TriplesView(permutationImpl, getExecutionContext()->getAllocator(), + ignoredRanges, isTripleIgnored); for (const auto& triple : triplesView) { if (i >= resultSize) { break; diff --git a/src/index/Index.cpp b/src/index/Index.cpp index 82c2ab5a0c..d413e54762 100644 --- a/src/index/Index.cpp +++ b/src/index/Index.cpp @@ -70,16 +70,12 @@ auto Index::getTextVocab() const -> const TextVocab& { // _____________________________________________________________________________ size_t Index::getCardinality(const TripleComponent& comp, Index::Permutation p) const { - return pimpl_->applyToPermutation(p, [&](const auto& permutation) { - return pimpl_->getCardinality(comp, permutation); - }); + return pimpl_->getCardinality(comp, p); } // _____________________________________________________________________________ size_t Index::getCardinality(Id id, Index::Permutation p) const { - return pimpl_->applyToPermutation(p, [&](const auto& permutation) { - return pimpl_->getCardinality(id, permutation); - }); + return pimpl_->getCardinality(id, p); } // _______________________________________________ @@ -345,34 +341,26 @@ bool Index::hasAllPermutations() const { return pimpl_->hasAllPermutations(); } // _____________________________________________________ vector Index::getMultiplicities(Permutation p) const { - return pimpl_->applyToPermutation(p, [this](const auto& permutation) { - return pimpl_->getMultiplicities(permutation); - }); + return pimpl_->getMultiplicities(p); } // _____________________________________________________ vector Index::getMultiplicities(const TripleComponent& key, - Permutation permutation) const { - return pimpl_->applyToPermutation(permutation, [this, key](const auto& p) { - return pimpl_->getMultiplicities(key, p); - }); + Permutation p) const { + return pimpl_->getMultiplicities(key, p); } // _____________________________________________________ void Index::scan(Id key, IdTable* result, Permutation p, ad_utility::SharedConcurrentTimeoutTimer timer) const { - return pimpl_->applyToPermutation(p, [&](const auto& perm) { - return pimpl_->scan(key, result, perm, std::move(timer)); - }); + return pimpl_->scan(key, result, p, std::move(timer)); } // _____________________________________________________ void Index::scan(const TripleComponent& key, IdTable* result, const Permutation& p, ad_utility::SharedConcurrentTimeoutTimer timer) const { - return pimpl_->applyToPermutation(p, [&](const auto& perm) { - return pimpl_->scan(key, result, perm, std::move(timer)); - }); + return pimpl_->scan(key, result, p, std::move(timer)); } // _____________________________________________________ @@ -380,7 +368,5 @@ void Index::scan(const TripleComponent& col0String, const TripleComponent& col1String, IdTable* result, Permutation p, ad_utility::SharedConcurrentTimeoutTimer timer) const { - return pimpl_->applyToPermutation(p, [&](const auto& perm) { - return pimpl_->scan(col0String, col1String, result, perm, std::move(timer)); - }); + return pimpl_->scan(col0String, col1String, result, p, std::move(timer)); } diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h index 6c5e4cafcb..28cbf0d4aa 100644 --- a/src/index/IndexImpl.h +++ b/src/index/IndexImpl.h @@ -194,6 +194,28 @@ class IndexImpl { const auto& OSP() const { return _OSP; } auto& OSP() { return _OSP; } + PermutationImpl& getPermutation(Index::Permutation p) { + using enum Index::Permutation; + switch (p) { + case PSO: + return _PSO; + case POS: + return _POS; + case SPO: + return _SPO; + case SOP: + return _SOP; + case OSP: + return _OSP; + case OPS: + return _OPS; + } + } + + const PermutationImpl& getPermutation(Index::Permutation p) const { + return const_cast(*this).getPermutation(p); + } + // Creates an index from a file. Parameter Parser must be able to split the // file's format into triples. // Will write vocabulary and on-disk index data. @@ -288,18 +310,18 @@ class IndexImpl { } } - template - size_t getCardinality(Id id, const Permutation& permutation) const { - if (permutation.metaData().col0IdExists(id)) { - return permutation.metaData().getMetaData(id).getNofElements(); + // ___________________________________________________________________________ + size_t getCardinality(Id id, Index::Permutation permutation) const { + const auto& p = getPermutation(permutation); + if (p.metaData().col0IdExists(id)) { + return p.metaData().getMetaData(id).getNofElements(); } return 0; } // ___________________________________________________________________________ - template size_t getCardinality(const TripleComponent& comp, - const Permutation& permutation) const { + Index::Permutation permutation) const { // TODO This special case is only relevant for the `PSO` and `POS` // permutations, but this internal predicate should never appear in subjects // or objects anyway. @@ -488,9 +510,9 @@ class IndexImpl { bool hasAllPermutations() const { return SPO()._isLoaded; } // _____________________________________________________________________________ - template vector getMultiplicities(const TripleComponent& key, - const PermutationImpl& p) const { + Index::Permutation permutation) const { + const auto& p = getPermutation(permutation); std::optional keyId = key.toValueId(getVocab()); vector res; if (keyId.has_value() && p._meta.col0IdExists(keyId.value())) { @@ -505,8 +527,8 @@ class IndexImpl { } // ___________________________________________________________________ - template - vector getMultiplicities(const PermutationImpl& p) const { + vector getMultiplicities(Index::Permutation permutation) const { + const auto& p = getPermutation(permutation); auto numTriples = static_cast(this->numTriples().normalAndInternal_()); std::array m{ @@ -527,10 +549,9 @@ class IndexImpl { * @param p The Permutation to use (in particularly POS(), SOP,... members of * IndexImpl class). */ - template - void scan(Id key, IdTable* result, const Permutation& p, + void scan(Id key, IdTable* result, const Index::Permutation& p, ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const { - p.scan(key, result, std::move(timer)); + getPermutation(p).scan(key, result, std::move(timer)); } /** @@ -544,15 +565,16 @@ class IndexImpl { * @param p The Permutation to use (in particularly POS(), SOP,... members of * IndexImpl class). */ - template - void scan(const TripleComponent& key, IdTable* result, const Permutation& p, + void scan(const TripleComponent& key, IdTable* result, + Index::Permutation permutation, ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const { + const auto& p = getPermutation(permutation); LOG(DEBUG) << "Performing " << p._readableName << " scan for full list for: " << key << "\n"; std::optional optionalId = key.toValueId(getVocab()); if (optionalId.has_value()) { LOG(TRACE) << "Successfully got key ID.\n"; - scan(optionalId.value(), result, p, std::move(timer)); + scan(optionalId.value(), result, permutation, std::move(timer)); } LOG(DEBUG) << "Scan done, got " << result->size() << " elements.\n"; } @@ -560,8 +582,6 @@ class IndexImpl { /** * @brief Perform a scan for two keys i.e. retrieve all Z from the XYZ * permutation for specific key values of X and Y. - * @tparam Permutation The permutations IndexImpl::POS()... have different - * types * @param col0String The first key (as a raw string that is yet to be * transformed to index space) for which to search, e.g. fixed value for O in * OSP permutation. @@ -573,13 +593,13 @@ class IndexImpl { * IndexImpl class). */ // _____________________________________________________________________________ - template void scan(const TripleComponent& col0String, const TripleComponent& col1String, IdTable* result, - const PermutationInfo& p, + const Index::Permutation& permutation, ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const { std::optional col0Id = col0String.toValueId(getVocab()); std::optional col1Id = col1String.toValueId(getVocab()); + const auto& p = getPermutation(permutation); if (!col0Id.has_value() || !col1Id.has_value()) { LOG(DEBUG) << "Key " << col0String << " or key " << col1String << " were not found in the vocabulary \n"; @@ -593,44 +613,6 @@ class IndexImpl { p.scan(col0Id.value(), col1Id.value(), result, timer); } - // Internal implementation for `applyToPermutation` (see below). - // TODO : Use "deducing this" - private: - static decltype(auto) applyToPermutationImpl(auto&& self, - Index::Permutation permutation, - auto&& F) { - using enum Index::Permutation; - switch (permutation) { - case POS: - return AD_FWD(F)(self._POS); - case PSO: - return AD_FWD(F)(self._PSO); - case SPO: - return AD_FWD(F)(self._SPO); - case SOP: - return AD_FWD(F)(self._SOP); - case OSP: - return AD_FWD(F)(self._OSP); - case OPS: - return AD_FWD(F)(self._OPS); - default: - AD_FAIL(); - } - } - - public: - // Apply the function `F` to the permutation that corresponds to the - // `permutation` argument. - decltype(auto) applyToPermutation(Index::Permutation permutation, auto&& F) { - return applyToPermutationImpl(*this, permutation, AD_FWD(F)); - } - - // TODO reduce code duplication here using `deducing this`. - decltype(auto) applyToPermutation(Index::Permutation permutation, - auto&& F) const { - return applyToPermutationImpl(*this, permutation, AD_FWD(F)); - } - private: // Private member functions @@ -705,7 +687,6 @@ class IndexImpl { // the SPO permutation is also needed for patterns (see usage in // IndexImpl::createFromFile function) - template <> void createPermutationPair(auto&& sortedTriples, const PermutationImpl& p1, const PermutationImpl& p2, auto&&... perTripleCallbacks); diff --git a/test/IndexTest.cpp b/test/IndexTest.cpp index 97fe4b4d3d..37682282e4 100644 --- a/test/IndexTest.cpp +++ b/test/IndexTest.cpp @@ -27,7 +27,10 @@ auto lit = ad_utility::testing::tripleComponentLiteral; // scan matches `expected`. auto makeTestScanWidthOne = [](const IndexImpl& index) { return [&index](const std::string& c0, const std::string& c1, - const auto& permutation, const VectorTable& expected) { + Index::Permutation permutation, const VectorTable& expected, + ad_utility::source_location l = + ad_utility::source_location::current()) { + auto t = generateLocationTrace(l); IdTable result(1, makeAllocator()); index.scan(c0, c1, &result, permutation); ASSERT_EQ(result, makeIdTableFromVector(expected)); @@ -39,7 +42,7 @@ auto makeTestScanWidthOne = [](const IndexImpl& index) { // of the `index` and checks whether the result of the // scan matches `expected`. auto makeTestScanWidthTwo = [](const IndexImpl& index) { - return [&index](const std::string& c0, const auto& permutation, + return [&index](const std::string& c0, Index::Permutation permutation, const VectorTable& expected, ad_utility::source_location l = ad_utility::source_location::current()) { @@ -88,11 +91,11 @@ TEST(IndexTest, createFromTurtleTest) { // Relation b // Pair index auto testTwo = makeTestScanWidthTwo(index); - testTwo("", index.PSO(), {{a, c}, {a, c2}}); + testTwo("", Index::Permutation::PSO, {{a, c}, {a, c2}}); std::vector> buffer; // Relation b2 - testTwo("", index.PSO(), {{a, c}, {a2, c2}}); + testTwo("", Index::Permutation::PSO, {{a, c}, {a2, c2}}); { // Test for a previous bug in the scan of two fixed elements: An assertion @@ -102,7 +105,7 @@ TEST(IndexTest, createFromTurtleTest) { // predicate that occurs and is larger than the largest subject that // appears with . auto testOne = makeTestScanWidthOne(index); - testOne("", "", index.PSO(), {}); + testOne("", "", Index::Permutation::PSO, {}); } } { @@ -136,7 +139,7 @@ TEST(IndexTest, createFromTurtleTest) { ASSERT_FALSE(index.POS().metaData().getMetaData(isA).isFunctional()); auto testTwo = makeTestScanWidthTwo(index); - testTwo("", index.PSO(), + testTwo("", Index::Permutation::PSO, {{a, zero}, {a, one}, {a, two}, @@ -146,7 +149,7 @@ TEST(IndexTest, createFromTurtleTest) { {c, two}}); // is-a for POS - testTwo("", index.POS(), + testTwo("", Index::Permutation::POS, {{zero, a}, {zero, b}, {one, a}, @@ -231,6 +234,7 @@ TEST(IndexTest, createFromOnDiskIndexTest) { }; TEST(IndexTest, scanTest) { + using enum Index::Permutation; std::string kb = " . \n" " . \n" @@ -249,19 +253,19 @@ TEST(IndexTest, scanTest) { Id c2 = getId(""); auto testTwo = makeTestScanWidthTwo(index); - testTwo("", index._PSO, {{a, c}, {a, c2}}); - testTwo("", index._PSO, {}); - testTwo("", index._PSO, {}); - testTwo("", index._POS, {{c, a}, {c2, a}}); - testTwo("", index._POS, {}); - testTwo("", index._POS, {}); + testTwo("", PSO, {{a, c}, {a, c2}}); + testTwo("", PSO, {}); + testTwo("", PSO, {}); + testTwo("", POS, {{c, a}, {c2, a}}); + testTwo("", POS, {}); + testTwo("", POS, {}); auto testOne = makeTestScanWidthOne(index); - testOne("", "", index._PSO, {{c}, {c2}}); - testOne("", "", index._PSO, {}); - testOne("", "", index._POS, {{a2}}); - testOne("", "", index._PSO, {}); + testOne("", "", PSO, {{c}, {c2}}); + testOne("", "", PSO, {}); + testOne("", "", POS, {{a2}}); + testOne("", "", PSO, {}); } kb = " <1> . \n" " <2> . \n" @@ -285,7 +289,7 @@ TEST(IndexTest, scanTest) { Id three = getId("<3>"); auto testTwo = makeTestScanWidthTwo(index); - testTwo("", index._PSO, + testTwo("", PSO, {{{a, zero}, {a, one}, {a, two}, @@ -293,7 +297,7 @@ TEST(IndexTest, scanTest) { {b, three}, {c, one}, {c, two}}}); - testTwo("", index._POS, + testTwo("", POS, {{zero, a}, {zero, b}, {one, a}, @@ -304,13 +308,13 @@ TEST(IndexTest, scanTest) { auto testWidthOne = makeTestScanWidthOne(index); - testWidthOne("", "<0>", index._POS, {{a}, {b}}); - testWidthOne("", "<1>", index._POS, {{a}, {c}}); - testWidthOne("", "<2>", index._POS, {{a}, {c}}); - testWidthOne("", "<3>", index._POS, {{b}}); - testWidthOne("", "", index._PSO, {{zero}, {one}, {two}}); - testWidthOne("", "", index._PSO, {{zero}, {three}}); - testWidthOne("", "", index._PSO, {{one}, {two}}); + testWidthOne("", "<0>", POS, {{a}, {b}}); + testWidthOne("", "<1>", POS, {{a}, {c}}); + testWidthOne("", "<2>", POS, {{a}, {c}}); + testWidthOne("", "<3>", POS, {{b}}); + testWidthOne("", "", PSO, {{zero}, {one}, {two}}); + testWidthOne("", "", PSO, {{zero}, {three}}); + testWidthOne("", "", PSO, {{one}, {two}}); } }; From 31883bcaa2ba435b6cb2f7c71df5050a2f29383e Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 4 May 2023 12:32:05 +0200 Subject: [PATCH 009/150] Moved many functions from IndexImpl.h to IndexImpl.cpp --- src/index/IndexImpl.cpp | 226 +++++++++++++++++++++++++++++++++++++++- src/index/IndexImpl.h | 219 ++++---------------------------------- 2 files changed, 246 insertions(+), 199 deletions(-) diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index 59eea5d864..bed420041c 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -493,7 +493,7 @@ IndexImpl::createPermutationPairImpl(const string& fileName1, size_t c2, auto&&... perTripleCallbacks) { using MetaData = IndexMetaDataMmapDispatcher::WriteType; MetaData metaData1, metaData2; - if constexpr (metaData1._isMmapBased) { + if constexpr (MetaData::_isMmapBased) { metaData1.setup(fileName1 + MMAP_FILE_SUFFIX, ad_utility::CreateTag{}); metaData2.setup(fileName2 + MMAP_FILE_SUFFIX, ad_utility::CreateTag{}); } @@ -1134,3 +1134,227 @@ std::future IndexImpl::writeNextPartialVocabulary( IndexImpl::NumNormalAndInternal IndexImpl::numTriples() const { return {_numTriplesNormal, PSO()._meta.getNofTriples() - _numTriplesNormal}; } + +// ____________________________________________________________________________ +IndexImpl::PermutationImpl& IndexImpl::getPermutation(Index::Permutation p) { + using enum Index::Permutation; + switch (p) { + case PSO: + return _PSO; + case POS: + return _POS; + case SPO: + return _SPO; + case SOP: + return _SOP; + case OSP: + return _OSP; + case OPS: + return _OPS; + } + AD_FAIL(); +} + +// ____________________________________________________________________________ +const IndexImpl::PermutationImpl& IndexImpl::getPermutation( + Index::Permutation p) const { + return const_cast(*this).getPermutation(p); +} + +// __________________________________________________________________________ +Index::NumNormalAndInternal IndexImpl::numDistinctSubjects() const { + if (hasAllPermutations()) { + auto numActually = _numSubjectsNormal; + return {numActually, _SPO.metaData().getNofDistinctC1() - numActually}; + } else { + AD_THROW( + "Can only get # distinct subjects if all 6 permutations " + "have been registered on sever start (and index build time) " + "with the -a option."); + } +} + +// __________________________________________________________________________ +Index::NumNormalAndInternal IndexImpl::numDistinctObjects() const { + if (hasAllPermutations()) { + auto numActually = _numObjectsNormal; + return {numActually, _OSP.metaData().getNofDistinctC1() - numActually}; + } else { + AD_THROW( + "Can only get # distinct objects if all 6 permutations " + "have been registered on sever start (and index build time) " + "with the -a option."); + } +} + +// __________________________________________________________________________ +Index::NumNormalAndInternal IndexImpl::numDistinctPredicates() const { + auto numActually = _numPredicatesNormal; + return {numActually, _PSO.metaData().getNofDistinctC1() - numActually}; +} + +// __________________________________________________________________________ +Index::NumNormalAndInternal IndexImpl::numDistinctCol0( + Index::Permutation permutation) const { + switch (permutation) { + case Index::Permutation::SOP: + case Index::Permutation::SPO: + return numDistinctSubjects(); + case Index::Permutation::OPS: + case Index::Permutation::OSP: + return numDistinctObjects(); + case Index::Permutation::POS: + case Index::Permutation::PSO: + return numDistinctPredicates(); + default: + AD_FAIL(); + } +} + +// ___________________________________________________________________________ +size_t IndexImpl::getCardinality(Id id, Index::Permutation permutation) const { + const auto& p = getPermutation(permutation); + if (p.metaData().col0IdExists(id)) { + return p.metaData().getMetaData(id).getNofElements(); + } + return 0; +} + +// ___________________________________________________________________________ +size_t IndexImpl::getCardinality(const TripleComponent& comp, + Index::Permutation permutation) const { + // TODO This special case is only relevant for the `PSO` and `POS` + // permutations, but this internal predicate should never appear in subjects + // or objects anyway. + // TODO Find out what the effect of this special case is for the + // query planning. + if (comp == INTERNAL_TEXT_MATCH_PREDICATE) { + return TEXT_PREDICATE_CARDINALITY_ESTIMATE; + } + std::optional relId = comp.toValueId(getVocab()); + if (relId.has_value()) { + return getCardinality(relId.value(), permutation); + } + return 0; +} + +// TODO Once we have an overview over the folding this logic should +// probably not be in the index class. +std::optional IndexImpl::idToOptionalString(Id id) const { + switch (id.getDatatype()) { + case Datatype::Undefined: + return std::nullopt; + case Datatype::Double: + return std::to_string(id.getDouble()); + case Datatype::Int: + return std::to_string(id.getInt()); + case Datatype::VocabIndex: { + auto result = _vocab.indexToOptionalString(id.getVocabIndex()); + if (result.has_value() && result.value().starts_with(VALUE_PREFIX)) { + result = ad_utility::convertIndexWordToValueLiteral(result.value()); + } + return result; + } + case Datatype::LocalVocabIndex: + // TODO:: this is why this shouldn't be here + return std::nullopt; + case Datatype::TextRecordIndex: + return getTextExcerpt(id.getTextRecordIndex()); + } + // should be unreachable because the enum is exhaustive. + AD_FAIL(); +} + +// ___________________________________________________________________________ +bool IndexImpl::getId(const string& element, Id* id) const { + // TODO we should parse doubles correctly in the SparqlParser and + // then return the correct ids here or somewhere else. + VocabIndex vocabId; + auto success = getVocab().getId(element, &vocabId); + *id = Id::makeFromVocabIndex(vocabId); + return success; +} + +// ___________________________________________________________________________ +std::pair IndexImpl::prefix_range(const std::string& prefix) const { + // TODO Do we need prefix ranges for numbers? + auto [begin, end] = _vocab.prefix_range(prefix); + return {Id::makeFromVocabIndex(begin), Id::makeFromVocabIndex(end)}; +} + +// _____________________________________________________________________________ +vector IndexImpl::getMultiplicities( + const TripleComponent& key, Index::Permutation permutation) const { + const auto& p = getPermutation(permutation); + std::optional keyId = key.toValueId(getVocab()); + vector res; + if (keyId.has_value() && p._meta.col0IdExists(keyId.value())) { + auto metaData = p._meta.getMetaData(keyId.value()); + res.push_back(metaData.getCol1Multiplicity()); + res.push_back(metaData.getCol2Multiplicity()); + } else { + res.push_back(1); + res.push_back(1); + } + return res; +} + +// ___________________________________________________________________ +vector IndexImpl::getMultiplicities( + Index::Permutation permutation) const { + const auto& p = getPermutation(permutation); + auto numTriples = static_cast(this->numTriples().normalAndInternal_()); + std::array m{ + numTriples / numDistinctSubjects().normalAndInternal_(), + numTriples / numDistinctPredicates().normalAndInternal_(), + numTriples / numDistinctObjects().normalAndInternal_()}; + return {m[p._keyOrder[0]], m[p._keyOrder[1]], m[p._keyOrder[2]]}; +} + +// ___________________________________________________________________ +void IndexImpl::scan(Id key, IdTable* result, const Index::Permutation& p, + ad_utility::SharedConcurrentTimeoutTimer timer) const { + getPermutation(p).scan(key, result, std::move(timer)); +} + +// ___________________________________________________________________ +void IndexImpl::scan(const TripleComponent& key, IdTable* result, + Index::Permutation permutation, + ad_utility::SharedConcurrentTimeoutTimer timer) const { + const auto& p = getPermutation(permutation); + LOG(DEBUG) << "Performing " << p._readableName + << " scan for full list for: " << key << "\n"; + std::optional optionalId = key.toValueId(getVocab()); + if (optionalId.has_value()) { + LOG(TRACE) << "Successfully got key ID.\n"; + scan(optionalId.value(), result, permutation, std::move(timer)); + } + LOG(DEBUG) << "Scan done, got " << result->size() << " elements.\n"; +} + +// _____________________________________________________________________________ +void IndexImpl::scan(const TripleComponent& col0String, + const TripleComponent& col1String, IdTable* result, + const Index::Permutation& permutation, + ad_utility::SharedConcurrentTimeoutTimer timer) const { + std::optional col0Id = col0String.toValueId(getVocab()); + std::optional col1Id = col1String.toValueId(getVocab()); + const auto& p = getPermutation(permutation); + if (!col0Id.has_value() || !col1Id.has_value()) { + LOG(DEBUG) << "Key " << col0String << " or key " << col1String + << " were not found in the vocabulary \n"; + return; + } + + LOG(DEBUG) << "Performing " << p._readableName << " scan of relation " + << col0String << " with fixed subject: " << col1String << "...\n"; + + p.scan(col0Id.value(), col1Id.value(), result, timer); +} + +// _____________________________________________________________________________ +void IndexImpl::deleteTemporaryFile(const string& path) { + if (!_keepTempFiles) { + ad_utility::deleteFile(path); + } +} diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h index 28cbf0d4aa..3a6457a1fc 100644 --- a/src/index/IndexImpl.h +++ b/src/index/IndexImpl.h @@ -194,27 +194,10 @@ class IndexImpl { const auto& OSP() const { return _OSP; } auto& OSP() { return _OSP; } - PermutationImpl& getPermutation(Index::Permutation p) { - using enum Index::Permutation; - switch (p) { - case PSO: - return _PSO; - case POS: - return _POS; - case SPO: - return _SPO; - case SOP: - return _SOP; - case OSP: - return _OSP; - case OPS: - return _OPS; - } - } - - const PermutationImpl& getPermutation(Index::Permutation p) const { - return const_cast(*this).getPermutation(p); - } + // For a given `Permutation` (e.g. `PSO`) return the corresponding + // `PermutationImpl` object by reference (`_PSO`). + PermutationImpl& getPermutation(Index::Permutation p); + const PermutationImpl& getPermutation(Index::Permutation p) const; // Creates an index from a file. Parameter Parser must be able to split the // file's format into triples. @@ -250,134 +233,39 @@ class IndexImpl { // -------------------------------------------------------------------------- // -- RETRIEVAL --- // -------------------------------------------------------------------------- - typedef vector> WidthOneList; - typedef vector> WidthTwoList; - typedef vector> WidthThreeList; - typedef vector> WidthFourList; - typedef vector> WidthFiveList; - typedef vector> VarWidthList; // -------------------------------------------------------------------------- // RDF RETRIEVAL // -------------------------------------------------------------------------- // __________________________________________________________________________ - NumNormalAndInternal numDistinctSubjects() const { - if (hasAllPermutations()) { - auto numActually = _numSubjectsNormal; - return {numActually, _SPO.metaData().getNofDistinctC1() - numActually}; - } else { - AD_THROW( - "Can only get # distinct subjects if all 6 permutations " - "have been registered on sever start (and index build time) " - "with the -a option."); - } - } + NumNormalAndInternal numDistinctSubjects() const; // __________________________________________________________________________ - NumNormalAndInternal numDistinctObjects() const { - if (hasAllPermutations()) { - auto numActually = _numObjectsNormal; - return {numActually, _OSP.metaData().getNofDistinctC1() - numActually}; - } else { - AD_THROW( - "Can only get # distinct objects if all 6 permutations " - "have been registered on sever start (and index build time) " - "with the -a option."); - } - } + NumNormalAndInternal numDistinctObjects() const; // __________________________________________________________________________ - NumNormalAndInternal numDistinctPredicates() const { - auto numActually = _numPredicatesNormal; - return {numActually, _PSO.metaData().getNofDistinctC1() - numActually}; - } + NumNormalAndInternal numDistinctPredicates() const; // __________________________________________________________________________ - NumNormalAndInternal numDistinctCol0(Index::Permutation permutation) const { - switch (permutation) { - case Index::Permutation::SOP: - case Index::Permutation::SPO: - return numDistinctSubjects(); - case Index::Permutation::OPS: - case Index::Permutation::OSP: - return numDistinctObjects(); - case Index::Permutation::POS: - case Index::Permutation::PSO: - return numDistinctPredicates(); - default: - AD_FAIL(); - } - } + NumNormalAndInternal numDistinctCol0(Index::Permutation permutation) const; // ___________________________________________________________________________ - size_t getCardinality(Id id, Index::Permutation permutation) const { - const auto& p = getPermutation(permutation); - if (p.metaData().col0IdExists(id)) { - return p.metaData().getMetaData(id).getNofElements(); - } - return 0; - } + size_t getCardinality(Id id, Index::Permutation permutation) const; // ___________________________________________________________________________ size_t getCardinality(const TripleComponent& comp, - Index::Permutation permutation) const { - // TODO This special case is only relevant for the `PSO` and `POS` - // permutations, but this internal predicate should never appear in subjects - // or objects anyway. - // TODO Find out what the effect of this special case is for the - // query planning. - if (comp == INTERNAL_TEXT_MATCH_PREDICATE) { - return TEXT_PREDICATE_CARDINALITY_ESTIMATE; - } - std::optional relId = comp.toValueId(getVocab()); - if (relId.has_value()) { - return getCardinality(relId.value(), permutation); - } - return 0; - } + Index::Permutation permutation) const; // TODO Once we have an overview over the folding this logic should // probably not be in the index class. - std::optional idToOptionalString(Id id) const { - switch (id.getDatatype()) { - case Datatype::Undefined: - return std::nullopt; - case Datatype::Double: - return std::to_string(id.getDouble()); - case Datatype::Int: - return std::to_string(id.getInt()); - case Datatype::VocabIndex: { - auto result = _vocab.indexToOptionalString(id.getVocabIndex()); - if (result.has_value() && result.value().starts_with(VALUE_PREFIX)) { - result = ad_utility::convertIndexWordToValueLiteral(result.value()); - } - return result; - } - case Datatype::LocalVocabIndex: - // TODO:: this is why this shouldn't be here - return std::nullopt; - case Datatype::TextRecordIndex: - return getTextExcerpt(id.getTextRecordIndex()); - } - // should be unreachable because the enum is exhaustive. - AD_FAIL(); - } + std::optional idToOptionalString(Id id) const; - bool getId(const string& element, Id* id) const { - // TODO we should parse doubles correctly in the SparqlParser and - // then return the correct ids here or somewhere else. - VocabIndex vocabId; - auto success = getVocab().getId(element, &vocabId); - *id = Id::makeFromVocabIndex(vocabId); - return success; - } + // ___________________________________________________________________________ + bool getId(const string& element, Id* id) const; - std::pair prefix_range(const std::string& prefix) const { - // TODO Do we need prefix ranges for numbers? - auto [begin, end] = _vocab.prefix_range(prefix); - return {Id::makeFromVocabIndex(begin), Id::makeFromVocabIndex(end)}; - } + // ___________________________________________________________________________ + std::pair prefix_range(const std::string& prefix) const; const vector& getHasPattern() const; const CompactVectorOfStrings& getHasPredicate() const; @@ -511,32 +399,10 @@ class IndexImpl { // _____________________________________________________________________________ vector getMultiplicities(const TripleComponent& key, - Index::Permutation permutation) const { - const auto& p = getPermutation(permutation); - std::optional keyId = key.toValueId(getVocab()); - vector res; - if (keyId.has_value() && p._meta.col0IdExists(keyId.value())) { - auto metaData = p._meta.getMetaData(keyId.value()); - res.push_back(metaData.getCol1Multiplicity()); - res.push_back(metaData.getCol2Multiplicity()); - } else { - res.push_back(1); - res.push_back(1); - } - return res; - } + Index::Permutation permutation) const; // ___________________________________________________________________ - vector getMultiplicities(Index::Permutation permutation) const { - const auto& p = getPermutation(permutation); - auto numTriples = - static_cast(this->numTriples().normalAndInternal_()); - std::array m{ - numTriples / numDistinctSubjects().normalAndInternal_(), - numTriples / numDistinctPredicates().normalAndInternal_(), - numTriples / numDistinctObjects().normalAndInternal_()}; - return {m[p._keyOrder[0]], m[p._keyOrder[1]], m[p._keyOrder[2]]}; - } + vector getMultiplicities(Index::Permutation permutation) const; /** * @brief Perform a scan for one key i.e. retrieve all YZ from the XYZ @@ -550,9 +416,7 @@ class IndexImpl { * IndexImpl class). */ void scan(Id key, IdTable* result, const Index::Permutation& p, - ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const { - getPermutation(p).scan(key, result, std::move(timer)); - } + ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const; /** * @brief Perform a scan for one key i.e. retrieve all YZ from the XYZ @@ -567,17 +431,7 @@ class IndexImpl { */ void scan(const TripleComponent& key, IdTable* result, Index::Permutation permutation, - ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const { - const auto& p = getPermutation(permutation); - LOG(DEBUG) << "Performing " << p._readableName - << " scan for full list for: " << key << "\n"; - std::optional optionalId = key.toValueId(getVocab()); - if (optionalId.has_value()) { - LOG(TRACE) << "Successfully got key ID.\n"; - scan(optionalId.value(), result, permutation, std::move(timer)); - } - LOG(DEBUG) << "Scan done, got " << result->size() << " elements.\n"; - } + ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const; /** * @brief Perform a scan for two keys i.e. retrieve all Z from the XYZ @@ -596,22 +450,7 @@ class IndexImpl { void scan(const TripleComponent& col0String, const TripleComponent& col1String, IdTable* result, const Index::Permutation& permutation, - ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const { - std::optional col0Id = col0String.toValueId(getVocab()); - std::optional col1Id = col1String.toValueId(getVocab()); - const auto& p = getPermutation(permutation); - if (!col0Id.has_value() || !col1Id.has_value()) { - LOG(DEBUG) << "Key " << col0String << " or key " << col1String - << " were not found in the vocabulary \n"; - return; - } - - LOG(DEBUG) << "Performing " << p._readableName << " scan of relation " - << col0String << " with fixed subject: " << col1String - << "...\n"; - - p.scan(col0Id.value(), col1Id.value(), result, timer); - } + ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const; private: // Private member functions @@ -814,27 +653,11 @@ class IndexImpl { template void readIndexBuilderSettingsFromFile(); - // Helper function for Debugging during the index build. - // ExtVecs are not persistent, so we dump them to a mmapVector in a file with - // given filename - static void dumpExtVecToMmap(const TripleVec& vec, std::string filename) { - LOG(INFO) << "Dumping ext vec to mmap" << std::endl; - MmapVector mmapVec(vec.size(), filename); - for (size_t i = 0; i < vec.size(); ++i) { - mmapVec[i] = vec[i]; - } - LOG(INFO) << "Done" << std::endl; - } - /** * Delete a temporary file unless the _keepTempFiles flag is set * @param path */ - void deleteTemporaryFile(const string& path) { - if (!_keepTempFiles) { - ad_utility::deleteFile(path); - } - } + void deleteTemporaryFile(const string& path); public: // Count the number of "QLever-internal" triples (predicate ql:langtag or From e8388e64370bad4c280b3a66db587c920ea42040 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 4 May 2023 14:17:09 +0200 Subject: [PATCH 010/150] Added some unit tests. --- src/index/Index.h | 1 + test/IndexTest.cpp | 14 ++++++++++++++ test/IndexTestHelpers.h | 17 ++++++++++------- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/index/Index.h b/src/index/Index.h index e436775c1a..044cbfcefa 100644 --- a/src/index/Index.h +++ b/src/index/Index.h @@ -45,6 +45,7 @@ class Index { size_t normal_; size_t internal_; size_t normalAndInternal_() const { return normal_ + internal_; } + bool operator==(const NumNormalAndInternal&) const = default; }; /// Forbid copy and assignment. diff --git a/test/IndexTest.cpp b/test/IndexTest.cpp index 37682282e4..fd304fe1de 100644 --- a/test/IndexTest.cpp +++ b/test/IndexTest.cpp @@ -471,6 +471,8 @@ TEST(IndexTest, NumDistinctEntities) { EXPECT_EQ(subjects.normal_, 3); // All literals with language tags are added subjects. EXPECT_EQ(subjects.internal_, 1); + EXPECT_EQ(subjects, index.numDistinctCol0(Index::Permutation::SPO)); + EXPECT_EQ(subjects, index.numDistinctCol0(Index::Permutation::SOP)); auto predicates = index.numDistinctPredicates(); EXPECT_EQ(predicates.normal_, 2); @@ -478,14 +480,26 @@ TEST(IndexTest, NumDistinctEntities) { // each combination of predicate+language that is actually used (e.g. // `@en@label`). EXPECT_EQ(predicates.internal_, 2); + EXPECT_EQ(predicates, index.numDistinctCol0(Index::Permutation::PSO)); + EXPECT_EQ(predicates, index.numDistinctCol0(Index::Permutation::POS)); auto objects = index.numDistinctObjects(); EXPECT_EQ(objects.normal_, 7); // One added object for each language that is used EXPECT_EQ(objects.internal_, 1); + EXPECT_EQ(objects, index.numDistinctCol0(Index::Permutation::OSP)); + EXPECT_EQ(objects, index.numDistinctCol0(Index::Permutation::OPS)); auto numTriples = index.numTriples(); EXPECT_EQ(numTriples.normal_, 7); // Two added triples for each triple that has an object with a language tag. EXPECT_EQ(numTriples.internal_, 2); + +} + +TEST(IndexTest, NumDistinctEntitiesCornerCases) { + const IndexImpl& index = getQec("", false)->getIndex().getImpl(); + AD_EXPECT_THROW_WITH_MESSAGE(index.numDistinctSubjects(), ::testing::ContainsRegex("if all 6")); + AD_EXPECT_THROW_WITH_MESSAGE(index.numDistinctObjects(), ::testing::ContainsRegex("if all 6")); + AD_EXPECT_THROW_WITH_MESSAGE(index.numDistinctCol0(static_cast(42)), ::testing::ContainsRegex("should be unreachable")); } diff --git a/test/IndexTestHelpers.h b/test/IndexTestHelpers.h index ddd274524f..2d7bc46d08 100644 --- a/test/IndexTestHelpers.h +++ b/test/IndexTestHelpers.h @@ -60,7 +60,7 @@ inline std::vector getAllIndexFilenames( // for the subclasses of `SparqlExpression`. // The concrete triple contents are currently used in `GroupByTest.cpp`. inline Index makeTestIndex(const std::string& indexBasename, - std::string turtleInput = "") { + std::string turtleInput = "", bool loadAllPermutations = true) { // Ignore the (irrelevant) log output of the index building and loading during // these tests. static std::ostringstream ignoreLogStream; @@ -86,6 +86,7 @@ inline Index makeTestIndex(const std::string& indexBasename, } Index index; index.setUsePatterns(true); + index.setLoadAllPermutations(loadAllPermutations); index.createFromOnDiskIndex(indexBasename); return index; } @@ -94,7 +95,7 @@ inline Index makeTestIndex(const std::string& indexBasename, // build using `makeTestIndex` (see above). The index (most notably its // vocabulary) is the only part of the `QueryExecutionContext` that is actually // relevant for these tests, so the other members are defaulted. -inline QueryExecutionContext* getQec(std::string turtleInput = "") { +inline QueryExecutionContext* getQec(std::string turtleInput = "", bool loadAllPermutations = true) { // Similar to `absl::Cleanup`. Calls the `callback_` in the destructor, but // the callback is stored as a `std::function`, which allows to store // different types of callbacks in the same wrapper type. @@ -116,13 +117,15 @@ inline QueryExecutionContext* getQec(std::string turtleInput = "") { *index_, cache_.get(), makeAllocator(), SortPerformanceEstimator{}); }; - static ad_utility::HashMap contextMap; + static ad_utility::HashMap, Context> contextMap; - if (!contextMap.contains(turtleInput)) { + auto key = std::pair{turtleInput, loadAllPermutations}; + + if (!contextMap.contains(key)) { std::string testIndexBasename = "_staticGlobalTestIndex" + std::to_string(contextMap.size()); contextMap.emplace( - turtleInput, Context{TypeErasedCleanup{[testIndexBasename]() { + key, Context{TypeErasedCleanup{[testIndexBasename]() { for (const std::string& indexFilename : getAllIndexFilenames(testIndexBasename)) { // Don't log when a file can't be deleted, @@ -132,10 +135,10 @@ inline QueryExecutionContext* getQec(std::string turtleInput = "") { } }}, std::make_unique( - makeTestIndex(testIndexBasename, turtleInput)), + makeTestIndex(testIndexBasename, turtleInput, loadAllPermutations)), std::make_unique()}); } - return contextMap.at(turtleInput).qec_.get(); + return contextMap.at(key).qec_.get(); } // Return a lambda that takes a string and converts it into an ID by looking From c86d7d0c87a21bec62a38798d6fc85aa12472bd0 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 4 May 2023 14:40:22 +0200 Subject: [PATCH 011/150] Changed the members from _this to that_ in the IndexImpl class. --- src/index/IndexImpl.Text.cpp | 230 +++++++++++------------ src/index/IndexImpl.cpp | 354 +++++++++++++++++------------------ src/index/IndexImpl.h | 139 +++++++------- test/IndexTest.cpp | 31 +-- test/IndexTestHelpers.h | 28 +-- 5 files changed, 393 insertions(+), 389 deletions(-) diff --git a/src/index/IndexImpl.Text.cpp b/src/index/IndexImpl.Text.cpp index 1488f6f7ac..0b83dc8c9c 100644 --- a/src/index/IndexImpl.Text.cpp +++ b/src/index/IndexImpl.Text.cpp @@ -38,7 +38,7 @@ struct LiteralsTokenizationDelimiter { // _____________________________________________________________________________ cppcoro::generator IndexImpl::wordsInTextRecords( const std::string& contextFile, bool addWordsFromLiterals) { - auto localeManager = _textVocab.getLocaleManager(); + auto localeManager = textVocab_.getLocaleManager(); // ROUND 1: If context file aka wordsfile is not empty, read words from there. // Remember the last context id for the (optional) second round. TextRecordIndex contextId = TextRecordIndex::make(0); @@ -57,9 +57,9 @@ cppcoro::generator IndexImpl::wordsInTextRecords( // ROUND 2: Optionally, consider each literal from the interal vocabulary as a // text record. if (addWordsFromLiterals) { - for (VocabIndex index = VocabIndex::make(0); index.get() < _vocab.size(); + for (VocabIndex index = VocabIndex::make(0); index.get() < vocab_.size(); index = index.incremented()) { - auto text = _vocab.at(index); + auto text = vocab_.at(index); if (!isLiteral(text)) { continue; } @@ -84,7 +84,7 @@ void IndexImpl::addTextFromContextFile(const string& contextFile, bool addWordsFromLiterals) { LOG(INFO) << std::endl; LOG(INFO) << "Adding text index ..." << std::endl; - string indexFilename = _onDiskBase + ".text.index"; + string indexFilename = onDiskBase_ + ".text.index"; // Either read words from given file or consider each literal as text record // or both (but at least one of them, otherwise this function is not called). if (!contextFile.empty()) { @@ -103,16 +103,16 @@ void IndexImpl::addTextFromContextFile(const string& contextFile, // That is, when we now call call `processWordsForVocabulary` (which builds // the text vocabulary), we already have the KB vocabular in RAM as well. LOG(DEBUG) << "Reloading the RDF vocabulary ..." << std::endl; - _vocab = RdfsVocabulary{}; + vocab_ = RdfsVocabulary{}; readConfiguration(); - _vocab.readFromFile(_onDiskBase + INTERNAL_VOCAB_SUFFIX, - _onDiskBase + EXTERNAL_VOCAB_SUFFIX); + vocab_.readFromFile(onDiskBase_ + INTERNAL_VOCAB_SUFFIX, + onDiskBase_ + EXTERNAL_VOCAB_SUFFIX); // Build the text vocabulary (first scan over the text records). LOG(INFO) << "Building text vocabulary ..." << std::endl; size_t nofLines = processWordsForVocabulary(contextFile, addWordsFromLiterals); - _textVocab.writeToFile(_onDiskBase + ".text.vocabulary"); + textVocab_.writeToFile(onDiskBase_ + ".text.vocabulary"); // Build the half-inverted lists (second scan over the text records). LOG(INFO) << "Building the half-inverted index lists ..." << std::endl; @@ -131,7 +131,7 @@ void IndexImpl::addTextFromContextFile(const string& contextFile, void IndexImpl::buildDocsDB(const string& docsFileName) { LOG(INFO) << "Building DocsDB...\n"; ad_utility::File docsFile(docsFileName.c_str(), "r"); - std::ofstream ofs(_onDiskBase + ".text.docsDB", std::ios_base::out); + std::ofstream ofs(onDiskBase_ + ".text.docsDB", std::ios_base::out); // To avoid excessive use of RAM, // we write the offsets to and stxxl:vector first; typedef stxxl::vector OffVec; @@ -158,7 +158,7 @@ void IndexImpl::buildDocsDB(const string& docsFileName) { delete[] buf; ofs.close(); // Now append the tmp file to the docsDB file. - ad_utility::File out(string(_onDiskBase + ".text.docsDB").c_str(), "a"); + ad_utility::File out(string(onDiskBase_ + ".text.docsDB").c_str(), "a"); for (size_t i = 0; i < offsets.size(); ++i) { off_t cur = offsets[i]; out.write(&cur, sizeof(cur)); @@ -170,33 +170,33 @@ void IndexImpl::buildDocsDB(const string& docsFileName) { // _____________________________________________________________________________ void IndexImpl::addTextFromOnDiskIndex() { // Read the text vocabulary (into RAM). - _textVocab.readFromFile(_onDiskBase + ".text.vocabulary"); + textVocab_.readFromFile(onDiskBase_ + ".text.vocabulary"); // Initialize the text index. - std::string textIndexFileName = _onDiskBase + ".text.index"; + std::string textIndexFileName = onDiskBase_ + ".text.index"; LOG(INFO) << "Reading metadata from file " << textIndexFileName << " ..." << std::endl; - _textIndexFile.open(textIndexFileName.c_str(), "r"); - AD_CONTRACT_CHECK(_textIndexFile.isOpen()); + textIndexFile_.open(textIndexFileName.c_str(), "r"); + AD_CONTRACT_CHECK(textIndexFile_.isOpen()); off_t metaFrom; - [[maybe_unused]] off_t metaTo = _textIndexFile.getLastOffset(&metaFrom); + [[maybe_unused]] off_t metaTo = textIndexFile_.getLastOffset(&metaFrom); ad_utility::serialization::FileReadSerializer serializer( - std::move(_textIndexFile)); + std::move(textIndexFile_)); serializer.setSerializationPosition(metaFrom); - serializer >> _textMeta; - _textIndexFile = std::move(serializer).file(); - LOG(INFO) << "Registered text index: " << _textMeta.statistics() << std::endl; + serializer >> textMeta_; + textIndexFile_ = std::move(serializer).file(); + LOG(INFO) << "Registered text index: " << textMeta_.statistics() << std::endl; // Initialize the text records file aka docsDB. NOTE: The search also works // without this, but then there is no content to show when a text record // matches. This is perfectly fine when the text records come from IRIs or // literals from our RDF vocabulary. - std::string docsDbFileName = _onDiskBase + ".text.docsDB"; + std::string docsDbFileName = onDiskBase_ + ".text.docsDB"; std::ifstream f(docsDbFileName.c_str()); if (f.good()) { f.close(); - _docsDB.init(string(_onDiskBase + ".text.docsDB")); - LOG(INFO) << "Registered text records: #records = " << _docsDB._size + docsDB_.init(string(onDiskBase_ + ".text.docsDB")); + LOG(INFO) << "Registered text records: #records = " << docsDB_._size << std::endl; } else { LOG(DEBUG) << "No file \"" << docsDbFileName @@ -221,7 +221,7 @@ size_t IndexImpl::processWordsForVocabulary(string const& contextFile, distinctWords.insert(line._word); } } - _textVocab.createFromSet(distinctWords); + textVocab_.createFromSet(distinctWords); return numLines; } @@ -270,9 +270,9 @@ void IndexImpl::processWordsForInvertedLists(const string& contextFile, } } else { ++nofWordPostings; - // TODO Let the `_textVocab` return a `WordIndex` directly. + // TODO Let the `textVocab_` return a `WordIndex` directly. VocabIndex vid; - bool ret = _textVocab.getId(line._word, &vid); + bool ret = textVocab_.getId(line._word, &vid); WordIndex wid = vid.get(); if (!ret) { LOG(ERROR) << "ERROR: word \"" << line._word << "\" " @@ -290,9 +290,9 @@ void IndexImpl::processWordsForInvertedLists(const string& contextFile, << std::endl; ++nofContexts; addContextToVector(writer, currentContext, wordsInContext, entitiesInContext); - _textMeta.setNofTextRecords(nofContexts); - _textMeta.setNofWordPostings(nofWordPostings); - _textMeta.setNofEntityPostings(nofEntityPostings); + textMeta_.setNofTextRecords(nofContexts); + textMeta_.setNofWordPostings(nofWordPostings); + textMeta_.setNofEntityPostings(nofEntityPostings); writer.finish(); LOG(TRACE) << "END IndexImpl::passContextFileIntoVector" << std::endl; @@ -342,7 +342,7 @@ void IndexImpl::addContextToVector( void IndexImpl::createTextIndex(const string& filename, const IndexImpl::TextVec& vec) { ad_utility::File out(filename.c_str(), "w"); - _currentoff_t = 0; + currentoff_t_ = 0; // Detect block boundaries from the main key of the vec. // Write the data for each block. // First, there's the classic lists, then the additional entity ones. @@ -364,7 +364,7 @@ void IndexImpl::createTextIndex(const string& filename, } ContextListMetaData classic = writePostings(out, classicPostings, true); ContextListMetaData entity = writePostings(out, entityPostings, false); - _textMeta.addBlock( + textMeta_.addBlock( TextBlockMetaData(currentMinWordIndex, currentMaxWordIndex, classic, entity), isEntityBlock); @@ -397,22 +397,22 @@ void IndexImpl::createTextIndex(const string& filename, } ContextListMetaData classic = writePostings(out, classicPostings, true); ContextListMetaData entity = writePostings(out, entityPostings, false); - _textMeta.addBlock(TextBlockMetaData(currentMinWordIndex, currentMaxWordIndex, + textMeta_.addBlock(TextBlockMetaData(currentMinWordIndex, currentMaxWordIndex, classic, entity), isEntityBlockId(currentMaxWordIndex)); - _textMeta.setNofEntities(nofEntities); - _textMeta.setNofEntityContexts(nofEntityContexts); + textMeta_.setNofEntities(nofEntities); + textMeta_.setNofEntityContexts(nofEntityContexts); classicPostings.clear(); entityPostings.clear(); LOG(DEBUG) << "Done creating text index." << std::endl; - LOG(INFO) << "Statistics for text index: " << _textMeta.statistics() + LOG(INFO) << "Statistics for text index: " << textMeta_.statistics() << std::endl; LOG(DEBUG) << "Writing Meta data to index file ..." << std::endl; ad_utility::serialization::FileWriteSerializer serializer{std::move(out)}; - serializer << _textMeta; + serializer << textMeta_; out = std::move(serializer).file(); - off_t startOfMeta = _textMeta.getOffsetAfter(); + off_t startOfMeta = textMeta_.getOffsetAfter(); out.write(&startOfMeta, sizeof(startOfMeta)); out.close(); LOG(INFO) << "Text index build completed" << std::endl; @@ -425,10 +425,10 @@ ContextListMetaData IndexImpl::writePostings(ad_utility::File& out, ContextListMetaData meta; meta._nofElements = postings.size(); if (meta._nofElements == 0) { - meta._startContextlist = _currentoff_t; - meta._startWordlist = _currentoff_t; - meta._startScorelist = _currentoff_t; - meta._lastByte = _currentoff_t - 1; + meta._startContextlist = currentoff_t_; + meta._startWordlist = currentoff_t_; + meta._startScorelist = currentoff_t_; + meta._lastByte = currentoff_t_ - 1; return meta; } @@ -471,28 +471,28 @@ ContextListMetaData IndexImpl::writePostings(ad_utility::File& out, size_t bytes = 0; // Write context list: - meta._startContextlist = _currentoff_t; + meta._startContextlist = currentoff_t_; bytes = writeList(contextList, meta._nofElements, out); - _currentoff_t += bytes; + currentoff_t_ += bytes; // Write word list: // This can be skipped if we're writing classic lists and there // is only one distinct wordId in the block, since this Id is already // stored in the meta data. - meta._startWordlist = _currentoff_t; + meta._startWordlist = currentoff_t_; if (!skipWordlistIfAllTheSame || wordCodebook.size() > 1) { - _currentoff_t += writeCodebook(wordCodebook, out); + currentoff_t_ += writeCodebook(wordCodebook, out); bytes = writeList(wordList, meta._nofElements, out); - _currentoff_t += bytes; + currentoff_t_ += bytes; } // Write scores - meta._startScorelist = _currentoff_t; - _currentoff_t += writeCodebook(scoreCodebook, out); + meta._startScorelist = currentoff_t_; + currentoff_t_ += writeCodebook(scoreCodebook, out); bytes = writeList(scoreList, meta._nofElements, out); - _currentoff_t += bytes; + currentoff_t_ += bytes; - meta._lastByte = _currentoff_t - 1; + meta._lastByte = currentoff_t_ - 1; delete[] contextList; delete[] wordList; @@ -554,20 +554,20 @@ void IndexImpl::calculateBlockBoundariesImpl( // A block boundary is always the last WordId in the block. // this way std::lower_bound will point to the correct bracket. - if (!areFourLetterPrefixesSorted(index._textVocab.getCaseComparator())) { + if (!areFourLetterPrefixesSorted(index.textVocab_.getCaseComparator())) { LOG(ERROR) << "You have chosen a locale where the prefixes aaaa, aaab, " "..., zzzz are not alphabetically ordered. This is currently " "unsupported when building a text index"; AD_FAIL(); } - if (index._textVocab.size() == 0) { + if (index.textVocab_.size() == 0) { LOG(WARN) << "You are trying to call calculateBlockBoundaries on an empty " "text vocabulary\n"; return; } size_t numBlocks = 0; - const auto& locManager = index._textVocab.getLocaleManager(); + const auto& locManager = index.textVocab_.getLocaleManager(); // iterator over aaaa, ..., zzzz auto forcedBlockStarts = fourLetterPrefixes(); @@ -598,7 +598,7 @@ void IndexImpl::calculateBlockBoundariesImpl( }; auto getLengthAndPrefixSortKey = [&](VocabIndex i) { - auto word = index._textVocab[i].value(); + auto word = index.textVocab_[i].value(); auto [len, prefixSortKey] = locManager.getPrefixSortKey(word, MIN_WORD_PREFIX_SIZE); if (len > MIN_WORD_PREFIX_SIZE) { @@ -613,7 +613,7 @@ void IndexImpl::calculateBlockBoundariesImpl( }; auto [currentLen, prefixSortKey] = getLengthAndPrefixSortKey(VocabIndex::make(0)); - for (size_t i = 0; i < index._textVocab.size() - 1; ++i) { + for (size_t i = 0; i < index.textVocab_.size() - 1; ++i) { // we need foo.value().get() because the vocab returns // a std::optional> and the "." currently // doesn't implicitly convert to a true reference (unlike function calls) @@ -634,16 +634,16 @@ void IndexImpl::calculateBlockBoundariesImpl( prefixSortKey = nextPrefixSortKey; } } - blockBoundaryAction(index._textVocab.size() - 1); + blockBoundaryAction(index.textVocab_.size() - 1); numBlocks++; LOG(DEBUG) << "Block boundaries computed: #blocks = " << numBlocks - << ", #words = " << index._textVocab.size() << std::endl; + << ", #words = " << index.textVocab_.size() << std::endl; } // _____________________________________________________________________________ void IndexImpl::calculateBlockBoundaries() { - _blockBoundaries.clear(); + blockBoundaries_.clear(); auto addToBlockBoundaries = [this](size_t i) { - _blockBoundaries.push_back(i); + blockBoundaries_.push_back(i); }; return calculateBlockBoundariesImpl(*this, addToBlockBoundaries); } @@ -654,9 +654,9 @@ void IndexImpl::printBlockBoundariesToFile(const string& filename) const { of << "Printing block boundaries ot text vocabulary\n" << "Format: \n"; auto printBlockToFile = [this, &of](size_t i) { - of << _textVocab[VocabIndex::make(i)].value() << " "; - if (i + 1 < _textVocab.size()) { - of << _textVocab[VocabIndex::make(i + 1)].value() << '\n'; + of << textVocab_[VocabIndex::make(i)].value() << " "; + if (i + 1 < textVocab_.size()) { + of << textVocab_[VocabIndex::make(i + 1)].value() << '\n'; } }; return calculateBlockBoundariesImpl(*this, printBlockToFile); @@ -664,20 +664,20 @@ void IndexImpl::printBlockBoundariesToFile(const string& filename) const { // _____________________________________________________________________________ TextBlockIndex IndexImpl::getWordBlockId(WordIndex wordIndex) const { - return std::lower_bound(_blockBoundaries.begin(), _blockBoundaries.end(), + return std::lower_bound(blockBoundaries_.begin(), blockBoundaries_.end(), wordIndex) - - _blockBoundaries.begin(); + blockBoundaries_.begin(); } // _____________________________________________________________________________ TextBlockIndex IndexImpl::getEntityBlockId(Id entityId) const { AD_CONTRACT_CHECK(entityId.getDatatype() == Datatype::VocabIndex); - return entityId.getVocabIndex().get() + _blockBoundaries.size(); + return entityId.getVocabIndex().get() + blockBoundaries_.size(); } // _____________________________________________________________________________ bool IndexImpl::isEntityBlockId(TextBlockIndex blockIndex) const { - return blockIndex >= _blockBoundaries.size(); + return blockIndex >= blockBoundaries_.size(); } // _____________________________________________________________________________ @@ -757,13 +757,13 @@ size_t IndexImpl::writeCodebook(const vector& codebook, // _____________________________________________________________________________ void IndexImpl::openTextFileHandle() { - AD_CONTRACT_CHECK(_onDiskBase.size() > 0); - _textIndexFile.open(string(_onDiskBase + ".text.index").c_str(), "r"); + AD_CONTRACT_CHECK(onDiskBase_.size() > 0); + textIndexFile_.open(string(onDiskBase_ + ".text.index").c_str(), "r"); } // _____________________________________________________________________________ std::string_view IndexImpl::wordIdToString(WordIndex wordIndex) const { - return _textVocab[VocabIndex::make(wordIndex)].value(); + return textVocab_[VocabIndex::make(wordIndex)].value(); } // _____________________________________________________________________________ @@ -817,30 +817,30 @@ void IndexImpl::getWordPostingsForTerm(const string& term, IdRange idRange; bool entityTerm = (term[0] == '<' && term.back() == '>'); if (term[term.size() - 1] == PREFIX_CHAR) { - if (!_textVocab.getIdRangeForFullTextPrefix(term, &idRange)) { + if (!textVocab_.getIdRangeForFullTextPrefix(term, &idRange)) { LOG(INFO) << "Prefix: " << term << " not in vocabulary\n"; return; } } else { if (entityTerm) { - if (!_vocab.getId(term, &idRange._first)) { + if (!vocab_.getId(term, &idRange._first)) { LOG(INFO) << "Term: " << term << " not in entity vocabulary\n"; return; } - } else if (!_textVocab.getId(term, &idRange._first)) { + } else if (!textVocab_.getId(term, &idRange._first)) { LOG(INFO) << "Term: " << term << " not in vocabulary\n"; return; } idRange._last = idRange._first; } if (entityTerm && - !_textMeta.existsTextBlockForEntityId(idRange._first.get())) { + !textMeta_.existsTextBlockForEntityId(idRange._first.get())) { LOG(INFO) << "Entity " << term << " not contained in the text.\n"; return; } const auto& tbmd = - entityTerm ? _textMeta.getBlockInfoByEntityId(idRange._first.get()) - : _textMeta.getBlockInfoByWordRange(idRange._first.get(), + entityTerm ? textMeta_.getBlockInfoByEntityId(idRange._first.get()) + : textMeta_.getBlockInfoByWordRange(idRange._first.get(), idRange._last.get()); if (tbmd._cl.hasMultipleWords() && !(tbmd._firstWordId == idRange._first.get() && @@ -1052,17 +1052,17 @@ void IndexImpl::getEntityPostingsForTerm(const string& term, IdRange idRange; bool entityTerm = (term[0] == '<' && term.back() == '>'); if (term.back() == PREFIX_CHAR) { - if (!_textVocab.getIdRangeForFullTextPrefix(term, &idRange)) { + if (!textVocab_.getIdRangeForFullTextPrefix(term, &idRange)) { LOG(INFO) << "Prefix: " << term << " not in vocabulary\n"; return; } } else { if (entityTerm) { - if (!_vocab.getId(term, &idRange._first)) { + if (!vocab_.getId(term, &idRange._first)) { LOG(DEBUG) << "Term: " << term << " not in entity vocabulary\n"; return; } - } else if (!_textVocab.getId(term, &idRange._first)) { + } else if (!textVocab_.getId(term, &idRange._first)) { LOG(DEBUG) << "Term: " << term << " not in vocabulary\n"; return; } @@ -1072,8 +1072,8 @@ void IndexImpl::getEntityPostingsForTerm(const string& term, // TODO Find out which ID types the `getBlockInfo...` functions // should take. const auto& tbmd = - entityTerm ? _textMeta.getBlockInfoByEntityId(idRange._first.get()) - : _textMeta.getBlockInfoByWordRange(idRange._first.get(), + entityTerm ? textMeta_.getBlockInfoByEntityId(idRange._first.get()) + : textMeta_.getBlockInfoByWordRange(idRange._first.get(), idRange._last.get()); if (!tbmd._cl.hasMultipleWords() || @@ -1139,7 +1139,7 @@ void IndexImpl::readGapComprList(size_t nofElements, off_t from, << ", nofBytes: " << nofBytes << '\n'; result.resize(nofElements + 250); uint64_t* encoded = new uint64_t[nofBytes / 8]; - _textIndexFile.read(encoded, nofBytes, from); + textIndexFile_.read(encoded, nofBytes, from); LOG(DEBUG) << "Decoding Simple8b code...\n"; ad_utility::Simple8bCode::decode(encoded, nofElements, result.data(), makeFromUint64t); @@ -1179,15 +1179,15 @@ void IndexImpl::readFreqComprList(size_t nofElements, off_t from, uint64_t* encoded = new uint64_t[nofElements]; result.resize(nofElements + 250); off_t current = from; - size_t ret = _textIndexFile.read(&nofCodebookBytes, sizeof(off_t), current); + size_t ret = textIndexFile_.read(&nofCodebookBytes, sizeof(off_t), current); LOG(TRACE) << "Nof Codebook Bytes: " << nofCodebookBytes << '\n'; AD_CONTRACT_CHECK(sizeof(off_t) == ret); current += ret; T* codebook = new T[nofCodebookBytes / sizeof(T)]; - ret = _textIndexFile.read(codebook, nofCodebookBytes, current); + ret = textIndexFile_.read(codebook, nofCodebookBytes, current); current += ret; AD_CONTRACT_CHECK(ret == size_t(nofCodebookBytes)); - ret = _textIndexFile.read( + ret = textIndexFile_.read( encoded, static_cast(nofBytes - (current - from)), current); current += ret; AD_CONTRACT_CHECK(size_t(current - from) == nofBytes); @@ -1216,13 +1216,13 @@ void IndexImpl::readFreqComprList(size_t nofElements, off_t from, void IndexImpl::dumpAsciiLists(const vector& lists, bool decGapsFreq) const { if (lists.size() == 0) { - size_t nofBlocks = _textMeta.getBlockCount(); + size_t nofBlocks = textMeta_.getBlockCount(); for (size_t i = 0; i < nofBlocks; ++i) { - TextBlockMetaData tbmd = _textMeta.getBlockById(i); + TextBlockMetaData tbmd = textMeta_.getBlockById(i); LOG(INFO) << "At block: " << i << std::endl; auto nofWordElems = tbmd._cl._nofElements; if (nofWordElems < 1000000) continue; - if (tbmd._firstWordId > _textVocab.size()) return; + if (tbmd._firstWordId > textVocab_.size()) return; if (decGapsFreq) { AD_THROW(ad_utility::Exception::NOT_YET_IMPLEMENTED, "not yet impl."); } else { @@ -1232,9 +1232,9 @@ void IndexImpl::dumpAsciiLists(const vector& lists, } else { for (size_t i = 0; i < lists.size(); ++i) { IdRange idRange; - _textVocab.getIdRangeForFullTextPrefix(lists[i], &idRange); + textVocab_.getIdRangeForFullTextPrefix(lists[i], &idRange); TextBlockMetaData tbmd = - _textMeta.getBlockInfoByWordRange(idRange._first, idRange._last); + textMeta_.getBlockInfoByWordRange(idRange._first, idRange._last); if (decGapsFreq) { vector eids; vector cids; @@ -1242,7 +1242,7 @@ void IndexImpl::dumpAsciiLists(const vector& lists, getEntityPostingsForTerm(lists[i], cids, eids, scores); auto firstWord = wordIdToString(tbmd._firstWordId); auto lastWord = wordIdToString(tbmd._lastWordId); - string basename = _onDiskBase + ".list." + firstWord + "-" + lastWord; + string basename = onDiskBase_ + ".list." + firstWord + "-" + lastWord; string docIdsFn = basename + ".recIds.ent.ascii"; string wordIdsFn = basename + ".wordIds.ent.ascii"; string scoresFn = basename + ".scores.ent.ascii"; @@ -1264,7 +1264,7 @@ void IndexImpl::dumpAsciiLists(const TextBlockMetaData& tbmd) const { auto lastWord = wordIdToString(tbmd._lastWordId); LOG(INFO) << "This block is from " << firstWord << " to " << lastWord << std::endl; - string basename = _onDiskBase + ".list." + firstWord + "-" + lastWord; + string basename = onDiskBase_ + ".list." + firstWord + "-" + lastWord; size_t nofCodebookBytes; { string docIdsFn = basename + ".docids.noent.ascii"; @@ -1282,7 +1282,7 @@ void IndexImpl::dumpAsciiLists(const TextBlockMetaData& tbmd) const { ids.resize(nofElements + 250); uint64_t* encodedD = new uint64_t[nofBytes / 8]; - _textIndexFile.read(encodedD, nofBytes, from); + textIndexFile_.read(encodedD, nofBytes, from); LOG(DEBUG) << "Decoding Simple8b code...\n"; ad_utility::Simple8bCode::decode(encodedD, nofElements, ids.data()); ids.resize(nofElements); @@ -1300,15 +1300,15 @@ void IndexImpl::dumpAsciiLists(const TextBlockMetaData& tbmd) const { uint64_t* encodedW = new uint64_t[nofBytes / 8]; off_t current = from; size_t ret = - _textIndexFile.read(&nofCodebookBytes, sizeof(off_t), current); + textIndexFile_.read(&nofCodebookBytes, sizeof(off_t), current); LOG(DEBUG) << "Nof Codebook Bytes: " << nofCodebookBytes << '\n'; AD_CHECK_EQ(sizeof(off_t), ret); current += ret; Id* codebookW = new Id[nofCodebookBytes / sizeof(Id)]; - ret = _textIndexFile.read(codebookW, nofCodebookBytes, current); + ret = textIndexFile_.read(codebookW, nofCodebookBytes, current); current += ret; AD_CHECK_EQ(ret, size_t(nofCodebookBytes)); - ret = _textIndexFile.read( + ret = textIndexFile_.read( encodedW, static_cast(nofBytes - (current - from)), current); current += ret; AD_CHECK_EQ(size_t(current - from), nofBytes); @@ -1328,15 +1328,15 @@ void IndexImpl::dumpAsciiLists(const TextBlockMetaData& tbmd) const { ids.resize(nofElements + 250); uint64_t* encodedS = new uint64_t[nofBytes / 8]; off_t current = from; - size_t ret = _textIndexFile.read(&nofCodebookBytes, sizeof(off_t), current); + size_t ret = textIndexFile_.read(&nofCodebookBytes, sizeof(off_t), current); LOG(DEBUG) << "Nof Codebook Bytes: " << nofCodebookBytes << '\n'; AD_CHECK_EQ(sizeof(off_t), ret); current += ret; Score* codebookS = new Score[nofCodebookBytes / sizeof(Score)]; - ret = _textIndexFile.read(codebookS, nofCodebookBytes, current); + ret = textIndexFile_.read(codebookS, nofCodebookBytes, current); current += ret; AD_CHECK_EQ(ret, size_t(nofCodebookBytes)); - ret = _textIndexFile.read( + ret = textIndexFile_.read( encodedS, static_cast(nofBytes - (current - from)), current); current += ret; AD_CHECK_EQ(size_t(current - from), nofBytes); @@ -1362,7 +1362,7 @@ void IndexImpl::dumpAsciiLists(const TextBlockMetaData& tbmd) const { ids.clear(); ids.resize(nofElements + 250); uint64_t* encodedD = new uint64_t[nofBytes / 8]; - _textIndexFile.read(encodedD, nofBytes, from); + textIndexFile_.read(encodedD, nofBytes, from); LOG(DEBUG) << "Decoding Simple8b code...\n"; ad_utility::Simple8bCode::decode(encodedD, nofElements, ids.data()); ids.resize(nofElements); @@ -1380,15 +1380,15 @@ void IndexImpl::dumpAsciiLists(const TextBlockMetaData& tbmd) const { uint64_t* encodedW = new uint64_t[nofBytes / 8]; off_t current = from; size_t ret = - _textIndexFile.read(&nofCodebookBytes, sizeof(off_t), current); + textIndexFile_.read(&nofCodebookBytes, sizeof(off_t), current); LOG(DEBUG) << "Nof Codebook Bytes: " << nofCodebookBytes << '\n'; AD_CHECK_EQ(sizeof(off_t), ret); current += ret; Id* codebookW = new Id[nofCodebookBytes / sizeof(Id)]; - ret = _textIndexFile.read(codebookW, nofCodebookBytes, current); + ret = textIndexFile_.read(codebookW, nofCodebookBytes, current); current += ret; AD_CHECK_EQ(ret, size_t(nofCodebookBytes)); - ret = _textIndexFile.read( + ret = textIndexFile_.read( encodedW, static_cast(nofBytes - (current - from)), current); current += ret; AD_CHECK_EQ(size_t(current - from), nofBytes); @@ -1408,16 +1408,16 @@ void IndexImpl::dumpAsciiLists(const TextBlockMetaData& tbmd) const { ids.resize(nofElements + 250); uint64_t* encodedS = new uint64_t[nofBytes / 8]; off_t current = from; - size_t ret = _textIndexFile.read(&nofCodebookBytes, sizeof(off_t), current); + size_t ret = textIndexFile_.read(&nofCodebookBytes, sizeof(off_t), current); LOG(DEBUG) << "Nof Codebook Bytes: " << nofCodebookBytes << '\n'; AD_CHECK_EQ(sizeof(off_t), ret); current += ret; Score* codebookS = new Score[nofCodebookBytes / sizeof(Score)]; - ret = _textIndexFile.read(codebookS, nofCodebookBytes, current); + ret = textIndexFile_.read(codebookS, nofCodebookBytes, current); current += ret; AD_CHECK_EQ(ret, size_t(nofCodebookBytes)); - ret = _textIndexFile.read( + ret = textIndexFile_.read( encodedS, static_cast(nofBytes - (current - from)), current); current += ret; AD_CHECK_EQ(size_t(current - from), nofBytes); @@ -1450,23 +1450,23 @@ size_t IndexImpl::getIndexOfBestSuitedElTerm( bool entityTerm = (terms[i][0] == '<' && terms[i].back() == '>'); IdRange range; if (terms[i].back() == PREFIX_CHAR) { - _textVocab.getIdRangeForFullTextPrefix(terms[i], &range); + textVocab_.getIdRangeForFullTextPrefix(terms[i], &range); } else { if (entityTerm) { - if (!_vocab.getId(terms[i], &range._first)) { + if (!vocab_.getId(terms[i], &range._first)) { LOG(DEBUG) << "Term: " << terms[i] << " not in entity vocabulary\n"; return i; } else { } - } else if (!_textVocab.getId(terms[i], &range._first)) { + } else if (!textVocab_.getId(terms[i], &range._first)) { LOG(DEBUG) << "Term: " << terms[i] << " not in vocabulary\n"; return i; } range._last = range._first; } const auto& tbmd = - entityTerm ? _textMeta.getBlockInfoByEntityId(range._first.get()) - : _textMeta.getBlockInfoByWordRange(range._first.get(), + entityTerm ? textMeta_.getBlockInfoByEntityId(range._first.get()) + : textMeta_.getBlockInfoByWordRange(range._first.get(), range._last.get()); toBeSorted.emplace_back(std::make_tuple( i, tbmd._firstWordId == tbmd._lastWordId, tbmd._entityCl._nofElements)); @@ -1658,24 +1658,24 @@ size_t IndexImpl::getSizeEstimate(const string& words) const { IdRange range; bool entityTerm = (terms[i][0] == '<' && terms[i].back() == '>'); if (terms[i].back() == PREFIX_CHAR) { - if (!_textVocab.getIdRangeForFullTextPrefix(terms[i], &range)) { + if (!textVocab_.getIdRangeForFullTextPrefix(terms[i], &range)) { return 0; } } else { if (entityTerm) { - if (!_vocab.getId(terms[i], &range._first)) { + if (!vocab_.getId(terms[i], &range._first)) { LOG(DEBUG) << "Term: " << terms[i] << " not in entity vocabulary\n"; return 0; } - } else if (!_textVocab.getId(terms[i], &range._first)) { + } else if (!textVocab_.getId(terms[i], &range._first)) { LOG(DEBUG) << "Term: " << terms[i] << " not in vocabulary\n"; return 0; } range._last = range._first; } const auto& tbmd = - entityTerm ? _textMeta.getBlockInfoByEntityId(range._first.get()) - : _textMeta.getBlockInfoByWordRange(range._first.get(), + entityTerm ? textMeta_.getBlockInfoByEntityId(range._first.get()) + : textMeta_.getBlockInfoByWordRange(range._first.get(), range._last.get()); if (minElLength > tbmd._entityCl._nofElements) { minElLength = tbmd._entityCl._nofElements; @@ -1709,4 +1709,4 @@ void IndexImpl::getRhsForSingleLhs(const IdTable& in, Id lhsId, } // _____________________________________________________________________________ -void IndexImpl::setTextName(const string& name) { _textMeta.setName(name); } +void IndexImpl::setTextName(const string& name) { textMeta_.setName(name); } diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index bed420041c..bec38b5f11 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -30,30 +30,30 @@ using std::array; // _____________________________________________________________________________ -IndexImpl::IndexImpl() : _usePatterns(false) {} +IndexImpl::IndexImpl() : usePatterns_(false) {} // _____________________________________________________________________________ template IndexBuilderDataAsPsoSorter IndexImpl::createIdTriplesAndVocab( const string& ntFile) { auto indexBuilderData = - passFileForVocabulary(ntFile, _numTriplesPerBatch); + passFileForVocabulary(ntFile, numTriplesPerBatch_); // first save the total number of words, this is needed to initialize the // dense IndexMetaData variants - _totalVocabularySize = indexBuilderData.vocabularyMetaData_.numWordsTotal_; + totalVocabularySize_ = indexBuilderData.vocabularyMetaData_.numWordsTotal_; LOG(DEBUG) << "Number of words in internal and external vocabulary: " - << _totalVocabularySize << std::endl; + << totalVocabularySize_ << std::endl; LOG(INFO) << "Converting external vocabulary to binary format ..." << std::endl; - _vocab.externalizeLiteralsFromTextFile( - _onDiskBase + EXTERNAL_LITS_TEXT_FILE_NAME, - _onDiskBase + EXTERNAL_VOCAB_SUFFIX); - deleteTemporaryFile(_onDiskBase + EXTERNAL_LITS_TEXT_FILE_NAME); + vocab_.externalizeLiteralsFromTextFile( + onDiskBase_ + EXTERNAL_LITS_TEXT_FILE_NAME, + onDiskBase_ + EXTERNAL_VOCAB_SUFFIX); + deleteTemporaryFile(onDiskBase_ + EXTERNAL_LITS_TEXT_FILE_NAME); // clear vocabulary to save ram (only information from partial binary files // used from now on). This will preserve information about externalized // Prefixes etc. - _vocab.clear(); + vocab_.clear(); auto psoSorter = convertPartialToGlobalIds( *indexBuilderData.idTriples, indexBuilderData.actualPartialSizes, NUM_TRIPLES_PER_PARTIAL_VOCAB); @@ -79,13 +79,13 @@ void createPatternsFromSpoTriplesView(auto&& spoTriplesView, // _____________________________________________________________________________ template void IndexImpl::createFromFile(const string& filename) { - string indexFilename = _onDiskBase + ".index"; + string indexFilename = onDiskBase_ + ".index"; readIndexBuilderSettingsFromFile(); IndexBuilderDataAsPsoSorter indexBuilderData; if constexpr (std::is_same_v, TurtleParserAuto>) { - if (_onlyAsciiTurtlePrefixes) { + if (onlyAsciiTurtlePrefixes_) { LOG(DEBUG) << "Using the CTRE library for tokenization" << std::endl; indexBuilderData = createIdTriplesAndVocab>( @@ -104,21 +104,21 @@ void IndexImpl::createFromFile(const string& filename) { // If we have no compression, this will also copy the whole vocabulary. // but since we expect compression to be the default case, this should not // hurt. - string vocabFile = _onDiskBase + INTERNAL_VOCAB_SUFFIX; - string vocabFileTmp = _onDiskBase + ".vocabularyTmp"; + string vocabFile = onDiskBase_ + INTERNAL_VOCAB_SUFFIX; + string vocabFileTmp = onDiskBase_ + ".vocabularyTmp"; const std::vector& prefixes = indexBuilderData.prefixes_; - if (_vocabPrefixCompressed) { - auto prefixFile = ad_utility::makeOfstream(_onDiskBase + PREFIX_FILE); + if (vocabPrefixCompressed_) { + auto prefixFile = ad_utility::makeOfstream(onDiskBase_ + PREFIX_FILE); for (const auto& prefix : prefixes) { prefixFile << prefix << std::endl; } } - _configurationJson["prefixes"] = _vocabPrefixCompressed; + configurationJson_["prefixes"] = vocabPrefixCompressed_; LOG(INFO) << "Writing compressed vocabulary to disk ..." << std::endl; - _vocab.buildCodebookForPrefixCompression(prefixes); - auto wordReader = _vocab.makeUncompressedDiskIterator(vocabFile); - auto wordWriter = _vocab.makeCompressedWordWriter(vocabFileTmp); + vocab_.buildCodebookForPrefixCompression(prefixes); + auto wordReader = vocab_.makeUncompressedDiskIterator(vocabFile); + auto wordWriter = vocab_.makeCompressedWordWriter(vocabFileTmp); for (const auto& word : wordReader) { wordWriter.push(word); } @@ -172,52 +172,52 @@ void IndexImpl::createFromFile(const string& filename) { size_t numPredicatesNormal = 0; createPermutationPair( - std::move(uniqueSorter), _PSO, _POS, spoSorter.makePushCallback(), + std::move(uniqueSorter), PSO_, POS_, spoSorter.makePushCallback(), makeNumEntitiesCounter(numPredicatesNormal, 1), countActualTriples); - _configurationJson["num-predicates-normal"] = numPredicatesNormal; - _configurationJson["num-triples-normal"] = numTriplesNormal; + configurationJson_["num-predicates-normal"] = numPredicatesNormal; + configurationJson_["num-triples-normal"] = numTriplesNormal; writeConfiguration(); psoSorter.clear(); - if (_loadAllPermutations) { + if (loadAllPermutations_) { // After the SPO permutation, create patterns if so desired. StxxlSorter ospSorter{stxxlMemoryInBytes() / 5}; size_t numSubjectsNormal = 0; auto numSubjectCounter = makeNumEntitiesCounter(numSubjectsNormal, 0); - if (_usePatterns) { - PatternCreator patternCreator{_onDiskBase + ".index.patterns"}; + if (usePatterns_) { + PatternCreator patternCreator{onDiskBase_ + ".index.patterns"}; auto pushTripleToPatterns = [&patternCreator, &isInternalId](const auto& triple) { if (!std::ranges::any_of(triple, isInternalId)) { patternCreator.processTriple(triple); } }; - createPermutationPair(spoSorter.sortedView(), _SPO, _SOP, + createPermutationPair(spoSorter.sortedView(), SPO_, SOP_, ospSorter.makePushCallback(), pushTripleToPatterns, numSubjectCounter); patternCreator.finish(); } else { - createPermutationPair(spoSorter.sortedView(), _SPO, _SOP, + createPermutationPair(spoSorter.sortedView(), SPO_, SOP_, ospSorter.makePushCallback(), numSubjectCounter); } spoSorter.clear(); - _configurationJson["num-subjects-normal"] = numSubjectsNormal; + configurationJson_["num-subjects-normal"] = numSubjectsNormal; writeConfiguration(); // For the last pair of permutations we don't need a next sorter, so we have // no fourth argument. size_t numObjectsNormal = 0; - createPermutationPair(ospSorter.sortedView(), _OSP, _OPS, + createPermutationPair(ospSorter.sortedView(), OSP_, OPS_, makeNumEntitiesCounter(numObjectsNormal, 2)); - _configurationJson["num-objects-normal"] = numObjectsNormal; - _configurationJson["has-all-permutations"] = true; + configurationJson_["num-objects-normal"] = numObjectsNormal; + configurationJson_["has-all-permutations"] = true; } else { - if (_usePatterns) { + if (usePatterns_) { createPatternsFromSpoTriplesView(spoSorter.sortedView(), - _onDiskBase + ".index.patterns", + onDiskBase_ + ".index.patterns", isInternalId); } - _configurationJson["has-all-permutations"] = false; + configurationJson_["has-all-permutations"] = false; } LOG(DEBUG) << "Finished writing permutations" << std::endl; @@ -242,8 +242,8 @@ IndexBuilderDataAsStxxlVector IndexImpl::passFileForVocabulary( LOG(INFO) << "Processing input triples from " << filename << " ..." << std::endl; auto parser = std::make_shared(filename); - parser->integerOverflowBehavior() = _turtleParserIntegerOverflowBehavior; - parser->invalidLiteralsAreSkipped() = _turtleParserSkipIllegalLiterals; + parser->integerOverflowBehavior() = turtleParserIntegerOverflowBehavior_; + parser->invalidLiteralsAreSkipped() = turtleParserSkipIllegalLiterals_; std::unique_ptr idTriples(new TripleVec()); ad_utility::Synchronized writer(*idTriples); bool parserExhausted = false; @@ -268,7 +268,7 @@ IndexBuilderDataAsStxxlVector IndexImpl::passFileForVocabulary( { auto p = ad_pipeline::setupParallelPipeline<3, NUM_PARALLEL_ITEM_MAPS>( - _parserBatchSize, + parserBatchSize_, // when called, returns an optional to the next triple. If // `linesPerPartial` triples were parsed, return std::nullopt. when // the parser is unable to deliver triples, set parserExhausted to @@ -286,7 +286,7 @@ IndexBuilderDataAsStxxlVector IndexImpl::passFileForVocabulary( // Tag triples using the provided HashMaps via itemArray. See // documentation of the function for more details getIdMapLambdas( - &itemArray, linesPerPartial, &(_vocab.getCaseComparator()))); + &itemArray, linesPerPartial, &(vocab_.getCaseComparator()))); while (auto opt = p.getNextValue()) { i++; @@ -356,12 +356,12 @@ IndexBuilderDataAsStxxlVector IndexImpl::passFileForVocabulary( size_t sizeInternalVocabulary = 0; std::vector prefixes; - if (_vocabPrefixCompressed) { + if (vocabPrefixCompressed_) { LOG(INFO) << "Merging partial vocabularies in byte order " << "(internal only) ..." << std::endl; VocabularyMerger m; auto compressionOutfile = ad_utility::makeOfstream( - _onDiskBase + TMP_BASENAME_COMPRESSION + INTERNAL_VOCAB_SUFFIX); + onDiskBase_ + TMP_BASENAME_COMPRESSION + INTERNAL_VOCAB_SUFFIX); auto internalVocabularyActionCompression = [&compressionOutfile](const auto& word) { compressionOutfile << RdfEscaping::escapeNewlinesAndBackslashes(word) @@ -369,7 +369,7 @@ IndexBuilderDataAsStxxlVector IndexImpl::passFileForVocabulary( }; m._noIdMapsAndIgnoreExternalVocab = true; auto mergeResult = - m.mergeVocabulary(_onDiskBase + TMP_BASENAME_COMPRESSION, numFiles, + m.mergeVocabulary(onDiskBase_ + TMP_BASENAME_COMPRESSION, numFiles, std::less<>(), internalVocabularyActionCompression); sizeInternalVocabulary = mergeResult.numWordsTotal_; LOG(INFO) << "Number of words in internal vocabulary: " @@ -380,7 +380,7 @@ IndexBuilderDataAsStxxlVector IndexImpl::passFileForVocabulary( // We have to use the "normally" sorted vocabulary for the prefix // compression. std::string vocabFileForPrefixCalculation = - _onDiskBase + TMP_BASENAME_COMPRESSION + INTERNAL_VOCAB_SUFFIX; + onDiskBase_ + TMP_BASENAME_COMPRESSION + INTERNAL_VOCAB_SUFFIX; prefixes = calculatePrefixes(vocabFileForPrefixCalculation, NUM_COMPRESSION_PREFIXES, 1, true); } @@ -389,16 +389,16 @@ IndexBuilderDataAsStxxlVector IndexImpl::passFileForVocabulary( << "(internal and external) ..." << std::endl; const VocabularyMerger::VocabularyMetaData mergeRes = [&]() { VocabularyMerger v; - auto sortPred = [cmp = &(_vocab.getCaseComparator())](std::string_view a, + auto sortPred = [cmp = &(vocab_.getCaseComparator())](std::string_view a, std::string_view b) { - return (*cmp)(a, b, decltype(_vocab)::SortLevel::TOTAL); + return (*cmp)(a, b, decltype(vocab_)::SortLevel::TOTAL); }; auto wordWriter = - _vocab.makeUncompressingWordWriter(_onDiskBase + INTERNAL_VOCAB_SUFFIX); + vocab_.makeUncompressingWordWriter(onDiskBase_ + INTERNAL_VOCAB_SUFFIX); auto internalVocabularyAction = [&wordWriter](const auto& word) { wordWriter.push(word.data(), word.size()); }; - return v.mergeVocabulary(_onDiskBase, numFiles, sortPred, + return v.mergeVocabulary(onDiskBase_, numFiles, sortPred, internalVocabularyAction); }(); LOG(DEBUG) << "Finished merging partial vocabularies" << std::endl; @@ -415,10 +415,10 @@ IndexBuilderDataAsStxxlVector IndexImpl::passFileForVocabulary( LOG(INFO) << "Removing temporary files ..." << std::endl; for (size_t n = 0; n < numFiles; ++n) { string partialFilename = - _onDiskBase + PARTIAL_VOCAB_FILE_NAME + std::to_string(n); + onDiskBase_ + PARTIAL_VOCAB_FILE_NAME + std::to_string(n); deleteTemporaryFile(partialFilename); - if (_vocabPrefixCompressed) { - partialFilename = _onDiskBase + TMP_BASENAME_COMPRESSION + + if (vocabPrefixCompressed_) { + partialFilename = onDiskBase_ + TMP_BASENAME_COMPRESSION + PARTIAL_VOCAB_FILE_NAME + std::to_string(n); deleteTemporaryFile(partialFilename); } @@ -443,7 +443,7 @@ std::unique_ptr IndexImpl::convertPartialToGlobalIds( size_t i = 0; for (size_t partialNum = 0; partialNum < actualLinesPerPartial.size(); partialNum++) { - std::string mmapFilename(_onDiskBase + PARTIAL_MMAP_IDS + + std::string mmapFilename(onDiskBase_ + PARTIAL_MMAP_IDS + std::to_string(partialNum)); LOG(DEBUG) << "Reading ID map from: " << mmapFilename << std::endl; ad_utility::HashMap idMap = IdMapFromPartialIdMapFile(mmapFilename); @@ -587,8 +587,8 @@ IndexImpl::createPermutations(auto&& sortedTriples, const PermutationImpl& p1, const PermutationImpl& p2, auto&&... perTripleCallbacks) { auto metaData = createPermutationPairImpl( - _onDiskBase + ".index" + p1._fileSuffix, - _onDiskBase + ".index" + p2._fileSuffix, AD_FWD(sortedTriples), + onDiskBase_ + ".index" + p1._fileSuffix, + onDiskBase_ + ".index" + p2._fileSuffix, AD_FWD(sortedTriples), p1._keyOrder[0], p1._keyOrder[1], p1._keyOrder[2], AD_FWD(perTripleCallbacks)...); @@ -611,24 +611,24 @@ void IndexImpl::createPermutationPair(auto&& sortedTriples, auto metaData = createPermutations(AD_FWD(sortedTriples), p1, p2, AD_FWD(perTripleCallbacks)...); // Set the name of this newly created pair of `IndexMetaData` objects. - // NOTE: When `setKbName` was called, it set the name of _PSO._meta, - // _PSO._meta, ... which however are not used during index building. + // NOTE: When `setKbName` was called, it set the name of PSO_._meta, + // PSO_._meta, ... which however are not used during index building. // `getKbName` simple reads one of these names. metaData.value().first.setName(getKbName()); metaData.value().second.setName(getKbName()); if (metaData) { LOG(INFO) << "Writing meta data for " << p1._readableName << " and " << p2._readableName << " ..." << std::endl; - ad_utility::File f1(_onDiskBase + ".index" + p1._fileSuffix, "r+"); + ad_utility::File f1(onDiskBase_ + ".index" + p1._fileSuffix, "r+"); metaData.value().first.appendToFile(&f1); - ad_utility::File f2(_onDiskBase + ".index" + p2._fileSuffix, "r+"); + ad_utility::File f2(onDiskBase_ + ".index" + p2._fileSuffix, "r+"); metaData.value().second.appendToFile(&f2); } } // _____________________________________________________________________________ void IndexImpl::addPatternsToExistingIndex() { - // auto [langPredLowerBound, langPredUpperBound] = _vocab.prefix_range("@"); + // auto [langPredLowerBound, langPredUpperBound] = vocab_.prefix_range("@"); // We only iterate over the SPO permutation which typically only has few // triples per subject, so it should be safe to not apply a memory limit // here. @@ -637,8 +637,8 @@ void IndexImpl::addPatternsToExistingIndex() { ad_utility::AllocatorWithLimit allocator{ ad_utility::makeAllocationMemoryLeftThreadsafeObject( std::numeric_limits::max())}; - auto iterator = TriplesView(_SPO, allocator); - createPatternsFromSpoTriplesView(iterator, _onDiskBase + ".index.patterns", + auto iterator = TriplesView(SPO_, allocator); + createPatternsFromSpoTriplesView(iterator, onDiskBase_ + ".index.patterns", Id::makeFromVocabIndex(langPredLowerBound), Id::makeFromVocabIndex(langPredUpperBound)); */ @@ -649,37 +649,37 @@ void IndexImpl::addPatternsToExistingIndex() { void IndexImpl::createFromOnDiskIndex(const string& onDiskBase) { setOnDiskBase(onDiskBase); readConfiguration(); - _vocab.readFromFile(_onDiskBase + INTERNAL_VOCAB_SUFFIX, - _onDiskBase + EXTERNAL_VOCAB_SUFFIX); + vocab_.readFromFile(onDiskBase_ + INTERNAL_VOCAB_SUFFIX, + onDiskBase_ + EXTERNAL_VOCAB_SUFFIX); - _totalVocabularySize = _vocab.size() + _vocab.getExternalVocab().size(); + totalVocabularySize_ = vocab_.size() + vocab_.getExternalVocab().size(); LOG(DEBUG) << "Number of words in internal and external vocabulary: " - << _totalVocabularySize << std::endl; - _PSO.loadFromDisk(_onDiskBase); - _POS.loadFromDisk(_onDiskBase); - - if (_loadAllPermutations) { - _OPS.loadFromDisk(_onDiskBase); - _OSP.loadFromDisk(_onDiskBase); - _SPO.loadFromDisk(_onDiskBase); - _SOP.loadFromDisk(_onDiskBase); + << totalVocabularySize_ << std::endl; + PSO_.loadFromDisk(onDiskBase_); + POS_.loadFromDisk(onDiskBase_); + + if (loadAllPermutations_) { + OPS_.loadFromDisk(onDiskBase_); + OSP_.loadFromDisk(onDiskBase_); + SPO_.loadFromDisk(onDiskBase_); + SOP_.loadFromDisk(onDiskBase_); } else { LOG(INFO) << "Only the PSO and POS permutation were loaded, SPARQL queries " "with predicate variables will therefore not work" << std::endl; } - if (_usePatterns) { + if (usePatterns_) { PatternCreator::readPatternsFromFile( - _onDiskBase + ".index.patterns", _avgNumDistinctSubjectsPerPredicate, - _avgNumDistinctPredicatesPerSubject, _numDistinctSubjectPredicatePairs, - _patterns, _hasPattern); + onDiskBase_ + ".index.patterns", avgNumDistinctSubjectsPerPredicate_, + avgNumDistinctPredicatesPerSubject_, numDistinctSubjectPredicatePairs_, + patterns_, hasPattern_); } } // _____________________________________________________________________________ void IndexImpl::throwExceptionIfNoPatterns() const { - if (!_usePatterns) { + if (!usePatterns_) { AD_THROW( "The requested feature requires a loaded patterns file (" "do not specify the --no-patterns option for this to work)"); @@ -689,37 +689,37 @@ void IndexImpl::throwExceptionIfNoPatterns() const { // _____________________________________________________________________________ const vector& IndexImpl::getHasPattern() const { throwExceptionIfNoPatterns(); - return _hasPattern; + return hasPattern_; } // _____________________________________________________________________________ const CompactVectorOfStrings& IndexImpl::getHasPredicate() const { throwExceptionIfNoPatterns(); - return _hasPredicate; + return hasPredicate_; } // _____________________________________________________________________________ const CompactVectorOfStrings& IndexImpl::getPatterns() const { throwExceptionIfNoPatterns(); - return _patterns; + return patterns_; } // _____________________________________________________________________________ double IndexImpl::getAvgNumDistinctPredicatesPerSubject() const { throwExceptionIfNoPatterns(); - return _avgNumDistinctPredicatesPerSubject; + return avgNumDistinctPredicatesPerSubject_; } // _____________________________________________________________________________ double IndexImpl::getAvgNumDistinctSubjectsPerPredicate() const { throwExceptionIfNoPatterns(); - return _avgNumDistinctSubjectsPerPredicate; + return avgNumDistinctSubjectsPerPredicate_; } // _____________________________________________________________________________ size_t IndexImpl::getNumDistinctSubjectPredicatePairs() const { throwExceptionIfNoPatterns(); - return _numDistinctSubjectPredicatePairs; + return numDistinctSubjectPredicatePairs_; } // _____________________________________________________________________________ @@ -741,68 +741,68 @@ template void IndexImpl::writeAsciiListFile>( // _____________________________________________________________________________ bool IndexImpl::isLiteral(const string& object) const { - return decltype(_vocab)::isLiteral(object); + return decltype(vocab_)::isLiteral(object); } // _____________________________________________________________________________ bool IndexImpl::shouldBeExternalized(const string& object) { - return _vocab.shouldBeExternalized(object); + return vocab_.shouldBeExternalized(object); } // _____________________________________________________________________________ void IndexImpl::setKbName(const string& name) { - _POS.setKbName(name); - _PSO.setKbName(name); - _SOP.setKbName(name); - _SPO.setKbName(name); - _OPS.setKbName(name); - _OSP.setKbName(name); + POS_.setKbName(name); + PSO_.setKbName(name); + SOP_.setKbName(name); + SPO_.setKbName(name); + OPS_.setKbName(name); + OSP_.setKbName(name); } // ____________________________________________________________________________ void IndexImpl::setOnDiskBase(const std::string& onDiskBase) { - _onDiskBase = onDiskBase; + onDiskBase_ = onDiskBase; } // ____________________________________________________________________________ void IndexImpl::setKeepTempFiles(bool keepTempFiles) { - _keepTempFiles = keepTempFiles; + keepTempFiles_ = keepTempFiles; } // _____________________________________________________________________________ -void IndexImpl::setUsePatterns(bool usePatterns) { _usePatterns = usePatterns; } +void IndexImpl::setUsePatterns(bool usePatterns) { usePatterns_ = usePatterns; } // _____________________________________________________________________________ void IndexImpl::setLoadAllPermutations(bool loadAllPermutations) { - _loadAllPermutations = loadAllPermutations; + loadAllPermutations_ = loadAllPermutations; } // ____________________________________________________________________________ void IndexImpl::setSettingsFile(const std::string& filename) { - _settingsFileName = filename; + settingsFileName_ = filename; } // ____________________________________________________________________________ void IndexImpl::setPrefixCompression(bool compressed) { - _vocabPrefixCompressed = compressed; + vocabPrefixCompressed_ = compressed; } // ____________________________________________________________________________ void IndexImpl::writeConfiguration() const { // Copy the configuration and add the current commit hash. - auto configuration = _configurationJson; + auto configuration = configurationJson_; configuration["git_hash"] = std::string(qlever::version::GitHash); - auto f = ad_utility::makeOfstream(_onDiskBase + CONFIGURATION_FILE); + auto f = ad_utility::makeOfstream(onDiskBase_ + CONFIGURATION_FILE); f << configuration; } // ___________________________________________________________________________ void IndexImpl::readConfiguration() { - auto f = ad_utility::makeIfstream(_onDiskBase + CONFIGURATION_FILE); - f >> _configurationJson; - if (_configurationJson.find("git_hash") != _configurationJson.end()) { + auto f = ad_utility::makeIfstream(onDiskBase_ + CONFIGURATION_FILE); + f >> configurationJson_; + if (configurationJson_.find("git_hash") != configurationJson_.end()) { LOG(INFO) << "The git hash used to build this index was " - << std::string(_configurationJson["git_hash"]).substr(0, 6) + << std::string(configurationJson_["git_hash"]).substr(0, 6) << std::endl; } else { LOG(INFO) << "The index was built before git commit hashes were stored in " @@ -810,36 +810,36 @@ void IndexImpl::readConfiguration() { << std::endl; } - if (_configurationJson.find("prefixes") != _configurationJson.end()) { - if (_configurationJson["prefixes"]) { + if (configurationJson_.find("prefixes") != configurationJson_.end()) { + if (configurationJson_["prefixes"]) { vector prefixes; - auto prefixFile = ad_utility::makeIfstream(_onDiskBase + PREFIX_FILE); + auto prefixFile = ad_utility::makeIfstream(onDiskBase_ + PREFIX_FILE); for (string prefix; std::getline(prefixFile, prefix);) { prefixes.emplace_back(std::move(prefix)); } - _vocab.buildCodebookForPrefixCompression(prefixes); + vocab_.buildCodebookForPrefixCompression(prefixes); } else { - _vocab.buildCodebookForPrefixCompression(std::vector()); + vocab_.buildCodebookForPrefixCompression(std::vector()); } } - if (_configurationJson.find("prefixes-external") != - _configurationJson.end()) { - _vocab.initializeExternalizePrefixes( - _configurationJson["prefixes-external"]); + if (configurationJson_.find("prefixes-external") != + configurationJson_.end()) { + vocab_.initializeExternalizePrefixes( + configurationJson_["prefixes-external"]); } - if (_configurationJson.count("ignore-case")) { + if (configurationJson_.count("ignore-case")) { LOG(ERROR) << ERROR_IGNORE_CASE_UNSUPPORTED << '\n'; throw std::runtime_error("Deprecated key \"ignore-case\" in index build"); } - if (_configurationJson.count("locale")) { - std::string lang{_configurationJson["locale"]["language"]}; - std::string country{_configurationJson["locale"]["country"]}; - bool ignorePunctuation{_configurationJson["locale"]["ignore-punctuation"]}; - _vocab.setLocale(lang, country, ignorePunctuation); - _textVocab.setLocale(lang, country, ignorePunctuation); + if (configurationJson_.count("locale")) { + std::string lang{configurationJson_["locale"]["language"]}; + std::string country{configurationJson_["locale"]["country"]}; + bool ignorePunctuation{configurationJson_["locale"]["ignore-punctuation"]}; + vocab_.setLocale(lang, country, ignorePunctuation); + textVocab_.setLocale(lang, country, ignorePunctuation); } else { LOG(ERROR) << "Key \"locale\" is missing in the metadata. This is probably " "and old index build that is no longer supported by QLever. " @@ -848,22 +848,22 @@ void IndexImpl::readConfiguration() { "Missing required key \"locale\" in index build's metadata"); } - if (_configurationJson.find("languages-internal") != - _configurationJson.end()) { - _vocab.initializeInternalizedLangs( - _configurationJson["languages-internal"]); + if (configurationJson_.find("languages-internal") != + configurationJson_.end()) { + vocab_.initializeInternalizedLangs( + configurationJson_["languages-internal"]); } - if (_configurationJson.find("has-all-permutations") != - _configurationJson.end() && - _configurationJson["has-all-permutations"] == false) { + if (configurationJson_.find("has-all-permutations") != + configurationJson_.end() && + configurationJson_["has-all-permutations"] == false) { // If the permutations simply don't exist, then we can never load them. - _loadAllPermutations = false; + loadAllPermutations_ = false; } auto loadRequestedDataMember = [this](std::string_view key, auto& target) { - auto it = _configurationJson.find(key); - if (it == _configurationJson.end()) { + auto it = configurationJson_.find(key); + if (it == configurationJson_.end()) { throw std::runtime_error{absl::StrCat( "The required key \"", key, "\" was not found in the `meta-data.json`. Most likely this index " @@ -872,10 +872,10 @@ void IndexImpl::readConfiguration() { target = std::decay_t{*it}; }; - loadRequestedDataMember("num-predicates-normal", _numPredicatesNormal); - loadRequestedDataMember("num-subjects-normal", _numSubjectsNormal); - loadRequestedDataMember("num-objects-normal", _numObjectsNormal); - loadRequestedDataMember("num-triples-normal", _numTriplesNormal); + loadRequestedDataMember("num-predicates-normal", numPredicatesNormal_); + loadRequestedDataMember("num-subjects-normal", numSubjectsNormal_); + loadRequestedDataMember("num-objects-normal", numObjectsNormal_); + loadRequestedDataMember("num-triples-normal", numTriplesNormal_); } // ___________________________________________________________________________ @@ -907,13 +907,13 @@ LangtagAndTriple IndexImpl::tripleToInternalRepresentation( } auto& component = std::get(el); auto& iriOrLiteral = component._iriOrLiteral; - iriOrLiteral = _vocab.getLocaleManager().normalizeUtf8(iriOrLiteral); - if (_vocab.shouldBeExternalized(iriOrLiteral)) { + iriOrLiteral = vocab_.getLocaleManager().normalizeUtf8(iriOrLiteral); + if (vocab_.shouldBeExternalized(iriOrLiteral)) { component._isExternal = true; } // Only the third element (the object) might contain a language tag. if (i == 2 && isLiteral(iriOrLiteral)) { - result._langtag = decltype(_vocab)::getLanguage(iriOrLiteral); + result._langtag = decltype(vocab_)::getLanguage(iriOrLiteral); } } return result; @@ -924,14 +924,14 @@ template void IndexImpl::readIndexBuilderSettingsFromFile() { json j; // if we have no settings, we still have to initialize some default // values - if (!_settingsFileName.empty()) { - auto f = ad_utility::makeIfstream(_settingsFileName); + if (!settingsFileName_.empty()) { + auto f = ad_utility::makeIfstream(settingsFileName_); f >> j; } if (j.find("prefixes-external") != j.end()) { - _vocab.initializeExternalizePrefixes(j["prefixes-external"]); - _configurationJson["prefixes-external"] = j["prefixes-external"]; + vocab_.initializeExternalizePrefixes(j["prefixes-external"]); + configurationJson_["prefixes-external"] = j["prefixes-external"]; } if (j.count("ignore-case")) { @@ -972,25 +972,25 @@ void IndexImpl::readIndexBuilderSettingsFromFile() { "filing a bug report. Also note that changing the\n\t" << "locale requires to completely rebuild the index\n"; } - _vocab.setLocale(lang, country, ignorePunctuation); - _textVocab.setLocale(lang, country, ignorePunctuation); - _configurationJson["locale"]["language"] = lang; - _configurationJson["locale"]["country"] = country; - _configurationJson["locale"]["ignore-punctuation"] = ignorePunctuation; + vocab_.setLocale(lang, country, ignorePunctuation); + textVocab_.setLocale(lang, country, ignorePunctuation); + configurationJson_["locale"]["language"] = lang; + configurationJson_["locale"]["country"] = country; + configurationJson_["locale"]["ignore-punctuation"] = ignorePunctuation; } if (j.find("languages-internal") != j.end()) { - _vocab.initializeInternalizedLangs(j["languages-internal"]); - _configurationJson["languages-internal"] = j["languages-internal"]; + vocab_.initializeInternalizedLangs(j["languages-internal"]); + configurationJson_["languages-internal"] = j["languages-internal"]; } if (j.count("ascii-prefixes-only")) { if constexpr (std::is_same_v, TurtleParserAuto>) { bool v{j["ascii-prefixes-only"]}; if (v) { LOG(INFO) << WARNING_ASCII_ONLY_PREFIXES << std::endl; - _onlyAsciiTurtlePrefixes = true; + onlyAsciiTurtlePrefixes_ = true; } else { - _onlyAsciiTurtlePrefixes = false; + onlyAsciiTurtlePrefixes_ = false; } } else { LOG(WARN) << "You specified the ascii-prefixes-only but a parser that is " @@ -1001,16 +1001,16 @@ void IndexImpl::readIndexBuilderSettingsFromFile() { } if (j.count("num-triples-per-batch")) { - _numTriplesPerBatch = size_t{j["num-triples-per-batch"]}; + numTriplesPerBatch_ = size_t{j["num-triples-per-batch"]}; LOG(INFO) - << "You specified \"num-triples-per-batch = " << _numTriplesPerBatch + << "You specified \"num-triples-per-batch = " << numTriplesPerBatch_ << "\", choose a lower value if the index builder runs out of memory" << std::endl; } if (j.count("parser-batch-size")) { - _parserBatchSize = size_t{j["parser-batch-size"]}; - LOG(INFO) << "Overriding setting parser-batch-size to " << _parserBatchSize + parserBatchSize_ = size_t{j["parser-batch-size"]}; + LOG(INFO) << "Overriding setting parser-batch-size to " << parserBatchSize_ << " This might influence performance during index build." << std::endl; } @@ -1029,17 +1029,17 @@ void IndexImpl::readIndexBuilderSettingsFromFile() { LOG(INFO) << "Integers that cannot be represented by QLever will throw " "an exception" << std::endl; - _turtleParserIntegerOverflowBehavior = + turtleParserIntegerOverflowBehavior_ = TurtleParserIntegerOverflowBehavior::Error; } else if (value == overflowingIntegersBecomeDoubles) { LOG(INFO) << "Integers that cannot be represented by QLever will be " "converted to doubles" << std::endl; - _turtleParserIntegerOverflowBehavior = + turtleParserIntegerOverflowBehavior_ = TurtleParserIntegerOverflowBehavior::OverflowingToDouble; } else if (value == allIntegersBecomeDoubles) { LOG(INFO) << "All integers will be converted to doubles" << std::endl; - _turtleParserIntegerOverflowBehavior = + turtleParserIntegerOverflowBehavior_ = TurtleParserIntegerOverflowBehavior::OverflowingToDouble; } else { AD_CONTRACT_CHECK(std::find(allModes.begin(), allModes.end(), value) == @@ -1049,7 +1049,7 @@ void IndexImpl::readIndexBuilderSettingsFromFile() { << absl::StrJoin(allModes, ",") << std::endl; } } else { - _turtleParserIntegerOverflowBehavior = + turtleParserIntegerOverflowBehavior_ = TurtleParserIntegerOverflowBehavior::Error; LOG(INFO) << "Integers that cannot be represented by QLever will throw an " "exception (this is the default behavior)" @@ -1068,20 +1068,20 @@ std::future IndexImpl::writeNextPartialVocabulary( << actualCurrentPartialSize << std::endl; std::future resultFuture; string partialFilename = - _onDiskBase + PARTIAL_VOCAB_FILE_NAME + std::to_string(numFiles); - string partialCompressionFilename = _onDiskBase + TMP_BASENAME_COMPRESSION + + onDiskBase_ + PARTIAL_VOCAB_FILE_NAME + std::to_string(numFiles); + string partialCompressionFilename = onDiskBase_ + TMP_BASENAME_COMPRESSION + PARTIAL_VOCAB_FILE_NAME + std::to_string(numFiles); auto lambda = [localIds = std::move(localIds), globalWritePtr, - items = std::move(items), vocab = &_vocab, partialFilename, + items = std::move(items), vocab = &vocab_, partialFilename, partialCompressionFilename, numFiles, - vocabPrefixCompressed = _vocabPrefixCompressed]() mutable { + vocabPrefixCompressed = vocabPrefixCompressed_]() mutable { auto vec = vocabMapsToVector(std::move(items)); const auto identicalPred = [&c = vocab->getCaseComparator()]( const auto& a, const auto& b) { return c(a.second.m_splitVal, b.second.m_splitVal, - decltype(_vocab)::SortLevel::TOTAL); + decltype(vocab_)::SortLevel::TOTAL); }; LOG(TIMING) << "Start sorting of vocabulary with #elements: " << vec.size() << std::endl; @@ -1132,7 +1132,7 @@ std::future IndexImpl::writeNextPartialVocabulary( // ____________________________________________________________________________ IndexImpl::NumNormalAndInternal IndexImpl::numTriples() const { - return {_numTriplesNormal, PSO()._meta.getNofTriples() - _numTriplesNormal}; + return {numTriplesNormal_, PSO()._meta.getNofTriples() - numTriplesNormal_}; } // ____________________________________________________________________________ @@ -1140,17 +1140,17 @@ IndexImpl::PermutationImpl& IndexImpl::getPermutation(Index::Permutation p) { using enum Index::Permutation; switch (p) { case PSO: - return _PSO; + return PSO_; case POS: - return _POS; + return POS_; case SPO: - return _SPO; + return SPO_; case SOP: - return _SOP; + return SOP_; case OSP: - return _OSP; + return OSP_; case OPS: - return _OPS; + return OPS_; } AD_FAIL(); } @@ -1164,8 +1164,8 @@ const IndexImpl::PermutationImpl& IndexImpl::getPermutation( // __________________________________________________________________________ Index::NumNormalAndInternal IndexImpl::numDistinctSubjects() const { if (hasAllPermutations()) { - auto numActually = _numSubjectsNormal; - return {numActually, _SPO.metaData().getNofDistinctC1() - numActually}; + auto numActually = numSubjectsNormal_; + return {numActually, SPO_.metaData().getNofDistinctC1() - numActually}; } else { AD_THROW( "Can only get # distinct subjects if all 6 permutations " @@ -1177,8 +1177,8 @@ Index::NumNormalAndInternal IndexImpl::numDistinctSubjects() const { // __________________________________________________________________________ Index::NumNormalAndInternal IndexImpl::numDistinctObjects() const { if (hasAllPermutations()) { - auto numActually = _numObjectsNormal; - return {numActually, _OSP.metaData().getNofDistinctC1() - numActually}; + auto numActually = numObjectsNormal_; + return {numActually, OSP_.metaData().getNofDistinctC1() - numActually}; } else { AD_THROW( "Can only get # distinct objects if all 6 permutations " @@ -1189,8 +1189,8 @@ Index::NumNormalAndInternal IndexImpl::numDistinctObjects() const { // __________________________________________________________________________ Index::NumNormalAndInternal IndexImpl::numDistinctPredicates() const { - auto numActually = _numPredicatesNormal; - return {numActually, _PSO.metaData().getNofDistinctC1() - numActually}; + auto numActually = numPredicatesNormal_; + return {numActually, PSO_.metaData().getNofDistinctC1() - numActually}; } // __________________________________________________________________________ @@ -1249,7 +1249,7 @@ std::optional IndexImpl::idToOptionalString(Id id) const { case Datatype::Int: return std::to_string(id.getInt()); case Datatype::VocabIndex: { - auto result = _vocab.indexToOptionalString(id.getVocabIndex()); + auto result = vocab_.indexToOptionalString(id.getVocabIndex()); if (result.has_value() && result.value().starts_with(VALUE_PREFIX)) { result = ad_utility::convertIndexWordToValueLiteral(result.value()); } @@ -1278,7 +1278,7 @@ bool IndexImpl::getId(const string& element, Id* id) const { // ___________________________________________________________________________ std::pair IndexImpl::prefix_range(const std::string& prefix) const { // TODO Do we need prefix ranges for numbers? - auto [begin, end] = _vocab.prefix_range(prefix); + auto [begin, end] = vocab_.prefix_range(prefix); return {Id::makeFromVocabIndex(begin), Id::makeFromVocabIndex(end)}; } @@ -1354,7 +1354,7 @@ void IndexImpl::scan(const TripleComponent& col0String, // _____________________________________________________________________________ void IndexImpl::deleteTemporaryFile(const string& path) { - if (!_keepTempFiles) { + if (!keepTempFiles_) { ad_utility::deleteFile(path); } } diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h index 3a6457a1fc..d2741462ad 100644 --- a/src/index/IndexImpl.h +++ b/src/index/IndexImpl.h @@ -108,67 +108,66 @@ class IndexImpl { // Private data members. private: - string _onDiskBase; - string _settingsFileName; - bool _onlyAsciiTurtlePrefixes = false; - TurtleParserIntegerOverflowBehavior _turtleParserIntegerOverflowBehavior = + string onDiskBase_; + string settingsFileName_; + bool onlyAsciiTurtlePrefixes_ = false; + TurtleParserIntegerOverflowBehavior turtleParserIntegerOverflowBehavior_ = TurtleParserIntegerOverflowBehavior::Error; - bool _turtleParserSkipIllegalLiterals = false; - bool _keepTempFiles = false; - uint64_t _stxxlMemoryInBytes = DEFAULT_STXXL_MEMORY_IN_BYTES; - json _configurationJson; - Vocabulary _vocab; - size_t _totalVocabularySize = 0; - bool _vocabPrefixCompressed = true; - Vocabulary _textVocab; - - TextMetaData _textMeta; - DocsDB _docsDB; - vector _blockBoundaries; - off_t _currentoff_t; - mutable ad_utility::File _textIndexFile; + bool turtleParserSkipIllegalLiterals_ = false; + bool keepTempFiles_ = false; + uint64_t stxxlMemoryInBytes_ = DEFAULT_STXXL_MEMORY_IN_BYTES; + json configurationJson_; + Vocabulary vocab_; + size_t totalVocabularySize_ = 0; + bool vocabPrefixCompressed_ = true; + Vocabulary textVocab_; + + TextMetaData textMeta_; + DocsDB docsDB_; + vector blockBoundaries_; + off_t currentoff_t_; + mutable ad_utility::File textIndexFile_; // If false, only PSO and POS permutations are loaded and expected. - bool _loadAllPermutations = true; + bool loadAllPermutations_ = true; // Pattern trick data - bool _usePatterns; - double _avgNumDistinctPredicatesPerSubject; - double _avgNumDistinctSubjectsPerPredicate; - size_t _numDistinctSubjectPredicatePairs; + bool usePatterns_; + double avgNumDistinctPredicatesPerSubject_; + double avgNumDistinctSubjectsPerPredicate_; + size_t numDistinctSubjectPredicatePairs_; - size_t _parserBatchSize = PARSER_BATCH_SIZE; - size_t _numTriplesPerBatch = NUM_TRIPLES_PER_PARTIAL_VOCAB; + size_t parserBatchSize_ = PARSER_BATCH_SIZE; + size_t numTriplesPerBatch_ = NUM_TRIPLES_PER_PARTIAL_VOCAB; // These statistics all do *not* include the triples that are added by // QLever for more efficient query processing. - size_t _numSubjectsNormal = 0; - size_t _numPredicatesNormal = 0; - size_t _numObjectsNormal = 0; - size_t _numTriplesNormal = 0; + size_t numSubjectsNormal_ = 0; + size_t numPredicatesNormal_ = 0; + size_t numObjectsNormal_ = 0; + size_t numTriplesNormal_ = 0; /** * @brief Maps pattern ids to sets of predicate ids. */ - CompactVectorOfStrings _patterns; + CompactVectorOfStrings patterns_; /** * @brief Maps entity ids to pattern ids. */ - std::vector _hasPattern; + std::vector hasPattern_; /** * @brief Maps entity ids to sets of predicate ids */ - CompactVectorOfStrings _hasPredicate; + CompactVectorOfStrings hasPredicate_; - public: // TODO: make those private and allow only const access // instantiations for the six permutations used in QLever. // They simplify the creation of permutations in the index class. - PermutationImpl _POS{"POS", ".pos", {1, 2, 0}}; - PermutationImpl _PSO{"PSO", ".pso", {1, 0, 2}}; - PermutationImpl _SOP{"SOP", ".sop", {0, 2, 1}}; - PermutationImpl _SPO{"SPO", ".spo", {0, 1, 2}}; - PermutationImpl _OPS{"OPS", ".ops", {2, 1, 0}}; - PermutationImpl _OSP{"OSP", ".osp", {2, 0, 1}}; + PermutationImpl POS_{"POS", ".pos", {1, 2, 0}}; + PermutationImpl PSO_{"PSO", ".pso", {1, 0, 2}}; + PermutationImpl SOP_{"SOP", ".sop", {0, 2, 1}}; + PermutationImpl SPO_{"SPO", ".spo", {0, 1, 2}}; + PermutationImpl OPS_{"OPS", ".ops", {2, 1, 0}}; + PermutationImpl OSP_{"OSP", ".osp", {2, 0, 1}}; public: IndexImpl(); @@ -181,21 +180,21 @@ class IndexImpl { IndexImpl& operator=(IndexImpl&&) = delete; IndexImpl(IndexImpl&&) = delete; - const auto& POS() const { return _POS; } - auto& POS() { return _POS; } - const auto& PSO() const { return _PSO; } - auto& PSO() { return _PSO; } - const auto& SPO() const { return _SPO; } - auto& SPO() { return _SPO; } - const auto& SOP() const { return _SOP; } - auto& SOP() { return _SOP; } - const auto& OPS() const { return _OPS; } - auto& OPS() { return _OPS; } - const auto& OSP() const { return _OSP; } - auto& OSP() { return _OSP; } + const auto& POS() const { return POS_; } + auto& POS() { return POS_; } + const auto& PSO() const { return PSO_; } + auto& PSO() { return PSO_; } + const auto& SPO() const { return SPO_; } + auto& SPO() { return SPO_; } + const auto& SOP() const { return SOP_; } + auto& SOP() { return SOP_; } + const auto& OPS() const { return OPS_; } + auto& OPS() { return OPS_; } + const auto& OSP() const { return OSP_; } + auto& OSP() { return OSP_; } // For a given `Permutation` (e.g. `PSO`) return the corresponding - // `PermutationImpl` object by reference (`_PSO`). + // `PermutationImpl` object by reference (`PSO_`). PermutationImpl& getPermutation(Index::Permutation p); const PermutationImpl& getPermutation(Index::Permutation p) const; @@ -225,10 +224,10 @@ class IndexImpl { // Read necessary meta data into memory and opens file handles. void addTextFromOnDiskIndex(); - const auto& getVocab() const { return _vocab; }; - auto& getNonConstVocabForTesting() { return _vocab; } + const auto& getVocab() const { return vocab_; }; + auto& getNonConstVocabForTesting() { return vocab_; } - const auto& getTextVocab() const { return _textVocab; }; + const auto& getTextVocab() const { return textVocab_; }; // -------------------------------------------------------------------------- // -- RETRIEVAL --- @@ -346,10 +345,10 @@ class IndexImpl { vector& scores) const; string getTextExcerpt(TextRecordIndex cid) const { - if (cid.get() >= _docsDB._size) { + if (cid.get() >= docsDB_._size) { return ""; } - return _docsDB.getTextExcerpt(cid); + return docsDB_.getTextExcerpt(cid); } // Only for debug reasons and external encoding tests. @@ -359,7 +358,7 @@ class IndexImpl { void dumpAsciiLists(const TextBlockMetaData& tbmd) const; float getAverageNofEntityContexts() const { - return _textMeta.getAverageNofEntityContexts(); + return textMeta_.getAverageNofEntityContexts(); }; void setKbName(const string& name); @@ -372,8 +371,8 @@ class IndexImpl { void setKeepTempFiles(bool keepTempFiles); - uint64_t& stxxlMemoryInBytes() { return _stxxlMemoryInBytes; } - const uint64_t& stxxlMemoryInBytes() const { return _stxxlMemoryInBytes; } + uint64_t& stxxlMemoryInBytes() { return stxxlMemoryInBytes_; } + const uint64_t& stxxlMemoryInBytes() const { return stxxlMemoryInBytes_; } void setOnDiskBase(const std::string& onDiskBase); @@ -382,17 +381,17 @@ class IndexImpl { void setPrefixCompression(bool compressed); void setNumTriplesPerBatch(uint64_t numTriplesPerBatch) { - _numTriplesPerBatch = numTriplesPerBatch; + numTriplesPerBatch_ = numTriplesPerBatch; } - const string& getTextName() const { return _textMeta.getName(); } + const string& getTextName() const { return textMeta_.getName(); } - const string& getKbName() const { return _PSO.metaData().getName(); } + const string& getKbName() const { return PSO_.metaData().getName(); } - size_t getNofTextRecords() const { return _textMeta.getNofTextRecords(); } - size_t getNofWordPostings() const { return _textMeta.getNofWordPostings(); } + size_t getNofTextRecords() const { return textMeta_.getNofTextRecords(); } + size_t getNofWordPostings() const { return textMeta_.getNofWordPostings(); } size_t getNofEntityPostings() const { - return _textMeta.getNofEntityPostings(); + return textMeta_.getNofEntityPostings(); } bool hasAllPermutations() const { return SPO()._isLoaded; } @@ -457,7 +456,7 @@ class IndexImpl { // Create Vocabulary and directly write it to disk. Create TripleVec with all // the triples converted to id space. This Vec can be used for creating - // permutations. Member _vocab will be empty after this because it is not + // permutations. Member vocab_ will be empty after this because it is not // needed for index creation once the TripleVec is set up and it would be a // waste of RAM. template @@ -570,7 +569,7 @@ class IndexImpl { size_t getIndexOfBestSuitedElTerm(const vector& terms) const; /// Calculate the block boundaries for the text index. The boundary of a - /// block is the index in the `_textVocab` of the last word that belongs + /// block is the index in the `textVocab_` of the last word that belongs /// to this block. /// This implementation takes a reference to an `IndexImpl` and a callable, /// that is called once for each blockBoundary, with the `size_t` @@ -581,7 +580,7 @@ class IndexImpl { I&& index, const BlockBoundaryAction& blockBoundaryAction); /// Calculate the block boundaries for the text index, and store them in the - /// _blockBoundaries member. + /// blockBoundaries_ member. void calculateBlockBoundaries(); /// Calculate the block boundaries for the text index, and store the @@ -654,7 +653,7 @@ class IndexImpl { void readIndexBuilderSettingsFromFile(); /** - * Delete a temporary file unless the _keepTempFiles flag is set + * Delete a temporary file unless the keepTempFiles_ flag is set * @param path */ void deleteTemporaryFile(const string& path); diff --git a/test/IndexTest.cpp b/test/IndexTest.cpp index fd304fe1de..a15f27727b 100644 --- a/test/IndexTest.cpp +++ b/test/IndexTest.cpp @@ -72,14 +72,14 @@ TEST(IndexTest, createFromTurtleTest) { Id c2 = getId(""); // TODO We could also test the multiplicities here. - ASSERT_TRUE(index._PSO.metaData().col0IdExists(b)); - ASSERT_TRUE(index._PSO.metaData().col0IdExists(b2)); - ASSERT_FALSE(index._PSO.metaData().col0IdExists(a)); - ASSERT_FALSE(index._PSO.metaData().col0IdExists(c)); - ASSERT_FALSE(index._PSO.metaData().col0IdExists( + ASSERT_TRUE(index.PSO().metaData().col0IdExists(b)); + ASSERT_TRUE(index.PSO().metaData().col0IdExists(b2)); + ASSERT_FALSE(index.PSO().metaData().col0IdExists(a)); + ASSERT_FALSE(index.PSO().metaData().col0IdExists(c)); + ASSERT_FALSE(index.PSO().metaData().col0IdExists( Id::makeFromVocabIndex(VocabIndex::make(735)))); - ASSERT_FALSE(index._PSO.metaData().getMetaData(b).isFunctional()); - ASSERT_TRUE(index._PSO.metaData().getMetaData(b2).isFunctional()); + ASSERT_FALSE(index.PSO().metaData().getMetaData(b).isFunctional()); + ASSERT_TRUE(index.PSO().metaData().getMetaData(b2).isFunctional()); ASSERT_TRUE(index.POS().metaData().col0IdExists(b)); ASSERT_TRUE(index.POS().metaData().col0IdExists(b2)); @@ -130,10 +130,10 @@ TEST(IndexTest, createFromTurtleTest) { Id c = getId(""); Id isA = getId(""); - ASSERT_TRUE(index._PSO.metaData().col0IdExists(isA)); - ASSERT_FALSE(index._PSO.metaData().col0IdExists(a)); + ASSERT_TRUE(index.PSO().metaData().col0IdExists(isA)); + ASSERT_FALSE(index.PSO().metaData().col0IdExists(a)); - ASSERT_FALSE(index._PSO.metaData().getMetaData(isA).isFunctional()); + ASSERT_FALSE(index.PSO().metaData().getMetaData(isA).isFunctional()); ASSERT_TRUE(index.POS().metaData().col0IdExists(isA)); ASSERT_FALSE(index.POS().metaData().getMetaData(isA).isFunctional()); @@ -494,12 +494,15 @@ TEST(IndexTest, NumDistinctEntities) { EXPECT_EQ(numTriples.normal_, 7); // Two added triples for each triple that has an object with a language tag. EXPECT_EQ(numTriples.internal_, 2); - } TEST(IndexTest, NumDistinctEntitiesCornerCases) { const IndexImpl& index = getQec("", false)->getIndex().getImpl(); - AD_EXPECT_THROW_WITH_MESSAGE(index.numDistinctSubjects(), ::testing::ContainsRegex("if all 6")); - AD_EXPECT_THROW_WITH_MESSAGE(index.numDistinctObjects(), ::testing::ContainsRegex("if all 6")); - AD_EXPECT_THROW_WITH_MESSAGE(index.numDistinctCol0(static_cast(42)), ::testing::ContainsRegex("should be unreachable")); + AD_EXPECT_THROW_WITH_MESSAGE(index.numDistinctSubjects(), + ::testing::ContainsRegex("if all 6")); + AD_EXPECT_THROW_WITH_MESSAGE(index.numDistinctObjects(), + ::testing::ContainsRegex("if all 6")); + AD_EXPECT_THROW_WITH_MESSAGE( + index.numDistinctCol0(static_cast(42)), + ::testing::ContainsRegex("should be unreachable")); } diff --git a/test/IndexTestHelpers.h b/test/IndexTestHelpers.h index 2d7bc46d08..91b66fed5b 100644 --- a/test/IndexTestHelpers.h +++ b/test/IndexTestHelpers.h @@ -60,7 +60,8 @@ inline std::vector getAllIndexFilenames( // for the subclasses of `SparqlExpression`. // The concrete triple contents are currently used in `GroupByTest.cpp`. inline Index makeTestIndex(const std::string& indexBasename, - std::string turtleInput = "", bool loadAllPermutations = true) { + std::string turtleInput = "", + bool loadAllPermutations = true) { // Ignore the (irrelevant) log output of the index building and loading during // these tests. static std::ostringstream ignoreLogStream; @@ -95,7 +96,8 @@ inline Index makeTestIndex(const std::string& indexBasename, // build using `makeTestIndex` (see above). The index (most notably its // vocabulary) is the only part of the `QueryExecutionContext` that is actually // relevant for these tests, so the other members are defaulted. -inline QueryExecutionContext* getQec(std::string turtleInput = "", bool loadAllPermutations = true) { +inline QueryExecutionContext* getQec(std::string turtleInput = "", + bool loadAllPermutations = true) { // Similar to `absl::Cleanup`. Calls the `callback_` in the destructor, but // the callback is stored as a `std::function`, which allows to store // different types of callbacks in the same wrapper type. @@ -126,17 +128,17 @@ inline QueryExecutionContext* getQec(std::string turtleInput = "", bool loadAllP "_staticGlobalTestIndex" + std::to_string(contextMap.size()); contextMap.emplace( key, Context{TypeErasedCleanup{[testIndexBasename]() { - for (const std::string& indexFilename : - getAllIndexFilenames(testIndexBasename)) { - // Don't log when a file can't be deleted, - // because the logging might already be - // destroyed. - ad_utility::deleteFile(indexFilename, false); - } - }}, - std::make_unique( - makeTestIndex(testIndexBasename, turtleInput, loadAllPermutations)), - std::make_unique()}); + for (const std::string& indexFilename : + getAllIndexFilenames(testIndexBasename)) { + // Don't log when a file can't be deleted, + // because the logging might already be + // destroyed. + ad_utility::deleteFile(indexFilename, false); + } + }}, + std::make_unique(makeTestIndex( + testIndexBasename, turtleInput, loadAllPermutations)), + std::make_unique()}); } return contextMap.at(key).qec_.get(); } From 6079437aa09546c67b4921a873fd42739cfa79bb Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 4 May 2023 15:58:52 +0200 Subject: [PATCH 012/150] Fixed some code smells. --- src/index/IndexImpl.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index bec38b5f11..94578100b2 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -526,7 +526,7 @@ IndexImpl::createPermutationPairImpl(const string& fileName1, metaData1.add(md1); metaData2.add(md2); }; - for (auto triple : sortedTriples) { + for (auto triple : AD_FWD(sortedTriples)) { if (!currentRel.has_value()) { currentRel = triple[c0]; } @@ -1213,8 +1213,8 @@ Index::NumNormalAndInternal IndexImpl::numDistinctCol0( // ___________________________________________________________________________ size_t IndexImpl::getCardinality(Id id, Index::Permutation permutation) const { - const auto& p = getPermutation(permutation); - if (p.metaData().col0IdExists(id)) { + if (const auto& p = getPermutation(permutation); + p.metaData().col0IdExists(id)) { return p.metaData().getMetaData(id).getNofElements(); } return 0; @@ -1231,8 +1231,7 @@ size_t IndexImpl::getCardinality(const TripleComponent& comp, if (comp == INTERNAL_TEXT_MATCH_PREDICATE) { return TEXT_PREDICATE_CARDINALITY_ESTIMATE; } - std::optional relId = comp.toValueId(getVocab()); - if (relId.has_value()) { + if (std::optional relId = comp.toValueId(getVocab()); relId.has_value()) { return getCardinality(relId.value(), permutation); } return 0; @@ -1241,24 +1240,25 @@ size_t IndexImpl::getCardinality(const TripleComponent& comp, // TODO Once we have an overview over the folding this logic should // probably not be in the index class. std::optional IndexImpl::idToOptionalString(Id id) const { + using enum Datatype; switch (id.getDatatype()) { - case Datatype::Undefined: + case Undefined: return std::nullopt; - case Datatype::Double: + case Double: return std::to_string(id.getDouble()); - case Datatype::Int: + case Int: return std::to_string(id.getInt()); - case Datatype::VocabIndex: { + case VocabIndex: { auto result = vocab_.indexToOptionalString(id.getVocabIndex()); if (result.has_value() && result.value().starts_with(VALUE_PREFIX)) { result = ad_utility::convertIndexWordToValueLiteral(result.value()); } return result; } - case Datatype::LocalVocabIndex: + case LocalVocabIndex: // TODO:: this is why this shouldn't be here return std::nullopt; - case Datatype::TextRecordIndex: + case TextRecordIndex: return getTextExcerpt(id.getTextRecordIndex()); } // should be unreachable because the enum is exhaustive. @@ -1324,10 +1324,10 @@ void IndexImpl::scan(const TripleComponent& key, IdTable* result, const auto& p = getPermutation(permutation); LOG(DEBUG) << "Performing " << p._readableName << " scan for full list for: " << key << "\n"; - std::optional optionalId = key.toValueId(getVocab()); - if (optionalId.has_value()) { + + if (std::optional id = key.toValueId(getVocab()); id.has_value()) { LOG(TRACE) << "Successfully got key ID.\n"; - scan(optionalId.value(), result, permutation, std::move(timer)); + scan(id.value(), result, permutation, std::move(timer)); } LOG(DEBUG) << "Scan done, got " << result->size() << " elements.\n"; } From 75897aa60c233739f7a43e511008eef76e2740f1 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 4 May 2023 16:48:30 +0200 Subject: [PATCH 013/150] Continuing after merge. --- src/engine/IndexScan.cpp | 1 + src/index/CompressedRelation.cpp | 21 +++++++++++++-------- src/index/CompressedRelation.h | 2 ++ src/index/IndexImpl.cpp | 5 ----- src/index/Permutations.h | 2 +- src/util/JoinAlgorithms/JoinAlgorithms.h | 4 +++- 6 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 884cfad92b..ebc96a769d 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -279,4 +279,5 @@ void IndexScan::computeFullScan(IdTable* result, std::array, 2> IndexScan::lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2) { const auto& index = s1.getExecutionContext()->getIndex().getImpl(); + } diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index 4006baeccc..fd076ea877 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -301,7 +301,6 @@ std::vector CompressedRelationReader::getBlocksForJoin( return result; } - std::array, 2> CompressedRelationReader::getBlocksForJoin(const CompressedRelationMetadata& md1, const CompressedRelationMetadata& md2, std::span blockMetadata) { // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ struct KeyLhs { @@ -334,24 +333,28 @@ std::array, 2> CompressedRelationReader::ge return a.col0FirstId_ < b.col0FirstId_ && a.col0LastId_ < b.col0LastId_; }); - auto blockLessThanId = [](const CompressedBlockMetadata& block1, const CompressedBlockMetadata& block2) { + auto blockLessThanBlock = [](const CompressedBlockMetadata& block1, const CompressedBlockMetadata& block2) { if (block1.col0LastId_ < block2.col0FirstId_ || block1.col0FirstId_ > block2.col0LastId_) { return block1.col0LastId_ < block2.col0LastId_; + } else { + return false; } }; - auto lessThan = - ad_utility::OverloadCallOperator{idLessThanBlock, blockLessThanId}; - std::vector result; + std::array, 2> result; auto addRow = [&result]([[maybe_unused]] auto it1, auto it2) { - result.push_back(*it2); + result[0].push_back(*it1); + result[1].push_back(*it2); }; auto noop = ad_utility::noop; [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( - joinColum, std::span(beginBlock, endBlock), - lessThan, addRow, noop, noop); + std::span{beginBlock1, endBlock1}, std::span(beginBlock2, endBlock2), + blockLessThanBlock, addRow, noop, noop); + for (auto& vec : result) { + vec.erase(std::unique(vec.begin(), vec.end()), vec.end()); + } return result; } @@ -720,3 +723,5 @@ CompressedRelationWriter::compressAndWriteColumn(std::span column) { outfile_.write(compressedBlock.data(), compressedBlock.size()); return {offsetInFile, compressedSize}; }; + +//CompressedRelationWriter::getBlocksFromMetadata \ No newline at end of file diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h index bc2c625a59..5d124eef70 100644 --- a/src/index/CompressedRelation.h +++ b/src/index/CompressedRelation.h @@ -258,6 +258,8 @@ class CompressedRelationReader { ad_utility::File& file, IdTable* result, ad_utility::SharedConcurrentTimeoutTimer timer) const; + std::span getBlocksFromMetadata(const CompressedRelationMetadata& metadata, std::optional col1Id, std::span blockMetadata); + // Get all the blocks that can contain an Id from the `joinColumn`. // TODO Include a timeout check. std::vector getBlocksForJoin( diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index 94578100b2..36fd5b4e11 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -1322,14 +1322,9 @@ void IndexImpl::scan(const TripleComponent& key, IdTable* result, Index::Permutation permutation, ad_utility::SharedConcurrentTimeoutTimer timer) const { const auto& p = getPermutation(permutation); - LOG(DEBUG) << "Performing " << p._readableName - << " scan for full list for: " << key << "\n"; - if (std::optional id = key.toValueId(getVocab()); id.has_value()) { - LOG(TRACE) << "Successfully got key ID.\n"; scan(id.value(), result, permutation, std::move(timer)); } - LOG(DEBUG) << "Scan done, got " << result->size() << " elements.\n"; } // _____________________________________________________________________________ diff --git a/src/index/Permutations.h b/src/index/Permutations.h index 2b9cee300a..fc3e87d600 100644 --- a/src/index/Permutations.h +++ b/src/index/Permutations.h @@ -88,7 +88,7 @@ class PermutationImpl { if (!_meta.col0IdExists(col0Id)) { return std::nullopt; } - return {_meta.getMetaData(col0Id), _meta.blockData()}; + return MetaDataAndBlocks{_meta.getMetaData(col0Id), _meta.blockData()}; } cppcoro::generator lazyScan(Id col0Id, const std::vector& blocks, ad_utility::AllocatorWithLimit allocator, ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) { diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 4cea2c7668..771c9e36f1 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -165,7 +165,9 @@ template < // element of `left` or `right`) constains no UNDEF values. It is used inside // the following `mergeWithUndefRight` function. auto containsNoUndefined = [](const T& row) { - if constexpr (std::is_same_v) { + if constexpr (isSimilar && isSimilar) { + return true; + } else if constexpr (std::is_same_v) { return row != Id::makeUndefined(); } else { return (std::ranges::none_of( From 16b45a53d9a9d1d25f54d631b9e2dd9e691b4192 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 4 May 2023 18:10:36 +0200 Subject: [PATCH 014/150] Implemented the IndexScan::lazyScansForJoin method. Next step: actually implement the logic in the JOIN class. --- src/engine/IndexScan.cpp | 51 ++- src/engine/IndexScan.h | 3 +- src/engine/Join.cpp | 5 +- src/index/CompressedRelation.cpp | 437 +++++++++++++---------- src/index/CompressedRelation.h | 84 +++-- src/index/Permutations.h | 32 +- src/util/JoinAlgorithms/JoinAlgorithms.h | 3 +- 7 files changed, 385 insertions(+), 230 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index ebc96a769d..818b52fbf1 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -277,7 +277,56 @@ void IndexScan::computeFullScan(IdTable* result, *result = std::move(table).toDynamic(); } -std::array, 2> IndexScan::lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2) { +std::array, 2> IndexScan::lazyScanForJoinOfTwoScans( + const IndexScan& s1, const IndexScan& s2) { const auto& index = s1.getExecutionContext()->getIndex().getImpl(); + AD_CONTRACT_CHECK(s1._numVariables < 3 && s2._numVariables < 3); + const auto& s = s1; + auto f = [&](const IndexScan& s) + -> std::optional { + auto permutedTriple = s.getPermutedTriple(); + std::optional optId = + s.getPermutedTriple()[0]->toValueId(index.getVocab()); + std::optional optId2 = + s._numVariables == 2 + ? std::nullopt + : s.getPermutedTriple()[1]->toValueId(index.getVocab()); + if (!optId.has_value() || (!optId2.has_value() && s._numVariables == 1)) { + return std::nullopt; + } + return index.getPermutation(s.permutation()) + .getMetadataAndBlocks(optId.value(), optId2); + }; + + auto metaBlocks1 = f(s1); + auto metaBlocks2 = f(s2); + + if (!metaBlocks1.has_value() || !metaBlocks2.has_value()) { + return {{}}; + } + auto [blocks1, blocks2] = CompressedRelationReader::getBlocksForJoin( + metaBlocks1.value().relationMetadata_, + metaBlocks2.value().relationMetadata_, metaBlocks1.value().blockMetadata_, + metaBlocks2.value().blockMetadata_); + + // TODO include a timeout timer. + auto getScan = [&index](const IndexScan& s, const auto& blocks) { + // TODO pass the IDs here. + Id col0Id = s.getPermutedTriple()[0]->toValueId(index.getVocab()).value(); + std::optional col1Id; + if (s._numVariables == 1) { + col1Id = s.getPermutedTriple()[1]->toValueId(index.getVocab()).value(); + } + if (!col1Id.has_value()) { + return index.getPermutation(s.permutation()) + .lazyScan(col0Id, blocks, s.getExecutionContext()->getAllocator()); + } else { + return index.getPermutation(s.permutation()) + .lazyScan(col0Id, col1Id.value(), blocks, + s.getExecutionContext()->getAllocator()); + } + }; + + return {getScan(s1, blocks1), getScan(s2, blocks2)}; } diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index 268653a281..d2797a49ad 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -153,7 +153,8 @@ class IndexScan : public Operation { // can be read from the Metadata. size_t getExactSize() const { return _sizeEstimate; } - static std::array, 2> lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2); + static std::array, 2> lazyScanForJoinOfTwoScans( + const IndexScan& s1, const IndexScan& s2); private: // TODO Make the `getSizeEstimateBeforeLimit()` function `const` for diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp index c6ee1f0bee..fb0b311813 100644 --- a/src/engine/Join.cpp +++ b/src/engine/Join.cpp @@ -37,7 +37,7 @@ Join::Join(QueryExecutionContext* qec, std::shared_ptr t1, std::swap(t1, t2); std::swap(t1JoinCol, t2JoinCol); } - if (isFullScanDummy(t1) || t1->getType() == QueryExecutionTree::SCAN) { + if (isFullScanDummy(t1)) { AD_CONTRACT_CHECK(!isFullScanDummy(t2)); std::swap(t1, t2); std::swap(t1JoinCol, t2JoinCol); @@ -610,5 +610,6 @@ void Join::addCombinedRowToIdTable(const ROW_A& rowA, const ROW_B& rowB, // ______________________________________________________________________________________________________ ResultTable Join::computeResultForTwoIndexScans() { - AD_CORRECTNESS_CHECK(_left->getType() == QueryExecutionTree::SCAN && _right->getType() == QueryExecutionTree::SCAN); + AD_CORRECTNESS_CHECK(_left->getType() == QueryExecutionTree::SCAN && + _right->getType() == QueryExecutionTree::SCAN); } diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index fd076ea877..0c59cb339d 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -23,27 +23,11 @@ void CompressedRelationReader::scan( ad_utility::SharedConcurrentTimeoutTimer timer) const { AD_CONTRACT_CHECK(result->numColumns() == NumColumns); - // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ - struct KeyLhs { - Id col0FirstId_; - Id col0LastId_; - }; + auto relevantBlocks = + getBlocksFromMetadata(metadata, std::nullopt, blockMetadata); + auto beginBlock = relevantBlocks.begin(); + auto endBlock = relevantBlocks.end(); Id col0Id = metadata.col0Id_; - // TODO Use a structured binding. Structured bindings are - // currently not supported by clang when using OpenMP because clang internally - // transforms the `#pragma`s into lambdas, and capturing structured bindings - // is only supported in clang >= 16. - decltype(blockMetadata.begin()) beginBlock, endBlock; - std::tie(beginBlock, endBlock) = std::equal_range( - // TODO For some reason we can't use `std::ranges::equal_range`, - // find out why. Note: possibly it has something to do with the limited - // support of ranges in clang with versions < 16. Revisit this when - // we use clang 16. - blockMetadata.begin(), blockMetadata.end(), KeyLhs{col0Id, col0Id}, - [](const auto& a, const auto& b) { - return a.col0FirstId_ < b.col0FirstId_ && a.col0LastId_ < b.col0LastId_; - }); - // The total size of the result is now known. result->resize(metadata.getNofElements()); @@ -83,7 +67,7 @@ void CompressedRelationReader::scan( auto cacheKey = block.offsetsAndCompressedSize_.at(0).offsetInFile_; auto uncompressedBuffer = blockCache_ .computeOnce(cacheKey, - [&]() { + [&]() { return readAndDecompressBlock( block, file, std::nullopt); }) @@ -155,128 +139,192 @@ void CompressedRelationReader::scan( } // _____________________________________________________________________________ -cppcoro::generator CompressedRelationReader::lazyScan(const CompressedRelationMetadata& metadata, std::span blockMetadata, ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, ad_utility::SharedConcurrentTimeoutTimer timer) { - // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ - struct KeyLhs { - Id col0FirstId_; - Id col0LastId_; - }; - Id col0Id = metadata.col0Id_; - // TODO Use a structured binding. Structured bindings are - // currently not supported by clang when using OpenMP because clang internally - // transforms the `#pragma`s into lambdas, and capturing structured bindings - // is only supported in clang >= 16. - decltype(blockMetadata.begin()) beginBlock, endBlock; - std::tie(beginBlock, endBlock) = std::equal_range( - // TODO For some reason we can't use `std::ranges::equal_range`, - // find out why. Note: possibly it has something to do with the limited - // support of ranges in clang with versions < 16. Revisit this when - // we use clang 16. - blockMetadata.begin(), blockMetadata.end(), KeyLhs{col0Id, col0Id}, - [](const auto& a, const auto& b) { - return a.col0FirstId_ < b.col0FirstId_ && a.col0LastId_ < b.col0LastId_; - }); +cppcoro::generator CompressedRelationReader::lazyScan( + const CompressedRelationMetadata& metadata, + std::span blockMetadata, + ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, + ad_utility::SharedConcurrentTimeoutTimer timer) const { + auto relevantBlocks = + getBlocksFromMetadata(metadata, std::nullopt, blockMetadata); + auto beginBlock = relevantBlocks.begin(); + auto endBlock = relevantBlocks.end(); + Id col0Id = metadata.col0Id_; + // The first block might contain entries that are not part of our + // actual scan result. + bool firstBlockIsIncomplete = + beginBlock < endBlock && + (beginBlock->col0FirstId_ < col0Id || beginBlock->col0LastId_ > col0Id); + auto lastBlock = endBlock - 1; + + bool lastBlockIsIncomplete = + beginBlock < lastBlock && + (lastBlock->col0FirstId_ < col0Id || lastBlock->col0LastId_ > col0Id); - // The first block might contain entries that are not part of our - // actual scan result. - bool firstBlockIsIncomplete = - beginBlock < endBlock && - (beginBlock->col0FirstId_ < col0Id || beginBlock->col0LastId_ > col0Id); - auto lastBlock = endBlock - 1; - - bool lastBlockIsIncomplete = - beginBlock < lastBlock && - (lastBlock->col0FirstId_ < col0Id || lastBlock->col0LastId_ > col0Id); - - // Invariant: A relation spans multiple blocks exclusively or several - // entities are stored completely in the same Block. - AD_CORRECTNESS_CHECK(!firstBlockIsIncomplete || (beginBlock == lastBlock)); - AD_CORRECTNESS_CHECK(!lastBlockIsIncomplete); - if (firstBlockIsIncomplete) { - AD_CORRECTNESS_CHECK(metadata.offsetInBlock_ != - std::numeric_limits::max()); + // Invariant: A relation spans multiple blocks exclusively or several + // entities are stored completely in the same Block. + AD_CORRECTNESS_CHECK(!firstBlockIsIncomplete || (beginBlock == lastBlock)); + AD_CORRECTNESS_CHECK(!lastBlockIsIncomplete); + if (firstBlockIsIncomplete) { + AD_CORRECTNESS_CHECK(metadata.offsetInBlock_ != + std::numeric_limits::max()); + } + + // We have at most one block that is incomplete and thus requires trimming. + // Set up a lambda, that reads this block and decompresses it to + // the result. + auto readIncompleteBlock = [&](const auto& block) { + // A block is uniquely identified by its start position in the file. + auto cacheKey = block.offsetsAndCompressedSize_.at(0).offsetInFile_; + auto uncompressedBuffer = blockCache_ + .computeOnce(cacheKey, + [&]() { + return readAndDecompressBlock( + block, file, std::nullopt); + }) + ._resultPointer; + + // Extract the part of the block that actually belongs to the relation + auto numElements = metadata.numRows_; + AD_CORRECTNESS_CHECK(uncompressedBuffer->numColumns() == + metadata.numColumns()); + IdTable result(uncompressedBuffer->numColumns(), allocator); + result.resize(numElements); + for (size_t i = 0; i < uncompressedBuffer->numColumns(); ++i) { + const auto& inputCol = uncompressedBuffer->getColumn(i); + auto begin = inputCol.begin() + metadata.offsetInBlock_; + decltype(auto) resultColumn = result.getColumn(i); + std::copy(begin, begin + numElements, resultColumn.begin()); } + return result; + }; - // We have at most one block that is incomplete and thus requires trimming. - // Set up a lambda, that reads this block and decompresses it to - // the result. - auto readIncompleteBlock = [&](const auto& block) { - // A block is uniquely identified by its start position in the file. - auto cacheKey = block.offsetsAndCompressedSize_.at(0).offsetInFile_; - auto uncompressedBuffer = blockCache_ - .computeOnce(cacheKey, - [&]() { - return readAndDecompressBlock( - block, file, std::nullopt); - }) - ._resultPointer; - - // Extract the part of the block that actually belongs to the relation - auto numElements = metadata.numRows_; - AD_CORRECTNESS_CHECK(uncompressedBuffer->numColumns() == - metadata.numColumns()); - IdTable result(uncompressedBuffer->numColumns(), allocator); - result.resize(numElements); - for (size_t i = 0; i < uncompressedBuffer->numColumns(); ++i) { - const auto& inputCol = uncompressedBuffer->getColumn(i); - auto begin = inputCol.begin() + metadata.offsetInBlock_; - decltype(auto) resultColumn = result.getColumn(i); - std::copy(begin, begin + numElements, resultColumn.begin()); - } - return result; + // Read the first block if it is incomplete + if (firstBlockIsIncomplete) { + co_yield readIncompleteBlock(*beginBlock); + ++beginBlock; + if (timer) { + timer->wlock()->checkTimeoutAndThrow("IndexScan :"); + } + } + + // Read all the other (complete!) blocks in parallel + for (; beginBlock < endBlock; ++beginBlock) { + const auto& block = *beginBlock; + // Read a block from disk (serially). + + CompressedBlock compressedBuffer = + readCompressedBlockFromFile(block, file, std::nullopt); + co_yield decompressBlock(compressedBuffer, block.numRows_); + // The `decompressLambda` can now run in parallel + if (timer) { + timer->wlock()->checkTimeoutAndThrow(); }; + } +} - // Read the first block if it is incomplete - if (firstBlockIsIncomplete) { - co_yield readIncompleteBlock(*beginBlock); - ++beginBlock; - if (timer) { - timer->wlock()->checkTimeoutAndThrow("IndexScan :"); - } +cppcoro::generator CompressedRelationReader::lazyScan( + const CompressedRelationMetadata& metadata, Id col1Id, + std::span blockMetadata, + ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, + ad_utility::SharedConcurrentTimeoutTimer timer) const { + auto relevantBlocks = getBlocksFromMetadata(metadata, col1Id, blockMetadata); + auto beginBlock = relevantBlocks.begin(); + auto endBlock = relevantBlocks.end(); + + // Invariant: The col0Id is completely stored in a single block, or it is + // contained in multiple blocks that only contain this col0Id, + bool col0IdHasExclusiveBlocks = + metadata.offsetInBlock_ == std::numeric_limits::max(); + if (!col0IdHasExclusiveBlocks) { + // This might also be zero if no block was found at all. + AD_CORRECTNESS_CHECK(endBlock - beginBlock <= 1); + } + + // The first and the last block might be incomplete (that is, only + // a part of these blocks is actually part of the result, + // set up a lambda which allows us to read these blocks, and returns + // the result as a vector. + auto readPossiblyIncompleteBlock = [&](const auto& block) { + DecompressedBlock uncompressedBuffer = + readAndDecompressBlock(block, file, std::nullopt); + AD_CORRECTNESS_CHECK(uncompressedBuffer.numColumns() == 2); + const auto& col1Column = uncompressedBuffer.getColumn(0); + const auto& col2Column = uncompressedBuffer.getColumn(1); + AD_CORRECTNESS_CHECK(col1Column.size() == col2Column.size()); + + // Find the range in the block, that belongs to the same relation `col0Id` + bool containedInOnlyOneBlock = + metadata.offsetInBlock_ != std::numeric_limits::max(); + auto begin = col1Column.begin(); + if (containedInOnlyOneBlock) { + begin += metadata.offsetInBlock_; } + auto end = + containedInOnlyOneBlock ? begin + metadata.numRows_ : col1Column.end(); + + // Find the range in the block, where also the col1Id matches (the second + // ID in the `std::array` does not matter). + std::tie(begin, end) = std::equal_range(begin, end, col1Id); - // Read all the other (complete!) blocks in parallel - for (; beginBlock < endBlock; ++beginBlock) { - const auto& block = *beginBlock; - // Read a block from disk (serially). - - CompressedBlock compressedBuffer = - readCompressedBlockFromFile(block, file, std::nullopt); - co_yield decompressBlock(compressedBuffer, block.numRows_); - // The `decompressLambda` can now run in parallel - if (timer) { - timer->wlock()->checkTimeoutAndThrow(); - }; + size_t beginIndex = begin - col1Column.begin(); + size_t endIndex = end - col1Column.begin(); + + // Only extract the relevant portion of the second column. + IdTable result{1, allocator}; + result.resize(endIndex - beginIndex); + std::copy(col2Column.begin() + beginIndex, col2Column.begin() + endIndex, + result.getColumn(0).begin()); + return result; + }; + + // The first and the last block might be incomplete, compute + // and store the partial results from them. + std::optional firstBlockResult, lastBlockResult; + if (beginBlock < endBlock) { + firstBlockResult = readPossiblyIncompleteBlock(*beginBlock); + ++beginBlock; + if (timer) { + timer->wlock()->checkTimeoutAndThrow("IndexScan: "); + } + } + if (beginBlock < endBlock) { + lastBlockResult = readPossiblyIncompleteBlock(*(endBlock - 1)); + endBlock--; + if (timer) { + timer->wlock()->checkTimeoutAndThrow("IndexScan: "); } } + if (firstBlockResult.has_value()) { + co_yield std::move(firstBlockResult.value()); + } + + for (; beginBlock < endBlock; ++beginBlock) { + const auto& block = *beginBlock; + + // Read the block serially, only read the second column. + AD_CORRECTNESS_CHECK(block.offsetsAndCompressedSize_.size() == 2); + CompressedBlock compressedBuffer = + readCompressedBlockFromFile(block, file, std::vector{1ul}); + co_yield decompressBlock(compressedBuffer, block.numRows_); + } + timer->wlock()->checkTimeoutAndThrow(); + if (lastBlockResult.has_value()) { + co_yield lastBlockResult.value(); + } +} + // _____________________________________________________________________________ std::vector CompressedRelationReader::getBlocksForJoin( std::span joinColum, const CompressedRelationMetadata& metadata, std::span blockMetadata) { // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ - struct KeyLhs { - Id col0FirstId_; - Id col0LastId_; - }; - Id col0Id = metadata.col0Id_; - // TODO Use a structured binding. Structured bindings are - // currently not supported by clang when using OpenMP because clang internally - // transforms the `#pragma`s into lambdas, and capturing structured bindings - // is only supported in clang >= 16. - decltype(blockMetadata.begin()) beginBlock, endBlock; - std::tie(beginBlock, endBlock) = std::equal_range( - // TODO For some reason we can't use `std::ranges::equal_range`, - // find out why. Note: possibly it has something to do with the limited - // support of ranges in clang with versions < 16. Revisit this when - // we use clang 16. - blockMetadata.begin(), blockMetadata.end(), KeyLhs{col0Id, col0Id}, - [](const auto& a, const auto& b) { - return a.col0FirstId_ < b.col0FirstId_ && a.col0LastId_ < b.col0LastId_; - }); - + auto relevantBlocks = + getBlocksFromMetadata(metadata, std::nullopt, blockMetadata); + auto beginBlock = relevantBlocks.begin(); + auto endBlock = relevantBlocks.end(); if (endBlock - beginBlock < 2) { - return std::vector(beginBlock, endBlock); + return {beginBlock, endBlock}; } auto idLessThanBlock = [](Id id, const CompressedBlockMetadata& block) { @@ -301,47 +349,38 @@ std::vector CompressedRelationReader::getBlocksForJoin( return result; } -std::array, 2> CompressedRelationReader::getBlocksForJoin(const CompressedRelationMetadata& md1, const CompressedRelationMetadata& md2, std::span blockMetadata) { +std::array, 2> +CompressedRelationReader::getBlocksForJoin( + const CompressedRelationMetadata& md1, + const CompressedRelationMetadata& md2, + std::span blockMetadata1, + std::span blockMetadata2) { // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ struct KeyLhs { Id col0FirstId_; Id col0LastId_; }; - Id col0Id = md1.col0Id_; - // TODO Use a structured binding. Structured bindings are - // currently not supported by clang when using OpenMP because clang internally - // transforms the `#pragma`s into lambdas, and capturing structured bindings - // is only supported in clang >= 16. - decltype(blockMetadata.begin()) beginBlock1, endBlock1, beginBlock2, endBlock2; - std::tie(beginBlock1, endBlock1) = std::equal_range( - // TODO For some reason we can't use `std::ranges::equal_range`, - // find out why. Note: possibly it has something to do with the limited - // support of ranges in clang with versions < 16. Revisit this when - // we use clang 16. - blockMetadata.begin(), blockMetadata.end(), KeyLhs{col0Id, col0Id}, - [](const auto& a, const auto& b) { - return a.col0FirstId_ < b.col0FirstId_ && a.col0LastId_ < b.col0LastId_; - }); - col0Id = md2.col0Id_; - std::tie(beginBlock2, endBlock2) = std::equal_range( - // TODO For some reason we can't use `std::ranges::equal_range`, - // find out why. Note: possibly it has something to do with the limited - // support of ranges in clang with versions < 16. Revisit this when - // we use clang 16. - blockMetadata.begin(), blockMetadata.end(), KeyLhs{col0Id, col0Id}, - [](const auto& a, const auto& b) { - return a.col0FirstId_ < b.col0FirstId_ && a.col0LastId_ < b.col0LastId_; - }); - auto blockLessThanBlock = [](const CompressedBlockMetadata& block1, const CompressedBlockMetadata& block2) { - if (block1.col0LastId_ < block2.col0FirstId_ || block1.col0FirstId_ > block2.col0LastId_) { + auto relevantBlocks1 = + getBlocksFromMetadata(md1, std::nullopt, blockMetadata1); + auto beginBlock1 = relevantBlocks1.begin(); + auto endBlock1 = relevantBlocks1.end(); + + auto relevantBlocks2 = + getBlocksFromMetadata(md2, std::nullopt, blockMetadata2); + auto beginBlock2 = relevantBlocks2.begin(); + auto endBlock2 = relevantBlocks2.end(); + + auto blockLessThanBlock = [](const CompressedBlockMetadata& block1, + const CompressedBlockMetadata& block2) { + if (block1.col0LastId_ < block2.col0FirstId_ || + block1.col0FirstId_ > block2.col0LastId_) { return block1.col0LastId_ < block2.col0LastId_; } else { return false; } }; - std::array, 2> result; auto addRow = [&result]([[maybe_unused]] auto it1, auto it2) { result[0].push_back(*it1); @@ -350,13 +389,13 @@ std::array, 2> CompressedRelationReader::ge auto noop = ad_utility::noop; [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( - std::span{beginBlock1, endBlock1}, std::span(beginBlock2, endBlock2), + std::span{beginBlock1, endBlock1}, + std::span(beginBlock2, endBlock2), blockLessThanBlock, addRow, noop, noop); for (auto& vec : result) { vec.erase(std::unique(vec.begin(), vec.end()), vec.end()); } return result; - } // _____________________________________________________________________________ @@ -366,30 +405,9 @@ void CompressedRelationReader::scan( IdTable* result, ad_utility::SharedConcurrentTimeoutTimer timer) const { AD_CONTRACT_CHECK(result->numColumns() == 1); - // Get all the blocks that possibly might contain our pair of col0Id and - // col1Id - struct KeyLhs { - Id col0FirstId_; - Id col0LastId_; - Id col1FirstId_; - Id col1LastId_; - }; - - auto comp = [](const auto& a, const auto& b) { - bool endBeforeBegin = a.col0LastId_ < b.col0FirstId_; - endBeforeBegin |= - (a.col0LastId_ == b.col0FirstId_ && a.col1LastId_ < b.col1FirstId_); - return endBeforeBegin; - }; - - Id col0Id = metaData.col0Id_; - - // Note: See the comment in the other overload for `scan` above for the - // reason why we (currently) can't use a structured binding here. - decltype(blocks.begin()) beginBlock, endBlock; - std::tie(beginBlock, endBlock) = - std::equal_range(blocks.begin(), blocks.end(), - KeyLhs{col0Id, col0Id, col1Id, col1Id}, comp); + auto relevantBlocks = getBlocksFromMetadata(metaData, col1Id, blocks); + auto beginBlock = relevantBlocks.begin(); + auto endBlock = relevantBlocks.end(); // Invariant: The col0Id is completely stored in a single block, or it is // contained in multiple blocks that only contain this col0Id, @@ -724,4 +742,57 @@ CompressedRelationWriter::compressAndWriteColumn(std::span column) { return {offsetInFile, compressedSize}; }; -//CompressedRelationWriter::getBlocksFromMetadata \ No newline at end of file +// _____________________________________________________________________________ +std::span +CompressedRelationReader::getBlocksFromMetadata( + const CompressedRelationMetadata& metadata, std::optional col1Id, + std::span blockMetadata) { + if (!col1Id.has_value()) { + // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ + struct KeyLhs { + Id col0FirstId_; + Id col0LastId_; + }; + Id col0Id = metadata.col0Id_; + // TODO Use a structured binding. Structured bindings are + // currently not supported by clang when using OpenMP because clang + // internally transforms the `#pragma`s into lambdas, and capturing + // structured bindings is only supported in clang >= 16. + auto [beginBlock, endBlock] = std::equal_range( + // TODO For some reason we can't use + // `std::ranges::equal_range`, find out why. Note: possibly it has + // something to do with the limited support of ranges in clang with + // versions < 16. Revisit this when we use clang 16. + blockMetadata.begin(), blockMetadata.end(), KeyLhs{col0Id, col0Id}, + [](const auto& a, const auto& b) { + return a.col0FirstId_ < b.col0FirstId_ && + a.col0LastId_ < b.col0LastId_; + }); + return {beginBlock, endBlock}; + } else { + // Get all the blocks that possibly might contain our pair of col0Id and + // col1Id + struct KeyLhs { + Id col0FirstId_; + Id col0LastId_; + Id col1FirstId_; + Id col1LastId_; + }; + + auto comp = [](const auto& a, const auto& b) { + bool endBeforeBegin = a.col0LastId_ < b.col0FirstId_; + endBeforeBegin |= + (a.col0LastId_ == b.col0FirstId_ && a.col1LastId_ < b.col1FirstId_); + return endBeforeBegin; + }; + + Id col0Id = metadata.col0Id_; + + // Note: See the comment in the other overload for `scan` above for the + // reason why we (currently) can't use a structured binding here. + auto [beginBlock, endBlock] = std::equal_range( + blockMetadata.begin(), blockMetadata.end(), + KeyLhs{col0Id, col0Id, col1Id.value(), col1Id.value()}, comp); + return {beginBlock, endBlock}; + } +} diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h index 5d124eef70..049d29f8bb 100644 --- a/src/index/CompressedRelation.h +++ b/src/index/CompressedRelation.h @@ -15,12 +15,12 @@ #include "util/Cache.h" #include "util/ConcurrentCache.h" #include "util/File.h" +#include "util/Generator.h" #include "util/Serializer/ByteBufferSerializer.h" #include "util/Serializer/SerializeVector.h" #include "util/Serializer/Serializer.h" #include "util/Timer.h" #include "util/TypeTraits.h" -#include "util/Generator.h" // Forward declaration of the `IdTable` class. class IdTable; @@ -235,7 +235,9 @@ class CompressedRelationReader { blockCache_{20ul}; // TODO This should probably be the allocator of the global qec. - mutable ad_utility::AllocatorWithLimit allocator{ad_utility::makeAllocationMemoryLeftThreadsafeObject(std::numeric_limits::max())}; + mutable ad_utility::AllocatorWithLimit allocator{ + ad_utility::makeAllocationMemoryLeftThreadsafeObject( + std::numeric_limits::max())}; public: /** @@ -258,43 +260,53 @@ class CompressedRelationReader { ad_utility::File& file, IdTable* result, ad_utility::SharedConcurrentTimeoutTimer timer) const; - std::span getBlocksFromMetadata(const CompressedRelationMetadata& metadata, std::optional col1Id, std::span blockMetadata); + static std::span getBlocksFromMetadata( + const CompressedRelationMetadata& metadata, std::optional col1Id, + std::span blockMetadata); // Get all the blocks that can contain an Id from the `joinColumn`. // TODO Include a timeout check. - std::vector getBlocksForJoin( + static std::vector getBlocksForJoin( std::span joinColum, const CompressedRelationMetadata& metadata, std::span blockMetadata); - std::array, 2> getBlocksForJoin( - const CompressedRelationMetadata& md1, const CompressedRelationMetadata& md2, std::span blockMetadata - ); - - cppcoro::generator lazyScan(const CompressedRelationMetadata& metadata, -std::span blockMetadata, -ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, -ad_utility::SharedConcurrentTimeoutTimer timer); - - /** - * @brief For a permutation XYZ, retrieve all Z for given X and Y. - * - * @param metaData The metadata of the given X. - * @param col1Id The ID for Y. - * @param blocks The metadata of the on-disk blocks for the given - * permutation. - * @param file The file in which the permutation is stored. - * @param result The ID table to which we write the result. It must have - * exactly one column. - * @param timer If specified (!= nullptr) a `TimeoutException` will be - * thrown if the timer runs out during the exeuction of this function. - * - * The arguments `metaData`, `blocks`, and `file` must all be obtained - * from The same `CompressedRelationWriter` (see below). - */ - void - scan(const CompressedRelationMetadata& metaData, Id col1Id, - const vector& blocks, - ad_utility::File& file, IdTable* result, - ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const; + static std::array, 2> getBlocksForJoin( + const CompressedRelationMetadata& md1, + const CompressedRelationMetadata& md2, + std::span blockMetadata1, + std::span blockMetadata2); + + cppcoro::generator lazyScan( + const CompressedRelationMetadata& metadata, + std::span blockMetadata, + ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, + ad_utility::SharedConcurrentTimeoutTimer timer) const; + + cppcoro::generator lazyScan( + const CompressedRelationMetadata& metadata, Id col1Id, + std::span blockMetadata, + ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, + ad_utility::SharedConcurrentTimeoutTimer timer) const; + + /** + * @brief For a permutation XYZ, retrieve all Z for given X and Y. + * + * @param metaData The metadata of the given X. + * @param col1Id The ID for Y. + * @param blocks The metadata of the on-disk blocks for the given + * permutation. + * @param file The file in which the permutation is stored. + * @param result The ID table to which we write the result. It must have + * exactly one column. + * @param timer If specified (!= nullptr) a `TimeoutException` will be + * thrown if the timer runs out during the exeuction of this function. + * + * The arguments `metaData`, `blocks`, and `file` must all be obtained + * from The same `CompressedRelationWriter` (see below). + */ + void scan(const CompressedRelationMetadata& metaData, Id col1Id, + const vector& blocks, + ad_utility::File& file, IdTable* result, + ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const; private: // Read the block that is identified by the `blockMetaData` from the `file`. @@ -308,8 +320,8 @@ ad_utility::SharedConcurrentTimeoutTimer timer); // have after decompression must be passed in via the `numRowsToRead` // argument. It is typically obtained from the corresponding // `CompressedBlockMetaData`. - DecompressedBlock decompressBlock( - const CompressedBlock& compressedBlock, size_t numRowsToRead) const; + DecompressedBlock decompressBlock(const CompressedBlock& compressedBlock, + size_t numRowsToRead) const; // Similar to `decompressBlock`, but the block is directly decompressed into // the `table`, starting at the `offsetInTable`-th row. The `table` and the diff --git a/src/index/Permutations.h b/src/index/Permutations.h index fc3e87d600..f1d03dc685 100644 --- a/src/index/Permutations.h +++ b/src/index/Permutations.h @@ -80,22 +80,42 @@ class PermutationImpl { } struct MetaDataAndBlocks { - const CompressedRelationMetadata& relationMetadata_; - const std::vector& blockMetadata_; + const CompressedRelationMetadata relationMetadata_; + std::span blockMetadata_; }; - std::optional getMetadataAndBlocks(Id col0Id) { + std::optional getMetadataAndBlocks( + Id col0Id, std::optional col1Id) const { if (!_meta.col0IdExists(col0Id)) { return std::nullopt; } - return MetaDataAndBlocks{_meta.getMetaData(col0Id), _meta.blockData()}; + + auto metadata = _meta.getMetaData(col0Id); + return MetaDataAndBlocks{ + _meta.getMetaData(col0Id), + _reader.getBlocksFromMetadata(metadata, col1Id, _meta.blockData())}; + } + + cppcoro::generator lazyScan( + Id col0Id, const std::vector& blocks, + ad_utility::AllocatorWithLimit allocator, + ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const { + if (!_meta.col0IdExists(col0Id)) { + return {}; + } + return _reader.lazyScan(_meta.getMetaData(col0Id), blocks, _file, + std::move(allocator), timer); } - cppcoro::generator lazyScan(Id col0Id, const std::vector& blocks, ad_utility::AllocatorWithLimit allocator, ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) { + cppcoro::generator lazyScan( + Id col0Id, Id col1Id, const std::vector& blocks, + ad_utility::AllocatorWithLimit allocator, + ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const { if (!_meta.col0IdExists(col0Id)) { return {}; } - return _reader.lazyScan(_meta.getMetaData(col0Id), blocks, _file, std::move(allocator), timer); + return _reader.lazyScan(_meta.getMetaData(col0Id), col1Id, blocks, _file, + std::move(allocator), timer); } // _______________________________________________________ diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 771c9e36f1..bfd6dd8e34 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -165,7 +165,8 @@ template < // element of `left` or `right`) constains no UNDEF values. It is used inside // the following `mergeWithUndefRight` function. auto containsNoUndefined = [](const T& row) { - if constexpr (isSimilar && isSimilar) { + if constexpr (isSimilar && + isSimilar) { return true; } else if constexpr (std::is_same_v) { return row != Id::makeUndefined(); From b7ee48d7b570e9563ac45838c1e481954197d939 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Fri, 5 May 2023 09:16:22 +0200 Subject: [PATCH 015/150] Removed some code smells. --- src/index/IndexImpl.Text.cpp | 28 ++++++------- src/index/IndexImpl.cpp | 78 +++++++++++++++++------------------- src/index/IndexImpl.h | 44 ++++++++++---------- 3 files changed, 73 insertions(+), 77 deletions(-) diff --git a/src/index/IndexImpl.Text.cpp b/src/index/IndexImpl.Text.cpp index 0b83dc8c9c..516cdfad98 100644 --- a/src/index/IndexImpl.Text.cpp +++ b/src/index/IndexImpl.Text.cpp @@ -342,7 +342,7 @@ void IndexImpl::addContextToVector( void IndexImpl::createTextIndex(const string& filename, const IndexImpl::TextVec& vec) { ad_utility::File out(filename.c_str(), "w"); - currentoff_t_ = 0; + currenttOffset_ = 0; // Detect block boundaries from the main key of the vec. // Write the data for each block. // First, there's the classic lists, then the additional entity ones. @@ -425,10 +425,10 @@ ContextListMetaData IndexImpl::writePostings(ad_utility::File& out, ContextListMetaData meta; meta._nofElements = postings.size(); if (meta._nofElements == 0) { - meta._startContextlist = currentoff_t_; - meta._startWordlist = currentoff_t_; - meta._startScorelist = currentoff_t_; - meta._lastByte = currentoff_t_ - 1; + meta._startContextlist = currenttOffset_; + meta._startWordlist = currenttOffset_; + meta._startScorelist = currenttOffset_; + meta._lastByte = currenttOffset_ - 1; return meta; } @@ -471,28 +471,28 @@ ContextListMetaData IndexImpl::writePostings(ad_utility::File& out, size_t bytes = 0; // Write context list: - meta._startContextlist = currentoff_t_; + meta._startContextlist = currenttOffset_; bytes = writeList(contextList, meta._nofElements, out); - currentoff_t_ += bytes; + currenttOffset_ += bytes; // Write word list: // This can be skipped if we're writing classic lists and there // is only one distinct wordId in the block, since this Id is already // stored in the meta data. - meta._startWordlist = currentoff_t_; + meta._startWordlist = currenttOffset_; if (!skipWordlistIfAllTheSame || wordCodebook.size() > 1) { - currentoff_t_ += writeCodebook(wordCodebook, out); + currenttOffset_ += writeCodebook(wordCodebook, out); bytes = writeList(wordList, meta._nofElements, out); - currentoff_t_ += bytes; + currenttOffset_ += bytes; } // Write scores - meta._startScorelist = currentoff_t_; - currentoff_t_ += writeCodebook(scoreCodebook, out); + meta._startScorelist = currenttOffset_; + currenttOffset_ += writeCodebook(scoreCodebook, out); bytes = writeList(scoreList, meta._nofElements, out); - currentoff_t_ += bytes; + currenttOffset_ += bytes; - meta._lastByte = currentoff_t_ - 1; + meta._lastByte = currenttOffset_ - 1; delete[] contextList; delete[] wordList; diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index 94578100b2..5214612927 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -30,7 +30,7 @@ using std::array; // _____________________________________________________________________________ -IndexImpl::IndexImpl() : usePatterns_(false) {} +IndexImpl::IndexImpl() = default; // _____________________________________________________________________________ template @@ -172,7 +172,7 @@ void IndexImpl::createFromFile(const string& filename) { size_t numPredicatesNormal = 0; createPermutationPair( - std::move(uniqueSorter), PSO_, POS_, spoSorter.makePushCallback(), + std::move(uniqueSorter), pso_, pos_, spoSorter.makePushCallback(), makeNumEntitiesCounter(numPredicatesNormal, 1), countActualTriples); configurationJson_["num-predicates-normal"] = numPredicatesNormal; configurationJson_["num-triples-normal"] = numTriplesNormal; @@ -192,12 +192,12 @@ void IndexImpl::createFromFile(const string& filename) { patternCreator.processTriple(triple); } }; - createPermutationPair(spoSorter.sortedView(), SPO_, SOP_, + createPermutationPair(spoSorter.sortedView(), spo_, sop_, ospSorter.makePushCallback(), pushTripleToPatterns, numSubjectCounter); patternCreator.finish(); } else { - createPermutationPair(spoSorter.sortedView(), SPO_, SOP_, + createPermutationPair(spoSorter.sortedView(), spo_, sop_, ospSorter.makePushCallback(), numSubjectCounter); } spoSorter.clear(); @@ -207,7 +207,7 @@ void IndexImpl::createFromFile(const string& filename) { // For the last pair of permutations we don't need a next sorter, so we have // no fourth argument. size_t numObjectsNormal = 0; - createPermutationPair(ospSorter.sortedView(), OSP_, OPS_, + createPermutationPair(ospSorter.sortedView(), osp_, ops_, makeNumEntitiesCounter(numObjectsNormal, 2)); configurationJson_["num-objects-normal"] = numObjectsNormal; configurationJson_["has-all-permutations"] = true; @@ -414,13 +414,10 @@ IndexBuilderDataAsStxxlVector IndexImpl::passFileForVocabulary( LOG(INFO) << "Removing temporary files ..." << std::endl; for (size_t n = 0; n < numFiles; ++n) { - string partialFilename = - onDiskBase_ + PARTIAL_VOCAB_FILE_NAME + std::to_string(n); - deleteTemporaryFile(partialFilename); + deleteTemporaryFile(absl::StrCat(onDiskBase_, PARTIAL_VOCAB_FILE_NAME, n)); if (vocabPrefixCompressed_) { - partialFilename = onDiskBase_ + TMP_BASENAME_COMPRESSION + - PARTIAL_VOCAB_FILE_NAME + std::to_string(n); - deleteTemporaryFile(partialFilename); + deleteTemporaryFile(absl::StrCat(onDiskBase_, TMP_BASENAME_COMPRESSION, + PARTIAL_VOCAB_FILE_NAME, n)); } } @@ -443,8 +440,8 @@ std::unique_ptr IndexImpl::convertPartialToGlobalIds( size_t i = 0; for (size_t partialNum = 0; partialNum < actualLinesPerPartial.size(); partialNum++) { - std::string mmapFilename(onDiskBase_ + PARTIAL_MMAP_IDS + - std::to_string(partialNum)); + std::string mmapFilename = + absl::StrCat(onDiskBase_, PARTIAL_MMAP_IDS, partialNum); LOG(DEBUG) << "Reading ID map from: " << mmapFilename << std::endl; ad_utility::HashMap idMap = IdMapFromPartialIdMapFile(mmapFilename); // Delete the temporary file in which we stored this map @@ -611,8 +608,8 @@ void IndexImpl::createPermutationPair(auto&& sortedTriples, auto metaData = createPermutations(AD_FWD(sortedTriples), p1, p2, AD_FWD(perTripleCallbacks)...); // Set the name of this newly created pair of `IndexMetaData` objects. - // NOTE: When `setKbName` was called, it set the name of PSO_._meta, - // PSO_._meta, ... which however are not used during index building. + // NOTE: When `setKbName` was called, it set the name of pso_._meta, + // pso_._meta, ... which however are not used during index building. // `getKbName` simple reads one of these names. metaData.value().first.setName(getKbName()); metaData.value().second.setName(getKbName()); @@ -637,7 +634,7 @@ void IndexImpl::addPatternsToExistingIndex() { ad_utility::AllocatorWithLimit allocator{ ad_utility::makeAllocationMemoryLeftThreadsafeObject( std::numeric_limits::max())}; - auto iterator = TriplesView(SPO_, allocator); + auto iterator = TriplesView(spo_, allocator); createPatternsFromSpoTriplesView(iterator, onDiskBase_ + ".index.patterns", Id::makeFromVocabIndex(langPredLowerBound), Id::makeFromVocabIndex(langPredUpperBound)); @@ -655,14 +652,14 @@ void IndexImpl::createFromOnDiskIndex(const string& onDiskBase) { totalVocabularySize_ = vocab_.size() + vocab_.getExternalVocab().size(); LOG(DEBUG) << "Number of words in internal and external vocabulary: " << totalVocabularySize_ << std::endl; - PSO_.loadFromDisk(onDiskBase_); - POS_.loadFromDisk(onDiskBase_); + pso_.loadFromDisk(onDiskBase_); + pos_.loadFromDisk(onDiskBase_); if (loadAllPermutations_) { - OPS_.loadFromDisk(onDiskBase_); - OSP_.loadFromDisk(onDiskBase_); - SPO_.loadFromDisk(onDiskBase_); - SOP_.loadFromDisk(onDiskBase_); + ops_.loadFromDisk(onDiskBase_); + osp_.loadFromDisk(onDiskBase_); + spo_.loadFromDisk(onDiskBase_); + sop_.loadFromDisk(onDiskBase_); } else { LOG(INFO) << "Only the PSO and POS permutation were loaded, SPARQL queries " "with predicate variables will therefore not work" @@ -751,12 +748,12 @@ bool IndexImpl::shouldBeExternalized(const string& object) { // _____________________________________________________________________________ void IndexImpl::setKbName(const string& name) { - POS_.setKbName(name); - PSO_.setKbName(name); - SOP_.setKbName(name); - SPO_.setKbName(name); - OPS_.setKbName(name); - OSP_.setKbName(name); + pos_.setKbName(name); + pso_.setKbName(name); + sop_.setKbName(name); + spo_.setKbName(name); + ops_.setKbName(name); + osp_.setKbName(name); } // ____________________________________________________________________________ @@ -1068,10 +1065,9 @@ std::future IndexImpl::writeNextPartialVocabulary( << actualCurrentPartialSize << std::endl; std::future resultFuture; string partialFilename = - onDiskBase_ + PARTIAL_VOCAB_FILE_NAME + std::to_string(numFiles); - string partialCompressionFilename = onDiskBase_ + TMP_BASENAME_COMPRESSION + - PARTIAL_VOCAB_FILE_NAME + - std::to_string(numFiles); + absl::StrCat(onDiskBase_, PARTIAL_VOCAB_FILE_NAME, numFiles); + string partialCompressionFilename = absl::StrCat( + onDiskBase_, TMP_BASENAME_COMPRESSION, PARTIAL_VOCAB_FILE_NAME, numFiles); auto lambda = [localIds = std::move(localIds), globalWritePtr, items = std::move(items), vocab = &vocab_, partialFilename, @@ -1140,17 +1136,17 @@ IndexImpl::PermutationImpl& IndexImpl::getPermutation(Index::Permutation p) { using enum Index::Permutation; switch (p) { case PSO: - return PSO_; + return pso_; case POS: - return POS_; + return pos_; case SPO: - return SPO_; + return spo_; case SOP: - return SOP_; + return sop_; case OSP: - return OSP_; + return osp_; case OPS: - return OPS_; + return ops_; } AD_FAIL(); } @@ -1165,7 +1161,7 @@ const IndexImpl::PermutationImpl& IndexImpl::getPermutation( Index::NumNormalAndInternal IndexImpl::numDistinctSubjects() const { if (hasAllPermutations()) { auto numActually = numSubjectsNormal_; - return {numActually, SPO_.metaData().getNofDistinctC1() - numActually}; + return {numActually, spo_.metaData().getNofDistinctC1() - numActually}; } else { AD_THROW( "Can only get # distinct subjects if all 6 permutations " @@ -1178,7 +1174,7 @@ Index::NumNormalAndInternal IndexImpl::numDistinctSubjects() const { Index::NumNormalAndInternal IndexImpl::numDistinctObjects() const { if (hasAllPermutations()) { auto numActually = numObjectsNormal_; - return {numActually, OSP_.metaData().getNofDistinctC1() - numActually}; + return {numActually, osp_.metaData().getNofDistinctC1() - numActually}; } else { AD_THROW( "Can only get # distinct objects if all 6 permutations " @@ -1190,7 +1186,7 @@ Index::NumNormalAndInternal IndexImpl::numDistinctObjects() const { // __________________________________________________________________________ Index::NumNormalAndInternal IndexImpl::numDistinctPredicates() const { auto numActually = numPredicatesNormal_; - return {numActually, PSO_.metaData().getNofDistinctC1() - numActually}; + return {numActually, pso_.metaData().getNofDistinctC1() - numActually}; } // __________________________________________________________________________ diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h index d2741462ad..152b356434 100644 --- a/src/index/IndexImpl.h +++ b/src/index/IndexImpl.h @@ -125,14 +125,14 @@ class IndexImpl { TextMetaData textMeta_; DocsDB docsDB_; vector blockBoundaries_; - off_t currentoff_t_; + off_t currenttOffset_; mutable ad_utility::File textIndexFile_; // If false, only PSO and POS permutations are loaded and expected. bool loadAllPermutations_ = true; // Pattern trick data - bool usePatterns_; + bool usePatterns_ = false; double avgNumDistinctPredicatesPerSubject_; double avgNumDistinctSubjectsPerPredicate_; size_t numDistinctSubjectPredicatePairs_; @@ -162,12 +162,12 @@ class IndexImpl { // TODO: make those private and allow only const access // instantiations for the six permutations used in QLever. // They simplify the creation of permutations in the index class. - PermutationImpl POS_{"POS", ".pos", {1, 2, 0}}; - PermutationImpl PSO_{"PSO", ".pso", {1, 0, 2}}; - PermutationImpl SOP_{"SOP", ".sop", {0, 2, 1}}; - PermutationImpl SPO_{"SPO", ".spo", {0, 1, 2}}; - PermutationImpl OPS_{"OPS", ".ops", {2, 1, 0}}; - PermutationImpl OSP_{"OSP", ".osp", {2, 0, 1}}; + PermutationImpl pos_{"POS", ".pos", {1, 2, 0}}; + PermutationImpl pso_{"PSO", ".pso", {1, 0, 2}}; + PermutationImpl sop_{"SOP", ".sop", {0, 2, 1}}; + PermutationImpl spo_{"SPO", ".spo", {0, 1, 2}}; + PermutationImpl ops_{"OPS", ".ops", {2, 1, 0}}; + PermutationImpl osp_{"OSP", ".osp", {2, 0, 1}}; public: IndexImpl(); @@ -180,21 +180,21 @@ class IndexImpl { IndexImpl& operator=(IndexImpl&&) = delete; IndexImpl(IndexImpl&&) = delete; - const auto& POS() const { return POS_; } - auto& POS() { return POS_; } - const auto& PSO() const { return PSO_; } - auto& PSO() { return PSO_; } - const auto& SPO() const { return SPO_; } - auto& SPO() { return SPO_; } - const auto& SOP() const { return SOP_; } - auto& SOP() { return SOP_; } - const auto& OPS() const { return OPS_; } - auto& OPS() { return OPS_; } - const auto& OSP() const { return OSP_; } - auto& OSP() { return OSP_; } + const auto& POS() const { return pos_; } + auto& POS() { return pos_; } + const auto& PSO() const { return pso_; } + auto& PSO() { return pso_; } + const auto& SPO() const { return spo_; } + auto& SPO() { return spo_; } + const auto& SOP() const { return sop_; } + auto& SOP() { return sop_; } + const auto& OPS() const { return ops_; } + auto& OPS() { return ops_; } + const auto& OSP() const { return osp_; } + auto& OSP() { return osp_; } // For a given `Permutation` (e.g. `PSO`) return the corresponding - // `PermutationImpl` object by reference (`PSO_`). + // `PermutationImpl` object by reference (`pso_`). PermutationImpl& getPermutation(Index::Permutation p); const PermutationImpl& getPermutation(Index::Permutation p) const; @@ -386,7 +386,7 @@ class IndexImpl { const string& getTextName() const { return textMeta_.getName(); } - const string& getKbName() const { return PSO_.metaData().getName(); } + const string& getKbName() const { return pso_.metaData().getName(); } size_t getNofTextRecords() const { return textMeta_.getNofTextRecords(); } size_t getNofWordPostings() const { return textMeta_.getNofWordPostings(); } From a25914bca6f81f6bf0890736388167ef1bff248e Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Fri, 5 May 2023 09:30:10 +0200 Subject: [PATCH 016/150] Started to write some tests. --- test/IndexTest.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/IndexTest.cpp b/test/IndexTest.cpp index a15f27727b..04cff051ac 100644 --- a/test/IndexTest.cpp +++ b/test/IndexTest.cpp @@ -462,7 +462,12 @@ TEST(IndexTest, getIgnoredIdRanges) { } TEST(IndexTest, NumDistinctEntities) { - const IndexImpl& index = getQec()->getIndex().getImpl(); + std::string turtleInput = + " . \n" - " . \n" - " . \n" - " . "; - { - const IndexImpl& index = getQec(kb)->getIndex().getImpl(); - - IdTable wol(1, makeAllocator()); - IdTable wtl(2, makeAllocator()); - - auto getId = makeGetId(getQec(kb)->getIndex()); - Id a = getId(""); - Id c = getId(""); - Id a2 = getId(""); - Id c2 = getId(""); - auto testTwo = makeTestScanWidthTwo(index); - - testTwo("", PSO, {{a, c}, {a, c2}}); - testTwo("", PSO, {}); - testTwo("", PSO, {}); - testTwo("", POS, {{c, a}, {c2, a}}); - testTwo("", POS, {}); - testTwo("", POS, {}); - - auto testOne = makeTestScanWidthOne(index); - - testOne("", "", PSO, {{c}, {c2}}); - testOne("", "", PSO, {}); - testOne("", "", POS, {{a2}}); - testOne("", "", PSO, {}); - } - kb = " <1> . \n" - " <2> . \n" - " <0> . \n" - " <3> . \n" - " <0> . \n" - " <1> . \n" - " <2> . \n"; - - { - const IndexImpl& index = - ad_utility::testing::getQec(kb)->getIndex().getImpl(); - - auto getId = makeGetId(ad_utility::testing::getQec(kb)->getIndex()); - Id a = getId(""); - Id b = getId(""); - Id c = getId(""); - Id zero = getId("<0>"); - Id one = getId("<1>"); - Id two = getId("<2>"); - Id three = getId("<3>"); + auto testWithAndWithoutPrefixCompression = [](bool useCompression) { + using enum Index::Permutation; + std::string kb = + " . \n" + " . \n" + " . \n" + " . "; + { + const IndexImpl& index = + getQec(kb, true, true, useCompression)->getIndex().getImpl(); + + IdTable wol(1, makeAllocator()); + IdTable wtl(2, makeAllocator()); + + auto getId = makeGetId(getQec(kb)->getIndex()); + Id a = getId(""); + Id c = getId(""); + Id a2 = getId(""); + Id c2 = getId(""); + auto testTwo = makeTestScanWidthTwo(index); + + testTwo("", PSO, {{a, c}, {a, c2}}); + testTwo("", PSO, {}); + testTwo("", PSO, {}); + testTwo("", POS, {{c, a}, {c2, a}}); + testTwo("", POS, {}); + testTwo("", POS, {}); - auto testTwo = makeTestScanWidthTwo(index); - testTwo("", PSO, - {{{a, zero}, - {a, one}, - {a, two}, - {b, zero}, - {b, three}, - {c, one}, - {c, two}}}); - testTwo("", POS, - {{zero, a}, - {zero, b}, - {one, a}, - {one, c}, - {two, a}, - {two, c}, - {three, b}}); + auto testOne = makeTestScanWidthOne(index); - auto testWidthOne = makeTestScanWidthOne(index); + testOne("", "", PSO, {{c}, {c2}}); + testOne("", "", PSO, {}); + testOne("", "", POS, {{a2}}); + testOne("", "", PSO, {}); + } + kb = " <1> . \n" + " <2> . \n" + " <0> . \n" + " <3> . \n" + " <0> . \n" + " <1> . \n" + " <2> . \n"; - testWidthOne("", "<0>", POS, {{a}, {b}}); - testWidthOne("", "<1>", POS, {{a}, {c}}); - testWidthOne("", "<2>", POS, {{a}, {c}}); - testWidthOne("", "<3>", POS, {{b}}); - testWidthOne("", "", PSO, {{zero}, {one}, {two}}); - testWidthOne("", "", PSO, {{zero}, {three}}); - testWidthOne("", "", PSO, {{one}, {two}}); - } + { + const IndexImpl& index = + ad_utility::testing::getQec(kb, true, true, useCompression) + ->getIndex() + .getImpl(); + + auto getId = makeGetId(ad_utility::testing::getQec(kb)->getIndex()); + Id a = getId(""); + Id b = getId(""); + Id c = getId(""); + Id zero = getId("<0>"); + Id one = getId("<1>"); + Id two = getId("<2>"); + Id three = getId("<3>"); + + auto testTwo = makeTestScanWidthTwo(index); + testTwo("", PSO, + {{{a, zero}, + {a, one}, + {a, two}, + {b, zero}, + {b, three}, + {c, one}, + {c, two}}}); + testTwo("", POS, + {{zero, a}, + {zero, b}, + {one, a}, + {one, c}, + {two, a}, + {two, c}, + {three, b}}); + + auto testWidthOne = makeTestScanWidthOne(index); + + testWidthOne("", "<0>", POS, {{a}, {b}}); + testWidthOne("", "<1>", POS, {{a}, {c}}); + testWidthOne("", "<2>", POS, {{a}, {c}}); + testWidthOne("", "<3>", POS, {{b}}); + testWidthOne("", "", PSO, {{zero}, {one}, {two}}); + testWidthOne("", "", PSO, {{zero}, {three}}); + testWidthOne("", "", PSO, {{one}, {two}}); + } + }; + testWithAndWithoutPrefixCompression(true); + testWithAndWithoutPrefixCompression(false); }; // Returns true iff `arg` (the first argument of `EXPECT_THAT` below) holds a @@ -519,4 +526,24 @@ TEST(IndexTest, NumDistinctEntitiesCornerCases) { AD_EXPECT_THROW_WITH_MESSAGE( index.numDistinctCol0(static_cast(42)), ::testing::ContainsRegex("should be unreachable")); + + const IndexImpl& indexNoPatterns = + getQec("", true, false)->getIndex().getImpl(); + AD_EXPECT_THROW_WITH_MESSAGE( + indexNoPatterns.getAvgNumDistinctPredicatesPerSubject(), + ::testing::ContainsRegex("requires a loaded patterns file")); + AD_EXPECT_THROW_WITH_MESSAGE( + indexNoPatterns.getAvgNumDistinctSubjectsPerPredicate(), + ::testing::ContainsRegex("requires a loaded patterns file")); +} + +TEST(IndexTest, getPermutation) { + using enum Index::Permutation; + const IndexImpl& index = getQec()->getIndex().getImpl(); + EXPECT_EQ(&index.PSO(), &index.getPermutation(PSO)); + EXPECT_EQ(&index.POS(), &index.getPermutation(POS)); + EXPECT_EQ(&index.SOP(), &index.getPermutation(SOP)); + EXPECT_EQ(&index.SPO(), &index.getPermutation(SPO)); + EXPECT_EQ(&index.OPS(), &index.getPermutation(OPS)); + EXPECT_EQ(&index.OSP(), &index.getPermutation(OSP)); } diff --git a/test/IndexTestHelpers.h b/test/IndexTestHelpers.h index 91b66fed5b..06fb6c1a7d 100644 --- a/test/IndexTestHelpers.h +++ b/test/IndexTestHelpers.h @@ -61,7 +61,9 @@ inline std::vector getAllIndexFilenames( // The concrete triple contents are currently used in `GroupByTest.cpp`. inline Index makeTestIndex(const std::string& indexBasename, std::string turtleInput = "", - bool loadAllPermutations = true) { + bool loadAllPermutations = true, + bool usePatterns = true, + bool usePrefixCompression = true) { // Ignore the (irrelevant) log output of the index building and loading during // these tests. static std::ostringstream ignoreLogStream; @@ -82,11 +84,12 @@ inline Index makeTestIndex(const std::string& indexBasename, { Index index = makeIndexWithTestSettings(); index.setOnDiskBase(indexBasename); - index.setUsePatterns(true); + index.setUsePatterns(usePatterns); + index.setPrefixCompression(usePrefixCompression); index.createFromFile(inputFilename); } Index index; - index.setUsePatterns(true); + index.setUsePatterns(usePatterns); index.setLoadAllPermutations(loadAllPermutations); index.createFromOnDiskIndex(indexBasename); return index; @@ -97,7 +100,9 @@ inline Index makeTestIndex(const std::string& indexBasename, // vocabulary) is the only part of the `QueryExecutionContext` that is actually // relevant for these tests, so the other members are defaulted. inline QueryExecutionContext* getQec(std::string turtleInput = "", - bool loadAllPermutations = true) { + bool loadAllPermutations = true, + bool usePatterns = true, + bool usePrefixCompression = true) { // Similar to `absl::Cleanup`. Calls the `callback_` in the destructor, but // the callback is stored as a `std::function`, which allows to store // different types of callbacks in the same wrapper type. @@ -119,9 +124,11 @@ inline QueryExecutionContext* getQec(std::string turtleInput = "", *index_, cache_.get(), makeAllocator(), SortPerformanceEstimator{}); }; - static ad_utility::HashMap, Context> contextMap; + using Key = std::tuple; + static ad_utility::HashMap contextMap; - auto key = std::pair{turtleInput, loadAllPermutations}; + auto key = + Key{turtleInput, loadAllPermutations, usePatterns, usePrefixCompression}; if (!contextMap.contains(key)) { std::string testIndexBasename = @@ -137,7 +144,8 @@ inline QueryExecutionContext* getQec(std::string turtleInput = "", } }}, std::make_unique(makeTestIndex( - testIndexBasename, turtleInput, loadAllPermutations)), + testIndexBasename, turtleInput, loadAllPermutations, + usePatterns, usePrefixCompression)), std::make_unique()}); } return contextMap.at(key).qec_.get(); From f0cbb21ed826ff65f5d2359ae36197d2dc00567c Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Fri, 5 May 2023 11:11:55 +0200 Subject: [PATCH 018/150] Merge in the updated master. --- src/index/IndexImpl.Text.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index/IndexImpl.Text.cpp b/src/index/IndexImpl.Text.cpp index b2176b0e0b..f5ecddb7d5 100644 --- a/src/index/IndexImpl.Text.cpp +++ b/src/index/IndexImpl.Text.cpp @@ -758,7 +758,7 @@ size_t IndexImpl::writeCodebook(const vector& codebook, // _____________________________________________________________________________ void IndexImpl::openTextFileHandle() { AD_CONTRACT_CHECK(!onDiskBase_.empty()); - _textIndexFile.open(string(onDiskBase_ + ".text.index").c_str(), "r"); + textIndexFile_.open(string(onDiskBase_ + ".text.index").c_str(), "r"); } // _____________________________________________________________________________ From a5bd352a63946a2ce805019a6d3b5b9e7f8f7407 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Fri, 5 May 2023 16:18:40 +0200 Subject: [PATCH 019/150] In the middle of figuring stuff out. --- src/engine/IndexScan.cpp | 5 ++- src/index/IndexImpl.Text.cpp | 12 ++--- src/util/JoinAlgorithms/JoinAlgorithms.h | 57 ++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 818b52fbf1..9968e111dc 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -277,6 +277,7 @@ void IndexScan::computeFullScan(IdTable* result, *result = std::move(table).toDynamic(); } +// __________________________________________________________________________________________________________ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans( const IndexScan& s1, const IndexScan& s2) { const auto& index = s1.getExecutionContext()->getIndex().getImpl(); @@ -287,11 +288,11 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans( -> std::optional { auto permutedTriple = s.getPermutedTriple(); std::optional optId = - s.getPermutedTriple()[0]->toValueId(index.getVocab()); + permutedTriple[0]->toValueId(index.getVocab()); std::optional optId2 = s._numVariables == 2 ? std::nullopt - : s.getPermutedTriple()[1]->toValueId(index.getVocab()); + : permutedTriple[1]->toValueId(index.getVocab()); if (!optId.has_value() || (!optId2.has_value() && s._numVariables == 1)) { return std::nullopt; } diff --git a/src/index/IndexImpl.Text.cpp b/src/index/IndexImpl.Text.cpp index f5ecddb7d5..e7c516bdd4 100644 --- a/src/index/IndexImpl.Text.cpp +++ b/src/index/IndexImpl.Text.cpp @@ -4,7 +4,7 @@ // Johannes Kalmbach // Hannah Bast -#include "./IndexImpl.h" +#include "index/IndexImpl.h" #include @@ -13,11 +13,11 @@ #include #include -#include "../engine/CallFixedSize.h" -#include "../parser/ContextFileParser.h" -#include "../util/Conversions.h" -#include "../util/Simple8bCode.h" -#include "./FTSAlgorithms.h" +#include "engine/CallFixedSize.h" +#include "parser/ContextFileParser.h" +#include "util/Conversions.h" +#include "util/Simple8bCode.h" +#include "index/FTSAlgorithms.h" namespace { diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index bfd6dd8e34..6a3c6b9ec5 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -7,6 +7,7 @@ #include #include #include +#include #include "engine/idTable/IdTable.h" #include "global/Id.h" @@ -93,11 +94,11 @@ concept BinaryIteratorFunction = * be exploited to fix the result in a cheaper way than a full sort. */ template < - bool addDuplicatesFromLeft = true, std::ranges::random_access_range Range1, + bool addDuplicatesFromLeft = true, bool returnIteratorsOfLastMatch = false, std::ranges::random_access_range Range1, std::ranges::random_access_range Range2, typename LessThan, typename FindSmallerUndefRangesLeft, typename FindSmallerUndefRangesRight, typename ElFromFirstNotFoundAction = decltype(noop)> -[[nodiscard]] size_t zipperJoinWithUndef( +[[nodiscard]] auto zipperJoinWithUndef( const Range1& left, const Range2& right, const LessThan& lessThan, const auto& compatibleRowAction, const FindSmallerUndefRangesLeft& findSmallerUndefRangesLeft, @@ -117,6 +118,10 @@ template < auto it2 = std::begin(right); auto end2 = std::end(right); + // Keep track of the last iterators that had an exact match + auto exactMatchIt1 = it1; + auto exactMatchIt2 = it2; + // If this is an OPTIONAL join or a MINUS then we have to keep track of the // information for which elements from the left input we have already found a // match in the right input. We call these elements "covered". For all @@ -225,6 +230,7 @@ template < mergeWithUndefRight(it1, std::begin(right), it2, true); ++it1; if (it1 >= end1) { + exactMatchIt2 = it2; return; } } @@ -232,6 +238,7 @@ template < mergeWithUndefLeft(it2, std::begin(left), it1); ++it2; if (it2 >= end2) { + exactMatchIt1 = it1; return; } } @@ -249,6 +256,12 @@ template < auto endSame2 = std::find_if_not( it2, end2, [&](const auto& row) { return eq(*it1, row); }); + if constexpr (returnIteratorsOfLastMatch) { + if (endSame1 != it1) { + exactMatchIt1 = it1; + exactMatchIt2 = it2; + } + } for (auto it = it1; it != endSame1; ++it) { mergeWithUndefRight(it, std::begin(right), it2, false); } @@ -256,6 +269,7 @@ template < mergeWithUndefLeft(it, std::begin(left), it1); } + for (; it1 != endSame1; ++it1) { cover(it1); for (auto innerIt2 = it2; innerIt2 != endSame2; ++innerIt2) { @@ -297,8 +311,13 @@ template < // `max()` then we can provide no guarantees about the sorting of the result. // Otherwise, the result consists of two consecutive sorted ranges, the second // of which has length `returnValue`. - return outOfOrderFound ? std::numeric_limits::max() + size_t numOutOfOrder = outOfOrderFound ? std::numeric_limits::max() : numOutOfOrderAtEnd; + if constexpr (returnIteratorsOfLastMatch) { + return std::tuple{numOutOfOrder, exactMatchIt1, exactMatchIt2}; + } else { + return numOutOfOrder; + } } /** @@ -531,4 +550,36 @@ void specialOptionalJoin( elFromFirstNotFoundAction(it); } } + + +template < + typename LessThan> +void zipperJoinForBlocksWithoutUndef( + auto&& leftBlocks, auto&&rightBlocks, const LessThan& lessThan, + const auto& compatibleRowAction) { + auto it1 = leftBlocks.begin(); + auto end1 = leftBlocks.end(); + auto it2 = rightBlocks.begin(); + auto end2 = rightBlocks.end(); + std::optional lastMatch1; + std::optional lastMatch2; + while (it1 != end1 && it2 != end2) { + const auto& block1 = *it1; + const auto& block2 = *it2; + auto view1 = std::ranges::subrange(block1.begin(), block1.end()); + auto view2 = std::ranges::subrange(block2.begin(), block2.end()); + if (lastMatch1.has_value()) { + view1 = std::ranges::subrange(lastMatch1.value(), block1.end()); + } + if (lastMatch2.has_value()) { + view2 = std::ranges::subrange(lastMatch2.value(), block2.end()); + } + auto [numOutOfOrder, lastMatchNew1, lastMatchNew2] = zipperJoinWithUndef(view1, view2, lessThan, compatibleRowAction, noop, noop); + AD_CONTRACT_CHECK(numOutOfOrder == 0); + if (lastMatchNew1 != ) + + } + +} + } // namespace ad_utility From 2a42cfa2200b8d102c0065fd6f23b4f35a498c95 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 25 May 2023 16:09:04 +0200 Subject: [PATCH 020/150] I am beginning to understand --- src/util/JoinAlgorithms/JoinAlgorithms.h | 93 +++++++++++++++++++++++- 1 file changed, 89 insertions(+), 4 deletions(-) diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 6a3c6b9ec5..ecea0acd5e 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -552,15 +552,94 @@ void specialOptionalJoin( } -template < +template < typename LeftBlocks, typename RightBlocks, typename LessThan> void zipperJoinForBlocksWithoutUndef( - auto&& leftBlocks, auto&&rightBlocks, const LessThan& lessThan, + LeftBlocks&& leftBlocks, RightBlocks&&rightBlocks, const LessThan& lessThan, const auto& compatibleRowAction) { + using LeftBlock = typename LeftBlocks::value_type; + using RightBlock = typename RightBlocks::value_type; auto it1 = leftBlocks.begin(); auto end1 = leftBlocks.end(); auto it2 = rightBlocks.begin(); auto end2 = rightBlocks.end(); + auto eq = [&lessThan](const auto& el1, const auto& el2) { + return !lessThan(el1, el2) && !lessThan(el2, el1); + }; + + std::vector sameBlocksLeft; + std::vector sameBlocksRight; + auto fillBuffer = [&]() { + AD_CORRECTNESS_CHECK(sameBlocksLeft.size() == 1); + AD_CORRECTNESS_CHECK(sameBlocksRight.size() == 1); + while (it1 != end1) { + sameBlocksLeft.push_back(std::move(*it1)); + if (!eq(sameBlocksLeft.back().back, sameBlocksLeft.front().back())) { + break; + } + ++it1; + } + while (it2 != end2) { + sameBlocksRight.push_back(std::move(*it2)); + if (!eq(sameBlocksRight.back().back, sameBlocksRight.front().back())) { + break; + } + ++it2; + } + }; + + auto join = [&](const auto& l, const auto& r) { + return zipperJoinWithUndef(l , r, lessThan, compatibleRowAction, noop, noop); + }; + + auto addAll = [&](const auto& l, const auto& r) { + for (const auto& lBlock : l) { + for (const auto& rBlock : r) { + for (const auto& lEl : lBlock) { + for (const auto& rEl : rBlock) { + compatibleRowAction(&lBlock, &rBlock); + } + } + } + } + }; + + auto joinBuffers = [&]() { + join(sameBlocksLeft.at(0), sameBlocksRight.at(0)); + auto subrangeLeft = std::ranges::equal_range(sameBlocksLeft.front(), sameBlocksLeft.front().back(), lessThan); + auto subrangeRight = std::ranges::equal_range(sameBlocksRight.front(), sameBlocksRight.front().back(), lessThan); + using SubLeft = decltype(subrangeLeft); + using SubRight = decltype(subrangeRight); + std::vector l; + std::vector r; + l.push_back(subrangeLeft); + for (size_t i = 1; i < sameBlocksLeft.size() - 1; ++i) { + l.push_back(sameBlocksLeft[i]); + } + if (sameBlocksLeft.size() > 1) { + l.push_back(std::ranges::equal_range(sameBlocksLeft.back(), sameBlocksLeft.front().back(), lessThan)); + } + r.push_back(subrangeRight); + for (size_t i = 1; i < sameBlocksRight.size() - 1; ++i) { + r.push_back(sameBlocksRight[i]); + } + if (sameBlocksRight.size() > 1) { + r.push_back(std::ranges::equal_range( + sameBlocksRight.back(), sameBlocksRight.front().back(), lessThan)); + } + addAll(sameBlocksLeft, sameBlocksRight); + if (sameBlocksLeft.size() > 1) { + sameBlocksLeft.at(0) = std::move(sameBlocksLeft.back()); + sameBlocksLeft.resize(1); + auto& b = sameBlocksLeft.front(); + // TODO delete the range that has already been dealt with. + b.erase(b.begin(), std::upper_bound(b.begin(), b.end(), )) + + } + + }; + + std::optional lastMatch1; std::optional lastMatch2; while (it1 != end1 && it2 != end2) { @@ -576,10 +655,16 @@ void zipperJoinForBlocksWithoutUndef( } auto [numOutOfOrder, lastMatchNew1, lastMatchNew2] = zipperJoinWithUndef(view1, view2, lessThan, compatibleRowAction, noop, noop); AD_CONTRACT_CHECK(numOutOfOrder == 0); - if (lastMatchNew1 != ) + if (lessThan(block1.back(), block2.back())) { + ++it1; + } else if (lessThan(block2.back(), block1.back())) { + ++it2; + } else { + + } + } } - } // namespace ad_utility From 96c3829f5a7ac98070954bbfbdc824ce85591b6d Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Fri, 26 May 2023 13:28:20 +0200 Subject: [PATCH 021/150] Fixed the merge and compilation and tests --- src/engine/GroupBy.cpp | 2 +- src/engine/IndexScan.cpp | 14 +++--- src/engine/IndexScan.h | 46 ++++++------------- src/index/IndexImpl.Text.cpp | 2 +- src/util/JoinAlgorithms/JoinAlgorithms.h | 56 ++++++++++++------------ test/QueryPlannerTest.cpp | 6 +-- test/QueryPlannerTestHelpers.h | 4 +- 7 files changed, 54 insertions(+), 76 deletions(-) diff --git a/src/engine/GroupBy.cpp b/src/engine/GroupBy.cpp index fbe2759263..0d304d9a2a 100644 --- a/src/engine/GroupBy.cpp +++ b/src/engine/GroupBy.cpp @@ -543,7 +543,7 @@ std::optional GroupBy::getPermutationForThreeVariableTriple( if (variableByWhichToSort == indexScan->getSubject()) { return Permutation::SPO; - } else if (variableByWhichToSort.name() == indexScan->getPredicate()) { + } else if (variableByWhichToSort == indexScan->getPredicate()) { return Permutation::POS; } else if (variableByWhichToSort == indexScan->getObject()) { return Permutation::OSP; diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 7bf4275983..ac22cbc61e 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -85,7 +85,7 @@ vector IndexScan::resultSortedOn() const { case 2: return {ColumnIndex{0}, ColumnIndex{1}}; case 3: - return {ColumnIndex{0}, ColumnIndex{1}, ColumnIndex{2}; + return {ColumnIndex{0}, ColumnIndex{1}, ColumnIndex{2}}; default: AD_FAIL(); } @@ -284,15 +284,13 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans( AD_CONTRACT_CHECK(s1._numVariables < 3 && s2._numVariables < 3); const auto& s = s1; - auto f = [&](const IndexScan& s) - -> std::optional { + auto f = + [&](const IndexScan& s) -> std::optional { auto permutedTriple = s.getPermutedTriple(); - std::optional optId = - permutedTriple[0]->toValueId(index.getVocab()); + std::optional optId = permutedTriple[0]->toValueId(index.getVocab()); std::optional optId2 = - s._numVariables == 2 - ? std::nullopt - : permutedTriple[1]->toValueId(index.getVocab()); + s._numVariables == 2 ? std::nullopt + : permutedTriple[1]->toValueId(index.getVocab()); if (!optId.has_value() || (!optId2.has_value() && s._numVariables == 1)) { return std::nullopt; } diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index 1441c28888..b411722c80 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -54,35 +54,36 @@ class IndexScan : public Operation { } } - static Index::Permutation scanTypeToPermutation(ScanType scanType) { + static Permutation::Enum scanTypeToPermutation(ScanType scanType) { + using enum Permutation::Enum; switch (scanType) { case PSO_BOUND_S: case PSO_FREE_S: case FULL_INDEX_SCAN_PSO: - return Index::Permutation::PSO; + return PSO; case POS_FREE_O: case POS_BOUND_O: case FULL_INDEX_SCAN_POS: - return Index::Permutation::POS; + return POS; case SPO_FREE_P: case FULL_INDEX_SCAN_SPO: - return Index::Permutation::SPO; + return SPO; case SOP_FREE_O: case SOP_BOUND_O: case FULL_INDEX_SCAN_SOP: - return Index::Permutation::SOP; + return SOP; case OSP_FREE_S: case FULL_INDEX_SCAN_OSP: - return Index::Permutation::OSP; + return OSP; case OPS_FREE_P: case FULL_INDEX_SCAN_OPS: - return Index::Permutation::OPS; + return OPS; } } static std::array permutationToKeyOrder( - Index::Permutation permutation) { - using enum Index::Permutation; + Permutation::Enum permutation) { + using enum Permutation::Enum; switch (permutation) { case POS: return {1, 2, 0}; @@ -99,8 +100,8 @@ class IndexScan : public Operation { } } - static std::string_view permutationToString(Index::Permutation permutation) { - using enum Index::Permutation; + static std::string_view permutationToString(Permutation::Enum permutation) { + using enum Permutation::Enum; switch (permutation) { case POS: return "POS"; @@ -118,7 +119,7 @@ class IndexScan : public Operation { } private: - Index::Permutation _permutation; + Permutation::Enum _permutation; size_t _numVariables; TripleComponent _subject; TripleComponent _predicate; @@ -183,32 +184,13 @@ class IndexScan : public Operation { return getResultWidth() == 3; } - Index::Permutation permutation() const { return _permutation; } + Permutation::Enum permutation() const { return _permutation; } private: ResultTable computeResult() override; vector getChildren() override { return {}; } - void computeFullScan(IdTable* result, Index::Permutation permutation) const; - void computePSOboundS(IdTable* result) const; - - void computePSOfreeS(IdTable* result) const; - - void computePOSboundO(IdTable* result) const; - - void computePOSfreeO(IdTable* result) const; - - void computeSPOfreeP(IdTable* result) const; - - void computeSOPboundO(IdTable* result) const; - - void computeSOPfreeO(IdTable* result) const; - - void computeOPSfreeP(IdTable* result) const; - - void computeOSPfreeS(IdTable* result) const; - void computeFullScan(IdTable* result, Permutation::Enum permutation) const; size_t computeSizeEstimate(); diff --git a/src/index/IndexImpl.Text.cpp b/src/index/IndexImpl.Text.cpp index ed67e90830..b7c9b33a6a 100644 --- a/src/index/IndexImpl.Text.cpp +++ b/src/index/IndexImpl.Text.cpp @@ -15,10 +15,10 @@ #include #include "engine/CallFixedSize.h" +#include "index/FTSAlgorithms.h" #include "parser/ContextFileParser.h" #include "util/Conversions.h" #include "util/Simple8bCode.h" -#include "index/FTSAlgorithms.h" namespace { diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 6932cc11c3..7e85e9b02a 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -94,7 +94,8 @@ concept BinaryIteratorFunction = * be exploited to fix the result in a cheaper way than a full sort. */ template < - bool addDuplicatesFromLeft = true, bool returnIteratorsOfLastMatch = false, std::ranges::random_access_range Range1, + bool addDuplicatesFromLeft = true, bool returnIteratorsOfLastMatch = false, + std::ranges::random_access_range Range1, std::ranges::random_access_range Range2, typename LessThan, typename FindSmallerUndefRangesLeft, typename FindSmallerUndefRangesRight, typename ElFromFirstNotFoundAction = decltype(noop)> @@ -269,7 +270,6 @@ template < mergeWithUndefLeft(it, std::begin(left), it1); } - for (; it1 != endSame1; ++it1) { cover(it1); for (auto innerIt2 = it2; innerIt2 != endSame2; ++innerIt2) { @@ -311,8 +311,8 @@ template < // `max()` then we can provide no guarantees about the sorting of the result. // Otherwise, the result consists of two consecutive sorted ranges, the second // of which has length `returnValue`. - size_t numOutOfOrder = outOfOrderFound ? std::numeric_limits::max() - : numOutOfOrderAtEnd; + size_t numOutOfOrder = + outOfOrderFound ? std::numeric_limits::max() : numOutOfOrderAtEnd; if constexpr (returnIteratorsOfLastMatch) { return std::tuple{numOutOfOrder, exactMatchIt1, exactMatchIt2}; } else { @@ -569,12 +569,11 @@ void specialOptionalJoin( } } - -template < typename LeftBlocks, typename RightBlocks, - typename LessThan> -void zipperJoinForBlocksWithoutUndef( - LeftBlocks&& leftBlocks, RightBlocks&&rightBlocks, const LessThan& lessThan, - const auto& compatibleRowAction) { +template +void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, + RightBlocks&& rightBlocks, + const LessThan& lessThan, + const auto& compatibleRowAction) { using LeftBlock = typename LeftBlocks::value_type; using RightBlock = typename RightBlocks::value_type; auto it1 = leftBlocks.begin(); @@ -590,13 +589,13 @@ void zipperJoinForBlocksWithoutUndef( auto fillBuffer = [&]() { AD_CORRECTNESS_CHECK(sameBlocksLeft.size() == 1); AD_CORRECTNESS_CHECK(sameBlocksRight.size() == 1); - while (it1 != end1) { - sameBlocksLeft.push_back(std::move(*it1)); - if (!eq(sameBlocksLeft.back().back, sameBlocksLeft.front().back())) { - break; + while (it1 != end1) { + sameBlocksLeft.push_back(std::move(*it1)); + if (!eq(sameBlocksLeft.back().back, sameBlocksLeft.front().back())) { + break; + } + ++it1; } - ++it1; - } while (it2 != end2) { sameBlocksRight.push_back(std::move(*it2)); if (!eq(sameBlocksRight.back().back, sameBlocksRight.front().back())) { @@ -607,7 +606,8 @@ void zipperJoinForBlocksWithoutUndef( }; auto join = [&](const auto& l, const auto& r) { - return zipperJoinWithUndef(l , r, lessThan, compatibleRowAction, noop, noop); + return zipperJoinWithUndef(l, r, lessThan, compatibleRowAction, + noop, noop); }; auto addAll = [&](const auto& l, const auto& r) { @@ -624,8 +624,10 @@ void zipperJoinForBlocksWithoutUndef( auto joinBuffers = [&]() { join(sameBlocksLeft.at(0), sameBlocksRight.at(0)); - auto subrangeLeft = std::ranges::equal_range(sameBlocksLeft.front(), sameBlocksLeft.front().back(), lessThan); - auto subrangeRight = std::ranges::equal_range(sameBlocksRight.front(), sameBlocksRight.front().back(), lessThan); + auto subrangeLeft = std::ranges::equal_range( + sameBlocksLeft.front(), sameBlocksLeft.front().back(), lessThan); + auto subrangeRight = std::ranges::equal_range( + sameBlocksRight.front(), sameBlocksRight.front().back(), lessThan); using SubLeft = decltype(subrangeLeft); using SubRight = decltype(subrangeRight); std::vector l; @@ -635,7 +637,8 @@ void zipperJoinForBlocksWithoutUndef( l.push_back(sameBlocksLeft[i]); } if (sameBlocksLeft.size() > 1) { - l.push_back(std::ranges::equal_range(sameBlocksLeft.back(), sameBlocksLeft.front().back(), lessThan)); + l.push_back(std::ranges::equal_range( + sameBlocksLeft.back(), sameBlocksLeft.front().back(), lessThan)); } r.push_back(subrangeRight); for (size_t i = 1; i < sameBlocksRight.size() - 1; ++i) { @@ -651,13 +654,10 @@ void zipperJoinForBlocksWithoutUndef( sameBlocksLeft.resize(1); auto& b = sameBlocksLeft.front(); // TODO delete the range that has already been dealt with. - b.erase(b.begin(), std::upper_bound(b.begin(), b.end(), )) - + // b.erase(b.begin(), std::upper_bound(b.begin(), b.end(), )) } - }; - std::optional lastMatch1; std::optional lastMatch2; while (it1 != end1 && it2 != end2) { @@ -671,18 +671,16 @@ void zipperJoinForBlocksWithoutUndef( if (lastMatch2.has_value()) { view2 = std::ranges::subrange(lastMatch2.value(), block2.end()); } - auto [numOutOfOrder, lastMatchNew1, lastMatchNew2] = zipperJoinWithUndef(view1, view2, lessThan, compatibleRowAction, noop, noop); + auto [numOutOfOrder, lastMatchNew1, lastMatchNew2] = + zipperJoinWithUndef(view1, view2, lessThan, + compatibleRowAction, noop, noop); AD_CONTRACT_CHECK(numOutOfOrder == 0); if (lessThan(block1.back(), block2.back())) { ++it1; } else if (lessThan(block2.back(), block1.back())) { ++it2; } else { - } - - } - } } // namespace ad_utility diff --git a/test/QueryPlannerTest.cpp b/test/QueryPlannerTest.cpp index 1e43f813a2..a82922d7ec 100644 --- a/test/QueryPlannerTest.cpp +++ b/test/QueryPlannerTest.cpp @@ -1017,7 +1017,7 @@ TEST(QueryPlannerTest, testSimpleOptional) { } TEST(QueryPlannerTest, SimpleTripleOneVariable) { - using enum Index::Permutation; + using enum Permutation::Enum; // With only one variable, there are always two permutations that will yield // exactly the same result. The query planner consistently chosses one of @@ -1031,7 +1031,7 @@ TEST(QueryPlannerTest, SimpleTripleOneVariable) { } TEST(QueryPlannerTest, SimpleTripleTwoVariables) { - using enum Index::Permutation; + using enum Permutation::Enum; // Fixed predicate. @@ -1062,7 +1062,7 @@ TEST(QueryPlannerTest, SimpleTripleTwoVariables) { } TEST(QueryPlannerTest, SimpleTripleThreeVariables) { - using enum Index::Permutation; + using enum Permutation::Enum; // Fixed predicate. // Don't care about the sorting. diff --git a/test/QueryPlannerTestHelpers.h b/test/QueryPlannerTestHelpers.h index 836c69e302..585c48a7b7 100644 --- a/test/QueryPlannerTestHelpers.h +++ b/test/QueryPlannerTestHelpers.h @@ -35,10 +35,10 @@ auto RootOperation(auto matcher) -> Matcher { /// that the `ScanType` of this `IndexScan` is any of the given `scanTypes`. auto IndexScan(TripleComponent subject, TripleComponent predicate, TripleComponent object, size_t numVariables, - const std::vector& scanTypes = {}) + const std::vector& scanTypes = {}) -> Matcher { auto typeMatcher = - scanTypes.empty() ? A() : AnyOfArray(scanTypes); + scanTypes.empty() ? A() : AnyOfArray(scanTypes); return RootOperation<::IndexScan>( AllOf(Property(&IndexScan::permutation, typeMatcher), Property(&IndexScan::getResultWidth, Eq(numVariables)), From b138441d5840128792f04fd7de7e2a31a17c2e57 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Sun, 28 May 2023 18:55:08 +0200 Subject: [PATCH 022/150] In the middle of everything --- src/util/JoinAlgorithms/JoinAlgorithms.h | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 7e85e9b02a..4f592d5387 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -622,12 +622,22 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, } }; + auto joinAndRemoveBeginning = [&]() { + auto& l = sameBlocksLeft.at(0); + auto& r = sameBlocksRight.at(0); + auto itL = std::ranges::lower_bound(l, l.back(), lessThan); + auto itR = std::ranges::lower_bound(r, r.back(), lessThan); + join(std::ranges::subrange{l.begin(), itL}, + std::ranges::subrange{r.begin(), itR}); + return std::pair{std::ranges::subrange{itL, l.end()}, std::ranges::subrange{itR, r.end()}}; + }; + auto joinBuffers = [&]() { - join(sameBlocksLeft.at(0), sameBlocksRight.at(0)); - auto subrangeLeft = std::ranges::equal_range( - sameBlocksLeft.front(), sameBlocksLeft.front().back(), lessThan); - auto subrangeRight = std::ranges::equal_range( - sameBlocksRight.front(), sameBlocksRight.front().back(), lessThan); + if (sameBlocksLeft.size() == 1) { + AD_CORRECTNESS_CHECK(sameBlocksRight.size() == 1); + join (sameBlocksLeft.at(0), sameBlocksRight.at(0)); + } + auto [subrangeLeft, subrangeRight] = joinAndRemoveBeginning(); using SubLeft = decltype(subrangeLeft); using SubRight = decltype(subrangeRight); std::vector l; From d1de8ab0bdf3ac3b60184f1ec885337b00520947 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Fri, 9 Jun 2023 14:56:37 +0200 Subject: [PATCH 023/150] Continuing some work on this. --- src/util/JoinAlgorithms/JoinAlgorithms.h | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 4f592d5387..339b0af504 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -659,12 +659,19 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, sameBlocksRight.back(), sameBlocksRight.front().back(), lessThan)); } addAll(sameBlocksLeft, sameBlocksRight); - if (sameBlocksLeft.size() > 1) { - sameBlocksLeft.at(0) = std::move(sameBlocksLeft.back()); - sameBlocksLeft.resize(1); - auto& b = sameBlocksLeft.front(); - // TODO delete the range that has already been dealt with. - // b.erase(b.begin(), std::upper_bound(b.begin(), b.end(), )) + if (sameBlocksLeft.size() > 1 && (l.back().end() != sameBlocksLeft.back().end())) { + LeftBlock remainder(l.back().end(), sameBlocksLeft.back().end()); + sameBlocksLeft.clear(); + sameBlocksLeft.push_back(std::move(remainder)); + } else { + sameBlocksLeft.clear(); + } + if (sameBlocksLeft.size() > 1 && (l.back().end() != sameBlocksLeft.back().end())) { + LeftBlock remainder(l.back().end(), sameBlocksLeft.back().end()); + sameBlocksLeft.clear(); + sameBlocksLeft.push_back(std::move(remainder)); + } else { + sameBlocksLeft.clear() } }; From d70fe894e7830762dcc04e0f3240c200338fc3f5 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Mon, 12 Jun 2023 10:19:44 +0200 Subject: [PATCH 024/150] A first draft of the block joining function, We yet need to write unit tests for it. --- src/util/JoinAlgorithms/JoinAlgorithms.h | 85 ++++++++++++------------ 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 339b0af504..d0c5bcb8e4 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -587,18 +587,33 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, std::vector sameBlocksLeft; std::vector sameBlocksRight; auto fillBuffer = [&]() { - AD_CORRECTNESS_CHECK(sameBlocksLeft.size() == 1); - AD_CORRECTNESS_CHECK(sameBlocksRight.size() == 1); + AD_CORRECTNESS_CHECK(sameBlocksLeft.size() <= 1); + AD_CORRECTNESS_CHECK(sameBlocksRight.size() <= 1); + if (sameBlocksLeft.empty() && it1 != end1) { + sameBlocksLeft.push_back(std::move(*it1)); + ++it1; + } + if (sameBlocksLeft.empty() && it2 != end2) { + sameBlocksLeft.push_back(std::move(*it2)); + ++it2; + } + + if (sameBlocksLeft.empty() || sameBlocksRight.empty()) { + return; + } + const auto& lastLeft = sameBlocksLeft.front().back(); + const auto& lastRight = sameBlocksRight.front().back(); + while (it1 != end1) { sameBlocksLeft.push_back(std::move(*it1)); - if (!eq(sameBlocksLeft.back().back, sameBlocksLeft.front().back())) { + if (!eq(sameBlocksLeft.back().back, lastLeft)) { break; } ++it1; } while (it2 != end2) { sameBlocksRight.push_back(std::move(*it2)); - if (!eq(sameBlocksRight.back().back, sameBlocksRight.front().back())) { + if (!eq(sameBlocksRight.back().back, lastRight)) { break; } ++it2; @@ -629,13 +644,28 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, auto itR = std::ranges::lower_bound(r, r.back(), lessThan); join(std::ranges::subrange{l.begin(), itL}, std::ranges::subrange{r.begin(), itR}); - return std::pair{std::ranges::subrange{itL, l.end()}, std::ranges::subrange{itR, r.end()}}; + return std::pair{std::ranges::subrange{itL, l.end()}, + std::ranges::subrange{itR, r.end()}}; + }; + + auto removeAllButUnjoined = [lessThan]( + Blocks& blocks, auto lastHandledElement) { + AD_CORRECTNESS_CHECK(!blocks.empty()); + const auto& lastBlock = blocks.back(); + auto beginningOfUnjoined = + std::ranges::upper_bound(lastBlock, lastHandledElement, lessThan); + typename Blocks::value_type remainingBlock{beginningOfUnjoined, + lastBlock.end()}; + blocks.clear(); + if (!remainingBlock.empty()) { + blocks.push_back(remainingBlock); + } }; auto joinBuffers = [&]() { if (sameBlocksLeft.size() == 1) { AD_CORRECTNESS_CHECK(sameBlocksRight.size() == 1); - join (sameBlocksLeft.at(0), sameBlocksRight.at(0)); + join(sameBlocksLeft.at(0), sameBlocksRight.at(0)); } auto [subrangeLeft, subrangeRight] = joinAndRemoveBeginning(); using SubLeft = decltype(subrangeLeft); @@ -659,45 +689,16 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, sameBlocksRight.back(), sameBlocksRight.front().back(), lessThan)); } addAll(sameBlocksLeft, sameBlocksRight); - if (sameBlocksLeft.size() > 1 && (l.back().end() != sameBlocksLeft.back().end())) { - LeftBlock remainder(l.back().end(), sameBlocksLeft.back().end()); - sameBlocksLeft.clear(); - sameBlocksLeft.push_back(std::move(remainder)); - } else { - sameBlocksLeft.clear(); - } - if (sameBlocksLeft.size() > 1 && (l.back().end() != sameBlocksLeft.back().end())) { - LeftBlock remainder(l.back().end(), sameBlocksLeft.back().end()); - sameBlocksLeft.clear(); - sameBlocksLeft.push_back(std::move(remainder)); - } else { - sameBlocksLeft.clear() - } + removeAllButUnjoined(sameBlocksLeft, sameBlocksLeft.front().back()); + removeAllButUnjoined(sameBlocksRight, sameBlocksRight.front().back()); }; - std::optional lastMatch1; - std::optional lastMatch2; - while (it1 != end1 && it2 != end2) { - const auto& block1 = *it1; - const auto& block2 = *it2; - auto view1 = std::ranges::subrange(block1.begin(), block1.end()); - auto view2 = std::ranges::subrange(block2.begin(), block2.end()); - if (lastMatch1.has_value()) { - view1 = std::ranges::subrange(lastMatch1.value(), block1.end()); - } - if (lastMatch2.has_value()) { - view2 = std::ranges::subrange(lastMatch2.value(), block2.end()); - } - auto [numOutOfOrder, lastMatchNew1, lastMatchNew2] = - zipperJoinWithUndef(view1, view2, lessThan, - compatibleRowAction, noop, noop); - AD_CONTRACT_CHECK(numOutOfOrder == 0); - if (lessThan(block1.back(), block2.back())) { - ++it1; - } else if (lessThan(block2.back(), block1.back())) { - ++it2; - } else { + while (true) { + fillBuffer(); + if (sameBlocksLeft.empty() || sameBlocksRight.empty()) { + return; } + joinBuffers(); } } } // namespace ad_utility From ecde3a66e6ade6be3fce6cc2209d97562b5f145f Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Mon, 12 Jun 2023 11:40:21 +0200 Subject: [PATCH 025/150] The code now passes some unit tests. --- src/util/JoinAlgorithms/JoinAlgorithms.h | 38 +++++++++++----- test/CMakeLists.txt | 2 + test/JoinAlgorithmsTest.cpp | 57 ++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 test/JoinAlgorithmsTest.cpp diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index d0c5bcb8e4..2426f9f8e4 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -574,8 +574,8 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, RightBlocks&& rightBlocks, const LessThan& lessThan, const auto& compatibleRowAction) { - using LeftBlock = typename LeftBlocks::value_type; - using RightBlock = typename RightBlocks::value_type; + using LeftBlock = typename std::decay_t::value_type; + using RightBlock = typename std::decay_t::value_type; auto it1 = leftBlocks.begin(); auto end1 = leftBlocks.end(); auto it2 = rightBlocks.begin(); @@ -593,8 +593,8 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, sameBlocksLeft.push_back(std::move(*it1)); ++it1; } - if (sameBlocksLeft.empty() && it2 != end2) { - sameBlocksLeft.push_back(std::move(*it2)); + if (sameBlocksRight.empty() && it2 != end2) { + sameBlocksRight.push_back(std::move(*it2)); ++it2; } @@ -604,16 +604,20 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, const auto& lastLeft = sameBlocksLeft.front().back(); const auto& lastRight = sameBlocksRight.front().back(); + if (!eq(lastLeft, lastRight)) { + return; + } + while (it1 != end1) { sameBlocksLeft.push_back(std::move(*it1)); - if (!eq(sameBlocksLeft.back().back, lastLeft)) { + if (!eq(sameBlocksLeft.back().back(), lastLeft)) { break; } ++it1; } while (it2 != end2) { sameBlocksRight.push_back(std::move(*it2)); - if (!eq(sameBlocksRight.back().back, lastRight)) { + if (!eq(sameBlocksRight.back().back(), lastRight)) { break; } ++it2; @@ -630,7 +634,7 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, for (const auto& rBlock : r) { for (const auto& lEl : lBlock) { for (const auto& rEl : rBlock) { - compatibleRowAction(&lBlock, &rBlock); + compatibleRowAction(&lEl, &rEl); } } } @@ -640,8 +644,9 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, auto joinAndRemoveBeginning = [&]() { auto& l = sameBlocksLeft.at(0); auto& r = sameBlocksRight.at(0); - auto itL = std::ranges::lower_bound(l, l.back(), lessThan); - auto itR = std::ranges::lower_bound(r, r.back(), lessThan); + auto maxEl = std::max(l.back(), r.back(), lessThan); + auto itL = std::ranges::lower_bound(l, maxEl, lessThan); + auto itR = std::ranges::lower_bound(r, maxEl, lessThan); join(std::ranges::subrange{l.begin(), itL}, std::ranges::subrange{r.begin(), itR}); return std::pair{std::ranges::subrange{itL, l.end()}, @@ -666,6 +671,11 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, if (sameBlocksLeft.size() == 1) { AD_CORRECTNESS_CHECK(sameBlocksRight.size() == 1); join(sameBlocksLeft.at(0), sameBlocksRight.at(0)); + auto minEl = std::min(sameBlocksLeft.front().back(), + sameBlocksRight.front().back()); + removeAllButUnjoined(sameBlocksLeft, minEl); + removeAllButUnjoined(sameBlocksRight, minEl); + return; } auto [subrangeLeft, subrangeRight] = joinAndRemoveBeginning(); using SubLeft = decltype(subrangeLeft); @@ -688,9 +698,13 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, r.push_back(std::ranges::equal_range( sameBlocksRight.back(), sameBlocksRight.front().back(), lessThan)); } - addAll(sameBlocksLeft, sameBlocksRight); - removeAllButUnjoined(sameBlocksLeft, sameBlocksLeft.front().back()); - removeAllButUnjoined(sameBlocksRight, sameBlocksRight.front().back()); + addAll(l, r); + // TODO If we reach this part of the code, then the following two + // should be equal. + auto maxEl = + std::max(sameBlocksLeft.front().back(), sameBlocksRight.front().back()); + removeAllButUnjoined(sameBlocksLeft, maxEl); + removeAllButUnjoined(sameBlocksRight, maxEl); }; while (true) { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7d7411eea2..543c7cb5e8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -327,3 +327,5 @@ addLinkAndDiscoverTest(BenchmarkMeasurementContainerTest benchmark) addLinkAndDiscoverTest(FindUndefRangesTest) addLinkAndDiscoverTest(AddCombinedRowToTableTest) + +addLinkAndDiscoverTest(JoinAlgorithmsTest) \ No newline at end of file diff --git a/test/JoinAlgorithmsTest.cpp b/test/JoinAlgorithmsTest.cpp new file mode 100644 index 0000000000..ccab415629 --- /dev/null +++ b/test/JoinAlgorithmsTest.cpp @@ -0,0 +1,57 @@ +// Copyright 2023, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Author: Johannes Kalmbach + +#include +#include + +#include "util/JoinAlgorithms/JoinAlgorithms.h" + +using namespace ad_utility; +namespace { +using NestedBlock = std::vector>; + +auto makeRowAdder(auto& target) { + return [&target](auto it1, auto it2) { + AD_CONTRACT_CHECK(*it1 == *it2); + target.push_back(*it1); + }; +} +} // namespace + +TEST(JoinAlgorithms, JoinWithBlocksEmptyInput) { + NestedBlock a; + NestedBlock b; + std::vector result; + zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); + EXPECT_TRUE(result.empty()); + + a.push_back({13}); + zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); + EXPECT_TRUE(result.empty()); + + b.emplace_back(); + zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); + EXPECT_TRUE(result.empty()); + + a.clear(); + b.push_back({23, 35}); + zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); + EXPECT_TRUE(result.empty()); +} + +TEST(JoinAlgorithms, JoinWithBlocksSingleBlock) { + NestedBlock a{{1, 4, 18, 42}}; + NestedBlock b{{0, 4, 5, 19, 42}}; + std::vector result; + zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); + EXPECT_THAT(result, ::testing::ElementsAre(4, 42)); +} + +TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksOverlap) { + NestedBlock a{{1, 4, 18, 42}, {54, 57, 59}}; + NestedBlock b{{0, 4, 5, 19, 42, 54}, {56, 57, 58}}; + std::vector result; + zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); + EXPECT_THAT(result, ::testing::ElementsAre(4, 42, 54, 57)); +} From 79f291babd7b1f1726267afd09e520bb93fd91f0 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Mon, 12 Jun 2023 12:08:28 +0200 Subject: [PATCH 026/150] Fixed some more unit tests. --- src/util/JoinAlgorithms/JoinAlgorithms.h | 16 ++++++++++------ test/JoinAlgorithmsTest.cpp | 6 +++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 2426f9f8e4..011ba111a5 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -589,12 +589,16 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, auto fillBuffer = [&]() { AD_CORRECTNESS_CHECK(sameBlocksLeft.size() <= 1); AD_CORRECTNESS_CHECK(sameBlocksRight.size() <= 1); - if (sameBlocksLeft.empty() && it1 != end1) { - sameBlocksLeft.push_back(std::move(*it1)); + while (sameBlocksLeft.empty() && it1 != end1) { + if (!it1->empty()) { + sameBlocksLeft.push_back(std::move(*it1)); + } ++it1; } - if (sameBlocksRight.empty() && it2 != end2) { - sameBlocksRight.push_back(std::move(*it2)); + while (sameBlocksRight.empty() && it2 != end2) { + if (!it2->empty()) { + sameBlocksRight.push_back(std::move(*it2)); + } ++it2; } @@ -610,17 +614,17 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, while (it1 != end1) { sameBlocksLeft.push_back(std::move(*it1)); + ++it1; if (!eq(sameBlocksLeft.back().back(), lastLeft)) { break; } - ++it1; } while (it2 != end2) { sameBlocksRight.push_back(std::move(*it2)); + ++it2; if (!eq(sameBlocksRight.back().back(), lastRight)) { break; } - ++it2; } }; diff --git a/test/JoinAlgorithmsTest.cpp b/test/JoinAlgorithmsTest.cpp index ccab415629..baf634b14d 100644 --- a/test/JoinAlgorithmsTest.cpp +++ b/test/JoinAlgorithmsTest.cpp @@ -49,9 +49,9 @@ TEST(JoinAlgorithms, JoinWithBlocksSingleBlock) { } TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksOverlap) { - NestedBlock a{{1, 4, 18, 42}, {54, 57, 59}}; - NestedBlock b{{0, 4, 5, 19, 42, 54}, {56, 57, 58}}; + NestedBlock a{{1, 4, 18, 42}, {54, 57, 59}, {60, 67}}; + NestedBlock b{{0, 4, 5, 19, 42, 54}, {56, 57, 58, 59}, {61, 67}}; std::vector result; zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); - EXPECT_THAT(result, ::testing::ElementsAre(4, 42, 54, 57)); + EXPECT_THAT(result, ::testing::ElementsAre(4, 42, 54, 57, 59, 67)); } From 1ba3c79d09ff59653a07560f55e6101d1ed8b3d9 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Mon, 12 Jun 2023 15:45:32 +0200 Subject: [PATCH 027/150] More unit tests run through, but there's still some stuff that I don't understand. --- src/util/JoinAlgorithms/JoinAlgorithms.h | 49 +++++++++++++----------- test/JoinAlgorithmsTest.cpp | 29 +++++++++++++- 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 011ba111a5..3eafbc319c 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -608,22 +608,25 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, const auto& lastLeft = sameBlocksLeft.front().back(); const auto& lastRight = sameBlocksRight.front().back(); - if (!eq(lastLeft, lastRight)) { - return; - } - - while (it1 != end1) { - sameBlocksLeft.push_back(std::move(*it1)); - ++it1; - if (!eq(sameBlocksLeft.back().back(), lastLeft)) { - break; + /* + if (!eq(lastRight, lastLeft)) { + return; + } + */ + if (!lessThan(lastRight, lastLeft)) { + while (it1 != end1 && eq((*it1).at(0), lastLeft)) { + sameBlocksLeft.push_back(std::move(*it1)); + ++it1; } } - while (it2 != end2) { - sameBlocksRight.push_back(std::move(*it2)); - ++it2; - if (!eq(sameBlocksRight.back().back(), lastRight)) { - break; + + if (!lessThan(lastLeft, lastRight)) { + while (it2 != end2 && eq((*it2).at(0), lastRight)) { + sameBlocksRight.push_back(std::move(*it2)); + ++it2; + if (!eq(sameBlocksRight.back().back(), lastRight)) { + break; + } } } }; @@ -648,13 +651,12 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, auto joinAndRemoveBeginning = [&]() { auto& l = sameBlocksLeft.at(0); auto& r = sameBlocksRight.at(0); - auto maxEl = std::max(l.back(), r.back(), lessThan); - auto itL = std::ranges::lower_bound(l, maxEl, lessThan); - auto itR = std::ranges::lower_bound(r, maxEl, lessThan); - join(std::ranges::subrange{l.begin(), itL}, - std::ranges::subrange{r.begin(), itR}); - return std::pair{std::ranges::subrange{itL, l.end()}, - std::ranges::subrange{itR, r.end()}}; + auto minEl = std::min(l.back(), r.back(), lessThan); + auto itL = std::ranges::equal_range(l, minEl, lessThan); + auto itR = std::ranges::equal_range(r, minEl, lessThan); + join(std::ranges::subrange{l.begin(), itL.begin()}, + std::ranges::subrange{r.begin(), itR.begin()}); + return std::pair{itL, itR}; }; auto removeAllButUnjoined = [lessThan]( @@ -672,8 +674,9 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, }; auto joinBuffers = [&]() { - if (sameBlocksLeft.size() == 1) { - AD_CORRECTNESS_CHECK(sameBlocksRight.size() == 1); + // TODO Figure out why we need this base case to make the tests + // work. + if (sameBlocksLeft.size() == 1 && sameBlocksRight.size() == 1) { join(sameBlocksLeft.at(0), sameBlocksRight.at(0)); auto minEl = std::min(sameBlocksLeft.front().back(), sameBlocksRight.front().back()); diff --git a/test/JoinAlgorithmsTest.cpp b/test/JoinAlgorithmsTest.cpp index baf634b14d..0d62c60391 100644 --- a/test/JoinAlgorithmsTest.cpp +++ b/test/JoinAlgorithmsTest.cpp @@ -52,6 +52,33 @@ TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksOverlap) { NestedBlock a{{1, 4, 18, 42}, {54, 57, 59}, {60, 67}}; NestedBlock b{{0, 4, 5, 19, 42, 54}, {56, 57, 58, 59}, {61, 67}}; std::vector result; - zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); + zipperJoinForBlocksWithoutUndef(NestedBlock(a), NestedBlock(b), std::less<>{}, + makeRowAdder(result)); + EXPECT_THAT(result, ::testing::ElementsAre(4, 42, 54, 57, 59, 67)); + result.clear(); + zipperJoinForBlocksWithoutUndef(NestedBlock(b), NestedBlock(a), std::less<>{}, + makeRowAdder(result)); EXPECT_THAT(result, ::testing::ElementsAre(4, 42, 54, 57, 59, 67)); } + +TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksDuplicates) { + using NB = std::vector>>; + NB a{{{1, 0}, {42, 0}}, {{42, 1}, {42, 2}}, {{42, 3}, {43, 5}, {67, 0}}}; + NB b{{{2, 0}, {42, 12}, {43, 1}}, {{67, 13}, {69, 14}}}; + std::vector> result; + std::vector> expectedResult{ + {42, 0, 12}, {42, 1, 12}, {42, 2, 12}, {42, 3, 12}, {67, 0, 13}}; + auto compare = [](auto l, auto r) { return l[0] < r[0]; }; + auto add = [&result](auto it1, auto it2) { + AD_CORRECTNESS_CHECK((*it1)[0] == (*it1)[0]); + result.push_back(std::array{(*it1)[0], (*it1)[1], (*it2)[1]}); + }; + zipperJoinForBlocksWithoutUndef(NB{a}, NB{b}, compare, add); + EXPECT_THAT(result, ::testing::ElementsAreArray(expectedResult)); + result.clear(); + for (auto& [x, y, z] : expectedResult) { + std::swap(y, z); + } + zipperJoinForBlocksWithoutUndef(NB{b}, NB{a}, compare, add); + EXPECT_THAT(result, ::testing::ElementsAreArray(expectedResult)); +} From 712eb527217ff63e50e88ef91a1df9faa5cda53a Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Mon, 12 Jun 2023 17:21:10 +0200 Subject: [PATCH 028/150] Delete a comment. --- src/util/JoinAlgorithms/JoinAlgorithms.h | 25 ++++-------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 3eafbc319c..27527103e6 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -608,11 +608,6 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, const auto& lastLeft = sameBlocksLeft.front().back(); const auto& lastRight = sameBlocksRight.front().back(); - /* - if (!eq(lastRight, lastLeft)) { - return; - } - */ if (!lessThan(lastRight, lastLeft)) { while (it1 != end1 && eq((*it1).at(0), lastLeft)) { sameBlocksLeft.push_back(std::move(*it1)); @@ -674,16 +669,6 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, }; auto joinBuffers = [&]() { - // TODO Figure out why we need this base case to make the tests - // work. - if (sameBlocksLeft.size() == 1 && sameBlocksRight.size() == 1) { - join(sameBlocksLeft.at(0), sameBlocksRight.at(0)); - auto minEl = std::min(sameBlocksLeft.front().back(), - sameBlocksRight.front().back()); - removeAllButUnjoined(sameBlocksLeft, minEl); - removeAllButUnjoined(sameBlocksRight, minEl); - return; - } auto [subrangeLeft, subrangeRight] = joinAndRemoveBeginning(); using SubLeft = decltype(subrangeLeft); using SubRight = decltype(subrangeRight); @@ -706,12 +691,10 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, sameBlocksRight.back(), sameBlocksRight.front().back(), lessThan)); } addAll(l, r); - // TODO If we reach this part of the code, then the following two - // should be equal. - auto maxEl = - std::max(sameBlocksLeft.front().back(), sameBlocksRight.front().back()); - removeAllButUnjoined(sameBlocksLeft, maxEl); - removeAllButUnjoined(sameBlocksRight, maxEl); + auto minEl = + std::min(sameBlocksLeft.front().back(), sameBlocksRight.front().back()); + removeAllButUnjoined(sameBlocksLeft, minEl); + removeAllButUnjoined(sameBlocksRight, minEl); }; while (true) { From 8df3277e63b6e590438044b81d0dd53073ee43c3 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 13 Jun 2023 12:02:00 +0200 Subject: [PATCH 029/150] The `getBlocksForJoin` method is semantically wrong. --- src/engine/IndexScan.cpp | 1 - src/engine/Join.cpp | 43 ++++++++++++++++++++++-- src/engine/Join.h | 2 +- src/index/CompressedRelation.cpp | 23 ++++++++----- src/util/JoinAlgorithms/JoinAlgorithms.h | 23 +++++++------ test/JoinAlgorithmsTest.cpp | 31 +++++++++++++++-- 6 files changed, 98 insertions(+), 25 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index ac22cbc61e..ba6de33e67 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -283,7 +283,6 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans( const auto& index = s1.getExecutionContext()->getIndex().getImpl(); AD_CONTRACT_CHECK(s1._numVariables < 3 && s2._numVariables < 3); - const auto& s = s1; auto f = [&](const IndexScan& s) -> std::optional { auto permutedTriple = s.getPermutedTriple(); diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp index 0a0377734c..807e1a0323 100644 --- a/src/engine/Join.cpp +++ b/src/engine/Join.cpp @@ -119,8 +119,13 @@ ResultTable Join::computeResult() { LOG(DEBUG) << "Computing Join result..." << endl; - join(leftRes->idTable(), _leftJoinCol, rightRes->idTable(), _rightJoinCol, - &idTable); + if (_left->getType() == QueryExecutionTree::SCAN && + _right->getType() == QueryExecutionTree::SCAN) { + computeResultForTwoIndexScans(&idTable); + } else { + join(leftRes->idTable(), _leftJoinCol, rightRes->idTable(), _rightJoinCol, + &idTable); + } LOG(DEBUG) << "Join result computation done" << endl; @@ -609,7 +614,39 @@ void Join::addCombinedRowToIdTable(const ROW_A& rowA, const ROW_B& rowB, } // ______________________________________________________________________________________________________ -ResultTable Join::computeResultForTwoIndexScans() { +void Join::computeResultForTwoIndexScans(IdTable* resultPtr) { AD_CORRECTNESS_CHECK(_left->getType() == QueryExecutionTree::SCAN && _right->getType() == QueryExecutionTree::SCAN); + auto& result = *resultPtr; + result.setNumColumns(getResultWidth()); + + auto addResultRow = [&](auto itLeft, auto itRight) { + const auto& l = *itLeft; + const auto& r = *itRight; + AD_CORRECTNESS_CHECK(l[0] == r[0]); + result.emplace_back(); + IdTable::row_reference lastRow = result.back(); + lastRow[0] = l[0]; + size_t nextIndex = 1; + for (size_t i = 1; i < l.size(); ++i) { + lastRow[nextIndex] = l[i]; + ++nextIndex; + } + for (size_t i = 1; i < r.size(); ++i) { + lastRow[nextIndex] = r[i]; + ++nextIndex; + } + }; + + auto lessThan = [](const auto& a, const auto& b) { + return a[0] < b[0]; + }; + + auto [leftBlocks, rightBlocks] = IndexScan::lazyScanForJoinOfTwoScans(dynamic_cast(*_left->getRootOperation()), dynamic_cast(*_right->getRootOperation())); + + LOG(WARN) << "num blocks in first: " << std::ranges::distance(leftBlocks) << std::endl; + LOG(WARN) << "num blocks in second: " << std::ranges::distance(rightBlocks) << std::endl; + AD_FAIL(); + + ad_utility::zipperJoinForBlocksWithoutUndef(std::move(leftBlocks), std::move(rightBlocks), lessThan, addResultRow); } diff --git a/src/engine/Join.h b/src/engine/Join.h index fd97b730a7..170f88b1b4 100644 --- a/src/engine/Join.h +++ b/src/engine/Join.h @@ -133,7 +133,7 @@ class Join : public Operation { ResultTable computeResultForJoinWithFullScanDummy(); - ResultTable computeResultForTwoIndexScans(); + void computeResultForTwoIndexScans(IdTable* tablePtr); using ScanMethodType = std::function; diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index 0c59cb339d..608bcd82c2 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -308,7 +308,9 @@ cppcoro::generator CompressedRelationReader::lazyScan( readCompressedBlockFromFile(block, file, std::vector{1ul}); co_yield decompressBlock(compressedBuffer, block.numRows_); } - timer->wlock()->checkTimeoutAndThrow(); + if (timer) { + timer->wlock()->checkTimeoutAndThrow(); + } if (lastBlockResult.has_value()) { co_yield lastBlockResult.value(); } @@ -371,14 +373,18 @@ CompressedRelationReader::getBlocksForJoin( auto beginBlock2 = relevantBlocks2.begin(); auto endBlock2 = relevantBlocks2.end(); - auto blockLessThanBlock = [](const CompressedBlockMetadata& block1, - const CompressedBlockMetadata& block2) { - if (block1.col0LastId_ < block2.col0FirstId_ || - block1.col0FirstId_ > block2.col0LastId_) { - return block1.col0LastId_ < block2.col0LastId_; - } else { - return false; + for (size_t i = 0; i < relevantBlocks1.size(); ++i) { + LOG(WARN) << "firstIdOfBlockRel1 " << i << ':' + << relevantBlocks1[i].col0FirstId_ << ' ' << relevantBlocks1[i].col0FirstId_ << std::endl; + } + for (size_t i = 0; i < relevantBlocks2.size(); ++i) { + LOG(WARN) << "firstIdOfBlockRel2 " << i << ':' + << relevantBlocks2[i].col0FirstId_ << ' ' << relevantBlocks2[i].col0FirstId_ << std::endl; } + + auto blockLessThanBlock = [](const CompressedBlockMetadata& block1, + const CompressedBlockMetadata& block2) { + return block1.col0LastId_ < block2.col0FirstId_; }; std::array, 2> result; @@ -793,6 +799,7 @@ CompressedRelationReader::getBlocksFromMetadata( auto [beginBlock, endBlock] = std::equal_range( blockMetadata.begin(), blockMetadata.end(), KeyLhs{col0Id, col0Id, col1Id.value(), col1Id.value()}, comp); + LOG(WARN) << "number of found blocks: " << endBlock - beginBlock << std::endl; return {beginBlock, endBlock}; } } diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 27527103e6..43ee82908f 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -609,14 +609,15 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, const auto& lastRight = sameBlocksRight.front().back(); if (!lessThan(lastRight, lastLeft)) { - while (it1 != end1 && eq((*it1).at(0), lastLeft)) { + // TODO here and below: use `at`, but it needs to be implemented in the `Row` class. + while (it1 != end1 && eq((*it1)[0], lastLeft)) { sameBlocksLeft.push_back(std::move(*it1)); ++it1; } } if (!lessThan(lastLeft, lastRight)) { - while (it2 != end2 && eq((*it2).at(0), lastRight)) { + while (it2 != end2 && eq((*it2)[0], lastRight)) { sameBlocksRight.push_back(std::move(*it2)); ++it2; if (!eq(sameBlocksRight.back().back(), lastRight)) { @@ -646,7 +647,7 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, auto joinAndRemoveBeginning = [&]() { auto& l = sameBlocksLeft.at(0); auto& r = sameBlocksRight.at(0); - auto minEl = std::min(l.back(), r.back(), lessThan); + typename std::iterator_traits::value_type minEl = std::min(l.back(), r.back(), lessThan); auto itL = std::ranges::equal_range(l, minEl, lessThan); auto itR = std::ranges::equal_range(r, minEl, lessThan); join(std::ranges::subrange{l.begin(), itL.begin()}, @@ -657,14 +658,15 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, auto removeAllButUnjoined = [lessThan]( Blocks& blocks, auto lastHandledElement) { AD_CORRECTNESS_CHECK(!blocks.empty()); - const auto& lastBlock = blocks.back(); + typename Blocks::value_type remainingBlock = std::move(blocks.back()); auto beginningOfUnjoined = - std::ranges::upper_bound(lastBlock, lastHandledElement, lessThan); - typename Blocks::value_type remainingBlock{beginningOfUnjoined, - lastBlock.end()}; + std::ranges::upper_bound(remainingBlock, lastHandledElement, lessThan); + // TODO This is not the most efficient way, but currently necessary because of the + // interface of the `IdTable`. + remainingBlock.erase(remainingBlock.begin(), beginningOfUnjoined); blocks.clear(); if (!remainingBlock.empty()) { - blocks.push_back(remainingBlock); + blocks.push_back(std::move(remainingBlock)); } }; @@ -691,8 +693,9 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, sameBlocksRight.back(), sameBlocksRight.front().back(), lessThan)); } addAll(l, r); - auto minEl = - std::min(sameBlocksLeft.front().back(), sameBlocksRight.front().back()); + typename std::iterator_traits::value_type minEl = + std::min(sameBlocksLeft.front().back(), sameBlocksRight.front().back(), + lessThan); removeAllButUnjoined(sameBlocksLeft, minEl); removeAllButUnjoined(sameBlocksRight, minEl); }; diff --git a/test/JoinAlgorithmsTest.cpp b/test/JoinAlgorithmsTest.cpp index 0d62c60391..c1edf91699 100644 --- a/test/JoinAlgorithmsTest.cpp +++ b/test/JoinAlgorithmsTest.cpp @@ -6,6 +6,7 @@ #include #include "util/JoinAlgorithms/JoinAlgorithms.h" +#include "util/TransparentFunctors.h" using namespace ad_utility; namespace { @@ -61,9 +62,9 @@ TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksOverlap) { EXPECT_THAT(result, ::testing::ElementsAre(4, 42, 54, 57, 59, 67)); } -TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksDuplicates) { +TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksPerElement) { using NB = std::vector>>; - NB a{{{1, 0}, {42, 0}}, {{42, 1}, {42, 2}}, {{42, 3}, {43, 5}, {67, 0}}}; + NB a{{{1, 0}, {42, 0}}, {{42, 1}, {42, 2}}, {{42, 3}, {48, 5}, {67, 0}}}; NB b{{{2, 0}, {42, 12}, {43, 1}}, {{67, 13}, {69, 14}}}; std::vector> result; std::vector> expectedResult{ @@ -82,3 +83,29 @@ TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksDuplicates) { zipperJoinForBlocksWithoutUndef(NB{b}, NB{a}, compare, add); EXPECT_THAT(result, ::testing::ElementsAreArray(expectedResult)); } + +TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksPerElementBothSides) { + using NB = std::vector>>; + NB a{{{42, 0}}, {{42, 1}, {42, 2}}, {{42, 3}, {67, 0}}}; + NB b{{{2, 0}, {42, 12}}, {{42, 13}, {67, 14}}}; + std::vector> result; + std::vector> expectedResult{ + {42, 0, 12}, {42, 0, 13},{42, 1, 12}, {42, 2, 12}, {42, 1, 13}, {42, 2, 13},{42, 3, 12}, {42, 3, 13}, {67, 0, 14}}; + auto compare = [](auto l, auto r) { return l[0] < r[0]; }; + auto add = [&result](auto it1, auto it2) { + AD_CORRECTNESS_CHECK((*it1)[0] == (*it1)[0]); + result.push_back(std::array{(*it1)[0], (*it1)[1], (*it2)[1]}); + }; + zipperJoinForBlocksWithoutUndef(NB{a}, NB{b}, compare, add); + // The exact order of the elements with the same first column is not important and depends on implementation + // details. We therefore do not enforce it here. + EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); + EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expectedResult)); + result.clear(); + for (auto& [x, y, z] : expectedResult) { + std::swap(y, z); + } + zipperJoinForBlocksWithoutUndef(NB{b}, NB{a}, compare, add); + EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); + EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expectedResult)); +} From 76f73209bae14efd889f1f7be552abef9cc244e5 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 13 Jun 2023 18:57:18 +0200 Subject: [PATCH 030/150] Currently not yet working, but we are getting there. --- src/engine/IndexScan.cpp | 19 ++-- src/engine/Join.cpp | 8 +- src/index/CompressedRelation.cpp | 132 ++++++++++++++++----------- src/index/CompressedRelation.h | 28 +++--- src/util/Serializer/SerializeArray.h | 24 +++++ src/util/TypeTraits.h | 6 ++ 6 files changed, 135 insertions(+), 82 deletions(-) create mode 100644 src/util/Serializer/SerializeArray.h diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index ba6de33e67..6b2cb671a9 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -283,8 +283,9 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans( const auto& index = s1.getExecutionContext()->getIndex().getImpl(); AD_CONTRACT_CHECK(s1._numVariables < 3 && s2._numVariables < 3); - auto f = - [&](const IndexScan& s) -> std::optional { + auto f = [&](const IndexScan& s) + -> std::optional< + std::pair>> { auto permutedTriple = s.getPermutedTriple(); std::optional optId = permutedTriple[0]->toValueId(index.getVocab()); std::optional optId2 = @@ -293,8 +294,12 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans( if (!optId.has_value() || (!optId2.has_value() && s._numVariables == 1)) { return std::nullopt; } - return index.getPermutation(s.permutation()) - .getMetadataAndBlocks(optId.value(), optId2); + auto optionalBla = index.getPermutation(s.permutation()) + .getMetadataAndBlocks(optId.value(), optId2); + if (optionalBla.has_value()) { + return std::pair{std::move(optionalBla.value()), optId2}; + } + return std::nullopt; }; auto metaBlocks1 = f(s1); @@ -304,9 +309,9 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans( return {{}}; } auto [blocks1, blocks2] = CompressedRelationReader::getBlocksForJoin( - metaBlocks1.value().relationMetadata_, - metaBlocks2.value().relationMetadata_, metaBlocks1.value().blockMetadata_, - metaBlocks2.value().blockMetadata_); + metaBlocks1.value().first.relationMetadata_, + metaBlocks2.value().first.relationMetadata_, metaBlocks1.value().second, metaBlocks2.value().second, metaBlocks1.value().first.blockMetadata_, + metaBlocks2.value().first.blockMetadata_); // TODO include a timeout timer. auto getScan = [&index](const IndexScan& s, const auto& blocks) { diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp index 807e1a0323..81086e1532 100644 --- a/src/engine/Join.cpp +++ b/src/engine/Join.cpp @@ -644,9 +644,7 @@ void Join::computeResultForTwoIndexScans(IdTable* resultPtr) { auto [leftBlocks, rightBlocks] = IndexScan::lazyScanForJoinOfTwoScans(dynamic_cast(*_left->getRootOperation()), dynamic_cast(*_right->getRootOperation())); - LOG(WARN) << "num blocks in first: " << std::ranges::distance(leftBlocks) << std::endl; - LOG(WARN) << "num blocks in second: " << std::ranges::distance(rightBlocks) << std::endl; - AD_FAIL(); - - ad_utility::zipperJoinForBlocksWithoutUndef(std::move(leftBlocks), std::move(rightBlocks), lessThan, addResultRow); + //LOG(WARN) << "num blocks in first: " << std::ranges::distance(leftBlocks) << std::endl; + //LOG(WARN) << "num blocks in second: " << std::ranges::distance(rightBlocks) << std::endl; + ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks, lessThan, addResultRow); } diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index 608bcd82c2..cce1b9264c 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -43,12 +43,12 @@ void CompressedRelationReader::scan( // actual scan result. bool firstBlockIsIncomplete = beginBlock < endBlock && - (beginBlock->col0FirstId_ < col0Id || beginBlock->col0LastId_ > col0Id); + (beginBlock->firstTriple_[0] < col0Id || beginBlock->lastTriple_[0] > col0Id); auto lastBlock = endBlock - 1; bool lastBlockIsIncomplete = beginBlock < lastBlock && - (lastBlock->col0FirstId_ < col0Id || lastBlock->col0LastId_ > col0Id); + (lastBlock->firstTriple_[0] < col0Id || lastBlock->lastTriple_[0] > col0Id); // Invariant: A relation spans multiple blocks exclusively or several // entities are stored completely in the same Block. @@ -140,8 +140,8 @@ void CompressedRelationReader::scan( // _____________________________________________________________________________ cppcoro::generator CompressedRelationReader::lazyScan( - const CompressedRelationMetadata& metadata, - std::span blockMetadata, + CompressedRelationMetadata metadata, + std::vector blockMetadata, ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, ad_utility::SharedConcurrentTimeoutTimer timer) const { auto relevantBlocks = @@ -153,12 +153,12 @@ cppcoro::generator CompressedRelationReader::lazyScan( // actual scan result. bool firstBlockIsIncomplete = beginBlock < endBlock && - (beginBlock->col0FirstId_ < col0Id || beginBlock->col0LastId_ > col0Id); + (beginBlock->firstTriple_[0] < col0Id || beginBlock->lastTriple_[0] > col0Id); auto lastBlock = endBlock - 1; bool lastBlockIsIncomplete = beginBlock < lastBlock && - (lastBlock->col0FirstId_ < col0Id || lastBlock->col0LastId_ > col0Id); + (lastBlock->firstTriple_[0] < col0Id || lastBlock->lastTriple_[0] > col0Id); // Invariant: A relation spans multiple blocks exclusively or several // entities are stored completely in the same Block. @@ -223,8 +223,8 @@ cppcoro::generator CompressedRelationReader::lazyScan( } cppcoro::generator CompressedRelationReader::lazyScan( - const CompressedRelationMetadata& metadata, Id col1Id, - std::span blockMetadata, + CompressedRelationMetadata metadata, Id col1Id, + std::vector blockMetadata, ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, ad_utility::SharedConcurrentTimeoutTimer timer) const { auto relevantBlocks = getBlocksFromMetadata(metadata, col1Id, blockMetadata); @@ -330,10 +330,10 @@ std::vector CompressedRelationReader::getBlocksForJoin( } auto idLessThanBlock = [](Id id, const CompressedBlockMetadata& block) { - return id < block.col1FirstId_; + return id < block.firstTriple_[1]; }; auto blockLessThanId = [](const CompressedBlockMetadata& block, Id id) { - return block.col1LastId_ < id; + return block.lastTriple_[1] < id; }; auto lessThan = @@ -355,6 +355,8 @@ std::array, 2> CompressedRelationReader::getBlocksForJoin( const CompressedRelationMetadata& md1, const CompressedRelationMetadata& md2, + std::optional col1Id1, + std::optional col1Id2, std::span blockMetadata1, std::span blockMetadata2) { // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ @@ -373,30 +375,58 @@ CompressedRelationReader::getBlocksForJoin( auto beginBlock2 = relevantBlocks2.begin(); auto endBlock2 = relevantBlocks2.end(); - for (size_t i = 0; i < relevantBlocks1.size(); ++i) { - LOG(WARN) << "firstIdOfBlockRel1 " << i << ':' - << relevantBlocks1[i].col0FirstId_ << ' ' << relevantBlocks1[i].col0FirstId_ << std::endl; - } - for (size_t i = 0; i < relevantBlocks2.size(); ++i) { - LOG(WARN) << "firstIdOfBlockRel2 " << i << ':' - << relevantBlocks2[i].col0FirstId_ << ' ' << relevantBlocks2[i].col0FirstId_ << std::endl; - } + auto getJoinColumnRangeValue = [](std::array block, Id col0Id, + std::optional col1Id) { + auto minId = Id::makeUndefined(); + auto maxId = Id::fromBits(std::numeric_limits::max()); + if (block[0] < col0Id) { + return minId; + } + if (block[0] > col0Id) { + return maxId; + } + if (!col1Id.has_value()) { + return block[1]; + } - auto blockLessThanBlock = [](const CompressedBlockMetadata& block1, - const CompressedBlockMetadata& block2) { - return block1.col0LastId_ < block2.col0FirstId_; + if (block[1] < col1Id.value()) { + return minId; + } + if (block[1] > col1Id.value()) { + return maxId; + } + return block[2]; + }; + auto blocksToIdRanges = [&getJoinColumnRangeValue](const auto& blocks, Id col0Id, + std::optional col1Id) { + std::vector> result; + for (const auto& block: blocks) { + result.emplace_back( + getJoinColumnRangeValue(block.firstTriple_, + col0Id, col1Id), + getJoinColumnRangeValue(block.lastTriple_, + col0Id, col1Id)); + } + return result; + }; + + auto blockLessThanBlock = []( + const auto& block1, + const auto& block2) { + return block1.second < block2.first; }; std::array, 2> result; - auto addRow = [&result]([[maybe_unused]] auto it1, auto it2) { - result[0].push_back(*it1); - result[1].push_back(*it2); + auto idRanges1 = blocksToIdRanges(std::ranges::subrange{beginBlock1, endBlock1}, md1.col0Id_, col1Id1); + auto idRanges2 = blocksToIdRanges(std::ranges::subrange{beginBlock2, endBlock2}, md2.col0Id_, col1Id2); + auto addRow = [&result, &beginBlock1, &beginBlock2, &idRanges1, &idRanges2]([[maybe_unused]] auto it1, auto it2) { + result[0].push_back(*(beginBlock1 + (it1 - idRanges1.begin()))); + result[1].push_back(*(beginBlock2 + (it2 - idRanges2.begin()))); }; auto noop = ad_utility::noop; [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( - std::span{beginBlock1, endBlock1}, - std::span(beginBlock2, endBlock2), + idRanges1, idRanges2, blockLessThanBlock, addRow, noop, noop); for (auto& vec : result) { vec.erase(std::unique(vec.begin(), vec.end()), vec.end()); @@ -592,12 +622,9 @@ CompressedRelationMetadata CompressedRelationWriter::addRelation( metaData.offsetInBlock_ = buffer_.numRows(); static_assert(sizeof(col1And2Ids[0][0]) == sizeof(Id)); if (buffer_.numRows() == 0) { - currentBlockData_.col0FirstId_ = col0Id; - currentBlockData_.col1FirstId_ = col1And2Ids(0, 0); + currentBlockData_.firstTriple_ = {col0Id, col1And2Ids(0, 0), col1And2Ids(0, 1)}; } - currentBlockData_.col0LastId_ = col0Id; - currentBlockData_.col1LastId_ = col1And2Ids(col1And2Ids.numRows() - 1, 0); - currentBlockData_.col2LastId_ = col1And2Ids(col1And2Ids.numRows() - 1, 1); + currentBlockData_.lastTriple_ = {col0Id, col1And2Ids(col1And2Ids.numRows() - 1, 0), col1And2Ids(col1And2Ids.numRows() - 1, 1)}; AD_CORRECTNESS_CHECK(buffer_.numColumns() == col1And2Ids.numColumns()); auto bufferOldSize = buffer_.numRows(); buffer_.resize(buffer_.numRows() + col1And2Ids.numRows()); @@ -625,10 +652,12 @@ void CompressedRelationWriter::writeRelationToExclusiveBlocks( {column.begin() + i, column.begin() + i + actualNumRowsPerBlock})); } - blockBuffer_.push_back(CompressedBlockMetadata{ - std::move(offsets), actualNumRowsPerBlock, col0Id, col0Id, data[i][0], - data[i + actualNumRowsPerBlock - 1][0], - data[i + actualNumRowsPerBlock - 1][1]}); + blockBuffer_.push_back( + CompressedBlockMetadata{std::move(offsets), + actualNumRowsPerBlock, + {col0Id, data[i][0], data[i][1]}, + {col0Id, data[i + actualNumRowsPerBlock - 1][0], + data[i + actualNumRowsPerBlock - 1][1]}}); } } @@ -755,11 +784,10 @@ CompressedRelationReader::getBlocksFromMetadata( std::span blockMetadata) { if (!col1Id.has_value()) { // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ - struct KeyLhs { - Id col0FirstId_; - Id col0LastId_; - }; Id col0Id = metadata.col0Id_; + CompressedBlockMetadata key; + key.firstTriple_[0] = col0Id; + key.lastTriple_[0] = col0Id; // TODO Use a structured binding. Structured bindings are // currently not supported by clang when using OpenMP because clang // internally transforms the `#pragma`s into lambdas, and capturing @@ -769,37 +797,35 @@ CompressedRelationReader::getBlocksFromMetadata( // `std::ranges::equal_range`, find out why. Note: possibly it has // something to do with the limited support of ranges in clang with // versions < 16. Revisit this when we use clang 16. - blockMetadata.begin(), blockMetadata.end(), KeyLhs{col0Id, col0Id}, + blockMetadata.begin(), blockMetadata.end(), key, [](const auto& a, const auto& b) { - return a.col0FirstId_ < b.col0FirstId_ && - a.col0LastId_ < b.col0LastId_; + return a.firstTriple_[0] < b.firstTriple_[0] && + a.lastTriple_[0] < b.lastTriple_[0]; }); return {beginBlock, endBlock}; } else { // Get all the blocks that possibly might contain our pair of col0Id and // col1Id - struct KeyLhs { - Id col0FirstId_; - Id col0LastId_; - Id col1FirstId_; - Id col1LastId_; - }; + Id col0Id = metadata.col0Id_; + CompressedBlockMetadata key; + key.firstTriple_[0] = col0Id; + key.lastTriple_[0] = col0Id; + key.firstTriple_[1] = col1Id.value(); + key.lastTriple_[1] = col1Id.value(); auto comp = [](const auto& a, const auto& b) { - bool endBeforeBegin = a.col0LastId_ < b.col0FirstId_; + bool endBeforeBegin = a.lastTriple_[0] < b.firstTriple_[0]; endBeforeBegin |= - (a.col0LastId_ == b.col0FirstId_ && a.col1LastId_ < b.col1FirstId_); + (a.lastTriple_[0] == b.firstTriple_[0] && a.lastTriple_[1] < b.firstTriple_[1]); return endBeforeBegin; }; - Id col0Id = metadata.col0Id_; // Note: See the comment in the other overload for `scan` above for the // reason why we (currently) can't use a structured binding here. auto [beginBlock, endBlock] = std::equal_range( blockMetadata.begin(), blockMetadata.end(), - KeyLhs{col0Id, col0Id, col1Id.value(), col1Id.value()}, comp); - LOG(WARN) << "number of found blocks: " << endBlock - beginBlock << std::endl; + key, comp); return {beginBlock, endBlock}; } } diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h index 049d29f8bb..333585f684 100644 --- a/src/index/CompressedRelation.h +++ b/src/index/CompressedRelation.h @@ -18,6 +18,7 @@ #include "util/Generator.h" #include "util/Serializer/ByteBufferSerializer.h" #include "util/Serializer/SerializeVector.h" +#include "util/Serializer/SerializeArray.h" #include "util/Serializer/Serializer.h" #include "util/Timer.h" #include "util/TypeTraits.h" @@ -67,14 +68,8 @@ struct CompressedBlockMetadata { // space efficiency. For example, for Wikidata, we have only around 50K blocks // with block size 8M and around 5M blocks with block size 80K; even the // latter takes only half a GB in total. - Id col0FirstId_; - Id col0LastId_; - Id col1FirstId_; - Id col1LastId_; - - // For our `DeltaTriples` (https://github.com/ad-freiburg/qlever/pull/916), we - // need to know the least significant `Id` of the last triple as well. - Id col2LastId_; + std::array firstTriple_; + std::array lastTriple_; // Two of these are equal if all members are equal. bool operator==(const CompressedBlockMetadata&) const = default; @@ -90,11 +85,8 @@ AD_SERIALIZE_FUNCTION(CompressedBlockMetadata::OffsetAndCompressedSize) { AD_SERIALIZE_FUNCTION(CompressedBlockMetadata) { serializer | arg.offsetsAndCompressedSize_; serializer | arg.numRows_; - serializer | arg.col0FirstId_; - serializer | arg.col0LastId_; - serializer | arg.col1FirstId_; - serializer | arg.col1LastId_; - serializer | arg.col2LastId_; + serializer | arg.firstTriple_; + serializer | arg.lastTriple_; } // The metadata of a whole compressed "relation", where relation refers to a @@ -272,18 +264,20 @@ class CompressedRelationReader { static std::array, 2> getBlocksForJoin( const CompressedRelationMetadata& md1, const CompressedRelationMetadata& md2, + std::optional col1Id1, + std::optional col1Id2, std::span blockMetadata1, std::span blockMetadata2); cppcoro::generator lazyScan( - const CompressedRelationMetadata& metadata, - std::span blockMetadata, + CompressedRelationMetadata metadata, + std::vector blockMetadata, ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, ad_utility::SharedConcurrentTimeoutTimer timer) const; cppcoro::generator lazyScan( - const CompressedRelationMetadata& metadata, Id col1Id, - std::span blockMetadata, + CompressedRelationMetadata metadata, Id col1Id, + std::vector blockMetadata, ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, ad_utility::SharedConcurrentTimeoutTimer timer) const; diff --git a/src/util/Serializer/SerializeArray.h b/src/util/Serializer/SerializeArray.h new file mode 100644 index 0000000000..751951f1ed --- /dev/null +++ b/src/util/Serializer/SerializeArray.h @@ -0,0 +1,24 @@ +// Copyright 2023, University of Freiburg, +// Chair of Algorithms and Data Structures +// Author: Johannes Kalmbach + +#pragma once + +#include +#include "util/TypeTraits.h" +#include "util/Serializer/Serializer.h" + +namespace ad_utility::serialization { +AD_SERIALIZE_FUNCTION_WITH_CONSTRAINT((ad_utility::isArray>)) { + using V = typename std::decay_t::value_type; + if constexpr (TriviallySerializable) { + using CharPtr = std::conditional_t, char*, const char*>; + serializer.serializeBytes(reinterpret_cast(arg.data()), + arg.size() * sizeof(V)); + } else { + for (size_t i = 0; i < arg.size(); ++i) { + serializer | arg[i]; + } + } +} +} diff --git a/src/util/TypeTraits.h b/src/util/TypeTraits.h index e125b1db29..f633a2f5bf 100644 --- a/src/util/TypeTraits.h +++ b/src/util/TypeTraits.h @@ -97,6 +97,12 @@ constexpr static bool isTuple = isInstantiation; template constexpr static bool isVariant = isInstantiation; +template +constexpr static bool isArray = false; + +template +constexpr static bool isArray> = true; + /// Two types are similar, if they are the same when we remove all cv (const or /// volatile) qualifiers and all references template From 3b9bbeed44c7488a0c9317e995bd3ffd39e9a11f Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Wed, 14 Jun 2023 09:49:48 +0200 Subject: [PATCH 031/150] The first version seems to work (joining two index scans) --- src/engine/IndexScan.cpp | 3 +- src/engine/Join.cpp | 24 ++-- src/global/ValueId.h | 5 + src/index/CompressedRelation.cpp | 154 ++++++++++++----------- src/index/CompressedRelation.h | 5 +- src/util/JoinAlgorithms/JoinAlgorithms.h | 59 ++++----- src/util/Serializer/SerializeArray.h | 5 +- test/IndexMetaDataTest.cpp | 6 +- test/JoinAlgorithmsTest.cpp | 47 +++---- 9 files changed, 158 insertions(+), 150 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 6b2cb671a9..0ee4403e4f 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -310,7 +310,8 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans( } auto [blocks1, blocks2] = CompressedRelationReader::getBlocksForJoin( metaBlocks1.value().first.relationMetadata_, - metaBlocks2.value().first.relationMetadata_, metaBlocks1.value().second, metaBlocks2.value().second, metaBlocks1.value().first.blockMetadata_, + metaBlocks2.value().first.relationMetadata_, metaBlocks1.value().second, + metaBlocks2.value().second, metaBlocks1.value().first.blockMetadata_, metaBlocks2.value().first.blockMetadata_); // TODO include a timeout timer. diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp index 81086e1532..1c946436c9 100644 --- a/src/engine/Join.cpp +++ b/src/engine/Join.cpp @@ -632,19 +632,21 @@ void Join::computeResultForTwoIndexScans(IdTable* resultPtr) { lastRow[nextIndex] = l[i]; ++nextIndex; } - for (size_t i = 1; i < r.size(); ++i) { - lastRow[nextIndex] = r[i]; - ++nextIndex; - } + for (size_t i = 1; i < r.size(); ++i) { + lastRow[nextIndex] = r[i]; + ++nextIndex; + } }; - auto lessThan = [](const auto& a, const auto& b) { - return a[0] < b[0]; - }; + auto lessThan = [](const auto& a, const auto& b) { return a[0] < b[0]; }; - auto [leftBlocks, rightBlocks] = IndexScan::lazyScanForJoinOfTwoScans(dynamic_cast(*_left->getRootOperation()), dynamic_cast(*_right->getRootOperation())); + auto [leftBlocks, rightBlocks] = IndexScan::lazyScanForJoinOfTwoScans( + dynamic_cast(*_left->getRootOperation()), + dynamic_cast(*_right->getRootOperation())); - //LOG(WARN) << "num blocks in first: " << std::ranges::distance(leftBlocks) << std::endl; - //LOG(WARN) << "num blocks in second: " << std::ranges::distance(rightBlocks) << std::endl; - ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks, lessThan, addResultRow); + // LOG(WARN) << "num blocks in first: " << std::ranges::distance(leftBlocks) + // << std::endl; LOG(WARN) << "num blocks in second: " << + // std::ranges::distance(rightBlocks) << std::endl; + ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks, lessThan, + addResultRow); } diff --git a/src/global/ValueId.h b/src/global/ValueId.h index 96a726be53..01c17bfc9d 100644 --- a/src/global/ValueId.h +++ b/src/global/ValueId.h @@ -51,6 +51,7 @@ constexpr std::string_view toString(Datatype type) { case Datatype::TextRecordIndex: return "TextRecordIndex"; } + return "OutOfRange"; // This line is reachable if we cast an arbitrary invalid int to this enum AD_FAIL(); } @@ -269,6 +270,10 @@ class ValueId { /// human-readable representation. friend std::ostream& operator<<(std::ostream& ostr, const ValueId& id) { ostr << toString(id.getDatatype()) << ':'; + if (id.getDatatype() > Datatype::MaxValue) { + ostr << " MaxValue "; + return ostr; + } auto visitor = [&ostr](T&& value) { if constexpr (ad_utility::isSimilar) { ostr << "Undefined"; diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index cce1b9264c..c214baec1e 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -42,13 +42,13 @@ void CompressedRelationReader::scan( // The first block might contain entries that are not part of our // actual scan result. bool firstBlockIsIncomplete = - beginBlock < endBlock && - (beginBlock->firstTriple_[0] < col0Id || beginBlock->lastTriple_[0] > col0Id); + beginBlock < endBlock && (beginBlock->firstTriple_[0] < col0Id || + beginBlock->lastTriple_[0] > col0Id); auto lastBlock = endBlock - 1; bool lastBlockIsIncomplete = - beginBlock < lastBlock && - (lastBlock->firstTriple_[0] < col0Id || lastBlock->lastTriple_[0] > col0Id); + beginBlock < lastBlock && (lastBlock->firstTriple_[0] < col0Id || + lastBlock->lastTriple_[0] > col0Id); // Invariant: A relation spans multiple blocks exclusively or several // entities are stored completely in the same Block. @@ -141,8 +141,8 @@ void CompressedRelationReader::scan( // _____________________________________________________________________________ cppcoro::generator CompressedRelationReader::lazyScan( CompressedRelationMetadata metadata, - std::vector blockMetadata, - ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, + std::vector blockMetadata, ad_utility::File& file, + ad_utility::AllocatorWithLimit allocator, ad_utility::SharedConcurrentTimeoutTimer timer) const { auto relevantBlocks = getBlocksFromMetadata(metadata, std::nullopt, blockMetadata); @@ -152,13 +152,13 @@ cppcoro::generator CompressedRelationReader::lazyScan( // The first block might contain entries that are not part of our // actual scan result. bool firstBlockIsIncomplete = - beginBlock < endBlock && - (beginBlock->firstTriple_[0] < col0Id || beginBlock->lastTriple_[0] > col0Id); + beginBlock < endBlock && (beginBlock->firstTriple_[0] < col0Id || + beginBlock->lastTriple_[0] > col0Id); auto lastBlock = endBlock - 1; bool lastBlockIsIncomplete = - beginBlock < lastBlock && - (lastBlock->firstTriple_[0] < col0Id || lastBlock->lastTriple_[0] > col0Id); + beginBlock < lastBlock && (lastBlock->firstTriple_[0] < col0Id || + lastBlock->lastTriple_[0] > col0Id); // Invariant: A relation spans multiple blocks exclusively or several // entities are stored completely in the same Block. @@ -223,9 +223,9 @@ cppcoro::generator CompressedRelationReader::lazyScan( } cppcoro::generator CompressedRelationReader::lazyScan( - CompressedRelationMetadata metadata, Id col1Id, - std::vector blockMetadata, - ad_utility::File& file, ad_utility::AllocatorWithLimit allocator, + CompressedRelationMetadata metadata, Id col1Id, + std::vector blockMetadata, ad_utility::File& file, + ad_utility::AllocatorWithLimit allocator, ad_utility::SharedConcurrentTimeoutTimer timer) const { auto relevantBlocks = getBlocksFromMetadata(metadata, col1Id, blockMetadata); auto beginBlock = relevantBlocks.begin(); @@ -336,26 +336,38 @@ std::vector CompressedRelationReader::getBlocksForJoin( return block.lastTriple_[1] < id; }; - auto lessThan = - ad_utility::OverloadCallOperator{idLessThanBlock, blockLessThanId}; + auto idLessThanId = [](Id id, Id id2) { return id < id2; }; + + auto lessThan = ad_utility::OverloadCallOperator{ + idLessThanBlock, blockLessThanId, idLessThanId}; std::vector result; auto addRow = [&result]([[maybe_unused]] auto it1, auto it2) { result.push_back(*it2); }; + // TODO is the copy really necessary? + // TODO Should we respect the memory limit? + std::vector joinColumnUnique; + joinColumnUnique.reserve(joinColum.size()); + std::ranges::unique_copy(joinColum, std::back_inserter(joinColumnUnique)); + + if (joinColum.empty()) { + return result; + } + auto noop = ad_utility::noop; - [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( - joinColum, std::span(beginBlock, endBlock), - lessThan, addRow, noop, noop); + [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( + joinColumnUnique, + std::span(beginBlock, endBlock), lessThan, + addRow, noop, noop); return result; } std::array, 2> CompressedRelationReader::getBlocksForJoin( const CompressedRelationMetadata& md1, - const CompressedRelationMetadata& md2, - std::optional col1Id1, + const CompressedRelationMetadata& md2, std::optional col1Id1, std::optional col1Id2, std::span blockMetadata1, std::span blockMetadata2) { @@ -375,59 +387,58 @@ CompressedRelationReader::getBlocksForJoin( auto beginBlock2 = relevantBlocks2.begin(); auto endBlock2 = relevantBlocks2.end(); - auto getJoinColumnRangeValue = [](std::array block, Id col0Id, - std::optional col1Id) { - auto minId = Id::makeUndefined(); - auto maxId = Id::fromBits(std::numeric_limits::max()); - if (block[0] < col0Id) { - return minId; - } - if (block[0] > col0Id) { - return maxId; - } - if (!col1Id.has_value()) { - return block[1]; - } + auto getJoinColumnRangeValue = [](std::array block, Id col0Id, + std::optional col1Id) { + auto minId = Id::makeUndefined(); + auto maxId = Id::fromBits(std::numeric_limits::max()); + if (block[0] < col0Id) { + return minId; + } + if (block[0] > col0Id) { + return maxId; + } + if (!col1Id.has_value()) { + return block[1]; + } - if (block[1] < col1Id.value()) { - return minId; - } - if (block[1] > col1Id.value()) { - return maxId; - } - return block[2]; - }; - auto blocksToIdRanges = [&getJoinColumnRangeValue](const auto& blocks, Id col0Id, - std::optional col1Id) { - std::vector> result; - for (const auto& block: blocks) { - result.emplace_back( - getJoinColumnRangeValue(block.firstTriple_, - col0Id, col1Id), - getJoinColumnRangeValue(block.lastTriple_, - col0Id, col1Id)); - } - return result; - }; - - auto blockLessThanBlock = []( - const auto& block1, - const auto& block2) { + if (block[1] < col1Id.value()) { + return minId; + } + if (block[1] > col1Id.value()) { + return maxId; + } + return block[2]; + }; + auto blocksToIdRanges = [&getJoinColumnRangeValue](const auto& blocks, + Id col0Id, + std::optional col1Id) { + std::vector> result; + for (const auto& block : blocks) { + result.emplace_back( + getJoinColumnRangeValue(block.firstTriple_, col0Id, col1Id), + getJoinColumnRangeValue(block.lastTriple_, col0Id, col1Id)); + } + return result; + }; + + auto blockLessThanBlock = [](const auto& block1, const auto& block2) { return block1.second < block2.first; }; std::array, 2> result; - auto idRanges1 = blocksToIdRanges(std::ranges::subrange{beginBlock1, endBlock1}, md1.col0Id_, col1Id1); - auto idRanges2 = blocksToIdRanges(std::ranges::subrange{beginBlock2, endBlock2}, md2.col0Id_, col1Id2); - auto addRow = [&result, &beginBlock1, &beginBlock2, &idRanges1, &idRanges2]([[maybe_unused]] auto it1, auto it2) { + auto idRanges1 = blocksToIdRanges( + std::ranges::subrange{beginBlock1, endBlock1}, md1.col0Id_, col1Id1); + auto idRanges2 = blocksToIdRanges( + std::ranges::subrange{beginBlock2, endBlock2}, md2.col0Id_, col1Id2); + auto addRow = [&result, &beginBlock1, &beginBlock2, &idRanges1, &idRanges2]( + [[maybe_unused]] auto it1, auto it2) { result[0].push_back(*(beginBlock1 + (it1 - idRanges1.begin()))); result[1].push_back(*(beginBlock2 + (it2 - idRanges2.begin()))); }; auto noop = ad_utility::noop; - [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( - idRanges1, idRanges2, - blockLessThanBlock, addRow, noop, noop); + [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( + idRanges1, idRanges2, blockLessThanBlock, addRow, noop, noop); for (auto& vec : result) { vec.erase(std::unique(vec.begin(), vec.end()), vec.end()); } @@ -622,9 +633,12 @@ CompressedRelationMetadata CompressedRelationWriter::addRelation( metaData.offsetInBlock_ = buffer_.numRows(); static_assert(sizeof(col1And2Ids[0][0]) == sizeof(Id)); if (buffer_.numRows() == 0) { - currentBlockData_.firstTriple_ = {col0Id, col1And2Ids(0, 0), col1And2Ids(0, 1)}; + currentBlockData_.firstTriple_ = {col0Id, col1And2Ids(0, 0), + col1And2Ids(0, 1)}; } - currentBlockData_.lastTriple_ = {col0Id, col1And2Ids(col1And2Ids.numRows() - 1, 0), col1And2Ids(col1And2Ids.numRows() - 1, 1)}; + currentBlockData_.lastTriple_ = {col0Id, + col1And2Ids(col1And2Ids.numRows() - 1, 0), + col1And2Ids(col1And2Ids.numRows() - 1, 1)}; AD_CORRECTNESS_CHECK(buffer_.numColumns() == col1And2Ids.numColumns()); auto bufferOldSize = buffer_.numRows(); buffer_.resize(buffer_.numRows() + col1And2Ids.numRows()); @@ -815,17 +829,15 @@ CompressedRelationReader::getBlocksFromMetadata( auto comp = [](const auto& a, const auto& b) { bool endBeforeBegin = a.lastTriple_[0] < b.firstTriple_[0]; - endBeforeBegin |= - (a.lastTriple_[0] == b.firstTriple_[0] && a.lastTriple_[1] < b.firstTriple_[1]); + endBeforeBegin |= (a.lastTriple_[0] == b.firstTriple_[0] && + a.lastTriple_[1] < b.firstTriple_[1]); return endBeforeBegin; }; - // Note: See the comment in the other overload for `scan` above for the // reason why we (currently) can't use a structured binding here. - auto [beginBlock, endBlock] = std::equal_range( - blockMetadata.begin(), blockMetadata.end(), - key, comp); + auto [beginBlock, endBlock] = + std::equal_range(blockMetadata.begin(), blockMetadata.end(), key, comp); return {beginBlock, endBlock}; } } diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h index 333585f684..6caf9aaf9b 100644 --- a/src/index/CompressedRelation.h +++ b/src/index/CompressedRelation.h @@ -17,8 +17,8 @@ #include "util/File.h" #include "util/Generator.h" #include "util/Serializer/ByteBufferSerializer.h" -#include "util/Serializer/SerializeVector.h" #include "util/Serializer/SerializeArray.h" +#include "util/Serializer/SerializeVector.h" #include "util/Serializer/Serializer.h" #include "util/Timer.h" #include "util/TypeTraits.h" @@ -263,8 +263,7 @@ class CompressedRelationReader { std::span blockMetadata); static std::array, 2> getBlocksForJoin( const CompressedRelationMetadata& md1, - const CompressedRelationMetadata& md2, - std::optional col1Id1, + const CompressedRelationMetadata& md2, std::optional col1Id1, std::optional col1Id2, std::span blockMetadata1, std::span blockMetadata2); diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 43ee82908f..a86e8d379b 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -93,12 +93,11 @@ concept BinaryIteratorFunction = * described cases leads to two sorted ranges in the output, this can possibly * be exploited to fix the result in a cheaper way than a full sort. */ -template < - bool addDuplicatesFromLeft = true, bool returnIteratorsOfLastMatch = false, - std::ranges::random_access_range Range1, - std::ranges::random_access_range Range2, typename LessThan, - typename FindSmallerUndefRangesLeft, typename FindSmallerUndefRangesRight, - typename ElFromFirstNotFoundAction = decltype(noop)> +template [[nodiscard]] auto zipperJoinWithUndef( const Range1& left, const Range2& right, const LessThan& lessThan, const auto& compatibleRowAction, @@ -119,10 +118,6 @@ template < auto it2 = std::begin(right); auto end2 = std::end(right); - // Keep track of the last iterators that had an exact match - auto exactMatchIt1 = it1; - auto exactMatchIt2 = it2; - // If this is an OPTIONAL join or a MINUS then we have to keep track of the // information for which elements from the left input we have already found a // match in the right input. We call these elements "covered". For all @@ -231,7 +226,6 @@ template < mergeWithUndefRight(it1, std::begin(right), it2, true); ++it1; if (it1 >= end1) { - exactMatchIt2 = it2; return; } } @@ -239,7 +233,6 @@ template < mergeWithUndefLeft(it2, std::begin(left), it1); ++it2; if (it2 >= end2) { - exactMatchIt1 = it1; return; } } @@ -257,12 +250,6 @@ template < auto endSame2 = std::find_if_not( it2, end2, [&](const auto& row) { return eq(*it1, row); }); - if constexpr (returnIteratorsOfLastMatch) { - if (endSame1 != it1) { - exactMatchIt1 = it1; - exactMatchIt2 = it2; - } - } for (auto it = it1; it != endSame1; ++it) { mergeWithUndefRight(it, std::begin(right), it2, false); } @@ -275,9 +262,6 @@ template < for (auto innerIt2 = it2; innerIt2 != endSame2; ++innerIt2) { compatibleRowAction(it1, innerIt2); } - if constexpr (!addDuplicatesFromLeft) { - break; - } } it1 = endSame1; it2 = endSame2; @@ -313,11 +297,7 @@ template < // of which has length `returnValue`. size_t numOutOfOrder = outOfOrderFound ? std::numeric_limits::max() : numOutOfOrderAtEnd; - if constexpr (returnIteratorsOfLastMatch) { - return std::tuple{numOutOfOrder, exactMatchIt1, exactMatchIt2}; - } else { - return numOutOfOrder; - } + return numOutOfOrder; } /** @@ -576,6 +556,11 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, const auto& compatibleRowAction) { using LeftBlock = typename std::decay_t::value_type; using RightBlock = typename std::decay_t::value_type; + + using LeftEl = + typename std::iterator_traits::value_type; + using RightEl = + typename std::iterator_traits::value_type; auto it1 = leftBlocks.begin(); auto end1 = leftBlocks.end(); auto it2 = rightBlocks.begin(); @@ -605,11 +590,12 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, if (sameBlocksLeft.empty() || sameBlocksRight.empty()) { return; } - const auto& lastLeft = sameBlocksLeft.front().back(); - const auto& lastRight = sameBlocksRight.front().back(); + LeftEl lastLeft = sameBlocksLeft.front().back(); + RightEl lastRight = sameBlocksRight.front().back(); if (!lessThan(lastRight, lastLeft)) { - // TODO here and below: use `at`, but it needs to be implemented in the `Row` class. + // TODO here and below: use `at`, but it needs to be implemented + // in the `Row` class. while (it1 != end1 && eq((*it1)[0], lastLeft)) { sameBlocksLeft.push_back(std::move(*it1)); ++it1; @@ -628,8 +614,7 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, }; auto join = [&](const auto& l, const auto& r) { - return zipperJoinWithUndef(l, r, lessThan, compatibleRowAction, - noop, noop); + return zipperJoinWithUndef(l, r, lessThan, compatibleRowAction, noop, noop); }; auto addAll = [&](const auto& l, const auto& r) { @@ -647,7 +632,8 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, auto joinAndRemoveBeginning = [&]() { auto& l = sameBlocksLeft.at(0); auto& r = sameBlocksRight.at(0); - typename std::iterator_traits::value_type minEl = std::min(l.back(), r.back(), lessThan); + typename std::iterator_traits::value_type minEl = + std::min(l.back(), r.back(), lessThan); auto itL = std::ranges::equal_range(l, minEl, lessThan); auto itR = std::ranges::equal_range(r, minEl, lessThan); join(std::ranges::subrange{l.begin(), itL.begin()}, @@ -658,11 +644,11 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, auto removeAllButUnjoined = [lessThan]( Blocks& blocks, auto lastHandledElement) { AD_CORRECTNESS_CHECK(!blocks.empty()); - typename Blocks::value_type remainingBlock = std::move(blocks.back()); + typename Blocks::value_type remainingBlock = std::move(blocks.back()); auto beginningOfUnjoined = std::ranges::upper_bound(remainingBlock, lastHandledElement, lessThan); - // TODO This is not the most efficient way, but currently necessary because of the - // interface of the `IdTable`. + // TODO This is not the most efficient way, but currently necessary + // because of the interface of the `IdTable`. remainingBlock.erase(remainingBlock.begin(), beginningOfUnjoined); blocks.clear(); if (!remainingBlock.empty()) { @@ -693,7 +679,8 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, sameBlocksRight.back(), sameBlocksRight.front().back(), lessThan)); } addAll(l, r); - typename std::iterator_traits::value_type minEl = + typename std::iterator_traits< + decltype(sameBlocksLeft.front().begin())>::value_type minEl = std::min(sameBlocksLeft.front().back(), sameBlocksRight.front().back(), lessThan); removeAllButUnjoined(sameBlocksLeft, minEl); diff --git a/src/util/Serializer/SerializeArray.h b/src/util/Serializer/SerializeArray.h index 751951f1ed..df6a02d29d 100644 --- a/src/util/Serializer/SerializeArray.h +++ b/src/util/Serializer/SerializeArray.h @@ -5,8 +5,9 @@ #pragma once #include -#include "util/TypeTraits.h" + #include "util/Serializer/Serializer.h" +#include "util/TypeTraits.h" namespace ad_utility::serialization { AD_SERIALIZE_FUNCTION_WITH_CONSTRAINT((ad_utility::isArray>)) { @@ -21,4 +22,4 @@ AD_SERIALIZE_FUNCTION_WITH_CONSTRAINT((ad_utility::isArray>)) { } } } -} +} // namespace ad_utility::serialization diff --git a/test/IndexMetaDataTest.cpp b/test/IndexMetaDataTest.cpp index 72d9fe8a6f..97af5fcbb6 100644 --- a/test/IndexMetaDataTest.cpp +++ b/test/IndexMetaDataTest.cpp @@ -17,7 +17,7 @@ auto V = ad_utility::testing::VocabId; TEST(RelationMetaDataTest, writeReadTest) { CompressedBlockMetadata rmdB{ - {{12, 34}, {46, 11}}, 5, V(0), V(2), V(13), V(24), V(62)}; + {{12, 34}, {46, 11}}, 5, {V(0), V(2), V(13)}, {V(3), V(24), V(62)}}; CompressedRelationMetadata rmdF{V(1), 3, 2.0, 42.0, 16}; ad_utility::serialization::FileWriteSerializer f("_testtmp.rmd"); @@ -41,9 +41,9 @@ TEST(IndexMetaDataTest, writeReadTest2Mmap) { std::string mmapFilename = imdFilename + ".mmap"; vector bs; bs.push_back(CompressedBlockMetadata{ - {{12, 34}, {42, 17}}, 5, V(0), V(2), V(13), V(24), V(62)}); + {{12, 34}, {42, 17}}, 5, {V(0), V(2), V(13)}, {V(2), V(24), V(62)}}); bs.push_back(CompressedBlockMetadata{ - {{12, 34}, {16, 12}}, 5, V(0), V(2), V(13), V(24), V(62)}); + {{12, 34}, {16, 12}}, 5, {V(0), V(2), V(13)}, {V(3), V(24), V(62)}}); CompressedRelationMetadata rmdF{V(1), 3, 2.0, 42.0, 16}; CompressedRelationMetadata rmdF2{V(2), 5, 3.0, 43.0, 10}; // The index MetaData does not have an explicit clear, so we diff --git a/test/JoinAlgorithmsTest.cpp b/test/JoinAlgorithmsTest.cpp index c1edf91699..5e1b01f574 100644 --- a/test/JoinAlgorithmsTest.cpp +++ b/test/JoinAlgorithmsTest.cpp @@ -85,27 +85,28 @@ TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksPerElement) { } TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksPerElementBothSides) { - using NB = std::vector>>; - NB a{{{42, 0}}, {{42, 1}, {42, 2}}, {{42, 3}, {67, 0}}}; - NB b{{{2, 0}, {42, 12}}, {{42, 13}, {67, 14}}}; - std::vector> result; - std::vector> expectedResult{ - {42, 0, 12}, {42, 0, 13},{42, 1, 12}, {42, 2, 12}, {42, 1, 13}, {42, 2, 13},{42, 3, 12}, {42, 3, 13}, {67, 0, 14}}; - auto compare = [](auto l, auto r) { return l[0] < r[0]; }; - auto add = [&result](auto it1, auto it2) { - AD_CORRECTNESS_CHECK((*it1)[0] == (*it1)[0]); - result.push_back(std::array{(*it1)[0], (*it1)[1], (*it2)[1]}); - }; - zipperJoinForBlocksWithoutUndef(NB{a}, NB{b}, compare, add); - // The exact order of the elements with the same first column is not important and depends on implementation - // details. We therefore do not enforce it here. - EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); - EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expectedResult)); - result.clear(); - for (auto& [x, y, z] : expectedResult) { - std::swap(y, z); - } - zipperJoinForBlocksWithoutUndef(NB{b}, NB{a}, compare, add); - EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); - EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expectedResult)); + using NB = std::vector>>; + NB a{{{42, 0}}, {{42, 1}, {42, 2}}, {{42, 3}, {67, 0}}}; + NB b{{{2, 0}, {42, 12}}, {{42, 13}, {67, 14}}}; + std::vector> result; + std::vector> expectedResult{ + {42, 0, 12}, {42, 0, 13}, {42, 1, 12}, {42, 2, 12}, {42, 1, 13}, + {42, 2, 13}, {42, 3, 12}, {42, 3, 13}, {67, 0, 14}}; + auto compare = [](auto l, auto r) { return l[0] < r[0]; }; + auto add = [&result](auto it1, auto it2) { + AD_CORRECTNESS_CHECK((*it1)[0] == (*it1)[0]); + result.push_back(std::array{(*it1)[0], (*it1)[1], (*it2)[1]}); + }; + zipperJoinForBlocksWithoutUndef(NB{a}, NB{b}, compare, add); + // The exact order of the elements with the same first column is not important + // and depends on implementation details. We therefore do not enforce it here. + EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); + EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expectedResult)); + result.clear(); + for (auto& [x, y, z] : expectedResult) { + std::swap(y, z); + } + zipperJoinForBlocksWithoutUndef(NB{b}, NB{a}, compare, add); + EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); + EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expectedResult)); } From c171d6decdaade5632535c421a1a75e9d7efbdc7 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Wed, 14 Jun 2023 12:09:48 +0200 Subject: [PATCH 032/150] The tests compile again, but they fail. TODO handle that stuff. --- src/engine/IndexScan.cpp | 56 ++++++++++++ src/engine/IndexScan.h | 2 + src/engine/Join.cpp | 58 +++++++++++++ src/engine/Join.h | 5 ++ src/index/CompressedRelation.cpp | 105 +++++++++++++---------- src/index/CompressedRelation.h | 2 + src/util/JoinAlgorithms/JoinAlgorithms.h | 101 +++++++++++++++------- 7 files changed, 251 insertions(+), 78 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 0ee4403e4f..e3920e4484 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -334,3 +334,59 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans( return {getScan(s1, blocks1), getScan(s2, blocks2)}; } + +// TODO There is a lot of duplication between this and the previous +// function. +// __________________________________________________________________________________________________________ +cppcoro::generator IndexScan::lazyScanForJoinOfColumnWithScan( + std::span joinColumn, const IndexScan& s) { + const auto& index = s.getExecutionContext()->getIndex().getImpl(); + AD_CONTRACT_CHECK(s._numVariables < 3); + + auto f = [&](const IndexScan& s) + -> std::optional< + std::pair>> { + auto permutedTriple = s.getPermutedTriple(); + std::optional optId = permutedTriple[0]->toValueId(index.getVocab()); + std::optional optId2 = + s._numVariables == 2 ? std::nullopt + : permutedTriple[1]->toValueId(index.getVocab()); + if (!optId.has_value() || (!optId2.has_value() && s._numVariables == 1)) { + return std::nullopt; + } + auto optionalBla = index.getPermutation(s.permutation()) + .getMetadataAndBlocks(optId.value(), optId2); + if (optionalBla.has_value()) { + return std::pair{std::move(optionalBla.value()), optId2}; + } + return std::nullopt; + }; + + auto metaBlocks1 = f(s); + + if (!metaBlocks1.has_value()) { + return {}; + } + auto blocks = CompressedRelationReader::getBlocksForJoin( + joinColumn, metaBlocks1.value().first.relationMetadata_, + metaBlocks1.value().second, metaBlocks1.value().first.blockMetadata_); + + // TODO include a timeout timer. + auto getScan = [&index](const IndexScan& s, const auto& blocks) { + // TODO pass the IDs here. + Id col0Id = s.getPermutedTriple()[0]->toValueId(index.getVocab()).value(); + std::optional col1Id; + if (s._numVariables == 1) { + col1Id = s.getPermutedTriple()[1]->toValueId(index.getVocab()).value(); + } + if (!col1Id.has_value()) { + return index.getPermutation(s.permutation()) + .lazyScan(col0Id, blocks, s.getExecutionContext()->getAllocator()); + } else { + return index.getPermutation(s.permutation()) + .lazyScan(col0Id, col1Id.value(), blocks, + s.getExecutionContext()->getAllocator()); + } + }; + return getScan(s, blocks); +} diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index b411722c80..a41e529ccf 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -156,6 +156,8 @@ class IndexScan : public Operation { static std::array, 2> lazyScanForJoinOfTwoScans( const IndexScan& s1, const IndexScan& s2); + static cppcoro::generator lazyScanForJoinOfColumnWithScan( + std::span joinColumn, const IndexScan& s); private: // TODO Make the `getSizeEstimateBeforeLimit()` function `const` for diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp index 1c946436c9..bab11aa9ac 100644 --- a/src/engine/Join.cpp +++ b/src/engine/Join.cpp @@ -650,3 +650,61 @@ void Join::computeResultForTwoIndexScans(IdTable* resultPtr) { ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks, lessThan, addResultRow); } + +// ______________________________________________________________________________________________________ +void Join::computeResultForIndexScanAndColumn(const IdTable& inputTable, + ColumnIndex joinColumnIndexTable, + const IndexScan& scan, + ColumnIndex joinColumnIndexScan, + IdTable* resultPtr) { + auto& result = *resultPtr; + result.setNumColumns(getResultWidth()); + + AD_CORRECTNESS_CHECK(joinColumnIndexScan == 0); + + auto joinColumn = inputTable.getColumn(joinColumnIndexTable); + auto addResultRow = [&, beg = joinColumn.begin()](auto itLeft, auto itRight) { + const auto& l = *inputTable.begin() + (itLeft - beg); + const auto& r = *itRight; + result.emplace_back(); + IdTable::row_reference lastRow = result.back(); + size_t nextIndex = 0; + for (size_t i = 0; i < inputTable.numColumns(); ++i) { + lastRow[nextIndex] = l[i]; + ++nextIndex; + } + + for (size_t i = 0; i < r.size(); ++i) { + if (i != joinColumnIndexScan) { + lastRow[nextIndex] = r[i]; + ++nextIndex; + } + } + }; + + auto lessThan = [](const A& a, const B& b) { + static constexpr bool aIsId = ad_utility::isSimilar; + static constexpr bool bIsId = ad_utility::isSimilar; + + if constexpr (aIsId && bIsId) { + return a < b; + } else if constexpr (aIsId) { + return a < b[0]; + } else if constexpr (bIsId) { + return a[0] < b; + } else { + return a[0] < b[0]; + } + }; + + auto rightBlocks = + IndexScan::lazyScanForJoinOfColumnWithScan(joinColumn, scan); + + auto rightProjection = [](const auto& row) { return row[0]; }; + // LOG(WARN) << "num blocks in first: " << std::ranges::distance(leftBlocks) + // << std::endl; LOG(WARN) << "num blocks in second: " << + // std::ranges::distance(rightBlocks) << std::endl; + ad_utility::zipperJoinForBlocksWithoutUndef( + std::span{&joinColumn, 1}, rightBlocks, lessThan, addResultRow, + std::identity{}, rightProjection); +} diff --git a/src/engine/Join.h b/src/engine/Join.h index 170f88b1b4..8794d385d2 100644 --- a/src/engine/Join.h +++ b/src/engine/Join.h @@ -134,6 +134,11 @@ class Join : public Operation { ResultTable computeResultForJoinWithFullScanDummy(); void computeResultForTwoIndexScans(IdTable* tablePtr); + void computeResultForIndexScanAndColumn(const IdTable& inputTable, + ColumnIndex joinColumnIndexTable, + const IndexScan& scan, + ColumnIndex joinColumnIndexScan, + IdTable* resultPtr); using ScanMethodType = std::function; diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index c214baec1e..c0177fe67b 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -316,24 +316,66 @@ cppcoro::generator CompressedRelationReader::lazyScan( } } +// TODO Comment those helpers. Should we register them in the header as +// private static functions? +namespace { +auto getJoinColumnRangeValue = [](std::array block, Id col0Id, + std::optional col1Id) { + auto minId = Id::makeUndefined(); + auto maxId = Id::fromBits(std::numeric_limits::max()); + if (block[0] < col0Id) { + return minId; + } + if (block[0] > col0Id) { + return maxId; + } + if (!col1Id.has_value()) { + return block[1]; + } + + if (block[1] < col1Id.value()) { + return minId; + } + if (block[1] > col1Id.value()) { + return maxId; + } + return block[2]; +}; + +using IdPair = std::pair; +auto blocksToIdRanges = [](std::span blocks, + Id col0Id, std::optional col1Id) { + std::vector result; + for (const auto& block : blocks) { + result.emplace_back( + getJoinColumnRangeValue(block.firstTriple_, col0Id, col1Id), + getJoinColumnRangeValue(block.lastTriple_, col0Id, col1Id)); + } + return result; +}; +} // namespace + // _____________________________________________________________________________ std::vector CompressedRelationReader::getBlocksForJoin( std::span joinColum, const CompressedRelationMetadata& metadata, + std::optional col1Id, std::span blockMetadata) { // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ auto relevantBlocks = getBlocksFromMetadata(metadata, std::nullopt, blockMetadata); - auto beginBlock = relevantBlocks.begin(); - auto endBlock = relevantBlocks.end(); - if (endBlock - beginBlock < 2) { - return {beginBlock, endBlock}; + // TODO Is this condition necessary? + if (relevantBlocks.size() < 2) { + return {relevantBlocks.begin(), relevantBlocks.end()}; } - auto idLessThanBlock = [](Id id, const CompressedBlockMetadata& block) { - return id < block.firstTriple_[1]; + auto idRanges = blocksToIdRanges(relevantBlocks, metadata.col0Id_, col1Id); + + auto idLessThanBlock = [](Id id, const IdPair& block) { + return id < block.first; }; - auto blockLessThanId = [](const CompressedBlockMetadata& block, Id id) { - return block.lastTriple_[1] < id; + + auto blockLessThanId = [](const IdPair& block, Id id) { + return block.second < id; }; auto idLessThanId = [](Id id, Id id2) { return id < id2; }; @@ -342,12 +384,16 @@ std::vector CompressedRelationReader::getBlocksForJoin( idLessThanBlock, blockLessThanId, idLessThanId}; std::vector result; - auto addRow = [&result]([[maybe_unused]] auto it1, auto it2) { - result.push_back(*it2); + auto addRow = [&result, begBlocks = relevantBlocks.begin(), + begRanges = idRanges.begin()]([[maybe_unused]] auto it1, + auto it2) { + result.push_back(*(begBlocks + (it2 - begRanges))); }; // TODO is the copy really necessary? // TODO Should we respect the memory limit? + // TODO The correct way indeed WAS to remove the duplicates inside + // the `zipperJoinWithUndef` function. std::vector joinColumnUnique; joinColumnUnique.reserve(joinColum.size()); std::ranges::unique_copy(joinColum, std::back_inserter(joinColumnUnique)); @@ -358,12 +404,11 @@ std::vector CompressedRelationReader::getBlocksForJoin( auto noop = ad_utility::noop; [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( - joinColumnUnique, - std::span(beginBlock, endBlock), lessThan, - addRow, noop, noop); + joinColumnUnique, idRanges, lessThan, addRow, noop, noop); return result; } +// _____________________________________________________________________________ std::array, 2> CompressedRelationReader::getBlocksForJoin( const CompressedRelationMetadata& md1, @@ -387,40 +432,6 @@ CompressedRelationReader::getBlocksForJoin( auto beginBlock2 = relevantBlocks2.begin(); auto endBlock2 = relevantBlocks2.end(); - auto getJoinColumnRangeValue = [](std::array block, Id col0Id, - std::optional col1Id) { - auto minId = Id::makeUndefined(); - auto maxId = Id::fromBits(std::numeric_limits::max()); - if (block[0] < col0Id) { - return minId; - } - if (block[0] > col0Id) { - return maxId; - } - if (!col1Id.has_value()) { - return block[1]; - } - - if (block[1] < col1Id.value()) { - return minId; - } - if (block[1] > col1Id.value()) { - return maxId; - } - return block[2]; - }; - auto blocksToIdRanges = [&getJoinColumnRangeValue](const auto& blocks, - Id col0Id, - std::optional col1Id) { - std::vector> result; - for (const auto& block : blocks) { - result.emplace_back( - getJoinColumnRangeValue(block.firstTriple_, col0Id, col1Id), - getJoinColumnRangeValue(block.lastTriple_, col0Id, col1Id)); - } - return result; - }; - auto blockLessThanBlock = [](const auto& block1, const auto& block2) { return block1.second < block2.first; }; diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h index 6caf9aaf9b..41c0970fa1 100644 --- a/src/index/CompressedRelation.h +++ b/src/index/CompressedRelation.h @@ -260,7 +260,9 @@ class CompressedRelationReader { // TODO Include a timeout check. static std::vector getBlocksForJoin( std::span joinColum, const CompressedRelationMetadata& metadata, + std::optional col1Id, std::span blockMetadata); + static std::array, 2> getBlocksForJoin( const CompressedRelationMetadata& md1, const CompressedRelationMetadata& md2, std::optional col1Id1, diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index a86e8d379b..dc5bc63b8f 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -549,11 +549,38 @@ void specialOptionalJoin( } } -template +// TODO move to detail namespace +using Range = std::pair; +template +class BlockAndSubrange { + public: + BlockAndSubrange(Block block) + : block_{std::move(block)}, subrange_{0, block_.size()} {} + const auto& back() { return block_.at(subrange_.second - 1); } + auto subrange() { + return std::ranges::subrange{block_.begin() + subrange_.first, + block_.begin() + subrange_.second}; + } + void setSubrange(auto it1, auto it2) { + // TODO Add assertions that the iterators are valid. + subrange_.first = it1 - block_.begin(); + subrange_.second = it2 - block_.begin(); + } + + private: + Block block_; + Range subrange_; +}; + +template void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, RightBlocks&& rightBlocks, const LessThan& lessThan, - const auto& compatibleRowAction) { + const auto& compatibleRowAction, + LeftProjection leftProjection = {}, + RightProjection rightProjection = {}) { using LeftBlock = typename std::decay_t::value_type; using RightBlock = typename std::decay_t::value_type; @@ -561,6 +588,12 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, typename std::iterator_traits::value_type; using RightEl = typename std::iterator_traits::value_type; + + using ProjectedEl = + std::decay_t>; + static_assert( + ad_utility::isSimilar>); auto it1 = leftBlocks.begin(); auto end1 = leftBlocks.end(); auto it2 = rightBlocks.begin(); @@ -569,8 +602,9 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, return !lessThan(el1, el2) && !lessThan(el2, el1); }; - std::vector sameBlocksLeft; - std::vector sameBlocksRight; + std::vector> sameBlocksLeft; + std::vector> sameBlocksRight; + auto fillBuffer = [&]() { AD_CORRECTNESS_CHECK(sameBlocksLeft.size() <= 1); AD_CORRECTNESS_CHECK(sameBlocksRight.size() <= 1); @@ -582,7 +616,7 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, } while (sameBlocksRight.empty() && it2 != end2) { if (!it2->empty()) { - sameBlocksRight.push_back(std::move(*it2)); + sameBlocksRight.emplace_back(std::move(*it2)); } ++it2; } @@ -630,59 +664,64 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, }; auto joinAndRemoveBeginning = [&]() { - auto& l = sameBlocksLeft.at(0); - auto& r = sameBlocksRight.at(0); - typename std::iterator_traits::value_type minEl = - std::min(l.back(), r.back(), lessThan); + decltype(auto) l = sameBlocksLeft.at(0).subrange(); + decltype(auto) r = sameBlocksRight.at(0).subrange(); + ProjectedEl minEl = + std::min(leftProjection(l.back()), rightProjection(r.back()), lessThan); auto itL = std::ranges::equal_range(l, minEl, lessThan); auto itR = std::ranges::equal_range(r, minEl, lessThan); join(std::ranges::subrange{l.begin(), itL.begin()}, std::ranges::subrange{r.begin(), itR.begin()}); - return std::pair{itL, itR}; + sameBlocksLeft.at(0).setSubrange(itL.begin(), l.end()); + sameBlocksRight.at(0).setSubrange(itR.begin(), r.end()); }; auto removeAllButUnjoined = [lessThan]( Blocks& blocks, auto lastHandledElement) { + // TODO This method can be incorporated into the calling code much + // more simply. AD_CORRECTNESS_CHECK(!blocks.empty()); - typename Blocks::value_type remainingBlock = std::move(blocks.back()); + blocks.erase(blocks.begin(), blocks.end() - 1); + decltype(auto) remainingBlock = blocks.at(0).subrange(); auto beginningOfUnjoined = std::ranges::upper_bound(remainingBlock, lastHandledElement, lessThan); // TODO This is not the most efficient way, but currently necessary // because of the interface of the `IdTable`. - remainingBlock.erase(remainingBlock.begin(), beginningOfUnjoined); - blocks.clear(); + remainingBlock = + std::ranges::subrange{beginningOfUnjoined, remainingBlock.end()}; if (!remainingBlock.empty()) { - blocks.push_back(std::move(remainingBlock)); + blocks.at(0).setSubrange(remainingBlock.begin(), remainingBlock.end()); + } else { + blocks.clear(); } }; auto joinBuffers = [&]() { - auto [subrangeLeft, subrangeRight] = joinAndRemoveBeginning(); - using SubLeft = decltype(subrangeLeft); - using SubRight = decltype(subrangeRight); + joinAndRemoveBeginning(); + using SubLeft = decltype(sameBlocksLeft.front().subrange()); + using SubRight = decltype(sameBlocksRight.front().subrange()); std::vector l; std::vector r; - l.push_back(subrangeLeft); - for (size_t i = 1; i < sameBlocksLeft.size() - 1; ++i) { - l.push_back(sameBlocksLeft[i]); + for (size_t i = 0; i < sameBlocksLeft.size() - 1; ++i) { + l.push_back(sameBlocksLeft[i].subrange()); } if (sameBlocksLeft.size() > 1) { - l.push_back(std::ranges::equal_range( - sameBlocksLeft.back(), sameBlocksLeft.front().back(), lessThan)); + l.push_back(std::ranges::equal_range(sameBlocksLeft.back().subrange(), + sameBlocksLeft.front().back(), + lessThan)); } - r.push_back(subrangeRight); - for (size_t i = 1; i < sameBlocksRight.size() - 1; ++i) { - r.push_back(sameBlocksRight[i]); + for (size_t i = 0; i < sameBlocksRight.size() - 1; ++i) { + r.push_back(sameBlocksRight[i].subrange()); } if (sameBlocksRight.size() > 1) { - r.push_back(std::ranges::equal_range( - sameBlocksRight.back(), sameBlocksRight.front().back(), lessThan)); + r.push_back(std::ranges::equal_range(sameBlocksRight.back().subrange(), + sameBlocksRight.front().back(), + lessThan)); } addAll(l, r); - typename std::iterator_traits< - decltype(sameBlocksLeft.front().begin())>::value_type minEl = - std::min(sameBlocksLeft.front().back(), sameBlocksRight.front().back(), - lessThan); + ProjectedEl minEl = + std::min(leftProjection(sameBlocksLeft.front().back()), + rightProjection(sameBlocksRight.front().back()), lessThan); removeAllButUnjoined(sameBlocksLeft, minEl); removeAllButUnjoined(sameBlocksRight, minEl); }; From f826526c16a07ec89045036607f88e124b0ef44f Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Wed, 14 Jun 2023 16:02:17 +0200 Subject: [PATCH 033/150] Only unimportant unit tests fail now. There are quite some TODOs and some performance issues yet. --- src/engine/Join.cpp | 25 ++++++++---- src/engine/Join.h | 1 + src/engine/idTable/IdTable.h | 10 +++++ src/index/CompressedRelation.cpp | 10 +++++ src/util/JoinAlgorithms/JoinAlgorithms.h | 50 ++++++++++++++++-------- 5 files changed, 72 insertions(+), 24 deletions(-) diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp index bab11aa9ac..1a8ba30175 100644 --- a/src/engine/Join.cpp +++ b/src/engine/Join.cpp @@ -33,14 +33,19 @@ Join::Join(QueryExecutionContext* qec, std::shared_ptr t1, t2 = QueryExecutionTree::createSortedTree(std::move(t2), {t2JoinCol}); // Make sure subtrees are ordered so that identical queries can be identified. - if (t1->asString() > t2->asString()) { + auto swapChildren = [&]() { std::swap(t1, t2); std::swap(t1JoinCol, t2JoinCol); + }; + if (t1->asString() > t2->asString()) { + swapChildren(); } if (isFullScanDummy(t1)) { AD_CONTRACT_CHECK(!isFullScanDummy(t2)); - std::swap(t1, t2); - std::swap(t1JoinCol, t2JoinCol); + swapChildren(); + } else if (t1->getType() == QueryExecutionTree::SCAN && + t2->getType() != QueryExecutionTree::SCAN) { + swapChildren(); } _left = std::move(t1); _leftJoinCol = t1JoinCol; @@ -119,9 +124,16 @@ ResultTable Join::computeResult() { LOG(DEBUG) << "Computing Join result..." << endl; + // TODO For the specialized cases we don't need to materialize the + // results of the childdren... if (_left->getType() == QueryExecutionTree::SCAN && _right->getType() == QueryExecutionTree::SCAN) { computeResultForTwoIndexScans(&idTable); + } else if (_right->getType() == QueryExecutionTree::SCAN) { + computeResultForIndexScanAndColumn( + leftRes->idTable(), _leftJoinCol, + dynamic_cast(*_right->getRootOperation()), + _rightJoinCol, &idTable); } else { join(leftRes->idTable(), _leftJoinCol, rightRes->idTable(), _rightJoinCol, &idTable); @@ -644,9 +656,6 @@ void Join::computeResultForTwoIndexScans(IdTable* resultPtr) { dynamic_cast(*_left->getRootOperation()), dynamic_cast(*_right->getRootOperation())); - // LOG(WARN) << "num blocks in first: " << std::ranges::distance(leftBlocks) - // << std::endl; LOG(WARN) << "num blocks in second: " << - // std::ranges::distance(rightBlocks) << std::endl; ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks, lessThan, addResultRow); } @@ -663,8 +672,8 @@ void Join::computeResultForIndexScanAndColumn(const IdTable& inputTable, AD_CORRECTNESS_CHECK(joinColumnIndexScan == 0); auto joinColumn = inputTable.getColumn(joinColumnIndexTable); - auto addResultRow = [&, beg = joinColumn.begin()](auto itLeft, auto itRight) { - const auto& l = *inputTable.begin() + (itLeft - beg); + auto addResultRow = [&, beg = joinColumn.data()](auto itLeft, auto itRight) { + const auto& l = *(inputTable.begin() + (&(*itLeft) - beg)); const auto& r = *itRight; result.emplace_back(); IdTable::row_reference lastRow = result.back(); diff --git a/src/engine/Join.h b/src/engine/Join.h index 8794d385d2..fe523cc1b8 100644 --- a/src/engine/Join.h +++ b/src/engine/Join.h @@ -8,6 +8,7 @@ #include +#include "engine/IndexScan.h" #include "engine/Operation.h" #include "engine/QueryExecutionTree.h" #include "util/HashMap.h" diff --git a/src/engine/idTable/IdTable.h b/src/engine/idTable/IdTable.h index 8f96cbf82e..a1cea0308b 100644 --- a/src/engine/idTable/IdTable.h +++ b/src/engine/idTable/IdTable.h @@ -308,6 +308,16 @@ class IdTable { return data().at(column).at(row); } + row_reference_restricted at(size_t row) requires(!isView) { + AD_CONTRACT_CHECK(row < numRows()); + return operator[](row); + } + + const_row_reference_restricted at(size_t row) const { + AD_CONTRACT_CHECK(row < numRows()); + return operator[](row); + } + // Get a reference to the `i`-th row. The returned proxy objects can be // implicitly and trivially converted to `row_reference`. For the design // rationale behind those proxy types see above for their definition. diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index c0177fe67b..b4d02ac9ec 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -240,6 +240,14 @@ cppcoro::generator CompressedRelationReader::lazyScan( AD_CORRECTNESS_CHECK(endBlock - beginBlock <= 1); } + LOG(WARN) << "col0 " << metadata.col0Id_ << " col1 " << col1Id << std::endl; + for (const auto& block : relevantBlocks) { + LOG(WARN) << "first " << block.firstTriple_[0] << block.firstTriple_[1] + << block.lastTriple_[2] << std::endl; + LOG(WARN) << "last " << block.lastTriple_[0] << block.lastTriple_[1] + << block.lastTriple_[2] << std::endl; + } + // The first and the last block might be incomplete (that is, only // a part of these blocks is actually part of the result, // set up a lambda which allows us to read these blocks, and returns @@ -405,6 +413,8 @@ std::vector CompressedRelationReader::getBlocksForJoin( auto noop = ad_utility::noop; [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( joinColumnUnique, idRanges, lessThan, addRow, noop, noop); + // TODO This really has to be done inside the function + result.erase(std::unique(result.begin(), result.end()), result.end()); return result; } diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index dc5bc63b8f..22208ec3f7 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -554,9 +554,11 @@ using Range = std::pair; template class BlockAndSubrange { public: - BlockAndSubrange(Block block) + using reference_type = + std::iterator_traits::reference; + explicit BlockAndSubrange(Block block) : block_{std::move(block)}, subrange_{0, block_.size()} {} - const auto& back() { return block_.at(subrange_.second - 1); } + reference_type back() { return block_[subrange_.second - 1]; } auto subrange() { return std::ranges::subrange{block_.begin() + subrange_.first, block_.begin() + subrange_.second}; @@ -610,12 +612,14 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, AD_CORRECTNESS_CHECK(sameBlocksRight.size() <= 1); while (sameBlocksLeft.empty() && it1 != end1) { if (!it1->empty()) { - sameBlocksLeft.push_back(std::move(*it1)); + AD_CORRECTNESS_CHECK(std::ranges::is_sorted(*it1, lessThan)); + sameBlocksLeft.emplace_back(std::move(*it1)); } ++it1; } while (sameBlocksRight.empty() && it2 != end2) { if (!it2->empty()) { + AD_CORRECTNESS_CHECK(std::ranges::is_sorted(*it2, lessThan)); sameBlocksRight.emplace_back(std::move(*it2)); } ++it2; @@ -631,14 +635,16 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, // TODO here and below: use `at`, but it needs to be implemented // in the `Row` class. while (it1 != end1 && eq((*it1)[0], lastLeft)) { - sameBlocksLeft.push_back(std::move(*it1)); + AD_CORRECTNESS_CHECK(std::ranges::is_sorted(*it1, lessThan)); + sameBlocksLeft.emplace_back(std::move(*it1)); ++it1; } } if (!lessThan(lastLeft, lastRight)) { while (it2 != end2 && eq((*it2)[0], lastRight)) { - sameBlocksRight.push_back(std::move(*it2)); + AD_CORRECTNESS_CHECK(std::ranges::is_sorted(*it2, lessThan)); + sameBlocksRight.emplace_back(std::move(*it2)); ++it2; if (!eq(sameBlocksRight.back().back(), lastRight)) { break; @@ -666,12 +672,26 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, auto joinAndRemoveBeginning = [&]() { decltype(auto) l = sameBlocksLeft.at(0).subrange(); decltype(auto) r = sameBlocksRight.at(0).subrange(); + auto lBack = l.back(); + auto rBack = r.back(); + const auto& lprof = std::move(leftProjection(std::move(l.back()))); + const auto& rpoj = std::move(rightProjection(std::move(r.back()))); + auto lf = l.front(); + auto rf = r.front(); + const auto& lfp = std::move(leftProjection(l.front())); + const auto& rfp = std::move(rightProjection(r.front())); ProjectedEl minEl = std::min(leftProjection(l.back()), rightProjection(r.back()), lessThan); auto itL = std::ranges::equal_range(l, minEl, lessThan); auto itR = std::ranges::equal_range(r, minEl, lessThan); join(std::ranges::subrange{l.begin(), itL.begin()}, std::ranges::subrange{r.begin(), itR.begin()}); + auto lBeg = itL.begin() - l.begin(); + auto rBeg = itR.begin() - r.begin(); + auto lEnd = itL.end() - l.begin(); + auto rEnd = itR.end() - r.begin(); + auto lSize = l.size(); + auto rSize = r.size(); sameBlocksLeft.at(0).setSubrange(itL.begin(), l.end()); sameBlocksRight.at(0).setSubrange(itR.begin(), r.end()); }; @@ -685,8 +705,6 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, decltype(auto) remainingBlock = blocks.at(0).subrange(); auto beginningOfUnjoined = std::ranges::upper_bound(remainingBlock, lastHandledElement, lessThan); - // TODO This is not the most efficient way, but currently necessary - // because of the interface of the `IdTable`. remainingBlock = std::ranges::subrange{beginningOfUnjoined, remainingBlock.end()}; if (!remainingBlock.empty()) { @@ -702,26 +720,26 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, using SubRight = decltype(sameBlocksRight.front().subrange()); std::vector l; std::vector r; + + ProjectedEl minEl = + std::min(leftProjection(sameBlocksLeft.front().back()), + rightProjection(sameBlocksRight.front().back()), lessThan); + for (size_t i = 0; i < sameBlocksLeft.size() - 1; ++i) { l.push_back(sameBlocksLeft[i].subrange()); } - if (sameBlocksLeft.size() > 1) { + if (sameBlocksLeft.size() > 0) { l.push_back(std::ranges::equal_range(sameBlocksLeft.back().subrange(), - sameBlocksLeft.front().back(), - lessThan)); + minEl, lessThan)); } for (size_t i = 0; i < sameBlocksRight.size() - 1; ++i) { r.push_back(sameBlocksRight[i].subrange()); } - if (sameBlocksRight.size() > 1) { + if (sameBlocksRight.size() > 0) { r.push_back(std::ranges::equal_range(sameBlocksRight.back().subrange(), - sameBlocksRight.front().back(), - lessThan)); + minEl, lessThan)); } addAll(l, r); - ProjectedEl minEl = - std::min(leftProjection(sameBlocksLeft.front().back()), - rightProjection(sameBlocksRight.front().back()), lessThan); removeAllButUnjoined(sameBlocksLeft, minEl); removeAllButUnjoined(sameBlocksRight, minEl); }; From 89286841b8a90fc6ac421f6b1aa5f724a3711f81 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Wed, 14 Jun 2023 18:02:44 +0200 Subject: [PATCH 034/150] Started to cleanup the central routine. --- src/index/CompressedRelation.cpp | 27 +--- src/util/JoinAlgorithms/JoinAlgorithms.h | 180 ++++++++++++++--------- 2 files changed, 119 insertions(+), 88 deletions(-) diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index b4d02ac9ec..ff8122fdc4 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -371,10 +371,6 @@ std::vector CompressedRelationReader::getBlocksForJoin( // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_ auto relevantBlocks = getBlocksFromMetadata(metadata, std::nullopt, blockMetadata); - // TODO Is this condition necessary? - if (relevantBlocks.size() < 2) { - return {relevantBlocks.begin(), relevantBlocks.end()}; - } auto idRanges = blocksToIdRanges(relevantBlocks, metadata.col0Id_, col1Id); @@ -398,23 +394,14 @@ std::vector CompressedRelationReader::getBlocksForJoin( result.push_back(*(begBlocks + (it2 - begRanges))); }; - // TODO is the copy really necessary? - // TODO Should we respect the memory limit? - // TODO The correct way indeed WAS to remove the duplicates inside - // the `zipperJoinWithUndef` function. - std::vector joinColumnUnique; - joinColumnUnique.reserve(joinColum.size()); - std::ranges::unique_copy(joinColum, std::back_inserter(joinColumnUnique)); - - if (joinColum.empty()) { - return result; - } - auto noop = ad_utility::noop; - [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( - joinColumnUnique, idRanges, lessThan, addRow, noop, noop); - // TODO This really has to be done inside the function - result.erase(std::unique(result.begin(), result.end()), result.end()); + [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef( + joinColum, idRanges, lessThan, addRow, noop, noop); + + // The following check shouldn't be too expensive as there are only few + // blocks. + auto unique = std::unique(result.begin(), result.end()); + AD_CORRECTNESS_CHECK(unique == result.end()); return result; } diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 22208ec3f7..62ff661d92 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -85,6 +85,15 @@ concept BinaryIteratorFunction = * @param elFromFirstNotFoundAction This function is called for each iterator in * `left` for which no corresponding match in `right` was found. This is `noop` * for "normal` joins, but can be set to implement `OPTIONAL` or `MINUS`. + * @tparam addDuplicatesFromLeft If set to `false`, then if several inputs from + * `left` are compatible with several inputs from `right`, then only the matches + * for the first matching entry from `left` are added to the result. For example + * if `A` and `B` from left are both compatible with `C` and `D` from right, + * then the result for these elements will be `AC, AD`. With the default + * (`addDuplicatesFromLeft = true`) it would be `AC, AD, BC, BD`. Note that this + * currently only works for the exact matches and has no effects on the merging + * with undefined values. This is useful when there are no undefined values and + * we are only interested in the unique results from `right`. * @return 0 if the result is sorted, > 0 if the result is not sorted. `Sorted` * means that all the calls to `compatibleRowAction` were ordered wrt * `lessThan`. A result being out of order can happen if two rows with UNDEF @@ -93,11 +102,11 @@ concept BinaryIteratorFunction = * described cases leads to two sorted ranges in the output, this can possibly * be exploited to fix the result in a cheaper way than a full sort. */ -template +template < + bool addDuplicatesFromLeft = true, std::ranges::random_access_range Range1, + std::ranges::random_access_range Range2, typename LessThan, + typename FindSmallerUndefRangesLeft, typename FindSmallerUndefRangesRight, + typename ElFromFirstNotFoundAction = decltype(noop)> [[nodiscard]] auto zipperJoinWithUndef( const Range1& left, const Range2& right, const LessThan& lessThan, const auto& compatibleRowAction, @@ -262,6 +271,9 @@ template Add assertions that the iterators are valid. subrange_.first = it1 - block_.begin(); @@ -574,6 +590,29 @@ class BlockAndSubrange { Range subrange_; }; +/** + * @brief Perform a zipper/merge join between two sorted inputs that are given + * as blocks of inputs, e.g. `std::vector>` or + * `ad_utility::generator>`. The blocks can be specified via a + * input range (e.g. a generator), but each single block must be a random access + * range (e.g. vector). The inputs will be moved from. The space complexity is + * `O(size of result)`. The result is only correct if none of the inputs + * contains UNDEF values. + * @param leftBlocks The left input to the join algorithm. Typically a range of + * blocks of rows of IDs (e.g.`std::vector` or + * `ad_utility::generator`). + * @param rightBlocks The right input to the join algorithm. + * @param lessThan This function is called with one element from one of the + * blocks of `leftBlocks` and `rightBlocks` each and must return `true` if the + * first argument comes before the second one. The concatenation of all blocks + * in `leftBlocks` must be sorted according to this function. The same + * requirement holds for `rightBlocks`. + * @param compatibleRowAction When an element from a block from `leftBlock` and + * an element from a block from `rightBlock` match (because they compare equal + * wrt the `lessThan` relation), this function is called with the two + * *iterators* to the elements, i.e . `compatibleRowAction(itToLeft, + * itToRight)` + */ template @@ -583,74 +622,94 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, const auto& compatibleRowAction, LeftProjection leftProjection = {}, RightProjection rightProjection = {}) { + // Type aliases for a single block from the left/right input using LeftBlock = typename std::decay_t::value_type; using RightBlock = typename std::decay_t::value_type; + // Type aliases for a single element from a block from the left/right input. using LeftEl = typename std::iterator_traits::value_type; using RightEl = typename std::iterator_traits::value_type; + // Type alias for the result of the projection. Elements from the left and + // right input must be projected to the same type. using ProjectedEl = std::decay_t>; static_assert( ad_utility::isSimilar>); + // Iterators for the two inputs. These iterators work on blocks. auto it1 = leftBlocks.begin(); auto end1 = leftBlocks.end(); auto it2 = rightBlocks.begin(); auto end2 = rightBlocks.end(); + + // Create an equality comparison from the `lessThan` predicate. auto eq = [&lessThan](const auto& el1, const auto& el2) { return !lessThan(el1, el2) && !lessThan(el2, el1); }; + // In these buffers we will store blocks that all contain the same elements + // and thus their cartesian products match. std::vector> sameBlocksLeft; std::vector> sameBlocksRight; + // Read the minimal number of unread blocks from `leftBlocks` into + // `sameBlocksLeft` and from `rightBlocks` into `sameBlocksRight` until the + // following conditions hold: + // * at least one block is contained in `sameBlocksLeft` and `sameBlocksRight` + // each. + // * assume that the last element from sameBlocksLeft[0] is less than or equal + // to the last element of sameBlocksLeft[1] (otherwise the condition is + // switched) + // * Then all the blocks that contain elements less than or equal to this + // minimum are read into the respective buffers. For example the following + // scenario might occur: + // sameBlocksLeft: [0-3], [3-3], [3-5] + // sameBlocksRight: [0-3], [3-7] + // The consequence of these postconditions is that we can always join at least + // on block from each input in the next step and can correctly handle the case + // that there are duplicates of the last element of the block in the adjacent + // blocks. If either of `sameBlocksLeft` or `sameBlocksRight` are empty after + // calling this function, then there are no more blocks to join and the + // algorithm can terminate. auto fillBuffer = [&]() { AD_CORRECTNESS_CHECK(sameBlocksLeft.size() <= 1); AD_CORRECTNESS_CHECK(sameBlocksRight.size() <= 1); - while (sameBlocksLeft.empty() && it1 != end1) { - if (!it1->empty()) { - AD_CORRECTNESS_CHECK(std::ranges::is_sorted(*it1, lessThan)); - sameBlocksLeft.emplace_back(std::move(*it1)); - } - ++it1; - } - while (sameBlocksRight.empty() && it2 != end2) { - if (!it2->empty()) { - AD_CORRECTNESS_CHECK(std::ranges::is_sorted(*it2, lessThan)); - sameBlocksRight.emplace_back(std::move(*it2)); + + auto fillWithAtLeastOne = [](auto& targetBuffer, auto& it, + const auto& end) { + while (targetBuffer.empty() && it != end) { + if (!it->empty()) { + AD_EXPENSIVE_CHECK(std::ranges::is_sorted(*it, lessThan)); + targetBuffer.emplace_back(std::move(*it)); + } + ++it; } - ++it2; - } + }; + fillWithAtLeastOne(sameBlocksLeft, it1, end1); + fillWithAtLeastOne(sameBlocksRight, it2, end2); if (sameBlocksLeft.empty() || sameBlocksRight.empty()) { return; } LeftEl lastLeft = sameBlocksLeft.front().back(); RightEl lastRight = sameBlocksRight.front().back(); - - if (!lessThan(lastRight, lastLeft)) { - // TODO here and below: use `at`, but it needs to be implemented - // in the `Row` class. - while (it1 != end1 && eq((*it1)[0], lastLeft)) { - AD_CORRECTNESS_CHECK(std::ranges::is_sorted(*it1, lessThan)); - sameBlocksLeft.emplace_back(std::move(*it1)); - ++it1; + ProjectedEl minEl = std::min(leftProjection(lastLeft), + rightProjection(lastRight), lessThan); + + auto fillEqualToMinimum = [&minEl, &lessThan, &eq](auto& targetBuffer, + auto& it, + const auto& end) { + while (it != end && eq((*it)[0], minEl)) { + AD_CORRECTNESS_CHECK(std::ranges::is_sorted(*it, lessThan)); + targetBuffer.emplace_back(std::move(*it)); + ++it; } - } - - if (!lessThan(lastLeft, lastRight)) { - while (it2 != end2 && eq((*it2)[0], lastRight)) { - AD_CORRECTNESS_CHECK(std::ranges::is_sorted(*it2, lessThan)); - sameBlocksRight.emplace_back(std::move(*it2)); - ++it2; - if (!eq(sameBlocksRight.back().back(), lastRight)) { - break; - } - } - } + }; + fillEqualToMinimum(sameBlocksLeft, it1, end1); + fillEqualToMinimum(sameBlocksRight, it2, end2); }; auto join = [&](const auto& l, const auto& r) { @@ -672,26 +731,12 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, auto joinAndRemoveBeginning = [&]() { decltype(auto) l = sameBlocksLeft.at(0).subrange(); decltype(auto) r = sameBlocksRight.at(0).subrange(); - auto lBack = l.back(); - auto rBack = r.back(); - const auto& lprof = std::move(leftProjection(std::move(l.back()))); - const auto& rpoj = std::move(rightProjection(std::move(r.back()))); - auto lf = l.front(); - auto rf = r.front(); - const auto& lfp = std::move(leftProjection(l.front())); - const auto& rfp = std::move(rightProjection(r.front())); ProjectedEl minEl = std::min(leftProjection(l.back()), rightProjection(r.back()), lessThan); auto itL = std::ranges::equal_range(l, minEl, lessThan); auto itR = std::ranges::equal_range(r, minEl, lessThan); join(std::ranges::subrange{l.begin(), itL.begin()}, std::ranges::subrange{r.begin(), itR.begin()}); - auto lBeg = itL.begin() - l.begin(); - auto rBeg = itR.begin() - r.begin(); - auto lEnd = itL.end() - l.begin(); - auto rEnd = itR.end() - r.begin(); - auto lSize = l.size(); - auto rSize = r.size(); sameBlocksLeft.at(0).setSubrange(itL.begin(), l.end()); sameBlocksRight.at(0).setSubrange(itR.begin(), r.end()); }; @@ -716,8 +761,9 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, auto joinBuffers = [&]() { joinAndRemoveBeginning(); - using SubLeft = decltype(sameBlocksLeft.front().subrange()); - using SubRight = decltype(sameBlocksRight.front().subrange()); + using SubLeft = decltype(std::as_const(sameBlocksLeft.front()).subrange()); + using SubRight = + decltype(std::as_const(sameBlocksRight.front()).subrange()); std::vector l; std::vector r; @@ -725,20 +771,18 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, std::min(leftProjection(sameBlocksLeft.front().back()), rightProjection(sameBlocksRight.front().back()), lessThan); - for (size_t i = 0; i < sameBlocksLeft.size() - 1; ++i) { - l.push_back(sameBlocksLeft[i].subrange()); - } - if (sameBlocksLeft.size() > 0) { - l.push_back(std::ranges::equal_range(sameBlocksLeft.back().subrange(), - minEl, lessThan)); - } - for (size_t i = 0; i < sameBlocksRight.size() - 1; ++i) { - r.push_back(sameBlocksRight[i].subrange()); - } - if (sameBlocksRight.size() > 0) { - r.push_back(std::ranges::equal_range(sameBlocksRight.back().subrange(), - minEl, lessThan)); - } + auto pushRelevantSubranges = [&minEl, &lessThan](auto& target, + const auto& input) { + for (size_t i = 0; i < input.size() - 1; ++i) { + target.push_back(input[i].subrange()); + } + if (!input.empty()) { + target.push_back( + std::ranges::equal_range(input.back().subrange(), minEl, lessThan)); + } + }; + pushRelevantSubranges(l, sameBlocksLeft); + pushRelevantSubranges(r, sameBlocksRight); addAll(l, r); removeAllButUnjoined(sameBlocksLeft, minEl); removeAllButUnjoined(sameBlocksRight, minEl); From 551d5678f0c3104ca86380f1454351c8760b1a8e Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Wed, 14 Jun 2023 19:24:29 +0200 Subject: [PATCH 035/150] Several additional cleanups. --- src/util/JoinAlgorithms/JoinAlgorithms.h | 104 ++++++++++++++--------- 1 file changed, 62 insertions(+), 42 deletions(-) diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 62ff661d92..bec29c8a26 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -655,6 +655,12 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, std::vector> sameBlocksLeft; std::vector> sameBlocksRight; + auto getMinEl = [&leftProjection, &rightProjection, &sameBlocksLeft, + &sameBlocksRight, &lessThan]() -> ProjectedEl { + return std::min(leftProjection(sameBlocksLeft.front().back()), + rightProjection(sameBlocksRight.front().back()), lessThan); + }; + // Read the minimal number of unread blocks from `leftBlocks` into // `sameBlocksLeft` and from `rightBlocks` into `sameBlocksRight` until the // following conditions hold: @@ -694,14 +700,10 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, if (sameBlocksLeft.empty() || sameBlocksRight.empty()) { return; } - LeftEl lastLeft = sameBlocksLeft.front().back(); - RightEl lastRight = sameBlocksRight.front().back(); - ProjectedEl minEl = std::min(leftProjection(lastLeft), - rightProjection(lastRight), lessThan); - - auto fillEqualToMinimum = [&minEl, &lessThan, &eq](auto& targetBuffer, - auto& it, - const auto& end) { + + auto fillEqualToMinimum = [minEl = getMinEl(), &lessThan, &eq]( + auto& targetBuffer, auto& it, + const auto& end) { while (it != end && eq((*it)[0], minEl)) { AD_CORRECTNESS_CHECK(std::ranges::is_sorted(*it, lessThan)); targetBuffer.emplace_back(std::move(*it)); @@ -712,13 +714,13 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, fillEqualToMinimum(sameBlocksRight, it2, end2); }; - auto join = [&](const auto& l, const auto& r) { - return zipperJoinWithUndef(l, r, lessThan, compatibleRowAction, noop, noop); - }; - - auto addAll = [&](const auto& l, const auto& r) { - for (const auto& lBlock : l) { - for (const auto& rBlock : r) { + // Call `compatibleRowAction` for all pairs of elements in the cartesian + // product of the blocks in `blocksLeft` and `blocksRight`. + auto addAll = [&compatibleRowAction](const auto& blocksLeft, + const auto& blocksRight) { + // TODO use `std::views::cartesian_product`. + for (const auto& lBlock : blocksLeft) { + for (const auto& rBlock : blocksRight) { for (const auto& lEl : lBlock) { for (const auto& rEl : rBlock) { compatibleRowAction(&lEl, &rEl); @@ -728,30 +730,51 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, } }; + // Join the first block in `sameBlocksLeft` with the first block in + // `sameBlocksRight`, but leave the last element in each block untouched (it + // might have matches in subsequent blocks). The fully joined parts of the + // block are then removed from `sameBlocksLeft/Right`, as they are not needed + // anymore. auto joinAndRemoveBeginning = [&]() { + // Get the first blocks. + AD_CORRECTNESS_CHECK(!sameBlocksLeft.empty()); + AD_CORRECTNESS_CHECK(!sameBlocksRight.empty()); decltype(auto) l = sameBlocksLeft.at(0).subrange(); decltype(auto) r = sameBlocksRight.at(0).subrange(); - ProjectedEl minEl = - std::min(leftProjection(l.back()), rightProjection(r.back()), lessThan); - auto itL = std::ranges::equal_range(l, minEl, lessThan); - auto itR = std::ranges::equal_range(r, minEl, lessThan); - join(std::ranges::subrange{l.begin(), itL.begin()}, - std::ranges::subrange{r.begin(), itR.begin()}); - sameBlocksLeft.at(0).setSubrange(itL.begin(), l.end()); - sameBlocksRight.at(0).setSubrange(itR.begin(), r.end()); + + // Compute the range that is safe to join and perform the join. + ProjectedEl minEl = getMinEl(); + auto itL = std::ranges::lower_bound(l, minEl, lessThan); + auto itR = std::ranges::lower_bound(r, minEl, lessThan); + [[maybe_unused]] auto res = + zipperJoinWithUndef(std::ranges::subrange{l.begin(), itL}, + std::ranges::subrange{r.begin(), itR}, lessThan, + compatibleRowAction, noop, noop); + + // Remove the joined elements. + sameBlocksLeft.at(0).setSubrange(itL, l.end()); + sameBlocksRight.at(0).setSubrange(itR, r.end()); }; + // Cleanup the `blocks` (either `sameBlocksLeft` or `sameBlocksRight`, by + // removing blocks and parts of blocks, s.t. only elements `>= + // lastHandledElement` remain. This function expects that all but the last + // block can completely be deleted. auto removeAllButUnjoined = [lessThan]( - Blocks& blocks, auto lastHandledElement) { - // TODO This method can be incorporated into the calling code much - // more simply. + Blocks& blocks, + ProjectedEl lastHandledElement) { + // Erase all but the last block. AD_CORRECTNESS_CHECK(!blocks.empty()); blocks.erase(blocks.begin(), blocks.end() - 1); + + // Delete the part from the last block that is `<= lastHandledElement`. decltype(auto) remainingBlock = blocks.at(0).subrange(); auto beginningOfUnjoined = std::ranges::upper_bound(remainingBlock, lastHandledElement, lessThan); remainingBlock = std::ranges::subrange{beginningOfUnjoined, remainingBlock.end()}; + // If the last block also was already handled completely, delete it (this + // might happen at the very end). if (!remainingBlock.empty()) { blocks.at(0).setSubrange(remainingBlock.begin(), remainingBlock.end()); } else { @@ -759,30 +782,27 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, } }; + // Combine the above functionality and perform one round of joining. auto joinBuffers = [&]() { + // Join the beginning of the first blocks and remove it from the input. joinAndRemoveBeginning(); - using SubLeft = decltype(std::as_const(sameBlocksLeft.front()).subrange()); - using SubRight = - decltype(std::as_const(sameBlocksRight.front()).subrange()); - std::vector l; - std::vector r; - - ProjectedEl minEl = - std::min(leftProjection(sameBlocksLeft.front().back()), - rightProjection(sameBlocksRight.front().back()), lessThan); - - auto pushRelevantSubranges = [&minEl, &lessThan](auto& target, - const auto& input) { + // Prepare the subranges of the loaded blocks that are equal to the + // last element that we can safely join. + ProjectedEl minEl = getMinEl(); + auto pushRelevantSubranges = [&minEl, &lessThan](const auto& input) { + using Subrange = decltype(std::as_const(input.front()).subrange()); + std::vector result; for (size_t i = 0; i < input.size() - 1; ++i) { - target.push_back(input[i].subrange()); + result.push_back(input[i].subrange()); } if (!input.empty()) { - target.push_back( + result.push_back( std::ranges::equal_range(input.back().subrange(), minEl, lessThan)); } + return result; }; - pushRelevantSubranges(l, sameBlocksLeft); - pushRelevantSubranges(r, sameBlocksRight); + auto l = pushRelevantSubranges(sameBlocksLeft); + auto r = pushRelevantSubranges(sameBlocksRight); addAll(l, r); removeAllButUnjoined(sameBlocksLeft, minEl); removeAllButUnjoined(sameBlocksRight, minEl); From 8006784eb89aaf55bc3f679b700d8ce9300099a0 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 15 Jun 2023 09:23:17 +0200 Subject: [PATCH 036/150] Completely cleaned up the JoinAlgorithms.h and the corresponding tests. --- src/index/CompressedRelation.cpp | 8 -- src/util/JoinAlgorithms/JoinAlgorithms.h | 60 +++++++--- test/JoinAlgorithmsTest.cpp | 137 ++++++++++------------- 3 files changed, 108 insertions(+), 97 deletions(-) diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index ff8122fdc4..1f83ade4f7 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -240,14 +240,6 @@ cppcoro::generator CompressedRelationReader::lazyScan( AD_CORRECTNESS_CHECK(endBlock - beginBlock <= 1); } - LOG(WARN) << "col0 " << metadata.col0Id_ << " col1 " << col1Id << std::endl; - for (const auto& block : relevantBlocks) { - LOG(WARN) << "first " << block.firstTriple_[0] << block.firstTriple_[1] - << block.lastTriple_[2] << std::endl; - LOG(WARN) << "last " << block.lastTriple_[0] << block.lastTriple_[1] - << block.lastTriple_[2] << std::endl; - } - // The first and the last block might be incomplete (that is, only // a part of these blocks is actually part of the result, // set up a lambda which allows us to read these blocks, and returns diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index bec29c8a26..88bf7fb1f3 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -561,34 +561,66 @@ void specialOptionalJoin( } } -// TODO move to detail namespace +namespace detail { using Range = std::pair; + +// Store a contiguous random-access range (e.g. `std::vector`or `std::span`, +// together with a pair of indices +// `[beginIndex, endIndex)` that denote a contiguous subrange of the container. +// Note that this approach is more robust than storing iterators or subranges +// directly instead of indices, because many containers have their iterators +// invalidated when they are being moved (e.g. `std::string` or `IdTable`). template class BlockAndSubrange { + private: + Block block_; + Range subrange_; + public: - using reference_type = - std::iterator_traits::reference; + // The reference type of the underlying container. + using reference = std::iterator_traits::reference; + + // Construct from a container object, the initial subrange will represent the + // whole container. explicit BlockAndSubrange(Block block) : block_{std::move(block)}, subrange_{0, block_.size()} {} - reference_type back() { return block_[subrange_.second - 1]; } + + // Return a reference to the last element of the currently specified subrange. + reference back() { + AD_CORRECTNESS_CHECK(subrange_.second - 1 < block_.size()); + return block_[subrange_.second - 1]; + } + + // Return the currently specified subrange as a `std::ranges::subrange` + // object. auto subrange() { return std::ranges::subrange{block_.begin() + subrange_.first, block_.begin() + subrange_.second}; } + + // The const overload of the `subrange` method (see above). auto subrange() const { return std::ranges::subrange{block_.begin() + subrange_.first, block_.begin() + subrange_.second}; } - void setSubrange(auto it1, auto it2) { - // TODO Add assertions that the iterators are valid. - subrange_.first = it1 - block_.begin(); - subrange_.second = it2 - block_.begin(); - } - private: - Block block_; - Range subrange_; + // Specify the subrange by using two iterators `begin` and `end`. The + // iterators must be valid iterators that point into the container, this is + // checked by an assertion. The only legal way to obtain such iterators is to + // call `subrange()` and then to extract valid iterators from the returned + // subrange. + void setSubrange(auto begin, auto end) { + auto checkIt = [this](const auto& it) { + AD_CONTRACT_CHECK(block_.begin() <= it && it <= block_.end()); + }; + checkIt(begin); + checkIt(end); + AD_CONTRACT_CHECK(begin <= end); + subrange_.first = begin - block_.begin(); + subrange_.second = end - block_.begin(); + } }; +} // namespace detail /** * @brief Perform a zipper/merge join between two sorted inputs that are given @@ -652,8 +684,8 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks, // In these buffers we will store blocks that all contain the same elements // and thus their cartesian products match. - std::vector> sameBlocksLeft; - std::vector> sameBlocksRight; + std::vector> sameBlocksLeft; + std::vector> sameBlocksRight; auto getMinEl = [&leftProjection, &rightProjection, &sameBlocksLeft, &sameBlocksRight, &lessThan]() -> ProjectedEl { diff --git a/test/JoinAlgorithmsTest.cpp b/test/JoinAlgorithmsTest.cpp index 5e1b01f574..78509a3087 100644 --- a/test/JoinAlgorithmsTest.cpp +++ b/test/JoinAlgorithmsTest.cpp @@ -5,108 +5,95 @@ #include #include +#include "./util/GTestHelpers.h" #include "util/JoinAlgorithms/JoinAlgorithms.h" #include "util/TransparentFunctors.h" using namespace ad_utility; namespace { -using NestedBlock = std::vector>; -auto makeRowAdder(auto& target) { +// Some helpers for testing the joining of blocks of Integers. +using Block = std::vector>; +using NestedBlock = std::vector; +using JoinResult = std::vector>; + +auto makeRowAdder(JoinResult& target) { + // `it1, it2` must be (const) iterators to a `Block`. return [&target](auto it1, auto it2) { - AD_CONTRACT_CHECK(*it1 == *it2); - target.push_back(*it1); + auto [x1, x2] = *it1; + auto [y1, y2] = *it2; + AD_CONTRACT_CHECK(x1 == y1); + target.push_back(std::array{x1, x2, y2}); }; } + +using ad_utility::source_location; +// Test that when joining `a` and `b` on the first column then the result is +// equal to the `expected` result. +// TODO We could also resplit inputs into blocks randomly and thus add +// more test cases automatically. +void testJoin(const NestedBlock& a, const NestedBlock& b, JoinResult expected, + source_location l = source_location::current()) { + auto trace = generateLocationTrace(l); + JoinResult result; + auto compare = [](auto l, auto r) { return l[0] < r[0]; }; + zipperJoinForBlocksWithoutUndef(a, b, compare, makeRowAdder(result)); + // The result must be sorted on the first column + EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); + // The exact order of the elements with the same first column is not important + // and depends on implementation details. We therefore do not enforce it here. + EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expected)); + result.clear(); + for (auto& [x, y, z] : expected) { + std::swap(y, z); + } + zipperJoinForBlocksWithoutUndef(b, a, compare, makeRowAdder(result)); + EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); + EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expected)); +} } // namespace +// ________________________________________________________________________________________ TEST(JoinAlgorithms, JoinWithBlocksEmptyInput) { - NestedBlock a; - NestedBlock b; - std::vector result; - zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); - EXPECT_TRUE(result.empty()); - - a.push_back({13}); - zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); - EXPECT_TRUE(result.empty()); + testJoin({}, {}, {}); - b.emplace_back(); - zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); - EXPECT_TRUE(result.empty()); - - a.clear(); - b.push_back({23, 35}); - zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); - EXPECT_TRUE(result.empty()); + testJoin({{{13, 0}}}, {}, {}); + testJoin({{}, {{13, 0}}, {}}, {{}}, {}); } TEST(JoinAlgorithms, JoinWithBlocksSingleBlock) { - NestedBlock a{{1, 4, 18, 42}}; - NestedBlock b{{0, 4, 5, 19, 42}}; - std::vector result; - zipperJoinForBlocksWithoutUndef(a, b, std::less<>{}, makeRowAdder(result)); - EXPECT_THAT(result, ::testing::ElementsAre(4, 42)); + NestedBlock a{{{1, 11}, {4, 12}, {18, 13}, {42, 14}}}; + NestedBlock b{{{0, 24}, {4, 25}, {5, 25}, {19, 26}, {42, 27}}}; + JoinResult expectedResult{{4, 12, 25}, {42, 14, 27}}; + testJoin(a, b, expectedResult); } TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksOverlap) { - NestedBlock a{{1, 4, 18, 42}, {54, 57, 59}, {60, 67}}; - NestedBlock b{{0, 4, 5, 19, 42, 54}, {56, 57, 58, 59}, {61, 67}}; - std::vector result; - zipperJoinForBlocksWithoutUndef(NestedBlock(a), NestedBlock(b), std::less<>{}, - makeRowAdder(result)); - EXPECT_THAT(result, ::testing::ElementsAre(4, 42, 54, 57, 59, 67)); - result.clear(); - zipperJoinForBlocksWithoutUndef(NestedBlock(b), NestedBlock(a), std::less<>{}, - makeRowAdder(result)); - EXPECT_THAT(result, ::testing::ElementsAre(4, 42, 54, 57, 59, 67)); + NestedBlock a{{{1, 10}, {4, 11}, {18, 12}, {42, 13}}, + {{54, 14}, {57, 15}, {59, 16}}, + {{60, 17}, {67, 18}}}; + NestedBlock b{{{0, 20}, {4, 21}, {5, 22}, {19, 23}, {42, 24}, {54, 25}}, + {{56, 26}, {57, 27}, {58, 28}, {59, 29}}, + {{61, 30}, {67, 30}}}; + JoinResult expectedResult{{4, 11, 21}, {42, 13, 24}, {54, 14, 25}, + {57, 15, 27}, {59, 16, 29}, {67, 18, 30}}; + testJoin(a, b, expectedResult); } TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksPerElement) { - using NB = std::vector>>; - NB a{{{1, 0}, {42, 0}}, {{42, 1}, {42, 2}}, {{42, 3}, {48, 5}, {67, 0}}}; - NB b{{{2, 0}, {42, 12}, {43, 1}}, {{67, 13}, {69, 14}}}; - std::vector> result; - std::vector> expectedResult{ + NestedBlock a{ + {{1, 0}, {42, 0}}, {{42, 1}, {42, 2}}, {{42, 3}, {48, 5}, {67, 0}}}; + NestedBlock b{{{2, 0}, {42, 12}, {43, 1}}, {{67, 13}, {69, 14}}}; + JoinResult expectedResult{ {42, 0, 12}, {42, 1, 12}, {42, 2, 12}, {42, 3, 12}, {67, 0, 13}}; - auto compare = [](auto l, auto r) { return l[0] < r[0]; }; - auto add = [&result](auto it1, auto it2) { - AD_CORRECTNESS_CHECK((*it1)[0] == (*it1)[0]); - result.push_back(std::array{(*it1)[0], (*it1)[1], (*it2)[1]}); - }; - zipperJoinForBlocksWithoutUndef(NB{a}, NB{b}, compare, add); - EXPECT_THAT(result, ::testing::ElementsAreArray(expectedResult)); - result.clear(); - for (auto& [x, y, z] : expectedResult) { - std::swap(y, z); - } - zipperJoinForBlocksWithoutUndef(NB{b}, NB{a}, compare, add); - EXPECT_THAT(result, ::testing::ElementsAreArray(expectedResult)); + testJoin(a, b, expectedResult); } TEST(JoinAlgorithms, JoinWithBlocksMultipleBlocksPerElementBothSides) { - using NB = std::vector>>; - NB a{{{42, 0}}, {{42, 1}, {42, 2}}, {{42, 3}, {67, 0}}}; - NB b{{{2, 0}, {42, 12}}, {{42, 13}, {67, 14}}}; - std::vector> result; + NestedBlock a{{{42, 0}}, {{42, 1}, {42, 2}}, {{42, 3}, {67, 0}}}; + NestedBlock b{{{2, 0}, {42, 12}}, {{42, 13}, {67, 14}}}; std::vector> expectedResult{ {42, 0, 12}, {42, 0, 13}, {42, 1, 12}, {42, 2, 12}, {42, 1, 13}, {42, 2, 13}, {42, 3, 12}, {42, 3, 13}, {67, 0, 14}}; - auto compare = [](auto l, auto r) { return l[0] < r[0]; }; - auto add = [&result](auto it1, auto it2) { - AD_CORRECTNESS_CHECK((*it1)[0] == (*it1)[0]); - result.push_back(std::array{(*it1)[0], (*it1)[1], (*it2)[1]}); - }; - zipperJoinForBlocksWithoutUndef(NB{a}, NB{b}, compare, add); - // The exact order of the elements with the same first column is not important - // and depends on implementation details. We therefore do not enforce it here. - EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); - EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expectedResult)); - result.clear(); - for (auto& [x, y, z] : expectedResult) { - std::swap(y, z); - } - zipperJoinForBlocksWithoutUndef(NB{b}, NB{a}, compare, add); - EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); - EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expectedResult)); + testJoin(a, b, expectedResult); } From c98c5b369d54c61fec3ead5e0bbad47fc542c5a8 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 15 Jun 2023 11:35:24 +0200 Subject: [PATCH 037/150] It compiles and the tests run. TODO: remove the ScanTypes also from the query planner. Move stuff out of IndexScan.h and document it. --- src/engine/GroupBy.cpp | 4 +- src/engine/IndexScan.cpp | 402 +++++++-------------------------- src/engine/IndexScan.h | 133 ++++++++--- src/engine/Join.cpp | 27 +-- test/QueryPlannerTest.cpp | 60 +++-- test/QueryPlannerTestHelpers.h | 17 +- 6 files changed, 221 insertions(+), 422 deletions(-) diff --git a/src/engine/GroupBy.cpp b/src/engine/GroupBy.cpp index 4edd864cd6..51ec31c1f7 100644 --- a/src/engine/GroupBy.cpp +++ b/src/engine/GroupBy.cpp @@ -535,7 +535,7 @@ std::optional GroupBy::getPermutationForThreeVariableTriple( } { auto v = variableThatMustBeContained; - if (v != indexScan->getSubject() && v.name() != indexScan->getPredicate() && + if (v != indexScan->getSubject() && v != indexScan->getPredicate() && v != indexScan->getObject()) { return std::nullopt; } @@ -543,7 +543,7 @@ std::optional GroupBy::getPermutationForThreeVariableTriple( if (variableByWhichToSort == indexScan->getSubject()) { return Permutation::SPO; - } else if (variableByWhichToSort.name() == indexScan->getPredicate()) { + } else if (variableByWhichToSort == indexScan->getPredicate()) { return Permutation::POS; } else if (variableByWhichToSort == indexScan->getObject()) { return Permutation::OSP; diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 44ec6d4cf5..14ef2b4f7a 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -13,127 +13,78 @@ using std::string; +// _____________________________________________________________________________ IndexScan::IndexScan(QueryExecutionContext* qec, ScanType type, const SparqlTriple& triple) : Operation(qec), - _type(type), + _permutation(scanTypeToPermutation(type)), + _numVariables(scanTypeToNumVariables(type)), _subject(triple._s), - _predicate(triple._p.getIri()), + _predicate(triple._p.getIri().starts_with("?") + ? TripleComponent(Variable{triple._p.getIri()}) + : TripleComponent(triple._p.getIri())), _object(triple._o), _sizeEstimate(std::numeric_limits::max()) { precomputeSizeEstimate(); + + auto permutedTriple = getPermutedTriple(); + for (size_t i = 0; i < 3 - _numVariables; ++i) { + AD_CONTRACT_CHECK(!permutedTriple.at(i)->isVariable()); + } + for (size_t i = 3 - _numVariables; i < permutedTriple.size(); ++i) { + AD_CONTRACT_CHECK(permutedTriple.at(i)->isVariable()); + } } + // _____________________________________________________________________________ string IndexScan::asStringImpl(size_t indent) const { std::ostringstream os; for (size_t i = 0; i < indent; ++i) { os << ' '; } - switch (_type) { - case PSO_BOUND_S: - os << "SCAN PSO with P = \"" << _predicate << "\", S = \"" << _subject - << "\""; - break; - case POS_BOUND_O: - os << "SCAN POS with P = \"" << _predicate << "\", O = \"" - << _object.toRdfLiteral() << "\""; - break; - case SOP_BOUND_O: - os << "SCAN SOP with S = \"" << _subject << "\", O = \"" - << _object.toRdfLiteral() << "\""; - break; - case PSO_FREE_S: - os << "SCAN PSO with P = \"" << _predicate << "\""; - break; - case POS_FREE_O: - os << "SCAN POS with P = \"" << _predicate << "\""; - break; - case SPO_FREE_P: - os << "SCAN SPO with S = \"" << _subject << "\""; - break; - case SOP_FREE_O: - os << "SCAN SOP with S = \"" << _subject << "\""; - break; - case OPS_FREE_P: - os << "SCAN OPS with O = \"" << _object.toRdfLiteral() << "\""; - break; - case OSP_FREE_S: - os << "SCAN OSP with O = \"" << _object.toRdfLiteral() << "\""; - break; - case FULL_INDEX_SCAN_SPO: - os << "SCAN FOR FULL INDEX SPO (DUMMY OPERATION)"; - break; - case FULL_INDEX_SCAN_SOP: - os << "SCAN FOR FULL INDEX SOP (DUMMY OPERATION)"; - break; - case FULL_INDEX_SCAN_PSO: - os << "SCAN FOR FULL INDEX PSO (DUMMY OPERATION)"; - break; - case FULL_INDEX_SCAN_POS: - os << "SCAN FOR FULL INDEX POS (DUMMY OPERATION)"; - break; - case FULL_INDEX_SCAN_OSP: - os << "SCAN FOR FULL INDEX OSP (DUMMY OPERATION)"; - break; - case FULL_INDEX_SCAN_OPS: - os << "SCAN FOR FULL INDEX OPS (DUMMY OPERATION)"; - break; + + auto permutationString = permutationToString(_permutation); + + if (getResultWidth() == 3) { + AD_CORRECTNESS_CHECK(getResultWidth() == 3); + os << "SCAN FOR FULL INDEX " << permutationToString(_permutation) + << " (DUMMY OPERATION)"; + + } else { + auto firstKeyString = permutationString.at(0); + auto permutedTriple = getPermutedTriple(); + const auto& firstKey = permutedTriple.at(0)->toRdfLiteral(); + if (getResultWidth() == 1) { + auto secondKeyString = permutationString.at(1); + const auto& secondKey = permutedTriple.at(1)->toRdfLiteral(); + os << "SCAN " << permutationString << " with " << firstKeyString + << " = \"" << firstKey << "\", " << secondKeyString << " = \"" + << secondKey << "\""; + } else if (getResultWidth() == 2) { + os << "SCAN " << permutationString << " with " << firstKeyString + << " = \"" << firstKey << "\""; + } } return std::move(os).str(); } // _____________________________________________________________________________ string IndexScan::getDescriptor() const { - return "IndexScan " + _subject.toString() + " " + _predicate + " " + - _object.toString(); + return "IndexScan " + _subject.toString() + " " + _predicate.toString() + + " " + _object.toString(); } // _____________________________________________________________________________ -size_t IndexScan::getResultWidth() const { - switch (_type) { - case PSO_BOUND_S: - case POS_BOUND_O: - case SOP_BOUND_O: - return 1; - case PSO_FREE_S: - case POS_FREE_O: - case SPO_FREE_P: - case SOP_FREE_O: - case OSP_FREE_S: - case OPS_FREE_P: - return 2; - case FULL_INDEX_SCAN_SPO: - case FULL_INDEX_SCAN_SOP: - case FULL_INDEX_SCAN_PSO: - case FULL_INDEX_SCAN_POS: - case FULL_INDEX_SCAN_OSP: - case FULL_INDEX_SCAN_OPS: - return 3; - default: - AD_FAIL(); - } -} +size_t IndexScan::getResultWidth() const { return _numVariables; } // _____________________________________________________________________________ vector IndexScan::resultSortedOn() const { - switch (_type) { - case PSO_BOUND_S: - case POS_BOUND_O: - case SOP_BOUND_O: + switch (getResultWidth()) { + case 1: return {ColumnIndex{0}}; - case PSO_FREE_S: - case POS_FREE_O: - case SPO_FREE_P: - case SOP_FREE_O: - case OSP_FREE_S: - case OPS_FREE_P: + case 2: return {ColumnIndex{0}, ColumnIndex{1}}; - case FULL_INDEX_SCAN_SPO: - case FULL_INDEX_SCAN_SOP: - case FULL_INDEX_SCAN_PSO: - case FULL_INDEX_SCAN_POS: - case FULL_INDEX_SCAN_OSP: - case FULL_INDEX_SCAN_OPS: + case 3: return {ColumnIndex{0}, ColumnIndex{1}, ColumnIndex{2}}; default: AD_FAIL(); @@ -147,71 +98,13 @@ VariableToColumnMap IndexScan::computeVariableToColumnMap() const { auto makeCol = makeAlwaysDefinedColumn; auto col = ColumnIndex{0}; - // Helper lambdas that add the respective triple component as the next column. - auto addSubject = [&]() { - if (_subject.isVariable()) { - res[_subject.getVariable()] = makeCol(col); + for (const TripleComponent* const ptr : getPermutedTriple()) { + if (ptr->isVariable()) { + res[ptr->getVariable()] = makeCol(col); ++col; } - }; - // TODO Refactor the `PropertyPath` class s.t. it also has - //`isVariable` and `getVariable`, then those three lambdas can become one. - auto addPredicate = [&]() { - if (_predicate[0] == '?') { - res[Variable{_predicate}] = makeCol(col); - ++col; - } - }; - auto addObject = [&]() { - if (_object.isVariable()) { - res[_object.getVariable()] = makeCol(col); - ++col; - } - }; - - switch (_type) { - case SPO_FREE_P: - case FULL_INDEX_SCAN_SPO: - addSubject(); - addPredicate(); - addObject(); - return res; - case SOP_FREE_O: - case SOP_BOUND_O: - case FULL_INDEX_SCAN_SOP: - addSubject(); - addObject(); - addPredicate(); - return res; - case PSO_BOUND_S: - case PSO_FREE_S: - case FULL_INDEX_SCAN_PSO: - addPredicate(); - addSubject(); - addObject(); - return res; - case POS_BOUND_O: - case POS_FREE_O: - case FULL_INDEX_SCAN_POS: - addPredicate(); - addObject(); - addSubject(); - return res; - case OPS_FREE_P: - case FULL_INDEX_SCAN_OPS: - addObject(); - addPredicate(); - addSubject(); - return res; - case OSP_FREE_S: - case FULL_INDEX_SCAN_OSP: - addObject(); - addSubject(); - addPredicate(); - return res; - default: - AD_FAIL(); } + return res; } // _____________________________________________________________________________ ResultTable IndexScan::computeResult() { @@ -219,86 +112,23 @@ ResultTable IndexScan::computeResult() { IdTable idTable{getExecutionContext()->getAllocator()}; using enum Permutation::Enum; - switch (_type) { - case PSO_BOUND_S: - computePSOboundS(&idTable); - break; - case POS_BOUND_O: - computePOSboundO(&idTable); - break; - case PSO_FREE_S: - computePSOfreeS(&idTable); - break; - case POS_FREE_O: - computePOSfreeO(&idTable); - break; - case SOP_BOUND_O: - computeSOPboundO(&idTable); - break; - case SPO_FREE_P: - computeSPOfreeP(&idTable); - break; - case SOP_FREE_O: - computeSOPfreeO(&idTable); - break; - case OSP_FREE_S: - computeOSPfreeS(&idTable); - break; - case OPS_FREE_P: - computeOPSfreeP(&idTable); - break; - case FULL_INDEX_SCAN_SPO: - computeFullScan(&idTable, SPO); - break; - case FULL_INDEX_SCAN_SOP: - computeFullScan(&idTable, SOP); - break; - case FULL_INDEX_SCAN_PSO: - computeFullScan(&idTable, PSO); - break; - case FULL_INDEX_SCAN_POS: - computeFullScan(&idTable, POS); - break; - case FULL_INDEX_SCAN_OSP: - computeFullScan(&idTable, OSP); - break; - case FULL_INDEX_SCAN_OPS: - computeFullScan(&idTable, OPS); - break; + idTable.setNumColumns(_numVariables); + const auto& idx = _executionContext->getIndex(); + const auto permutedTriple = getPermutedTriple(); + if (_numVariables == 2) { + idx.scan(*permutedTriple[0], &idTable, _permutation, _timeoutTimer); + } else if (_numVariables == 1) { + idx.scan(*permutedTriple[0], *permutedTriple[1], &idTable, _permutation, + _timeoutTimer); + } else { + AD_CORRECTNESS_CHECK(_numVariables == 3); + computeFullScan(&idTable, _permutation); } LOG(DEBUG) << "IndexScan result computation done.\n"; return {std::move(idTable), resultSortedOn(), LocalVocab{}}; } -// _____________________________________________________________________________ -void IndexScan::computePSOboundS(IdTable* result) const { - result->setNumColumns(1); - const auto& idx = _executionContext->getIndex(); - idx.scan(_predicate, _subject, result, Permutation::PSO, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computePSOfreeS(IdTable* result) const { - result->setNumColumns(2); - const auto& idx = _executionContext->getIndex(); - idx.scan(_predicate, result, Permutation::PSO, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computePOSboundO(IdTable* result) const { - result->setNumColumns(1); - const auto& idx = _executionContext->getIndex(); - idx.scan(_predicate, _object, result, Permutation::POS, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computePOSfreeO(IdTable* result) const { - result->setNumColumns(2); - const auto& idx = _executionContext->getIndex(); - idx.scan(_predicate, result, Permutation::POS, _timeoutTimer); -} - // _____________________________________________________________________________ size_t IndexScan::computeSizeEstimate() { if (_executionContext) { @@ -330,23 +160,19 @@ size_t IndexScan::computeSizeEstimate() { getRuntimeInfo().costEstimate_ = sizeEstimate; return sizeEstimate; } + } else if (getResultWidth() == 2) { + const auto& firstKey = *getPermutedTriple()[0]; + return getIndex().getCardinality(firstKey, _permutation); + } else { + // The triple consists of three variables. + // TODO As soon as all implementations of a full index scan + // (Including the "dummy joins" in Join.cpp) consistently exclude the + // internal triples, this estimate should be changed to only return + // the number of triples in the actual knowledge graph (excluding the + // internal triples). + AD_CORRECTNESS_CHECK(getResultWidth() == 3); + return getIndex().numTriples().normalAndInternal_(); } - // TODO Should be a oneliner - // getIndex().cardinality(getPermutation(), getFirstKey()); - if (_type == SPO_FREE_P || _type == SOP_FREE_O) { - return getIndex().getCardinality(_subject, Permutation::SPO); - } else if (_type == POS_FREE_O || _type == PSO_FREE_S) { - return getIndex().getCardinality(_predicate, Permutation::PSO); - } else if (_type == OPS_FREE_P || _type == OSP_FREE_S) { - return getIndex().getCardinality(_object, Permutation::OSP); - } - // The triple consists of three variables. - // TODO As soon as all implementations of a full index scan - // (Including the "dummy joins" in Join.cpp) consistently exclude the - // internal triples, this estimate should be changed to only return - // the number of triples in the actual knowledge graph (excluding the - // internal triples). - return getIndex().numTriples().normalAndInternal_(); } else { // Only for test cases. The handling of the objects is to make the // strange query planner tests pass. @@ -355,7 +181,9 @@ size_t IndexScan::computeSizeEstimate() { _object.isString() ? _object.getString() : _object.toString(); std::string subjectStr = _subject.isString() ? _subject.getString() : _subject.toString(); - return 1000 + subjectStr.size() + _predicate.size() + objectStr.size(); + std::string predStr = + _predicate.isString() ? _predicate.getString() : _predicate.toString(); + return 1000 + subjectStr.size() + predStr.size() + objectStr.size(); } } @@ -386,89 +214,19 @@ size_t IndexScan::getCostEstimate() { } } -// _____________________________________________________________________________ -void IndexScan::computeSPOfreeP(IdTable* result) const { - result->setNumColumns(2); - const auto& idx = _executionContext->getIndex(); - idx.scan(_subject, result, Permutation::SPO, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computeSOPboundO(IdTable* result) const { - result->setNumColumns(1); - const auto& idx = _executionContext->getIndex(); - idx.scan(_subject, _object, result, Permutation::SOP, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computeSOPfreeO(IdTable* result) const { - result->setNumColumns(2); - const auto& idx = _executionContext->getIndex(); - idx.scan(_subject, result, Permutation::SOP, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computeOPSfreeP(IdTable* result) const { - result->setNumColumns(2); - const auto& idx = _executionContext->getIndex(); - idx.scan(_object, result, Permutation::OPS, _timeoutTimer); -} - -// _____________________________________________________________________________ -void IndexScan::computeOSPfreeS(IdTable* result) const { - result->setNumColumns(2); - const auto& idx = _executionContext->getIndex(); - idx.scan(_object, result, Permutation::OSP, _timeoutTimer); -} - // _____________________________________________________________________________ void IndexScan::determineMultiplicities() { _multiplicity.clear(); if (_executionContext) { + const auto& idx = getIndex(); if (getResultWidth() == 1) { _multiplicity.emplace_back(1); + } else if (getResultWidth() == 2) { + const auto permutedTriple = getPermutedTriple(); + _multiplicity = idx.getMultiplicities(*permutedTriple[0], _permutation); } else { - const auto& idx = getIndex(); - switch (_type) { - case PSO_FREE_S: - _multiplicity = idx.getMultiplicities(_predicate, Permutation::PSO); - break; - case POS_FREE_O: - _multiplicity = idx.getMultiplicities(_predicate, Permutation::POS); - break; - case SPO_FREE_P: - _multiplicity = idx.getMultiplicities(_subject, Permutation::SPO); - break; - case SOP_FREE_O: - _multiplicity = idx.getMultiplicities(_subject, Permutation::SOP); - break; - case OSP_FREE_S: - _multiplicity = idx.getMultiplicities(_object, Permutation::OSP); - break; - case OPS_FREE_P: - _multiplicity = idx.getMultiplicities(_object, Permutation::OPS); - break; - case FULL_INDEX_SCAN_SPO: - _multiplicity = idx.getMultiplicities(Permutation::SPO); - break; - case FULL_INDEX_SCAN_SOP: - _multiplicity = idx.getMultiplicities(Permutation::SOP); - break; - case FULL_INDEX_SCAN_PSO: - _multiplicity = idx.getMultiplicities(Permutation::PSO); - break; - case FULL_INDEX_SCAN_POS: - _multiplicity = idx.getMultiplicities(Permutation::POS); - break; - case FULL_INDEX_SCAN_OSP: - _multiplicity = idx.getMultiplicities(Permutation::OSP); - break; - case FULL_INDEX_SCAN_OPS: - _multiplicity = idx.getMultiplicities(Permutation::OPS); - break; - default: - AD_FAIL(); - } + AD_CORRECTNESS_CHECK(getResultWidth() == 3); + _multiplicity = idx.getMultiplicities(_permutation); } } else { _multiplicity.emplace_back(1); diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index 047a9c9d62..ef178a8019 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -31,10 +31,98 @@ class IndexScan : public Operation { FULL_INDEX_SCAN_OPS = 14 }; + static size_t scanTypeToNumVariables(ScanType scanType) { + switch (scanType) { + case PSO_BOUND_S: + case POS_BOUND_O: + case SOP_BOUND_O: + return 1; + case PSO_FREE_S: + case POS_FREE_O: + case SOP_FREE_O: + case SPO_FREE_P: + case OSP_FREE_S: + case OPS_FREE_P: + return 2; + case FULL_INDEX_SCAN_SPO: + case FULL_INDEX_SCAN_SOP: + case FULL_INDEX_SCAN_PSO: + case FULL_INDEX_SCAN_POS: + case FULL_INDEX_SCAN_OSP: + case FULL_INDEX_SCAN_OPS: + return 3; + } + } + + static Permutation::Enum scanTypeToPermutation(ScanType scanType) { + using enum Permutation::Enum; + switch (scanType) { + case PSO_BOUND_S: + case PSO_FREE_S: + case FULL_INDEX_SCAN_PSO: + return PSO; + case POS_FREE_O: + case POS_BOUND_O: + case FULL_INDEX_SCAN_POS: + return POS; + case SPO_FREE_P: + case FULL_INDEX_SCAN_SPO: + return SPO; + case SOP_FREE_O: + case SOP_BOUND_O: + case FULL_INDEX_SCAN_SOP: + return SOP; + case OSP_FREE_S: + case FULL_INDEX_SCAN_OSP: + return OSP; + case OPS_FREE_P: + case FULL_INDEX_SCAN_OPS: + return OPS; + } + } + + static std::array permutationToKeyOrder( + Permutation::Enum permutation) { + using enum Permutation::Enum; + switch (permutation) { + case POS: + return {1, 2, 0}; + case PSO: + return {1, 0, 2}; + case SOP: + return {0, 2, 1}; + case SPO: + return {0, 1, 2}; + case OPS: + return {2, 1, 0}; + case OSP: + return {2, 0, 1}; + } + } + + static std::string_view permutationToString(Permutation::Enum permutation) { + using enum Permutation::Enum; + switch (permutation) { + case POS: + return "POS"; + case PSO: + return "PSO"; + case SOP: + return "SOP"; + case SPO: + return "SPO"; + case OPS: + return "OPS"; + case OSP: + return "OSP"; + } + } + private: - ScanType _type; + Permutation::Enum _permutation; + size_t _numVariables; TripleComponent _subject; - string _predicate; + TripleComponent _predicate; TripleComponent _object; size_t _sizeEstimate; vector _multiplicity; @@ -50,7 +138,7 @@ class IndexScan : public Operation { virtual ~IndexScan() = default; - const string& getPredicate() const { return _predicate; } + const TripleComponent& getPredicate() const { return _predicate; } const TripleComponent& getSubject() const { return _subject; } const TripleComponent& getObject() const { return _object; } @@ -90,44 +178,16 @@ class IndexScan : public Operation { // Currently only the full scans support a limit clause. [[nodiscard]] bool supportsLimit() const override { - switch (_type) { - case FULL_INDEX_SCAN_SPO: - case FULL_INDEX_SCAN_SOP: - case FULL_INDEX_SCAN_PSO: - case FULL_INDEX_SCAN_POS: - case FULL_INDEX_SCAN_OSP: - case FULL_INDEX_SCAN_OPS: - return true; - default: - return false; - } + return getResultWidth() == 3; } - ScanType getType() const { return _type; } + Permutation::Enum permutation() const { return _permutation; } private: ResultTable computeResult() override; vector getChildren() override { return {}; } - void computePSOboundS(IdTable* result) const; - - void computePSOfreeS(IdTable* result) const; - - void computePOSboundO(IdTable* result) const; - - void computePOSfreeO(IdTable* result) const; - - void computeSPOfreeP(IdTable* result) const; - - void computeSOPboundO(IdTable* result) const; - - void computeSOPfreeO(IdTable* result) const; - - void computeOPSfreeP(IdTable* result) const; - - void computeOSPfreeS(IdTable* result) const; - void computeFullScan(IdTable* result, Permutation::Enum permutation) const; size_t computeSizeEstimate(); @@ -140,4 +200,11 @@ class IndexScan : public Operation { getPrecomputedResultFromQueryPlanning() override { return _precomputedResult; } + + std::array getPermutedTriple() const { + using Arr = std::array; + Arr inp{&_subject, &_predicate, &_object}; + auto permutation = permutationToKeyOrder(_permutation); + return {inp[permutation[0]], inp[permutation[1]], inp[permutation[2]]}; + } }; diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp index e8d0a8e0eb..f5413ad275 100644 --- a/src/engine/Join.cpp +++ b/src/engine/Join.cpp @@ -232,31 +232,8 @@ Join::ScanMethodType Join::getScanMethod( return [&idx, perm](Id id, IdTable* idTable) { idx.scan(id, idTable, perm); }; }; - - using enum Permutation::Enum; - switch (scan.getType()) { - case IndexScan::FULL_INDEX_SCAN_SPO: - scanMethod = scanLambda(SPO); - break; - case IndexScan::FULL_INDEX_SCAN_SOP: - scanMethod = scanLambda(SOP); - break; - case IndexScan::FULL_INDEX_SCAN_PSO: - scanMethod = scanLambda(PSO); - break; - case IndexScan::FULL_INDEX_SCAN_POS: - scanMethod = scanLambda(POS); - break; - case IndexScan::FULL_INDEX_SCAN_OSP: - scanMethod = scanLambda(OSP); - break; - case IndexScan::FULL_INDEX_SCAN_OPS: - scanMethod = scanLambda(OPS); - break; - default: - AD_THROW("Found non-dummy scan where one was expected."); - } - return scanMethod; + AD_CORRECTNESS_CHECK(scan.getResultWidth() == 3); + return scanLambda(scan.permutation()); } // _____________________________________________________________________________ diff --git a/test/QueryPlannerTest.cpp b/test/QueryPlannerTest.cpp index 7d5d47a001..a82922d7ec 100644 --- a/test/QueryPlannerTest.cpp +++ b/test/QueryPlannerTest.cpp @@ -1017,84 +1017,78 @@ TEST(QueryPlannerTest, testSimpleOptional) { } TEST(QueryPlannerTest, SimpleTripleOneVariable) { - using enum IndexScan::ScanType; + using enum Permutation::Enum; // With only one variable, there are always two permutations that will yield // exactly the same result. The query planner consistently chosses one of // them. h::expect("SELECT * WHERE { ?s

}", - h::IndexScan(Var{"?s"}, "

", "", {POS_BOUND_O})); + h::IndexScan(Var{"?s"}, "

", "", 1, {POS})); h::expect("SELECT * WHERE { ?p }", - h::IndexScan("", "?p", "", {SOP_BOUND_O})); + h::IndexScan("", Var{"?p"}, "", 1, {SOP})); h::expect("SELECT * WHERE {

?o }", - h::IndexScan("", "

", Var{"?o"}, {PSO_BOUND_S})); + h::IndexScan("", "

", Var{"?o"}, 1, {PSO})); } TEST(QueryPlannerTest, SimpleTripleTwoVariables) { - using enum IndexScan::ScanType; + using enum Permutation::Enum; // Fixed predicate. // Without `Order By`, two orderings are possible, both are fine. - h::expect( - "SELECT * WHERE { ?s

?o }", - h::IndexScan(Var{"?s"}, "

", Var{"?o"}, {POS_FREE_O, PSO_FREE_S})); + h::expect("SELECT * WHERE { ?s

?o }", + h::IndexScan(Var{"?s"}, "

", Var{"?o"}, 2, {POS, PSO})); // Must always be a single index scan, never index scan + sorting. h::expect("SELECT * WHERE { ?s

?o } INTERNAL SORT BY ?o", - h::IndexScan(Var{"?s"}, "

", Var{"?o"}, {POS_FREE_O})); + h::IndexScan(Var{"?s"}, "

", Var{"?o"}, 2, {POS})); h::expect("SELECT * WHERE { ?s

?o } INTERNAL SORT BY ?s", - h::IndexScan(Var{"?s"}, "

", Var{"?o"}, {PSO_FREE_S})); + h::IndexScan(Var{"?s"}, "

", Var{"?o"}, 2, {PSO})); // Fixed subject. h::expect("SELECT * WHERE { ?p ?o }", - h::IndexScan("", "?p", Var{"?o"}, {SOP_FREE_O, SPO_FREE_P})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, 2, {SOP, SPO})); h::expect("SELECT * WHERE { ?p ?o } INTERNAL SORT BY ?o", - h::IndexScan("", "?p", Var{"?o"}, {SOP_FREE_O})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, 2, {SOP})); h::expect("SELECT * WHERE { ?p ?o } INTERNAL SORT BY ?p", - h::IndexScan("", "?p", Var{"?o"}, {SPO_FREE_P})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, 2, {SPO})); // Fixed object. h::expect("SELECT * WHERE { ?p ?o }", - h::IndexScan("", "?p", Var{"?o"}, {SOP_FREE_O, SPO_FREE_P})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, 2, {SOP, SPO})); h::expect("SELECT * WHERE { ?p ?o } INTERNAL SORT BY ?o", - h::IndexScan("", "?p", Var{"?o"}, {SOP_FREE_O})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, 2, {SOP})); h::expect("SELECT * WHERE { ?p ?o } INTERNAL SORT BY ?p", - h::IndexScan("", "?p", Var{"?o"}, {SPO_FREE_P})); + h::IndexScan("", Var{"?p"}, Var{"?o"}, 2, {SPO})); } TEST(QueryPlannerTest, SimpleTripleThreeVariables) { - using enum IndexScan::ScanType; + using enum Permutation::Enum; // Fixed predicate. // Don't care about the sorting. h::expect("SELECT * WHERE { ?s ?p ?o }", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, - {FULL_INDEX_SCAN_SPO, FULL_INDEX_SCAN_SOP, - FULL_INDEX_SCAN_PSO, FULL_INDEX_SCAN_POS, - FULL_INDEX_SCAN_OSP, FULL_INDEX_SCAN_OPS})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, + {SPO, SOP, PSO, POS, OSP, OPS})); // Sorted by one variable, two possible permutations remain. h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, - {FULL_INDEX_SCAN_SPO, FULL_INDEX_SCAN_SOP})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {SPO, SOP})); h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, - {FULL_INDEX_SCAN_POS, FULL_INDEX_SCAN_PSO})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {POS, PSO})); h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, - {FULL_INDEX_SCAN_OSP, FULL_INDEX_SCAN_OPS})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {OSP, OPS})); // Sorted by two variables, this makes the permutation unique. h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s ?o", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, {FULL_INDEX_SCAN_SOP})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {SOP})); h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?s ?p", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, {FULL_INDEX_SCAN_SPO})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {SPO})); h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o ?s", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, {FULL_INDEX_SCAN_OSP})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {OSP})); h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?o ?p", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, {FULL_INDEX_SCAN_OPS})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {OPS})); h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p ?s", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, {FULL_INDEX_SCAN_PSO})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {PSO})); h::expect("SELECT * WHERE { ?s ?p ?o } INTERNAL SORT BY ?p ?o", - h::IndexScan(Var{"?s"}, "?p", Var{"?o"}, {FULL_INDEX_SCAN_POS})); + h::IndexScan(Var{"?s"}, Var{"?p"}, Var{"?o"}, 3, {POS})); } diff --git a/test/QueryPlannerTestHelpers.h b/test/QueryPlannerTestHelpers.h index cbb23f699a..2bfc4b602b 100644 --- a/test/QueryPlannerTestHelpers.h +++ b/test/QueryPlannerTestHelpers.h @@ -32,15 +32,18 @@ auto RootOperation(auto matcher) -> Matcher { /// Return a matcher that checks that a given `QueryExecutionTree` consists of a /// single `IndexScan` with the given `subject`, `predicate`, and `object`, and -/// that the `ScanType` of this `IndexScan` is any of the given `scanTypes`. -auto IndexScan(TripleComponent subject, std::string predicate, - TripleComponent object, - const std::vector& scanTypes = {}) +/// that the `ScanType` of this `IndexScan` is any of the given +/// `allowedPermutations`. +auto IndexScan(TripleComponent subject, TripleComponent predicate, + TripleComponent object, size_t numVariables, + const std::vector& allowedPermutations = {}) -> Matcher { - auto typeMatcher = - scanTypes.empty() ? A() : AnyOfArray(scanTypes); + auto permutationMatcher = allowedPermutations.empty() + ? A() + : AnyOfArray(allowedPermutations); return RootOperation<::IndexScan>( - AllOf(Property(&IndexScan::getType, typeMatcher), + AllOf(Property(&IndexScan::permutation, permutationMatcher), + Property(&IndexScan::getResultWidth, Eq(numVariables)), Property(&IndexScan::getSubject, Eq(subject)), Property(&IndexScan::getPredicate, Eq(predicate)), Property(&IndexScan::getObject, Eq(object)))); From 520c98feb9210a81dd496e6876f8855dba5aad43 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 15 Jun 2023 12:15:26 +0200 Subject: [PATCH 038/150] Several simplifications and moved a lot of stuff into the Permutation.cpp --- src/engine/IndexScan.cpp | 16 +++- src/engine/IndexScan.h | 82 ++------------------- src/engine/QueryPlanner.cpp | 39 +++++----- src/index/CMakeLists.txt | 1 + src/index/Index.h | 2 +- src/index/IndexImpl.h | 2 +- src/index/Permutation.cpp | 59 +++++++++++++++ src/index/{Permutations.h => Permutation.h} | 57 +++----------- test/CompressedRelationsTest.cpp | 2 +- test/GroupByTest.cpp | 14 ++-- test/JoinTest.cpp | 2 +- 11 files changed, 119 insertions(+), 157 deletions(-) create mode 100644 src/index/Permutation.cpp rename src/index/{Permutations.h => Permutation.h} (57%) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 14ef2b4f7a..c189d0832b 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -14,16 +14,17 @@ using std::string; // _____________________________________________________________________________ -IndexScan::IndexScan(QueryExecutionContext* qec, ScanType type, +IndexScan::IndexScan(QueryExecutionContext* qec, Permutation::Enum permutation, const SparqlTriple& triple) : Operation(qec), - _permutation(scanTypeToPermutation(type)), - _numVariables(scanTypeToNumVariables(type)), + _permutation(permutation), _subject(triple._s), _predicate(triple._p.getIri().starts_with("?") ? TripleComponent(Variable{triple._p.getIri()}) : TripleComponent(triple._p.getIri())), _object(triple._o), + _numVariables(_subject.isVariable() + _predicate.isVariable() + + _object.isVariable()), _sizeEstimate(std::numeric_limits::max()) { precomputeSizeEstimate(); @@ -276,3 +277,12 @@ void IndexScan::computeFullScan(IdTable* result, } *result = std::move(table).toDynamic(); } + +// ___________________________________________________________________________ +std::array IndexScan::getPermutedTriple() + const { + using Arr = std::array; + Arr inp{&_subject, &_predicate, &_object}; + auto permutation = permutationToKeyOrder(_permutation); + return {inp[permutation[0]], inp[permutation[1]], inp[permutation[2]]}; +} diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index ef178a8019..60a47486d9 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -13,74 +13,6 @@ class SparqlTriple; class IndexScan : public Operation { public: - enum ScanType { - PSO_BOUND_S = 0, - POS_BOUND_O = 1, - PSO_FREE_S = 2, - POS_FREE_O = 3, - SPO_FREE_P = 4, - SOP_BOUND_O = 5, - SOP_FREE_O = 6, - OPS_FREE_P = 7, - OSP_FREE_S = 8, - FULL_INDEX_SCAN_SPO = 9, - FULL_INDEX_SCAN_SOP = 10, - FULL_INDEX_SCAN_PSO = 11, - FULL_INDEX_SCAN_POS = 12, - FULL_INDEX_SCAN_OSP = 13, - FULL_INDEX_SCAN_OPS = 14 - }; - - static size_t scanTypeToNumVariables(ScanType scanType) { - switch (scanType) { - case PSO_BOUND_S: - case POS_BOUND_O: - case SOP_BOUND_O: - return 1; - case PSO_FREE_S: - case POS_FREE_O: - case SOP_FREE_O: - case SPO_FREE_P: - case OSP_FREE_S: - case OPS_FREE_P: - return 2; - case FULL_INDEX_SCAN_SPO: - case FULL_INDEX_SCAN_SOP: - case FULL_INDEX_SCAN_PSO: - case FULL_INDEX_SCAN_POS: - case FULL_INDEX_SCAN_OSP: - case FULL_INDEX_SCAN_OPS: - return 3; - } - } - - static Permutation::Enum scanTypeToPermutation(ScanType scanType) { - using enum Permutation::Enum; - switch (scanType) { - case PSO_BOUND_S: - case PSO_FREE_S: - case FULL_INDEX_SCAN_PSO: - return PSO; - case POS_FREE_O: - case POS_BOUND_O: - case FULL_INDEX_SCAN_POS: - return POS; - case SPO_FREE_P: - case FULL_INDEX_SCAN_SPO: - return SPO; - case SOP_FREE_O: - case SOP_BOUND_O: - case FULL_INDEX_SCAN_SOP: - return SOP; - case OSP_FREE_S: - case FULL_INDEX_SCAN_OSP: - return OSP; - case OPS_FREE_P: - case FULL_INDEX_SCAN_OPS: - return OPS; - } - } - static std::array permutationToKeyOrder( Permutation::Enum permutation) { using enum Permutation::Enum; @@ -120,10 +52,10 @@ class IndexScan : public Operation { private: Permutation::Enum _permutation; - size_t _numVariables; TripleComponent _subject; TripleComponent _predicate; TripleComponent _object; + size_t _numVariables; size_t _sizeEstimate; vector _multiplicity; @@ -133,7 +65,7 @@ class IndexScan : public Operation { public: string getDescriptor() const override; - IndexScan(QueryExecutionContext* qec, ScanType type, + IndexScan(QueryExecutionContext* qec, Permutation::Enum permutation, const SparqlTriple& triple); virtual ~IndexScan() = default; @@ -201,10 +133,8 @@ class IndexScan : public Operation { return _precomputedResult; } - std::array getPermutedTriple() const { - using Arr = std::array; - Arr inp{&_subject, &_predicate, &_object}; - auto permutation = permutationToKeyOrder(_permutation); - return {inp[permutation[0]], inp[permutation[1]], inp[permutation[2]]}; - } + // Return the stored triple in the order that corresponds to the + // `_permutation`. For example if `_permutation == PSO` then the result is + // {&_predicate, &_subject, &_object} + std::array getPermutedTriple() const; }; diff --git a/src/engine/QueryPlanner.cpp b/src/engine/QueryPlanner.cpp index 746123824c..491b37f78f 100644 --- a/src/engine/QueryPlanner.cpp +++ b/src/engine/QueryPlanner.cpp @@ -715,11 +715,11 @@ vector QueryPlanner::seedWithScansAndText( seeds.push_back(std::move(plan)); }; - auto addIndexScan = [&](IndexScan::ScanType type) { - pushPlan(makeSubtreePlan(_qec, type, node._triple)); + auto addIndexScan = [&](Permutation::Enum permutation) { + pushPlan(makeSubtreePlan(_qec, permutation, node._triple)); }; - using enum IndexScan::ScanType; + using enum Permutation::Enum; if (node._cvar.has_value()) { seeds.push_back(getTextLeafPlan(node)); @@ -768,8 +768,7 @@ vector QueryPlanner::seedWithScansAndText( Variable filterVar = generateUniqueVarName(); auto scanTriple = node._triple; scanTriple._o = filterVar; - auto scanTree = - makeExecutionTree(_qec, PSO_FREE_S, scanTriple); + auto scanTree = makeExecutionTree(_qec, PSO, scanTriple); // The simplest way to set up the filtering expression is to use the // parser. std::string filterString = @@ -782,35 +781,35 @@ vector QueryPlanner::seedWithScansAndText( std::move(filter.expression_)); pushPlan(std::move(plan)); } else if (isVariable(node._triple._s)) { - addIndexScan(POS_BOUND_O); + addIndexScan(POS); } else if (isVariable(node._triple._o)) { - addIndexScan(PSO_BOUND_S); + addIndexScan(PSO); } else { AD_CONTRACT_CHECK(isVariable(node._triple._p)); - addIndexScan(SOP_BOUND_O); + addIndexScan(SOP); } } else if (node._variables.size() == 2) { // Add plans for both possible scan directions. if (!isVariable(node._triple._p._iri)) { - addIndexScan(PSO_FREE_S); - addIndexScan(POS_FREE_O); + addIndexScan(PSO); + addIndexScan(POS); } else if (!isVariable(node._triple._s)) { - addIndexScan(SPO_FREE_P); - addIndexScan(SOP_FREE_O); + addIndexScan(SPO); + addIndexScan(SOP); } else if (!isVariable(node._triple._o)) { - addIndexScan(OSP_FREE_S); - addIndexScan(OPS_FREE_P); + addIndexScan(OSP); + addIndexScan(OPS); } } else { // The current triple contains three distinct variables. if (!_qec || _qec->getIndex().hasAllPermutations()) { // Add plans for all six permutations. - addIndexScan(FULL_INDEX_SCAN_OPS); - addIndexScan(FULL_INDEX_SCAN_OSP); - addIndexScan(FULL_INDEX_SCAN_PSO); - addIndexScan(FULL_INDEX_SCAN_POS); - addIndexScan(FULL_INDEX_SCAN_SPO); - addIndexScan(FULL_INDEX_SCAN_SOP); + addIndexScan(OPS); + addIndexScan(OSP); + addIndexScan(PSO); + addIndexScan(POS); + addIndexScan(SPO); + addIndexScan(SOP); } else { AD_THROW( "With only 2 permutations registered (no -a option), " diff --git a/src/index/CMakeLists.txt b/src/index/CMakeLists.txt index 4bbf53f647..d437fe4a13 100644 --- a/src/index/CMakeLists.txt +++ b/src/index/CMakeLists.txt @@ -9,6 +9,7 @@ add_library(index IndexMetaData.h IndexMetaDataImpl.h MetaDataHandler.h StxxlSortFunctors.h + Permutation.cpp TextMetaData.cpp TextMetaData.h DocsDB.cpp DocsDB.h FTSAlgorithms.cpp FTSAlgorithms.h diff --git a/src/index/Index.h b/src/index/Index.h index a071c7b730..1ab9d33ca4 100644 --- a/src/index/Index.h +++ b/src/index/Index.h @@ -12,7 +12,7 @@ #include "global/Id.h" #include "index/CompressedString.h" -#include "index/Permutations.h" +#include "index/Permutation.h" #include "index/StringSortComparator.h" #include "index/Vocabulary.h" #include "parser/TripleComponent.h" diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h index a920579302..d0b5566201 100644 --- a/src/index/IndexImpl.h +++ b/src/index/IndexImpl.h @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp new file mode 100644 index 0000000000..23c4467171 --- /dev/null +++ b/src/index/Permutation.cpp @@ -0,0 +1,59 @@ +// Copyright 2023, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Author: Johannes Kalmbach + +#include "index/Permutation.h" + +// _____________________________________________________________________ +Permutation::Permutation(string name, string suffix, + array order) + : _readableName(std::move(name)), + _fileSuffix(std::move(suffix)), + _keyOrder(order) {} + +// _____________________________________________________________________ +void Permutation::loadFromDisk(const std::string& onDiskBase) { + if constexpr (MetaData::_isMmapBased) { + _meta.setup(onDiskBase + ".index" + _fileSuffix + MMAP_FILE_SUFFIX, + ad_utility::ReuseTag(), ad_utility::AccessPattern::Random); + } + auto filename = string(onDiskBase + ".index" + _fileSuffix); + try { + _file.open(filename, "r"); + } catch (const std::runtime_error& e) { + AD_THROW("Could not open the index file " + filename + + " for reading. Please check that you have read access to " + "this file. If it does not exist, your index is broken."); + } + _meta.readFromFile(&_file); + LOG(INFO) << "Registered " << _readableName + << " permutation: " << _meta.statistics() << std::endl; + _isLoaded = true; +} + +// _____________________________________________________________________ +void Permutation::scan(Id col0Id, IdTable* result, + ad_utility::SharedConcurrentTimeoutTimer timer) const { + if (!_isLoaded) { + throw std::runtime_error("This query requires the permutation " + + _readableName + ", which was not loaded"); + } + if (!_meta.col0IdExists(col0Id)) { + return; + } + const auto& metaData = _meta.getMetaData(col0Id); + return _reader.scan(metaData, _meta.blockData(), _file, result, + std::move(timer)); +} + +// _____________________________________________________________________ +void Permutation::scan(Id col0Id, Id col1Id, IdTable* result, + ad_utility::SharedConcurrentTimeoutTimer timer) const { + if (!_meta.col0IdExists(col0Id)) { + return; + } + const auto& metaData = _meta.getMetaData(col0Id); + + return _reader.scan(metaData, col1Id, _meta.blockData(), _file, result, + timer); +} diff --git a/src/index/Permutations.h b/src/index/Permutation.h similarity index 57% rename from src/index/Permutations.h rename to src/index/Permutation.h index 63578d746e..99b0b87779 100644 --- a/src/index/Permutations.h +++ b/src/index/Permutation.h @@ -11,6 +11,9 @@ #include "util/File.h" #include "util/Log.h" +// Forward declaration of `IdTable` +class IdTable; + // Helper class to store static properties of the different permutations to // avoid code duplication. The first template parameter is a search functor for // STXXL. @@ -27,62 +30,22 @@ class Permutation { static constexpr auto OPS = Enum::OPS; static constexpr auto OSP = Enum::OSP; using MetaData = IndexMetaDataMmapView; - Permutation(string name, string suffix, array order) - : _readableName(std::move(name)), - _fileSuffix(std::move(suffix)), - _keyOrder(order) {} + Permutation(string name, string suffix, array order); // everything that has to be done when reading an index from disk - void loadFromDisk(const std::string& onDiskBase) { - if constexpr (MetaData::_isMmapBased) { - _meta.setup(onDiskBase + ".index" + _fileSuffix + MMAP_FILE_SUFFIX, - ad_utility::ReuseTag(), ad_utility::AccessPattern::Random); - } - auto filename = string(onDiskBase + ".index" + _fileSuffix); - try { - _file.open(filename, "r"); - } catch (const std::runtime_error& e) { - AD_THROW("Could not open the index file " + filename + - " for reading. Please check that you have read access to " - "this file. If it does not exist, your index is broken."); - } - _meta.readFromFile(&_file); - LOG(INFO) << "Registered " << _readableName - << " permutation: " << _meta.statistics() << std::endl; - _isLoaded = true; - } + void loadFromDisk(const std::string& onDiskBase); /// For a given ID for the first column, retrieve all IDs of the second and /// third column, and store them in `result`. This is just a thin wrapper /// around `CompressedRelationMetaData::scan`. - template - void scan(Id col0Id, IdTableImpl* result, - ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const { - if (!_isLoaded) { - throw std::runtime_error("This query requires the permutation " + - _readableName + ", which was not loaded"); - } - if (!_meta.col0IdExists(col0Id)) { - return; - } - const auto& metaData = _meta.getMetaData(col0Id); - return _reader.scan(metaData, _meta.blockData(), _file, result, - std::move(timer)); - } + void scan(Id col0Id, IdTable* result, + ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const; + /// For given IDs for the first and second column, retrieve all IDs of the /// third column, and store them in `result`. This is just a thin wrapper /// around `CompressedRelationMetaData::scan`. - template - void scan(Id col0Id, Id col1Id, IdTableImpl* result, - ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const { - if (!_meta.col0IdExists(col0Id)) { - return; - } - const auto& metaData = _meta.getMetaData(col0Id); - - return _reader.scan(metaData, col1Id, _meta.blockData(), _file, result, - timer); - } + void scan(Id col0Id, Id col1Id, IdTable* result, + ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const; // _______________________________________________________ void setKbName(const string& name) { _meta.setName(name); } diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp index 64e40d80de..006685330e 100644 --- a/test/CompressedRelationsTest.cpp +++ b/test/CompressedRelationsTest.cpp @@ -6,7 +6,7 @@ #include "./IndexTestHelpers.h" #include "index/CompressedRelation.h" -#include "index/Permutations.h" +#include "index/Permutation.h" #include "util/Serializer/ByteBufferSerializer.h" namespace { diff --git a/test/GroupByTest.cpp b/test/GroupByTest.cpp index 42da40f6b0..3ef19c93d2 100644 --- a/test/GroupByTest.cpp +++ b/test/GroupByTest.cpp @@ -330,18 +330,18 @@ struct GroupBySpecialCount : ::testing::Test { Variable varA{"?a"}; QueryExecutionContext* qec = getQec(); SparqlTriple xyzTriple{Variable{"?x"}, "?y", Variable{"?z"}}; - Tree xyzScanSortedByX = makeExecutionTree( - qec, IndexScan::FULL_INDEX_SCAN_SOP, xyzTriple); - Tree xyzScanSortedByY = makeExecutionTree( - qec, IndexScan::FULL_INDEX_SCAN_POS, xyzTriple); + Tree xyzScanSortedByX = + makeExecutionTree(qec, Permutation::Enum::SOP, xyzTriple); + Tree xyzScanSortedByY = + makeExecutionTree(qec, Permutation::Enum::POS, xyzTriple); Tree xScan = makeExecutionTree( - qec, IndexScan::PSO_BOUND_S, + qec, Permutation::Enum::PSO, SparqlTriple{{""}, {"

\", O = \"
\"\n      "
-      "qet-width: 1 \n    } join-column: [0]\n    |X|\n    {\n      "
-      "SORT(internal) on columns:asc(1) \n      {\n        JOIN\n        {\n "
-      "         SCAN POS with P = \"
\"\n          qet-width: 2 "
-      "\n        } join-column: [0]\n        |X|\n        {\n          SCAN "
-      "POS with P = \"
\", O = \"
\"\n          qet-width: "
-      "1 \n        } join-column: [0]\n        qet-width: 2 \n      }\n      "
-      "qet-width: 2 \n    } join-column: [1]\n    qet-width: 2 \n  }\n  "
-      "qet-width: 2 \n}",
+      "{\n  ORDER BY on columns:asc(0) \n  {\n    JOIN\n    {\n      "
+      "SORT(internal) on columns:asc(1) \n      {\n        JOIN\n        {\n   "
+      "       SCAN POS with P = \"
\", O = \"
\"\n    "
+      "      qet-width: 1 \n        } join-column: [0]\n        |X|\n        "
+      "{\n          SCAN PSO with P = \"
\"\n          qet-width: "
+      "2 \n        } join-column: [0]\n        qet-width: 2 \n      }\n      "
+      "qet-width: 2 \n    } join-column: [1]\n    |X|\n    {\n      SCAN POS "
+      "with P = \"
\", O = \"
\"\n      qet-width: 1 \n    } "
+      "join-column: [0]\n    qet-width: 2 \n  }\n  qet-width: 2 \n}",
       qet.asString());
 }
 
@@ -644,11 +643,11 @@ TEST(QueryPlannerTest, threeVarTriples) {
     QueryPlanner qp(nullptr);
     QueryExecutionTree qet = qp.createExecutionTree(pq);
     ASSERT_EQ(
-        "{\n  JOIN\n  {\n    SCAN PSO with P = \"
\", S = \"\"\n    "
-        "qet-width: 1 \n  } join-column: [0]\n  |X|\n  {\n    SORT(internal) "
-        "on columns:asc(1) \n    {\n      SCAN FOR FULL INDEX OSP (DUMMY "
-        "OPERATION)\n      qet-width: 3 \n    }\n    qet-width: 3 \n  } "
-        "join-column: [1]\n  qet-width: 3 \n}",
+        "{\n  JOIN\n  {\n    SORT(internal) on columns:asc(1) \n    {\n      "
+        "SCAN FOR FULL INDEX OSP (DUMMY OPERATION)\n      qet-width: 3 \n    "
+        "}\n    qet-width: 3 \n  } join-column: [1]\n  |X|\n  {\n    SCAN PSO "
+        "with P = \"
\", S = \"\"\n    qet-width: 1 \n  } join-column: "
+        "[0]\n  qet-width: 3 \n}",
         qet.asString());
   }
   {
@@ -658,11 +657,11 @@ TEST(QueryPlannerTest, threeVarTriples) {
     QueryPlanner qp(nullptr);
     QueryExecutionTree qet = qp.createExecutionTree(pq);
     ASSERT_EQ(
-        "{\n  JOIN\n  {\n    SCAN SOP with S = \"\", O = \"\"\n    "
-        "qet-width: 1 \n  } join-column: [0]\n  |X|\n  {\n    SORT(internal) "
-        "on columns:asc(1) \n    {\n      SCAN FOR FULL INDEX OSP (DUMMY "
-        "OPERATION)\n      qet-width: 3 \n    }\n    qet-width: 3 \n  } "
-        "join-column: [1]\n  qet-width: 3 \n}",
+        "{\n  JOIN\n  {\n    SORT(internal) on columns:asc(1) \n    {\n      "
+        "SCAN FOR FULL INDEX OSP (DUMMY OPERATION)\n      qet-width: 3 \n    "
+        "}\n    qet-width: 3 \n  } join-column: [1]\n  |X|\n  {\n    SCAN SOP "
+        "with S = \"\", O = \"\"\n    qet-width: 1 \n  } join-column: "
+        "[0]\n  qet-width: 3 \n}",
         qet.asString());
   }
   {
@@ -672,11 +671,11 @@ TEST(QueryPlannerTest, threeVarTriples) {
     QueryPlanner qp(nullptr);
     QueryExecutionTree qet = qp.createExecutionTree(pq);
     ASSERT_EQ(
-        "{\n  JOIN\n  {\n    SCAN PSO with P = \"
\", S = \"\"\n    "
-        "qet-width: 1 \n  } join-column: [0]\n  |X|\n  {\n    SORT(internal) "
-        "on columns:asc(1) \n    {\n      SCAN FOR FULL INDEX OPS (DUMMY "
-        "OPERATION)\n      qet-width: 3 \n    }\n    qet-width: 3 \n  } "
-        "join-column: [1]\n  qet-width: 3 \n}",
+        "{\n  JOIN\n  {\n    SORT(internal) on columns:asc(1) \n    {\n      "
+        "SCAN FOR FULL INDEX OPS (DUMMY OPERATION)\n      qet-width: 3 \n    "
+        "}\n    qet-width: 3 \n  } join-column: [1]\n  |X|\n  {\n    SCAN PSO "
+        "with P = \"
\", S = \"\"\n    qet-width: 1 \n  } join-column: "
+        "[0]\n  qet-width: 3 \n}",
         qet.asString());
   }
 }
@@ -841,19 +840,17 @@ TEST(QueryExecutionTreeTest, testPoliticiansFriendWithScieManHatProj) {
   QueryPlanner qp(nullptr);
   QueryExecutionTree qet = qp.createExecutionTree(pq);
   ASSERT_EQ(
-
-      "{\n  TEXT OPERATION WITH FILTER: co-occurrence with words: "
-      "\"manhattan project\" and 1 variables with textLimit = 1 filtered "
-      "by\n  {\n    JOIN\n    {\n      SCAN POS with P = \"\", O = "
-      "\"\"\n      qet-width: 1 \n    } join-column: [0]\n    "
-      "|X|\n    {\n      SORT(internal) on columns:asc(2) \n      {\n       "
-      " TEXT OPERATION "
-      "WITH FILTER: co-occurrence with words: \"friend*\" and 2 variables "
-      "with textLimit = 1 filtered by\n        {\n          SCAN POS with P "
-      "= \"\", O = \"\"\n          qet-width: 1 \n        "
-      "}\n         filtered on column 0\n        qet-width: 4 \n      }\n    "
-      "  qet-width: 4 \n    } join-column: [2]\n    qet-width: 4 \n  }\n   "
-      "filtered on column 0\n  qet-width: 6 \n}",
+      "{\n  TEXT OPERATION WITH FILTER: co-occurrence with words: \"manhattan "
+      "project\" and 1 variables with textLimit = 1 filtered by\n  {\n    "
+      "JOIN\n    {\n      SORT(internal) on columns:asc(2) \n      {\n        "
+      "TEXT OPERATION WITH FILTER: co-occurrence with words: \"friend*\" and 2 "
+      "variables with textLimit = 1 filtered by\n        {\n          SCAN POS "
+      "with P = \"\", O = \"\"\n          qet-width: 1 \n    "
+      "    }\n         filtered on column 0\n        qet-width: 4 \n      }\n  "
+      "    qet-width: 4 \n    } join-column: [2]\n    |X|\n    {\n      SCAN "
+      "POS with P = \"\", O = \"\"\n      qet-width: 1 \n    "
+      "} join-column: [0]\n    qet-width: 4 \n  }\n   filtered on column 2\n  "
+      "qet-width: 6 \n}",
       qet.asString());
 }
 
diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 741e6519e2..6c36903728 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -3,133 +3,177 @@
 // Author: Johannes Kalmbach (kalmbach@cs.uni-freiburg.de)
 
 #include 
-#include 
+
 #include 
 
 #include "util/ThreadSafeQueue.h"
+#include "util/TypeTraits.h"
 #include "util/jthread.h"
 
-
 using namespace ad_utility::data_structures;
 
+namespace {
+
+template 
+auto makePush(Queue& queue) {
+  return [&queue](size_t i) {
+    if constexpr (ad_utility::similarToInstantiation) {
+      return queue.push(i);
+    } else {
+      return queue.push(i, i);
+    }
+  };
+}
+
+constexpr size_t queueSize = 5;
+
+void runWithBothQueueTypes(const auto& testFunction) {
+  testFunction(ThreadSafeQueue{queueSize});
+  testFunction(OrderedThreadSafeQueue{queueSize});
+}
+}  // namespace
+
 // ________________________________________________________________
 TEST(ThreadSafeQueue, BufferSizeIsRespected) {
-  std::atomic numPushed = 0;
+  auto runTest = [](auto queue) {
+    std::atomic numPushed = 0;
+    auto push = makePush(queue);
 
-  ThreadSafeQueue queue{5};
+    ad_utility::JThread t([&numPushed, &push, &queue] {
+      while (numPushed < 200) {
+        push(numPushed++);
+      }
+      queue.signalLastElementWasPushed();
+    });
 
-  ad_utility::JThread t([&numPushed, &queue]{
-    while(numPushed < 200) {
-      queue.push(numPushed++);
+    size_t numPopped = 0;
+    while (auto opt = queue.pop()) {
+      EXPECT_EQ(opt.value(), numPopped);
+      ++numPopped;
+      EXPECT_LE(numPushed, numPopped + queueSize + 1);
     }
-    queue.signalLastElementWasPushed();
-  });
-
-  size_t numPopped = 0;
-  while (auto opt = queue.pop()) {
-    EXPECT_EQ(opt.value(), numPopped);
-    ++numPopped;
-    EXPECT_LE(numPushed, numPopped + 6);
-  }
+  };
+  runWithBothQueueTypes(runTest);
 }
 
 // _______________________________________________________________
 TEST(ThreadSafeQueue, ReturnValueOfPush) {
-  ThreadSafeQueue queue{3};
-  EXPECT_TRUE(queue.push(42));
-  EXPECT_EQ(queue.pop(), 42);
-  queue.disablePush();
-  EXPECT_FALSE(queue.push(15));
+  auto runTest = [](auto queue) {
+    auto push = makePush(queue);
+    EXPECT_TRUE(push(0));
+    EXPECT_EQ(queue.pop(), 0);
+    queue.disablePush();
+    EXPECT_FALSE(push(1));
+  };
+  runWithBothQueueTypes(runTest);
 }
 
 // ________________________________________________________________
 TEST(ThreadSafeQueue, Concurrency) {
+  auto runTest = [](Queue queue) {
     std::atomic numPushed = 0;
-
-    ThreadSafeQueue queue{5};
+    std::atomic numThreadsDone = 0;
+    auto push = makePush(queue);
 
     size_t numThreads = 20;
-    auto push = [&numPushed, &queue, numThreads]{
-        for (size_t i = 0; i < 200; ++i) {
-            queue.push(numPushed++);
-        }
-        if (numPushed == 200 * numThreads) {
-            queue.signalLastElementWasPushed();
-        }
+    auto threadFunction = [&numPushed, &queue, numThreads, &push,
+                           &numThreadsDone] {
+      for (size_t i = 0; i < 200; ++i) {
+        push(numPushed++);
+      }
+      numThreadsDone++;
+      if (numThreadsDone == numThreads) {
+        queue.signalLastElementWasPushed();
+      }
     };
 
     std::vector threads;
     for (size_t i = 0; i < numThreads; ++i) {
-      threads.emplace_back(push);
+      threads.emplace_back(threadFunction);
     }
 
     size_t numPopped = 0;
     std::vector result;
     while (auto opt = queue.pop()) {
-        ++numPopped;
-        result.push_back(opt.value());
-        EXPECT_LE(numPushed, numPopped + 6 + numThreads);
+      ++numPopped;
+      result.push_back(opt.value());
+      EXPECT_LE(numPushed, numPopped + 6 + numThreads);
     }
 
-    std::ranges::sort(result);
+    if (ad_utility::isInstantiation) {
+      std::ranges::sort(result);
+    }
     std::vector expected;
     for (size_t i = 0; i < 200 * numThreads; ++i) {
-        expected.push_back(i);
+      expected.push_back(i);
     }
     EXPECT_EQ(result, expected);
+  };
+  runWithBothQueueTypes(runTest);
 }
 
 // ________________________________________________________________
 TEST(ThreadSafeQueue, PushException) {
+  auto runTest = [](auto queue) {
     std::atomic numPushed = 0;
     std::atomic threadIndex = 0;
 
-    ThreadSafeQueue queue{5};
+    auto push = makePush(queue);
+
+    struct IntegerException : public std::exception {
+      int value_;
+      explicit IntegerException(int value) : value_{value} {}
+    };
 
     size_t numThreads = 20;
-    auto push = [&numPushed, &queue, &threadIndex]{
-        for (size_t i = 0; i < 200; ++i) {
-            if (numPushed > 300) {
-              try {
-                throw static_cast(threadIndex++);
-              } catch(...) {
-                queue.pushException(std::current_exception());
-              }
-              EXPECT_FALSE(queue.push(numPushed++));
-            } else {
-                queue.push(numPushed++);
-            }
+    auto threadFunction = [&numPushed, &queue, &threadIndex, &push] {
+      bool hasThrown = false;
+      for (size_t i = 0; i < 200; ++i) {
+        if (numPushed > 300 && !hasThrown) {
+          hasThrown = true;
+          try {
+            int idx = threadIndex++;
+            throw IntegerException(idx);
+          } catch (...) {
+            queue.pushException(std::current_exception());
+          }
+          EXPECT_FALSE(push(numPushed++));
+        } else {
+          push(numPushed++);
         }
+      }
     };
 
     std::vector threads;
     for (size_t i = 0; i < numThreads; ++i) {
-        threads.emplace_back(push);
+      threads.emplace_back(threadFunction);
     }
 
     size_t numPopped = 0;
 
     try {
-        while (auto opt = queue.pop()) {
-          ++numPopped;
-          EXPECT_LE(numPushed, numPopped + 6 + numThreads);
-        }
-        FAIL() << "Should have thrown" << std::endl;
-    } catch(int i) {
-        EXPECT_LT(i, numThreads);
+      while (auto opt = queue.pop()) {
+        ++numPopped;
+        EXPECT_LE(numPushed, numPopped + 6 + numThreads);
+      }
+      FAIL() << "Should have thrown" << std::endl;
+    } catch (const IntegerException& i) {
+      EXPECT_LT(i.value_, numThreads);
     }
+  };
+  runWithBothQueueTypes(runTest);
 }
 
 // ________________________________________________________________
 TEST(ThreadSafeQueue, DisablePush) {
+  auto runTest = [](auto queue) {
     std::atomic numPushed = 0;
-
-    ThreadSafeQueue queue{5};
+    auto push = makePush(queue);
 
     size_t numThreads = 20;
-    auto push = [&numPushed, &queue, numThreads]{
+    auto threadFunction = [&numPushed, &push] {
       for (size_t i = 0; i < 200; ++i) {
-        if (!queue.push(numPushed++)) {
+        if (!push(numPushed++)) {
           return;
         }
       }
@@ -137,24 +181,25 @@ TEST(ThreadSafeQueue, DisablePush) {
 
     std::vector threads;
     for (size_t i = 0; i < numThreads; ++i) {
-        threads.emplace_back(push);
+      threads.emplace_back(threadFunction);
     }
 
     size_t numPopped = 0;
     std::vector result;
     while (auto opt = queue.pop()) {
-        ++numPopped;
-        result.push_back(opt.value());
-        EXPECT_LE(numPushed, numPopped + 6 + numThreads);
+      ++numPopped;
+      result.push_back(opt.value());
+      EXPECT_LE(numPushed, numPopped + 6 + numThreads);
 
-        if (numPopped == 400) {
-          queue.disablePush();
-          break;
-        }
+      if (numPopped == 400) {
+        queue.disablePush();
+        break;
+      }
     }
 
     // When terminating early, we cannot actually say much about the result.
     std::ranges::sort(result);
     EXPECT_TRUE(std::unique(result.begin(), result.end()) == result.end());
+  };
+  runWithBothQueueTypes(runTest);
 }
-

From c07e8d4b1717ba3e868719c72017e92ad5617d5f Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 21 Jun 2023 20:45:33 +0200
Subject: [PATCH 056/150] Fixed warnings

---
 src/engine/IndexScan.cpp         | 12 ++++--------
 src/index/CompressedRelation.cpp |  4 +---
 src/index/CompressedRelation.h   |  4 ++--
 src/index/Permutation.h          |  7 ++-----
 4 files changed, 9 insertions(+), 18 deletions(-)

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index b35e81b769..122604b014 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -316,12 +316,10 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans(
       col1Id = s.getPermutedTriple()[1]->toValueId(index.getVocab()).value();
     }
     if (!col1Id.has_value()) {
-      return index.getPermutation(s.permutation())
-          .lazyScan(col0Id, blocks, s.getExecutionContext()->getAllocator());
+      return index.getPermutation(s.permutation()).lazyScan(col0Id, blocks);
     } else {
       return index.getPermutation(s.permutation())
-          .lazyScan(col0Id, col1Id.value(), blocks,
-                    s.getExecutionContext()->getAllocator());
+          .lazyScan(col0Id, col1Id.value(), blocks);
     }
   };
 
@@ -373,12 +371,10 @@ cppcoro::generator IndexScan::lazyScanForJoinOfColumnWithScan(
       col1Id = s.getPermutedTriple()[1]->toValueId(index.getVocab()).value();
     }
     if (!col1Id.has_value()) {
-      return index.getPermutation(s.permutation())
-          .lazyScan(col0Id, blocks, s.getExecutionContext()->getAllocator());
+      return index.getPermutation(s.permutation()).lazyScan(col0Id, blocks);
     } else {
       return index.getPermutation(s.permutation())
-          .lazyScan(col0Id, col1Id.value(), blocks,
-                    s.getExecutionContext()->getAllocator());
+          .lazyScan(col0Id, col1Id.value(), blocks);
     }
   };
   return getScan(s, blocks);
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index f0826c42f0..d400105f2e 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -144,7 +144,6 @@ void CompressedRelationReader::scan(
 cppcoro::generator CompressedRelationReader::lazyScan(
     CompressedRelationMetadata metadata,
     std::vector blockMetadata, ad_utility::File& file,
-    ad_utility::AllocatorWithLimit allocator,
     ad_utility::SharedConcurrentTimeoutTimer timer) const {
   auto relevantBlocks =
       getBlocksFromMetadata(metadata, std::nullopt, blockMetadata);
@@ -192,7 +191,7 @@ cppcoro::generator CompressedRelationReader::lazyScan(
     auto numElements = metadata.numRows_;
     AD_CORRECTNESS_CHECK(uncompressedBuffer->numColumns() ==
                          metadata.numColumns());
-    IdTable result(uncompressedBuffer->numColumns(), allocator);
+    IdTable result(uncompressedBuffer->numColumns(), allocator_);
     result.resize(numElements);
     for (size_t i = 0; i < uncompressedBuffer->numColumns(); ++i) {
       const auto& inputCol = uncompressedBuffer->getColumn(i);
@@ -263,7 +262,6 @@ cppcoro::generator CompressedRelationReader::lazyScan(
 cppcoro::generator CompressedRelationReader::lazyScan(
     CompressedRelationMetadata metadata, Id col1Id,
     std::vector blockMetadata, ad_utility::File& file,
-    ad_utility::AllocatorWithLimit allocator,
     ad_utility::SharedConcurrentTimeoutTimer timer) const {
   auto relevantBlocks = getBlocksFromMetadata(metadata, col1Id, blockMetadata);
   auto beginBlock = relevantBlocks.begin();
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index 1c368a4e44..9f201949a6 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -285,13 +285,13 @@ class CompressedRelationReader {
   cppcoro::generator lazyScan(
       CompressedRelationMetadata metadata,
       std::vector blockMetadata,
-      ad_utility::File& file, ad_utility::AllocatorWithLimit allocator,
+      ad_utility::File& file,
       ad_utility::SharedConcurrentTimeoutTimer timer) const;
 
   cppcoro::generator lazyScan(
       CompressedRelationMetadata metadata, Id col1Id,
       std::vector blockMetadata,
-      ad_utility::File& file, ad_utility::AllocatorWithLimit allocator,
+      ad_utility::File& file,
       ad_utility::SharedConcurrentTimeoutTimer timer) const;
 
   /**
diff --git a/src/index/Permutation.h b/src/index/Permutation.h
index a730a6c0b4..1a4b628cfd 100644
--- a/src/index/Permutation.h
+++ b/src/index/Permutation.h
@@ -64,24 +64,21 @@ class Permutation {
 
   cppcoro::generator lazyScan(
       Id col0Id, const std::vector& blocks,
-      ad_utility::AllocatorWithLimit allocator,
       ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const {
     if (!meta_.col0IdExists(col0Id)) {
       return {};
     }
-    return reader_.lazyScan(meta_.getMetaData(col0Id), blocks, file_,
-                            std::move(allocator), timer);
+    return reader_.lazyScan(meta_.getMetaData(col0Id), blocks, file_, timer);
   }
 
   cppcoro::generator lazyScan(
       Id col0Id, Id col1Id, const std::vector& blocks,
-      ad_utility::AllocatorWithLimit allocator,
       ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const {
     if (!meta_.col0IdExists(col0Id)) {
       return {};
     }
     return reader_.lazyScan(meta_.getMetaData(col0Id), col1Id, blocks, file_,
-                            std::move(allocator), timer);
+                            timer);
   }
   /// For a given ID for the first column, retrieve all IDs of the second and
   /// third column, and store them in `result`. This is just a thin wrapper

From 92ac4d97faf8330383f1d90f622d5e903c90d04e Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Thu, 22 Jun 2023 11:48:41 +0200
Subject: [PATCH 057/150] Fixed several bugs, this should now work.

---
 src/engine/Join.cpp              |  2 +-
 src/engine/MultiColumnJoin.cpp   |  2 +-
 src/engine/OptionalJoin.cpp      |  2 +-
 src/engine/idTable/IdTable.h     | 31 +++++++------
 src/index/CompressedRelation.cpp | 75 ++++++++++++++++----------------
 test/CompressedRelationsTest.cpp | 26 ++++++++++-
 6 files changed, 84 insertions(+), 54 deletions(-)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index bbb1e5b658..e4da7c9a3f 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -491,7 +491,7 @@ void Join::join(const IdTable& a, ColumnIndex jc1, const IdTable& b,
   // algorithms above easier), be the order that is expected by the rest of
   // the code is [columns-a, non-join-columns-b]. Permute the columns to fix
   // the order.
-  result->permuteColumns(joinColumnData.permutationResult());
+  result->setColumnSubset(joinColumnData.permutationResult());
 
   LOG(DEBUG) << "Join done.\n";
   LOG(DEBUG) << "Result: width = " << result->numColumns()
diff --git a/src/engine/MultiColumnJoin.cpp b/src/engine/MultiColumnJoin.cpp
index 7eaeef5d1a..c7e6e48bdf 100644
--- a/src/engine/MultiColumnJoin.cpp
+++ b/src/engine/MultiColumnJoin.cpp
@@ -288,5 +288,5 @@ void MultiColumnJoin::computeMultiColumnJoin(
   // The result that `zipperJoinWithUndef` produces has a different order of
   // columns than expected, permute them. See the documentation of
   // `JoinColumnMapping` for details.
-  result->permuteColumns(joinColumnData.permutationResult());
+  result->setColumnSubset(joinColumnData.permutationResult());
 }
diff --git a/src/engine/OptionalJoin.cpp b/src/engine/OptionalJoin.cpp
index 2ad984a427..f5f5feeb87 100644
--- a/src/engine/OptionalJoin.cpp
+++ b/src/engine/OptionalJoin.cpp
@@ -364,5 +364,5 @@ void OptionalJoin::optionalJoin(
     }
     Engine::sort(*result, cols);
   }
-  result->permuteColumns(joinColumnData.permutationResult());
+  result->setColumnSubset(joinColumnData.permutationResult());
 }
diff --git a/src/engine/idTable/IdTable.h b/src/engine/idTable/IdTable.h
index a1cea0308b..dc8c8a4ba8 100644
--- a/src/engine/idTable/IdTable.h
+++ b/src/engine/idTable/IdTable.h
@@ -545,26 +545,31 @@ class IdTable {
         std::move(viewSpans), columnIndices.size(), numRows_, allocator_};
   }
 
-  // Apply the `permutation` to the columns of the table. The permutation must
-  // be a permutation of the values `[0, 1, ..., numColumns - 1 ]`. The column
-  // with the old index `permutation[i]` will become the `i`-th column after the
-  // permutation. For example, `permuteColumns({1, 2, 0})` rotates the columns
-  // of a table with three columns left by one element.
-  void permuteColumns(std::span permutation) {
-    // First check that the `permutation` is indeed a permutation of the column
+  // Modify the table, s.t. it contains only the specified `subset` of the
+  // original columns in the specified order. Each index in the `subset`
+  // must be `< numColumns()` and must appear at most once in the subset.
+  // The column with the old index `subset[i]` will become the `i`-th column
+  // after the subset. For example `setColumnSubset({2, 1})` will result in a
+  // table with 2 columns,// with the original columns with index 2 and 1, with
+  // their order switched. The special case where `subset.size() ==
+  // numColumns()` implies that the function applies a permutation to the table.
+  // For example `setColumnSubset({1, 2, 0})` rotates the columns of a table
+  // with three columns left by one element.
+  void setColumnSubset(std::span subset) requires isDynamic {
+    // First check that the `subset` is indeed a subset of the column
     // indices.
-    std::vector check{permutation.begin(), permutation.end()};
+    std::vector check{subset.begin(), subset.end()};
     std::ranges::sort(check);
-    std::vector expected(numColumns());
-    std::iota(expected.begin(), expected.end(), ColumnIndex{0});
-    AD_CONTRACT_CHECK(check == expected);
+    AD_CONTRACT_CHECK(std::unique(check.begin(), check.end()) == check.end());
+    AD_CONTRACT_CHECK(!subset.empty() && subset.back() < numColumns());
 
     Data newData;
-    newData.reserve(numColumns());
-    for (auto colIdx : permutation) {
+    newData.reserve(subset.size());
+    for (auto colIdx : subset) {
       newData.push_back(std::move(data().at(colIdx)));
     }
     data() = std::move(newData);
+    numColumns_ = subset.size();
   }
 
   // Helper `struct` that stores a pointer to this table and has an `operator()`
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index d400105f2e..0568d4fd62 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -302,49 +302,49 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   }
 
   if (firstBlockResult.has_value()) {
+    firstBlockResult.value().setColumnSubset(std::array{1});
     co_yield std::move(firstBlockResult.value());
   }
 
-  ad_utility::data_structures::OrderedThreadSafeQueue queue{
-      5};
-  // TODO We can configure whether we want to allow async access to the
-  // file.
-  std::mutex fileMutex;
-  size_t blockIndex = 0;
-  auto readAndDecompressBlock = [&]() {
-    // LOG(WARN) << "Starting a thread" << std::endl;
-    while (true) {
-      std::unique_lock lock{fileMutex};
-      if (beginBlock == endBlock) {
-        queue.signalLastElementWasPushed();
-        return;
-      }
-      auto block = *beginBlock;
-      CompressedBlock compressedBuffer =
-          readCompressedBlockFromFile(block, file, std::vector{1ul});
-      ++beginBlock;
-      size_t myIndex = blockIndex;
-      ++blockIndex;
-      bool isLastBlock = beginBlock == endBlock;
-      lock.unlock();
-      bool success = queue.push(
-          myIndex, decompressBlock(compressedBuffer, block.numRows_));
-      if (!success) {
-        return;
-      }
-      if (isLastBlock) {
-        queue.signalLastElementWasPushed();
+  if (beginBlock != endBlock) {
+    ad_utility::data_structures::OrderedThreadSafeQueue
+        queue{5};
+    // TODO We can configure whether we want to allow async access to
+    // the file.
+    std::mutex fileMutex;
+    size_t blockIndex = 0;
+    auto readAndDecompressBlock = [&]() {
+      while (true) {
+        std::unique_lock lock{fileMutex};
+        if (beginBlock == endBlock) {
+          return;
+        }
+        auto block = *beginBlock;
+        CompressedBlock compressedBuffer =
+            readCompressedBlockFromFile(block, file, std::vector{1ul});
+        ++beginBlock;
+        size_t myIndex = blockIndex;
+        ++blockIndex;
+        bool isLastBlock = beginBlock == endBlock;
+        lock.unlock();
+        bool success = queue.push(
+            myIndex, decompressBlock(compressedBuffer, block.numRows_));
+        if (!success) {
+          return;
+        }
+        if (isLastBlock) {
+          queue.signalLastElementWasPushed();
+        }
       }
-    }
-  };
+    };
 
-  std::vector threads;
-  // TODO get the number of optimal threads from the operating system.
-  for (size_t j = 0; j < 5; ++j) {
-    threads.emplace_back(readAndDecompressBlock);
-  }
+    std::vector threads;
+    // TODO get the number of optimal threads from the operating
+    // system.
+    for (size_t j = 0; j < 5; ++j) {
+      threads.emplace_back(readAndDecompressBlock);
+    }
 
-  if (beginBlock != endBlock) {
     while (auto opt = queue.pop()) {
       co_yield opt.value();
     }
@@ -365,6 +365,7 @@ cppcoro::generator CompressedRelationReader::lazyScan(
     timer->wlock()->checkTimeoutAndThrow();
   }
   if (lastBlockResult.has_value()) {
+    lastBlockResult.value().setColumnSubset(std::array{1});
     co_yield lastBlockResult.value();
   }
 }
diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp
index 985e993df4..75b8e3aadf 100644
--- a/test/CompressedRelationsTest.cpp
+++ b/test/CompressedRelationsTest.cpp
@@ -7,9 +7,14 @@
 #include "./IndexTestHelpers.h"
 #include "index/CompressedRelation.h"
 #include "index/Permutation.h"
+#include "util/GTestHelpers.h"
 #include "util/Serializer/ByteBufferSerializer.h"
+#include "util/SourceLocation.h"
 
 namespace {
+
+using ad_utility::source_location;
+
 // Return an `ID` of type `VocabIndex` from `index`. Assert that `index`
 // is `>= 0`.
 Id V(int64_t index) {
@@ -31,8 +36,14 @@ struct RelationInput {
 template 
 void checkThatTablesAreEqual(
     const std::vector> expected,
-    const IdTable& actual) {
+    const IdTable& actual, source_location l = source_location::current()) {
+  auto trace = generateLocationTrace(l);
   ASSERT_EQ(NumColumns, actual.numColumns());
+  if (actual.numRows() != expected.size()) {
+    LOG(WARN) << actual.numRows() << "vs " << expected.size() << std::endl;
+    LOG(WARN) << "mismatch" << std::endl;
+  }
+  ASSERT_EQ(actual.numRows(), expected.size());
   for (size_t i = 0; i < actual.numRows(); ++i) {
     for (size_t j = 0; j < actual.numColumns(); ++j) {
       ASSERT_EQ(V(expected[i][j]), actual(i, j));
@@ -125,6 +136,13 @@ void testCompressedRelations(const std::vector& inputs,
     const auto& col1And2 = inputs[i].col1And2_;
     checkThatTablesAreEqual(col1And2, table);
 
+    table.clear();
+    for (const auto& block :
+         reader.lazyScan(metaData[i], blocks, file, timer)) {
+      table.insertAtEnd(block.begin(), block.end());
+    }
+    checkThatTablesAreEqual(col1And2, table);
+
     // Check for all distinct combinations of `(col0, col1)` and check that
     // we get the expected result.
     // TODO, C++23 use views::chunk_by
@@ -139,6 +157,12 @@ void testCompressedRelations(const std::vector& inputs,
                   timer);
       EXPECT_EQ(size, tableWidthOne.numRows());
       checkThatTablesAreEqual(col3, tableWidthOne);
+      tableWidthOne.clear();
+      for (const auto& block :
+           reader.lazyScan(metaData[i], V(lastCol1Id), blocks, file, timer)) {
+        tableWidthOne.insertAtEnd(block.begin(), block.end());
+      }
+      checkThatTablesAreEqual(col3, tableWidthOne);
       {
         IdTable wrongNumCols{2, ad_utility::testing::makeAllocator()};
         ASSERT_THROW(reader.scan(metaData[i], V(lastCol1Id), blocks, file,

From 35bbf4fa1c39d6febb548fc16960d31681eada62 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Thu, 22 Jun 2023 19:14:19 +0200
Subject: [PATCH 058/150] Commented the tests for the threadsafe queue.

---
 test/ThreadSafeQueueTest.cpp | 91 +++++++++++++++++++++++++-----------
 1 file changed, 64 insertions(+), 27 deletions(-)

diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 6c36903728..07e42c99b1 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -2,9 +2,11 @@
 //                 Chair of Algorithms and Data Structures.
 // Author: Johannes Kalmbach (kalmbach@cs.uni-freiburg.de)
 
+#include 
 #include 
 
 #include 
+#include 
 
 #include "util/ThreadSafeQueue.h"
 #include "util/TypeTraits.h"
@@ -14,6 +16,10 @@ using namespace ad_utility::data_structures;
 
 namespace {
 
+// Create a lambda, that pushes a given `size_t i` to the given `queue`. If the
+// `Queue` is a `OrderedThreadSafeQueue`, then `i` is also used as the index for
+// the call to `push`. This imposes requirements on the values that are pushed
+// to avoid deadlocks, see `ThreadSafeQueue.h` for details.
 template 
 auto makePush(Queue& queue) {
   return [&queue](size_t i) {
@@ -25,8 +31,14 @@ auto makePush(Queue& queue) {
   };
 }
 
+// Some constants that are used in almost every test case.
 constexpr size_t queueSize = 5;
+constexpr size_t numThreads = 20;
+constexpr size_t numValues = 200;
 
+// Run the `test` function with a `ThreadSafeQueue` and an
+// `OrderedThreadSafeQueue`. Both queues have a size of `queueSize` and `size_t`
+// as their value type.
 void runWithBothQueueTypes(const auto& testFunction) {
   testFunction(ThreadSafeQueue{queueSize});
   testFunction(OrderedThreadSafeQueue{queueSize});
@@ -39,6 +51,7 @@ TEST(ThreadSafeQueue, BufferSizeIsRespected) {
     std::atomic numPushed = 0;
     auto push = makePush(queue);
 
+    // Asynchronous worker thread that pushes incremental values to the queue.
     ad_utility::JThread t([&numPushed, &push, &queue] {
       while (numPushed < 200) {
         push(numPushed++);
@@ -48,8 +61,14 @@ TEST(ThreadSafeQueue, BufferSizeIsRespected) {
 
     size_t numPopped = 0;
     while (auto opt = queue.pop()) {
+      // We have only one thread pushing, so the elements in the queue are
+      // ordered.
       EXPECT_EQ(opt.value(), numPopped);
       ++numPopped;
+      // Check that the size of the queue is respected. The pushing thread must
+      // only continue to push once enough elements have been `pop`ped. The `+1`
+      // is necessary because the calls to `pop` and `push` are not synchronized
+      // with the atomic value `numPushed`.
       EXPECT_LE(numPushed, numPopped + queueSize + 1);
     }
   };
@@ -60,6 +79,8 @@ TEST(ThreadSafeQueue, BufferSizeIsRespected) {
 TEST(ThreadSafeQueue, ReturnValueOfPush) {
   auto runTest = [](auto queue) {
     auto push = makePush(queue);
+    // Test that `push` always returns true until `disablePush()` has been
+    // called.
     EXPECT_TRUE(push(0));
     EXPECT_EQ(queue.pop(), 0);
     queue.disablePush();
@@ -68,17 +89,18 @@ TEST(ThreadSafeQueue, ReturnValueOfPush) {
   runWithBothQueueTypes(runTest);
 }
 
-// ________________________________________________________________
+// Test the case that multiple workers are pushing concurrently.
 TEST(ThreadSafeQueue, Concurrency) {
   auto runTest = [](Queue queue) {
     std::atomic numPushed = 0;
     std::atomic numThreadsDone = 0;
     auto push = makePush(queue);
 
-    size_t numThreads = 20;
-    auto threadFunction = [&numPushed, &queue, numThreads, &push,
-                           &numThreadsDone] {
+    // Set up the worker threads.
+    auto threadFunction = [&numPushed, &queue, &push, &numThreadsDone] {
       for (size_t i = 0; i < 200; ++i) {
+        // push the next available value that hasn't been pushed yet by another
+        // thread.
         push(numPushed++);
       }
       numThreadsDone++;
@@ -92,22 +114,26 @@ TEST(ThreadSafeQueue, Concurrency) {
       threads.emplace_back(threadFunction);
     }
 
+    // Pop the values from the queue and store them.
     size_t numPopped = 0;
     std::vector result;
     while (auto opt = queue.pop()) {
       ++numPopped;
       result.push_back(opt.value());
-      EXPECT_LE(numPushed, numPopped + 6 + numThreads);
+      // The ` + numThreads` is because the atomic increment of `numPushed` is
+      // done before the actual call to `push`. The `+ 1` is because another
+      // element might have been pushed since our last call to `pop()`.
+      EXPECT_LE(numPushed, numPopped + queueSize + 1 + numThreads);
     }
 
+    // For the `OrderedThreadSafeQueue` we expect the result to already be in
+    // order, for the `ThreadSafeQueue` the order is unspecified and we only
+    // check the content.
     if (ad_utility::isInstantiation) {
       std::ranges::sort(result);
     }
-    std::vector expected;
-    for (size_t i = 0; i < 200 * numThreads; ++i) {
-      expected.push_back(i);
-    }
-    EXPECT_EQ(result, expected);
+    EXPECT_THAT(result, ::testing::ElementsAreArray(
+                            std::views::iota(0UL, 200UL * numThreads)));
   };
   runWithBothQueueTypes(runTest);
 }
@@ -121,24 +147,25 @@ TEST(ThreadSafeQueue, PushException) {
     auto push = makePush(queue);
 
     struct IntegerException : public std::exception {
-      int value_;
-      explicit IntegerException(int value) : value_{value} {}
+      size_t value_;
+      explicit IntegerException(size_t value) : value_{value} {}
     };
 
-    size_t numThreads = 20;
     auto threadFunction = [&numPushed, &queue, &threadIndex, &push] {
       bool hasThrown = false;
       for (size_t i = 0; i < 200; ++i) {
         if (numPushed > 300 && !hasThrown) {
           hasThrown = true;
-          try {
-            int idx = threadIndex++;
-            throw IntegerException(idx);
-          } catch (...) {
-            queue.pushException(std::current_exception());
-          }
+          // At some point, each thread pushes an Exception. After pushing the
+          // exception, all calls to `push` return false.
+          queue.pushException(
+              std::make_exception_ptr(IntegerException{threadIndex++}));
+          EXPECT_FALSE(push(numPushed++));
+        } else if (hasThrown) {
           EXPECT_FALSE(push(numPushed++));
         } else {
+          // We cannot know whether this returns true or false, because another
+          // thread already might have thrown an exception.
           push(numPushed++);
         }
       }
@@ -152,9 +179,11 @@ TEST(ThreadSafeQueue, PushException) {
     size_t numPopped = 0;
 
     try {
+      // The usual check as always, but at some point `pop` will throw, because
+      // exceptions were pushed to the queue.
       while (auto opt = queue.pop()) {
         ++numPopped;
-        EXPECT_LE(numPushed, numPopped + 6 + numThreads);
+        EXPECT_LE(numPushed, numPopped + queueSize + 1 + numThreads);
       }
       FAIL() << "Should have thrown" << std::endl;
     } catch (const IntegerException& i) {
@@ -166,13 +195,13 @@ TEST(ThreadSafeQueue, PushException) {
 
 // ________________________________________________________________
 TEST(ThreadSafeQueue, DisablePush) {
-  auto runTest = [](auto queue) {
+  auto runTest = [](Queue queue) {
     std::atomic numPushed = 0;
     auto push = makePush(queue);
 
-    size_t numThreads = 20;
     auto threadFunction = [&numPushed, &push] {
-      for (size_t i = 0; i < 200; ++i) {
+      while (true) {
+        // Push until the consumer calls `disablePush`.
         if (!push(numPushed++)) {
           return;
         }
@@ -191,15 +220,23 @@ TEST(ThreadSafeQueue, DisablePush) {
       result.push_back(opt.value());
       EXPECT_LE(numPushed, numPopped + 6 + numThreads);
 
+      // Disable the push, make the consumers finish.
       if (numPopped == 400) {
         queue.disablePush();
         break;
       }
     }
-
-    // When terminating early, we cannot actually say much about the result.
-    std::ranges::sort(result);
-    EXPECT_TRUE(std::unique(result.begin(), result.end()) == result.end());
+    if (ad_utility::similarToInstantiation) {
+      // When terminating early, we cannot actually say much about the result,
+      // other than that it contains no duplicate values
+      std::ranges::sort(result);
+      EXPECT_TRUE(std::unique(result.begin(), result.end()) == result.end());
+    } else {
+      // For the ordered queue we have the guarantee that all the pushed values
+      // were in order.
+      EXPECT_THAT(result,
+                  ::testing::ElementsAreArray(std::views::iota(0U, 400U)));
+    }
   };
   runWithBothQueueTypes(runTest);
 }

From 79ad8ca6962ed0a4314dc1b050645521c7c0e8b0 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 26 Jun 2023 09:33:54 +0200
Subject: [PATCH 059/150] Fixed some tests

---
 test/CMakeLists.txt          |  2 +-
 test/ThreadSafeQueueTest.cpp | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 3a7d8bf94c..013fb1ad28 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -20,7 +20,7 @@ endfunction()
 # if a test binary requires multiple sources
 function(linkAndDiscoverTest basename)
     linkTest(${basename} ${ARGN})
-    gtest_discover_tests(${basename} ${basename})
+    gtest_discover_tests(${basename} ${basename} DISCOVERY_TIMEOUT 600)
 endfunction()
 
 # Usage: `linkAndDiscoverTestSerial(basename, [additionalLibraries...]`
diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 07e42c99b1..7281ed1e25 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -48,12 +48,12 @@ void runWithBothQueueTypes(const auto& testFunction) {
 // ________________________________________________________________
 TEST(ThreadSafeQueue, BufferSizeIsRespected) {
   auto runTest = [](auto queue) {
-    std::atomic numPushed = 0;
+    std::atomic numPushed = 0;
     auto push = makePush(queue);
 
     // Asynchronous worker thread that pushes incremental values to the queue.
     ad_utility::JThread t([&numPushed, &push, &queue] {
-      while (numPushed < 200) {
+      while (numPushed < numValues) {
         push(numPushed++);
       }
       queue.signalLastElementWasPushed();
@@ -98,7 +98,7 @@ TEST(ThreadSafeQueue, Concurrency) {
 
     // Set up the worker threads.
     auto threadFunction = [&numPushed, &queue, &push, &numThreadsDone] {
-      for (size_t i = 0; i < 200; ++i) {
+      for (size_t i = 0; i < numValues; ++i) {
         // push the next available value that hasn't been pushed yet by another
         // thread.
         push(numPushed++);
@@ -133,7 +133,7 @@ TEST(ThreadSafeQueue, Concurrency) {
       std::ranges::sort(result);
     }
     EXPECT_THAT(result, ::testing::ElementsAreArray(
-                            std::views::iota(0UL, 200UL * numThreads)));
+                            std::views::iota(0UL, numValues * numThreads)));
   };
   runWithBothQueueTypes(runTest);
 }
@@ -153,7 +153,7 @@ TEST(ThreadSafeQueue, PushException) {
 
     auto threadFunction = [&numPushed, &queue, &threadIndex, &push] {
       bool hasThrown = false;
-      for (size_t i = 0; i < 200; ++i) {
+      for (size_t i = 0; i < numValues; ++i) {
         if (numPushed > 300 && !hasThrown) {
           hasThrown = true;
           // At some point, each thread pushes an Exception. After pushing the

From 82adab5b52bff608ece90eae206ba9782de33dd6 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 26 Jun 2023 09:46:21 +0200
Subject: [PATCH 060/150] Implement an ordered threadsafe queue and test it.

---
 src/util/ThreadSafeQueue.h   | 101 ++++++++++++++-
 test/CMakeLists.txt          |   2 +
 test/ThreadSafeQueueTest.cpp | 242 +++++++++++++++++++++++++++++++++++
 3 files changed, 338 insertions(+), 7 deletions(-)
 create mode 100644 test/ThreadSafeQueueTest.cpp

diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index ac50ec03c4..a82b0bc0d3 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -1,20 +1,23 @@
 // Copyright 2022, University of Freiburg,
 // Chair of Algorithms and Data Structures.
-// Author: Robin Textor-Falconi (textorr@informatik.uni-freiburg.de)
+// Authors: Robin Textor-Falconi (textorr@informatik.uni-freiburg.de)
+//          Johannes Kalmbach (kalmbach@cs.uni-freiburg.de)
 
-#ifndef QLEVER_THREADSAFEQUEUE_H
-#define QLEVER_THREADSAFEQUEUE_H
+#pragma once
 
 #include 
 #include 
 #include 
 #include 
 
+#include "absl/synchronization/mutex.h"
+
 namespace ad_utility::data_structures {
 
 /// A thread safe, multi-consumer, multi-producer queue.
 template 
 class ThreadSafeQueue {
+  std::exception_ptr pushedException_;
   std::queue _queue;
   std::mutex _mutex;
   std::condition_variable _pushNotification;
@@ -43,6 +46,18 @@ class ThreadSafeQueue {
     return true;
   }
 
+  // The semantics of pushing an exception are as follows: All subsequent calls
+  // to `pop` will throw the `exception`, and all subsequent calls to `push`
+  // will return `false`.
+  void pushException(std::exception_ptr exception) {
+    std::unique_lock lock{_mutex};
+    pushedException_ = std::move(exception);
+    _pushDisabled = true;
+    lock.unlock();
+    _pushNotification.notify_all();
+    _popNotification.notify_all();
+  }
+
   /// Signals all threads waiting for pop() to return that data transmission
   /// has ended and it should stop processing.
   void signalLastElementWasPushed() {
@@ -71,8 +86,12 @@ class ThreadSafeQueue {
   /// empty optional, whatever happens first
   std::optional pop() {
     std::unique_lock lock{_mutex};
-    _pushNotification.wait(
-        lock, [&] { return !_queue.empty() || _lastElementPushed; });
+    _pushNotification.wait(lock, [&] {
+      return !_queue.empty() || _lastElementPushed || pushedException_;
+    });
+    if (pushedException_) {
+      std::rethrow_exception(pushedException_);
+    }
     if (_lastElementPushed && _queue.empty()) {
       return {};
     }
@@ -84,6 +103,74 @@ class ThreadSafeQueue {
   }
 };
 
-}  // namespace ad_utility::data_structures
+// A thread safe queue that is similar (wrt the interface and the behavior) to
+// the `ThreadSafeQueue` above, with the following differente: Each element that
+// is pushed is associated with a unique index `n`. A call to `push(n,
+// someValue)` will block until other threads have pushed all indices in the
+// range [0, ..., n - 1]. This can be used to enforce the ordering of values
+// that are asynchronously created by multiple threads. Note that great care has
+// to be taken that all the indices will be pushed eventually by some thread,
+// and that for each thread individually the indices are increasing, otherwise
+// the queue will lead to a deadlock.
+template 
+class OrderedThreadSafeQueue {
+ private:
+  std::mutex mutex_;
+  std::condition_variable cv_;
+  ThreadSafeQueue queue_;
+  size_t nextIndex_ = 0;
+  bool pushWasDisabled_ = false;
 
-#endif  // QLEVER_THREADSAFEQUEUE_H
+ public:
+  // Construct from the maximal queue size (see `ThreadSafeQueue` for details).
+  explicit OrderedThreadSafeQueue(size_t maxSize) : queue_{maxSize} {}
+
+  // Push the `value` to the queue that is associated with the `index`. This
+  // call blocks, until `push` has been called for all indices in `[0, ...,
+  // index - 1]` or until `disablePush` was called. The remaining behavior is
+  // equal to `ThreadSafeQueue::push`.
+  bool push(size_t index, T value) {
+    std::unique_lock lock{mutex_};
+    cv_.wait(lock, [this, index]() {
+      return index == nextIndex_ || pushWasDisabled_;
+    });
+    if (pushWasDisabled_) {
+      return false;
+    }
+    ++nextIndex_;
+    bool result = queue_.push(std::move(value));
+    lock.unlock();
+    cv_.notify_all();
+    return result;
+  }
+
+  // See `ThreadSafeQueue` for details.
+  void pushException(std::exception_ptr exception) {
+    std::unique_lock l{mutex_};
+    queue_.pushException(std::move(exception));
+    pushWasDisabled_ = true;
+    l.unlock();
+    cv_.notify_all();
+  }
+
+  // See `ThreadSafeQueue` for details.
+  void signalLastElementWasPushed() { queue_.signalLastElementWasPushed(); }
+
+  // See `ThreadSafeQueue` for details.
+  void disablePush() {
+    queue_.disablePush();
+    std::unique_lock lock{mutex_};
+    pushWasDisabled_ = true;
+    lock.unlock();
+    cv_.notify_all();
+  }
+
+  // See `ThreadSafeQueue` for details.
+  ~OrderedThreadSafeQueue() { disablePush(); }
+
+  // See `ThreadSafeQueue` for details. All the returned values will be in
+  // ascending consecutive order wrt the index with which they were pushed.
+  std::optional pop() { return queue_.pop(); }
+};
+
+}  // namespace ad_utility::data_structures
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 5b6804fe85..eaa23b3f75 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -329,3 +329,5 @@ addLinkAndDiscoverTest(AddCombinedRowToTableTest util)
 addLinkAndDiscoverTest(CtreHelpersTest)
 
 addLinkAndDiscoverTest(ComparisonWithNanTest)
+
+addLinkAndDiscoverTest(ThreadSafeQueueTest)
\ No newline at end of file
diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
new file mode 100644
index 0000000000..7281ed1e25
--- /dev/null
+++ b/test/ThreadSafeQueueTest.cpp
@@ -0,0 +1,242 @@
+// Copyright 2023, University of Freiburg,
+//                 Chair of Algorithms and Data Structures.
+// Author: Johannes Kalmbach (kalmbach@cs.uni-freiburg.de)
+
+#include 
+#include 
+
+#include 
+#include 
+
+#include "util/ThreadSafeQueue.h"
+#include "util/TypeTraits.h"
+#include "util/jthread.h"
+
+using namespace ad_utility::data_structures;
+
+namespace {
+
+// Create a lambda, that pushes a given `size_t i` to the given `queue`. If the
+// `Queue` is a `OrderedThreadSafeQueue`, then `i` is also used as the index for
+// the call to `push`. This imposes requirements on the values that are pushed
+// to avoid deadlocks, see `ThreadSafeQueue.h` for details.
+template 
+auto makePush(Queue& queue) {
+  return [&queue](size_t i) {
+    if constexpr (ad_utility::similarToInstantiation) {
+      return queue.push(i);
+    } else {
+      return queue.push(i, i);
+    }
+  };
+}
+
+// Some constants that are used in almost every test case.
+constexpr size_t queueSize = 5;
+constexpr size_t numThreads = 20;
+constexpr size_t numValues = 200;
+
+// Run the `test` function with a `ThreadSafeQueue` and an
+// `OrderedThreadSafeQueue`. Both queues have a size of `queueSize` and `size_t`
+// as their value type.
+void runWithBothQueueTypes(const auto& testFunction) {
+  testFunction(ThreadSafeQueue{queueSize});
+  testFunction(OrderedThreadSafeQueue{queueSize});
+}
+}  // namespace
+
+// ________________________________________________________________
+TEST(ThreadSafeQueue, BufferSizeIsRespected) {
+  auto runTest = [](auto queue) {
+    std::atomic numPushed = 0;
+    auto push = makePush(queue);
+
+    // Asynchronous worker thread that pushes incremental values to the queue.
+    ad_utility::JThread t([&numPushed, &push, &queue] {
+      while (numPushed < numValues) {
+        push(numPushed++);
+      }
+      queue.signalLastElementWasPushed();
+    });
+
+    size_t numPopped = 0;
+    while (auto opt = queue.pop()) {
+      // We have only one thread pushing, so the elements in the queue are
+      // ordered.
+      EXPECT_EQ(opt.value(), numPopped);
+      ++numPopped;
+      // Check that the size of the queue is respected. The pushing thread must
+      // only continue to push once enough elements have been `pop`ped. The `+1`
+      // is necessary because the calls to `pop` and `push` are not synchronized
+      // with the atomic value `numPushed`.
+      EXPECT_LE(numPushed, numPopped + queueSize + 1);
+    }
+  };
+  runWithBothQueueTypes(runTest);
+}
+
+// _______________________________________________________________
+TEST(ThreadSafeQueue, ReturnValueOfPush) {
+  auto runTest = [](auto queue) {
+    auto push = makePush(queue);
+    // Test that `push` always returns true until `disablePush()` has been
+    // called.
+    EXPECT_TRUE(push(0));
+    EXPECT_EQ(queue.pop(), 0);
+    queue.disablePush();
+    EXPECT_FALSE(push(1));
+  };
+  runWithBothQueueTypes(runTest);
+}
+
+// Test the case that multiple workers are pushing concurrently.
+TEST(ThreadSafeQueue, Concurrency) {
+  auto runTest = [](Queue queue) {
+    std::atomic numPushed = 0;
+    std::atomic numThreadsDone = 0;
+    auto push = makePush(queue);
+
+    // Set up the worker threads.
+    auto threadFunction = [&numPushed, &queue, &push, &numThreadsDone] {
+      for (size_t i = 0; i < numValues; ++i) {
+        // push the next available value that hasn't been pushed yet by another
+        // thread.
+        push(numPushed++);
+      }
+      numThreadsDone++;
+      if (numThreadsDone == numThreads) {
+        queue.signalLastElementWasPushed();
+      }
+    };
+
+    std::vector threads;
+    for (size_t i = 0; i < numThreads; ++i) {
+      threads.emplace_back(threadFunction);
+    }
+
+    // Pop the values from the queue and store them.
+    size_t numPopped = 0;
+    std::vector result;
+    while (auto opt = queue.pop()) {
+      ++numPopped;
+      result.push_back(opt.value());
+      // The ` + numThreads` is because the atomic increment of `numPushed` is
+      // done before the actual call to `push`. The `+ 1` is because another
+      // element might have been pushed since our last call to `pop()`.
+      EXPECT_LE(numPushed, numPopped + queueSize + 1 + numThreads);
+    }
+
+    // For the `OrderedThreadSafeQueue` we expect the result to already be in
+    // order, for the `ThreadSafeQueue` the order is unspecified and we only
+    // check the content.
+    if (ad_utility::isInstantiation) {
+      std::ranges::sort(result);
+    }
+    EXPECT_THAT(result, ::testing::ElementsAreArray(
+                            std::views::iota(0UL, numValues * numThreads)));
+  };
+  runWithBothQueueTypes(runTest);
+}
+
+// ________________________________________________________________
+TEST(ThreadSafeQueue, PushException) {
+  auto runTest = [](auto queue) {
+    std::atomic numPushed = 0;
+    std::atomic threadIndex = 0;
+
+    auto push = makePush(queue);
+
+    struct IntegerException : public std::exception {
+      size_t value_;
+      explicit IntegerException(size_t value) : value_{value} {}
+    };
+
+    auto threadFunction = [&numPushed, &queue, &threadIndex, &push] {
+      bool hasThrown = false;
+      for (size_t i = 0; i < numValues; ++i) {
+        if (numPushed > 300 && !hasThrown) {
+          hasThrown = true;
+          // At some point, each thread pushes an Exception. After pushing the
+          // exception, all calls to `push` return false.
+          queue.pushException(
+              std::make_exception_ptr(IntegerException{threadIndex++}));
+          EXPECT_FALSE(push(numPushed++));
+        } else if (hasThrown) {
+          EXPECT_FALSE(push(numPushed++));
+        } else {
+          // We cannot know whether this returns true or false, because another
+          // thread already might have thrown an exception.
+          push(numPushed++);
+        }
+      }
+    };
+
+    std::vector threads;
+    for (size_t i = 0; i < numThreads; ++i) {
+      threads.emplace_back(threadFunction);
+    }
+
+    size_t numPopped = 0;
+
+    try {
+      // The usual check as always, but at some point `pop` will throw, because
+      // exceptions were pushed to the queue.
+      while (auto opt = queue.pop()) {
+        ++numPopped;
+        EXPECT_LE(numPushed, numPopped + queueSize + 1 + numThreads);
+      }
+      FAIL() << "Should have thrown" << std::endl;
+    } catch (const IntegerException& i) {
+      EXPECT_LT(i.value_, numThreads);
+    }
+  };
+  runWithBothQueueTypes(runTest);
+}
+
+// ________________________________________________________________
+TEST(ThreadSafeQueue, DisablePush) {
+  auto runTest = [](Queue queue) {
+    std::atomic numPushed = 0;
+    auto push = makePush(queue);
+
+    auto threadFunction = [&numPushed, &push] {
+      while (true) {
+        // Push until the consumer calls `disablePush`.
+        if (!push(numPushed++)) {
+          return;
+        }
+      }
+    };
+
+    std::vector threads;
+    for (size_t i = 0; i < numThreads; ++i) {
+      threads.emplace_back(threadFunction);
+    }
+
+    size_t numPopped = 0;
+    std::vector result;
+    while (auto opt = queue.pop()) {
+      ++numPopped;
+      result.push_back(opt.value());
+      EXPECT_LE(numPushed, numPopped + 6 + numThreads);
+
+      // Disable the push, make the consumers finish.
+      if (numPopped == 400) {
+        queue.disablePush();
+        break;
+      }
+    }
+    if (ad_utility::similarToInstantiation) {
+      // When terminating early, we cannot actually say much about the result,
+      // other than that it contains no duplicate values
+      std::ranges::sort(result);
+      EXPECT_TRUE(std::unique(result.begin(), result.end()) == result.end());
+    } else {
+      // For the ordered queue we have the guarantee that all the pushed values
+      // were in order.
+      EXPECT_THAT(result,
+                  ::testing::ElementsAreArray(std::views::iota(0U, 400U)));
+    }
+  };
+  runWithBothQueueTypes(runTest);
+}

From 00ea579aeec67c128ec22531df20f9048c6838a7 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 26 Jun 2023 14:06:10 +0200
Subject: [PATCH 061/150] Got rid of some code smells in the threadsafe queue.

---
 src/util/ThreadSafeQueue.h | 75 ++++++++++++++++++++++----------------
 1 file changed, 44 insertions(+), 31 deletions(-)

diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index a82b0bc0d3..108c376710 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -18,31 +18,37 @@ namespace ad_utility::data_structures {
 template 
 class ThreadSafeQueue {
   std::exception_ptr pushedException_;
-  std::queue _queue;
-  std::mutex _mutex;
-  std::condition_variable _pushNotification;
-  std::condition_variable _popNotification;
-  bool _lastElementPushed = false;
-  bool _pushDisabled = false;
-  size_t _maxSize;
+  std::queue queue_;
+  std::mutex mutex_;
+  std::condition_variable pushNotification_;
+  std::condition_variable popNotification_;
+  bool lastElementPushed_ = false;
+  bool pushDisabled_ = false;
+  size_t maxSize_;
 
  public:
-  explicit ThreadSafeQueue(size_t maxSize) : _maxSize{maxSize} {}
+  explicit ThreadSafeQueue(size_t maxSize) : maxSize_{maxSize} {}
+
+  // We can neither copy nor move this class
+  ThreadSafeQueue(const ThreadSafeQueue&) = delete;
+  const ThreadSafeQueue& operator=(const ThreadSafeQueue&) = delete;
+  ThreadSafeQueue(ThreadSafeQueue&&) = delete;
+  const ThreadSafeQueue& operator=(ThreadSafeQueue&&) = delete;
 
   /// Push an element into the queue. Block until there is free space in the
   /// queue or until disablePush() was called. Return false if disablePush()
   /// was called. In this case the current element element abd akk future
   /// elements are not added to the queue.
   bool push(T value) {
-    std::unique_lock lock{_mutex};
-    _popNotification.wait(
-        lock, [&] { return _queue.size() < _maxSize || _pushDisabled; });
-    if (_pushDisabled) {
+    std::unique_lock lock{mutex_};
+    popNotification_.wait(
+        lock, [this] { return queue_.size() < maxSize_ || pushDisabled_; });
+    if (pushDisabled_) {
       return false;
     }
-    _queue.push(std::move(value));
+    queue_.push(std::move(value));
     lock.unlock();
-    _pushNotification.notify_one();
+    pushNotification_.notify_one();
     return true;
   }
 
@@ -50,29 +56,29 @@ class ThreadSafeQueue {
   // to `pop` will throw the `exception`, and all subsequent calls to `push`
   // will return `false`.
   void pushException(std::exception_ptr exception) {
-    std::unique_lock lock{_mutex};
+    std::unique_lock lock{mutex_};
     pushedException_ = std::move(exception);
-    _pushDisabled = true;
+    pushDisabled_ = true;
     lock.unlock();
-    _pushNotification.notify_all();
-    _popNotification.notify_all();
+    pushNotification_.notify_all();
+    popNotification_.notify_all();
   }
 
   /// Signals all threads waiting for pop() to return that data transmission
   /// has ended and it should stop processing.
   void signalLastElementWasPushed() {
-    std::unique_lock lock{_mutex};
-    _lastElementPushed = true;
+    std::unique_lock lock{mutex_};
+    lastElementPushed_ = true;
     lock.unlock();
-    _pushNotification.notify_all();
+    pushNotification_.notify_all();
   }
 
   /// Wakes up all blocked threads waiting for push, cancelling execution
   void disablePush() {
-    std::unique_lock lock{_mutex};
-    _pushDisabled = true;
+    std::unique_lock lock{mutex_};
+    pushDisabled_ = true;
     lock.unlock();
-    _popNotification.notify_all();
+    popNotification_.notify_all();
   }
 
   /// Always call `disablePush` on destruction. This makes sure that worker
@@ -85,20 +91,20 @@ class ThreadSafeQueue {
   /// hen returned or signalLastElementWasPushed() is called resulting in an
   /// empty optional, whatever happens first
   std::optional pop() {
-    std::unique_lock lock{_mutex};
-    _pushNotification.wait(lock, [&] {
-      return !_queue.empty() || _lastElementPushed || pushedException_;
+    std::unique_lock lock{mutex_};
+    pushNotification_.wait(lock, [this] {
+      return !queue_.empty() || lastElementPushed_ || pushedException_;
     });
     if (pushedException_) {
       std::rethrow_exception(pushedException_);
     }
-    if (_lastElementPushed && _queue.empty()) {
+    if (lastElementPushed_ && queue_.empty()) {
       return {};
     }
-    std::optional value = std::move(_queue.front());
-    _queue.pop();
+    std::optional value = std::move(queue_.front());
+    queue_.pop();
     lock.unlock();
-    _popNotification.notify_one();
+    popNotification_.notify_one();
     return value;
   }
 };
@@ -125,6 +131,13 @@ class OrderedThreadSafeQueue {
   // Construct from the maximal queue size (see `ThreadSafeQueue` for details).
   explicit OrderedThreadSafeQueue(size_t maxSize) : queue_{maxSize} {}
 
+  // We can neither copy nor move this class
+  OrderedThreadSafeQueue(const OrderedThreadSafeQueue&) = delete;
+  const OrderedThreadSafeQueue& operator=(const OrderedThreadSafeQueue&) =
+      delete;
+  OrderedThreadSafeQueue(OrderedThreadSafeQueue&&) = delete;
+  const OrderedThreadSafeQueue& operator=(OrderedThreadSafeQueue&&) = delete;
+
   // Push the `value` to the queue that is associated with the `index`. This
   // call blocks, until `push` has been called for all indices in `[0, ...,
   // index - 1]` or until `disablePush` was called. The remaining behavior is

From 723c0431f773718142e4a63a0d7f335d4e3e2a27 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 26 Jun 2023 15:29:56 +0200
Subject: [PATCH 062/150] Got rid of a lot of code duplication

---
 src/engine/IndexScan.cpp         | 149 +++++++++--------------
 src/engine/IndexScan.h           |   4 +
 src/index/CompressedRelation.cpp | 198 ++++++++++++-------------------
 src/index/CompressedRelation.h   |  20 ++--
 src/index/Permutation.h          |   8 +-
 5 files changed, 149 insertions(+), 230 deletions(-)

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index 122604b014..264bb9d431 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -270,121 +270,78 @@ void IndexScan::computeFullScan(IdTable* result,
   *result = std::move(table).toDynamic();
 }
 
-// __________________________________________________________________________________________________________
+// ___________________________________________________________________________
+std::array IndexScan::getPermutedTriple()
+    const {
+  std::array triple{&subject_, &predicate_, &object_};
+  auto permutation = Permutation::toKeyOrder(permutation_);
+  return {triple[permutation[0]], triple[permutation[1]],
+          triple[permutation[2]]};
+}
+
+// TODO include a timeout timer.
+cppcoro::generator IndexScan::getLazyScan(const IndexScan& s,
+                                                   const auto& blocks) {
+  const IndexImpl& index = s.getIndex().getImpl();
+  Id col0Id = s.getPermutedTriple()[0]->toValueId(index.getVocab()).value();
+  std::optional col1Id;
+  if (s.numVariables_ == 1) {
+    col1Id = s.getPermutedTriple()[1]->toValueId(index.getVocab()).value();
+  }
+  if (!col1Id.has_value()) {
+    return index.getPermutation(s.permutation()).lazyScan(col0Id, blocks);
+  } else {
+    return index.getPermutation(s.permutation())
+        .lazyScan(col0Id, col1Id.value(), blocks);
+  }
+};
+
+// ________________________________________________________________
+auto IndexScan::getMetadataForScan(const IndexScan& s)
+    -> std::optional {
+  auto permutedTriple = s.getPermutedTriple();
+  const IndexImpl& index = s.getIndex().getImpl();
+  std::optional optId = permutedTriple[0]->toValueId(index.getVocab());
+  std::optional optId2 =
+      s.numVariables_ == 2 ? std::nullopt
+                           : permutedTriple[1]->toValueId(index.getVocab());
+  if (!optId.has_value() || (!optId2.has_value() && s.numVariables_ == 1)) {
+    return std::nullopt;
+  }
+
+  return index.getPermutation(s.permutation())
+      .getMetadataAndBlocks(optId.value(), optId2);
+};
+
+// ________________________________________________________________
 std::array, 2> IndexScan::lazyScanForJoinOfTwoScans(
     const IndexScan& s1, const IndexScan& s2) {
-  const auto& index = s1.getExecutionContext()->getIndex().getImpl();
   AD_CONTRACT_CHECK(s1.numVariables_ < 3 && s2.numVariables_ < 3);
 
-  auto f = [&](const IndexScan& s)
-      -> std::optional<
-          std::pair>> {
-    auto permutedTriple = s.getPermutedTriple();
-    std::optional optId = permutedTriple[0]->toValueId(index.getVocab());
-    std::optional optId2 =
-        s.numVariables_ == 2 ? std::nullopt
-                             : permutedTriple[1]->toValueId(index.getVocab());
-    if (!optId.has_value() || (!optId2.has_value() && s.numVariables_ == 1)) {
-      return std::nullopt;
-    }
-    auto optionalBla = index.getPermutation(s.permutation())
-                           .getMetadataAndBlocks(optId.value(), optId2);
-    if (optionalBla.has_value()) {
-      return std::pair{std::move(optionalBla.value()), optId2};
-    }
-    return std::nullopt;
-  };
-
-  auto metaBlocks1 = f(s1);
-  auto metaBlocks2 = f(s2);
+  auto metaBlocks1 = getMetadataForScan(s1);
+  auto metaBlocks2 = getMetadataForScan(s2);
 
   if (!metaBlocks1.has_value() || !metaBlocks2.has_value()) {
     return {{}};
   }
   auto [blocks1, blocks2] = CompressedRelationReader::getBlocksForJoin(
-      metaBlocks1.value().first.relationMetadata_,
-      metaBlocks2.value().first.relationMetadata_, metaBlocks1.value().second,
-      metaBlocks2.value().second, metaBlocks1.value().first.blockMetadata_,
-      metaBlocks2.value().first.blockMetadata_);
-
-  // TODO include a timeout timer.
-  auto getScan = [&index](const IndexScan& s, const auto& blocks) {
-    // TODO pass the IDs here.
-    Id col0Id = s.getPermutedTriple()[0]->toValueId(index.getVocab()).value();
-    std::optional col1Id;
-    if (s.numVariables_ == 1) {
-      col1Id = s.getPermutedTriple()[1]->toValueId(index.getVocab()).value();
-    }
-    if (!col1Id.has_value()) {
-      return index.getPermutation(s.permutation()).lazyScan(col0Id, blocks);
-    } else {
-      return index.getPermutation(s.permutation())
-          .lazyScan(col0Id, col1Id.value(), blocks);
-    }
-  };
+      metaBlocks1.value(), metaBlocks2.value());
 
-  return {getScan(s1, blocks1), getScan(s2, blocks2)};
+  return {getLazyScan(s1, blocks1), getLazyScan(s2, blocks2)};
 }
 
-// TODO There is a lot of duplication between this and the previous
-// function.
-// __________________________________________________________________________________________________________
+// ________________________________________________________________
 cppcoro::generator IndexScan::lazyScanForJoinOfColumnWithScan(
     std::span joinColumn, const IndexScan& s) {
-  const auto& index = s.getExecutionContext()->getIndex().getImpl();
   AD_CONTRACT_CHECK(s.numVariables_ < 3);
 
-  auto f = [&](const IndexScan& s)
-      -> std::optional<
-          std::pair>> {
-    auto permutedTriple = s.getPermutedTriple();
-    std::optional optId = permutedTriple[0]->toValueId(index.getVocab());
-    std::optional optId2 =
-        s.numVariables_ == 2 ? std::nullopt
-                             : permutedTriple[1]->toValueId(index.getVocab());
-    if (!optId.has_value() || (!optId2.has_value() && s.numVariables_ == 1)) {
-      return std::nullopt;
-    }
-    auto optionalBla = index.getPermutation(s.permutation())
-                           .getMetadataAndBlocks(optId.value(), optId2);
-    if (optionalBla.has_value()) {
-      return std::pair{std::move(optionalBla.value()), optId2};
-    }
-    return std::nullopt;
-  };
-
-  auto metaBlocks1 = f(s);
+  auto metaBlocks1 = getMetadataForScan(s);
 
   if (!metaBlocks1.has_value()) {
     return {};
   }
-  auto blocks = CompressedRelationReader::getBlocksForJoin(
-      joinColumn, metaBlocks1.value().first.relationMetadata_,
-      metaBlocks1.value().second, metaBlocks1.value().first.blockMetadata_);
-
-  // TODO include a timeout timer.
-  auto getScan = [&index](const IndexScan& s, const auto& blocks) {
-    // TODO pass the IDs here.
-    Id col0Id = s.getPermutedTriple()[0]->toValueId(index.getVocab()).value();
-    std::optional col1Id;
-    if (s.numVariables_ == 1) {
-      col1Id = s.getPermutedTriple()[1]->toValueId(index.getVocab()).value();
-    }
-    if (!col1Id.has_value()) {
-      return index.getPermutation(s.permutation()).lazyScan(col0Id, blocks);
-    } else {
-      return index.getPermutation(s.permutation())
-          .lazyScan(col0Id, col1Id.value(), blocks);
-    }
-  };
-  return getScan(s, blocks);
-}
+  auto blocks = CompressedRelationReader::getBlocksForJoin(joinColumn,
+                                                           metaBlocks1.value());
 
-// ___________________________________________________________________________
-std::array IndexScan::getPermutedTriple()
-    const {
-  std::array triple{&subject_, &predicate_, &object_};
-  auto permutation = Permutation::toKeyOrder(permutation_);
-  return {triple[permutation[0]], triple[permutation[1]],
-          triple[permutation[2]]};
+  return getLazyScan(s, blocks);
 }
diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h
index 4868f5a7cd..4f16db8c75 100644
--- a/src/engine/IndexScan.h
+++ b/src/engine/IndexScan.h
@@ -49,6 +49,10 @@ class IndexScan : public Operation {
       const IndexScan& s1, const IndexScan& s2);
   static cppcoro::generator lazyScanForJoinOfColumnWithScan(
       std::span joinColumn, const IndexScan& s);
+  static cppcoro::generator getLazyScan(const IndexScan& s,
+                                                 const auto& blocks);
+  static auto getMetadataForScan(const IndexScan& s)
+      -> std::optional;
 
  private:
   // TODO Make the `getSizeEstimateBeforeLimit()` function `const` for
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 0568d4fd62..44a703ce35 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -140,6 +140,56 @@ void CompressedRelationReader::scan(
   }
 }
 
+cppcoro::generator
+CompressedRelationReader::asyncParallelBlockGenerator(
+    auto beginBlock, auto endBlock, ad_utility::File& file,
+    std::optional> columnIndices) const {
+  if (beginBlock == endBlock) {
+    co_return;
+  }
+  ad_utility::data_structures::OrderedThreadSafeQueue queue{
+      5};
+  // TODO We can configure whether we want to allow async access to
+  // the file.
+  std::mutex fileMutex;
+  size_t blockIndex = 0;
+  auto readAndDecompressBlock = [&]() {
+    while (true) {
+      std::unique_lock lock{fileMutex};
+      if (beginBlock == endBlock) {
+        return;
+      }
+      auto block = *beginBlock;
+      CompressedBlock compressedBuffer =
+          readCompressedBlockFromFile(block, file, columnIndices);
+      ++beginBlock;
+      size_t myIndex = blockIndex;
+      ++blockIndex;
+      bool isLastBlock = beginBlock == endBlock;
+      lock.unlock();
+      bool success = queue.push(
+          myIndex, decompressBlock(compressedBuffer, block.numRows_));
+      if (!success) {
+        return;
+      }
+      if (isLastBlock) {
+        queue.signalLastElementWasPushed();
+      }
+    }
+  };
+
+  std::vector threads;
+  // TODO get the number of optimal threads from the operating
+  // system.
+  for (size_t j = 0; j < 5; ++j) {
+    threads.emplace_back(readAndDecompressBlock);
+  }
+
+  while (auto opt = queue.pop()) {
+    co_yield opt.value();
+  }
+}
+
 // _____________________________________________________________________________
 cppcoro::generator CompressedRelationReader::lazyScan(
     CompressedRelationMetadata metadata,
@@ -211,49 +261,9 @@ cppcoro::generator CompressedRelationReader::lazyScan(
     }
   }
 
-  if (beginBlock == endBlock) {
-    co_return;
-  }
-
-  ad_utility::data_structures::OrderedThreadSafeQueue queue{
-      5};
-  // TODO We can configure whether we want to allow async access to the
-  // file.
-  std::mutex fileMutex;
-  size_t blockIndex = 0;
-  auto readAndDecompressBlock = [&]() {
-    while (true) {
-      std::unique_lock lock{fileMutex};
-      if (beginBlock == endBlock) {
-        return;
-      }
-      auto block = *beginBlock;
-      CompressedBlock compressedBuffer =
-          readCompressedBlockFromFile(block, file, std::nullopt);
-      ++beginBlock;
-      size_t myIndex = blockIndex;
-      ++blockIndex;
-      bool isLastBlock = beginBlock == endBlock;
-      lock.unlock();
-      bool success = queue.push(
-          myIndex, decompressBlock(compressedBuffer, block.numRows_));
-      if (!success) {
-        return;
-      }
-      if (isLastBlock) {
-        queue.signalLastElementWasPushed();
-      }
-    }
-  };
-
-  std::vector threads;
-  // TODO get the number of optimal threads from the operating system.
-  for (size_t j = 0; j < 5; ++j) {
-    threads.emplace_back(readAndDecompressBlock);
-  }
-
-  while (auto opt = queue.pop()) {
-    co_yield opt.value();
+  for (auto& block :
+       asyncParallelBlockGenerator(beginBlock, endBlock, file, std::nullopt)) {
+    co_yield block;
   }
 
   // TODO Timeout checks.
@@ -306,61 +316,11 @@ cppcoro::generator CompressedRelationReader::lazyScan(
     co_yield std::move(firstBlockResult.value());
   }
 
-  if (beginBlock != endBlock) {
-    ad_utility::data_structures::OrderedThreadSafeQueue
-        queue{5};
-    // TODO We can configure whether we want to allow async access to
-    // the file.
-    std::mutex fileMutex;
-    size_t blockIndex = 0;
-    auto readAndDecompressBlock = [&]() {
-      while (true) {
-        std::unique_lock lock{fileMutex};
-        if (beginBlock == endBlock) {
-          return;
-        }
-        auto block = *beginBlock;
-        CompressedBlock compressedBuffer =
-            readCompressedBlockFromFile(block, file, std::vector{1ul});
-        ++beginBlock;
-        size_t myIndex = blockIndex;
-        ++blockIndex;
-        bool isLastBlock = beginBlock == endBlock;
-        lock.unlock();
-        bool success = queue.push(
-            myIndex, decompressBlock(compressedBuffer, block.numRows_));
-        if (!success) {
-          return;
-        }
-        if (isLastBlock) {
-          queue.signalLastElementWasPushed();
-        }
-      }
-    };
-
-    std::vector threads;
-    // TODO get the number of optimal threads from the operating
-    // system.
-    for (size_t j = 0; j < 5; ++j) {
-      threads.emplace_back(readAndDecompressBlock);
-    }
-
-    while (auto opt = queue.pop()) {
-      co_yield opt.value();
-    }
+  for (auto& block : asyncParallelBlockGenerator(beginBlock, endBlock, file,
+                                                 std::vector{1ul})) {
+    co_yield block;
   }
 
-  /*
-  for (; beginBlock < endBlock; ++beginBlock) {
-    const auto& block = *beginBlock;
-
-    // Read the block serially, only read the second column.
-    AD_CORRECTNESS_CHECK(block.offsetsAndCompressedSize_.size() == 2);
-    CompressedBlock compressedBuffer =
-        readCompressedBlockFromFile(block, file, std::vector{1ul});
-    co_yield decompressBlock(compressedBuffer, block.numRows_);
-  }
-   */
   if (timer) {
     timer->wlock()->checkTimeoutAndThrow();
   }
@@ -411,14 +371,14 @@ auto blocksToIdRanges = [](std::span blocks,
 
 // _____________________________________________________________________________
 std::vector CompressedRelationReader::getBlocksForJoin(
-    std::span joinColum, const CompressedRelationMetadata& metadata,
-    std::optional col1Id,
-    std::span blockMetadata) {
+    std::span joinColum, const MetadataAndBlocks& scanMetadata) {
   // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_
+  const auto& [relationMetadata, blockMetadata, col1Id] = scanMetadata;
   auto relevantBlocks =
-      getBlocksFromMetadata(metadata, std::nullopt, blockMetadata);
+      getBlocksFromMetadata(relationMetadata, std::nullopt, blockMetadata);
 
-  auto idRanges = blocksToIdRanges(relevantBlocks, metadata.col0Id_, col1Id);
+  auto idRanges =
+      blocksToIdRanges(relevantBlocks, relationMetadata.col0Id_, col1Id);
 
   auto idLessThanBlock = [](Id id, const IdPair& block) {
     return id < block.first;
@@ -454,11 +414,8 @@ std::vector CompressedRelationReader::getBlocksForJoin(
 // _____________________________________________________________________________
 std::array, 2>
 CompressedRelationReader::getBlocksForJoin(
-    const CompressedRelationMetadata& md1,
-    const CompressedRelationMetadata& md2, std::optional col1Id1,
-    std::optional col1Id2,
-    std::span blockMetadata1,
-    std::span blockMetadata2) {
+    const MetadataAndBlocks& scanMetadata1,
+    const MetadataAndBlocks& scanMetadata2) {
   // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_
   struct KeyLhs {
     Id col0FirstId_;
@@ -466,28 +423,27 @@ CompressedRelationReader::getBlocksForJoin(
   };
 
   auto relevantBlocks1 =
-      getBlocksFromMetadata(md1, std::nullopt, blockMetadata1);
-  auto beginBlock1 = relevantBlocks1.begin();
-  auto endBlock1 = relevantBlocks1.end();
-
+      getBlocksFromMetadata(scanMetadata1.relationMetadata_, std::nullopt,
+                            scanMetadata1.blockMetadata_);
   auto relevantBlocks2 =
-      getBlocksFromMetadata(md2, std::nullopt, blockMetadata2);
-  auto beginBlock2 = relevantBlocks2.begin();
-  auto endBlock2 = relevantBlocks2.end();
+      getBlocksFromMetadata(scanMetadata2.relationMetadata_, std::nullopt,
+                            scanMetadata2.blockMetadata_);
 
   auto blockLessThanBlock = [](const auto& block1, const auto& block2) {
     return block1.second < block2.first;
   };
 
   std::array, 2> result;
-  auto idRanges1 = blocksToIdRanges(
-      std::ranges::subrange{beginBlock1, endBlock1}, md1.col0Id_, col1Id1);
-  auto idRanges2 = blocksToIdRanges(
-      std::ranges::subrange{beginBlock2, endBlock2}, md2.col0Id_, col1Id2);
-  auto addRow = [&result, &beginBlock1, &beginBlock2, &idRanges1, &idRanges2](
-                    [[maybe_unused]] auto it1, auto it2) {
-    result[0].push_back(*(beginBlock1 + (it1 - idRanges1.begin())));
-    result[1].push_back(*(beginBlock2 + (it2 - idRanges2.begin())));
+  auto idRanges1 =
+      blocksToIdRanges(relevantBlocks1, scanMetadata1.relationMetadata_.col0Id_,
+                       scanMetadata1.col1Id_);
+  auto idRanges2 =
+      blocksToIdRanges(relevantBlocks2, scanMetadata2.relationMetadata_.col0Id_,
+                       scanMetadata2.col1Id_);
+  auto addRow = [&result, &relevantBlocks1, &relevantBlocks2, &idRanges1,
+                 &idRanges2]([[maybe_unused]] auto it1, auto it2) {
+    result[0].push_back(*(relevantBlocks1.begin() + (it1 - idRanges1.begin())));
+    result[1].push_back(*(relevantBlocks2.begin() + (it2 - idRanges2.begin())));
   };
 
   auto noop = ad_utility::noop;
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index 9f201949a6..be2feb7a32 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -270,17 +270,18 @@ class CompressedRelationReader {
 
   // Get all the blocks that can contain an Id from the `joinColumn`.
   // TODO Include a timeout check.
+
+  struct MetadataAndBlocks {
+    const CompressedRelationMetadata relationMetadata_;
+    std::span blockMetadata_;
+    std::optional col1Id_;
+  };
   static std::vector getBlocksForJoin(
-      std::span joinColum, const CompressedRelationMetadata& metadata,
-      std::optional col1Id,
-      std::span blockMetadata);
+      std::span joinColum, const MetadataAndBlocks& scanMetadata);
 
   static std::array, 2> getBlocksForJoin(
-      const CompressedRelationMetadata& md1,
-      const CompressedRelationMetadata& md2, std::optional col1Id1,
-      std::optional col1Id2,
-      std::span blockMetadata1,
-      std::span blockMetadata2);
+      const MetadataAndBlocks& scanMetadata1,
+      const MetadataAndBlocks& scanMetadata2);
 
   cppcoro::generator lazyScan(
       CompressedRelationMetadata metadata,
@@ -293,6 +294,9 @@ class CompressedRelationReader {
       std::vector blockMetadata,
       ad_utility::File& file,
       ad_utility::SharedConcurrentTimeoutTimer timer) const;
+  cppcoro::generator asyncParallelBlockGenerator(
+      auto beginBlock, auto endBlock, ad_utility::File& file,
+      std::optional> columnIndices) const;
 
   /**
    * @brief For a permutation XYZ, retrieve all Z for given X and Y.
diff --git a/src/index/Permutation.h b/src/index/Permutation.h
index 1a4b628cfd..02d7febfc7 100644
--- a/src/index/Permutation.h
+++ b/src/index/Permutation.h
@@ -45,10 +45,7 @@ class Permutation {
   // everything that has to be done when reading an index from disk
   void loadFromDisk(const std::string& onDiskBase);
 
-  struct MetaDataAndBlocks {
-    const CompressedRelationMetadata relationMetadata_;
-    std::span blockMetadata_;
-  };
+  using MetaDataAndBlocks = CompressedRelationReader::MetadataAndBlocks;
 
   std::optional getMetadataAndBlocks(
       Id col0Id, std::optional col1Id) const {
@@ -59,7 +56,8 @@ class Permutation {
     auto metadata = meta_.getMetaData(col0Id);
     return MetaDataAndBlocks{
         meta_.getMetaData(col0Id),
-        reader_.getBlocksFromMetadata(metadata, col1Id, meta_.blockData())};
+        reader_.getBlocksFromMetadata(metadata, col1Id, meta_.blockData()),
+        col1Id};
   }
 
   cppcoro::generator lazyScan(

From 08463691867b13af967ecce68a9a0a792fa20618 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 26 Jun 2023 15:59:20 +0200
Subject: [PATCH 063/150] Use 10 blocks.

---
 src/index/CompressedRelation.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 44a703ce35..2c1167fd66 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -181,7 +181,7 @@ CompressedRelationReader::asyncParallelBlockGenerator(
   std::vector threads;
   // TODO get the number of optimal threads from the operating
   // system.
-  for (size_t j = 0; j < 5; ++j) {
+  for (size_t j = 0; j < 10; ++j) {
     threads.emplace_back(readAndDecompressBlock);
   }
 

From 31e2c501bb364ebb32e915a0d8c4da768122abca Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 26 Jun 2023 18:07:17 +0200
Subject: [PATCH 064/150] Make the tests more robust for the threadsafe queue.

---
 test/ThreadSafeQueueTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 7281ed1e25..064f79f14a 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -162,7 +162,7 @@ TEST(ThreadSafeQueue, PushException) {
               std::make_exception_ptr(IntegerException{threadIndex++}));
           EXPECT_FALSE(push(numPushed++));
         } else if (hasThrown) {
-          EXPECT_FALSE(push(numPushed++));
+          EXPECT_FALSE(push(0));
         } else {
           // We cannot know whether this returns true or false, because another
           // thread already might have thrown an exception.

From ce5839824fd3be91107a85651a422f1141c7f101 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 26 Jun 2023 18:16:49 +0200
Subject: [PATCH 065/150] Make the tests pass consistently.

---
 test/ThreadSafeQueueTest.cpp | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 7281ed1e25..2d706dfa90 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -162,7 +162,15 @@ TEST(ThreadSafeQueue, PushException) {
               std::make_exception_ptr(IntegerException{threadIndex++}));
           EXPECT_FALSE(push(numPushed++));
         } else if (hasThrown) {
-          EXPECT_FALSE(push(numPushed++));
+          // In the case that we have previously thrown an exception, we know
+          // that the queue is disabled. This means that we can safely push an
+          // out-of-order value even to the ordered queue. Note that we
+          // deliberately do not push `numPushed++` as usual, because otherwise
+          // we cannot say much about the value of `numPushed` after throwing
+          // the first exception. Note that this pattern is only for testing,
+          // and that a thread that has pushed an exception to a queue should
+          // stop pushing to the same queue in real life.
+          EXPECT_FALSE(push(0));
         } else {
           // We cannot know whether this returns true or false, because another
           // thread already might have thrown an exception.

From 519d7978ec0244daea9fa320b70793f87af4b377 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 26 Jun 2023 20:24:49 +0200
Subject: [PATCH 066/150] Small changes

---
 src/util/ThreadSafeQueue.h   | 4 +---
 test/CMakeLists.txt          | 2 +-
 test/ThreadSafeQueueTest.cpp | 2 +-
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index 108c376710..8ea1bfd4d7 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -10,8 +10,6 @@
 #include 
 #include 
 
-#include "absl/synchronization/mutex.h"
-
 namespace ad_utility::data_structures {
 
 /// A thread safe, multi-consumer, multi-producer queue.
@@ -110,7 +108,7 @@ class ThreadSafeQueue {
 };
 
 // A thread safe queue that is similar (wrt the interface and the behavior) to
-// the `ThreadSafeQueue` above, with the following differente: Each element that
+// the `ThreadSafeQueue` above, with the following difference: Each element that
 // is pushed is associated with a unique index `n`. A call to `push(n,
 // someValue)` will block until other threads have pushed all indices in the
 // range [0, ..., n - 1]. This can be used to enforce the ordering of values
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index eaa23b3f75..55ae0711b7 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -330,4 +330,4 @@ addLinkAndDiscoverTest(CtreHelpersTest)
 
 addLinkAndDiscoverTest(ComparisonWithNanTest)
 
-addLinkAndDiscoverTest(ThreadSafeQueueTest)
\ No newline at end of file
+addLinkAndDiscoverTest(ThreadSafeQueueTest)
diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 2d706dfa90..753c9ace91 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -226,7 +226,7 @@ TEST(ThreadSafeQueue, DisablePush) {
     while (auto opt = queue.pop()) {
       ++numPopped;
       result.push_back(opt.value());
-      EXPECT_LE(numPushed, numPopped + 6 + numThreads);
+      EXPECT_LE(numPushed, numPopped + queueSize + 1 + numThreads);
 
       // Disable the push, make the consumers finish.
       if (numPopped == 400) {

From b6e94ee3bf233c4eaea948815b46091f7c7f1f66 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 27 Jun 2023 09:01:13 +0200
Subject: [PATCH 067/150] Fixed the error from yesterday.

---
 src/index/CompressedRelation.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 2c1167fd66..0b2fcee121 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -147,6 +147,9 @@ CompressedRelationReader::asyncParallelBlockGenerator(
   if (beginBlock == endBlock) {
     co_return;
   }
+  // Note: It is important to define the `threads` before the `queue`. That way
+  // the joining destructor of the threads will see that the queue is finished and join.
+  std::vector threads;
   ad_utility::data_structures::OrderedThreadSafeQueue queue{
       5};
   // TODO We can configure whether we want to allow async access to
@@ -178,7 +181,6 @@ CompressedRelationReader::asyncParallelBlockGenerator(
     }
   };
 
-  std::vector threads;
   // TODO get the number of optimal threads from the operating
   // system.
   for (size_t j = 0; j < 10; ++j) {

From b164415c658a17011d9d7fd97fdbbe6b3cbe2912 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 27 Jun 2023 10:30:51 +0200
Subject: [PATCH 068/150] Simplified the interface.

---
 src/util/ThreadSafeQueue.h   | 54 ++++++++++++++++--------------------
 test/ThreadSafeQueueTest.cpp | 10 +++----
 2 files changed, 29 insertions(+), 35 deletions(-)

diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index 8ea1bfd4d7..fe9ec1f42a 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -20,8 +20,7 @@ class ThreadSafeQueue {
   std::mutex mutex_;
   std::condition_variable pushNotification_;
   std::condition_variable popNotification_;
-  bool lastElementPushed_ = false;
-  bool pushDisabled_ = false;
+  bool queueDisabled_ = false;
   size_t maxSize_;
 
  public:
@@ -34,14 +33,14 @@ class ThreadSafeQueue {
   const ThreadSafeQueue& operator=(ThreadSafeQueue&&) = delete;
 
   /// Push an element into the queue. Block until there is free space in the
-  /// queue or until disablePush() was called. Return false if disablePush()
-  /// was called. In this case the current element element abd akk future
+  /// queue or until disableQueue() was called. Return false if disableQueue()
+  /// was called. In this case the current element element and all future
   /// elements are not added to the queue.
   bool push(T value) {
     std::unique_lock lock{mutex_};
     popNotification_.wait(
-        lock, [this] { return queue_.size() < maxSize_ || pushDisabled_; });
-    if (pushDisabled_) {
+        lock, [this] { return queue_.size() < maxSize_ || queueDisabled_; });
+    if (queueDisabled_) {
       return false;
     }
     queue_.push(std::move(value));
@@ -56,34 +55,32 @@ class ThreadSafeQueue {
   void pushException(std::exception_ptr exception) {
     std::unique_lock lock{mutex_};
     pushedException_ = std::move(exception);
-    pushDisabled_ = true;
+    queueDisabled_ = true;
     lock.unlock();
     pushNotification_.notify_all();
     popNotification_.notify_all();
   }
 
-  /// Signals all threads waiting for pop() to return that data transmission
-  /// has ended and it should stop processing.
-  void signalLastElementWasPushed() {
+  // After calling this function, all calls to `push` will return `false` and no
+  // further elements will be added to the queue. Calls to `pop` will yield the
+  // elements that were already stored in the queue before the call to
+  // `disableQueue`, after those were drained, `pop` will return `nullopt`. This
+  // function can be called from the producing/pushing threads to signal that
+  // all elements have been pushed, or from the consumers to signal that they
+  // will not pop further elements from the queue.
+  void disableQueue() {
     std::unique_lock lock{mutex_};
-    lastElementPushed_ = true;
+    queueDisabled_ = true;
     lock.unlock();
     pushNotification_.notify_all();
-  }
-
-  /// Wakes up all blocked threads waiting for push, cancelling execution
-  void disablePush() {
-    std::unique_lock lock{mutex_};
-    pushDisabled_ = true;
-    lock.unlock();
     popNotification_.notify_all();
   }
 
-  /// Always call `disablePush` on destruction. This makes sure that worker
+  /// Always call `disableQueue` on destruction. This makes sure that worker
   /// threads that pop from the queue always see std::nullopt, even if the
   /// threads that push to the queue exit via an exception or if the explicit
-  /// call to `disablePush` is missing.
-  ~ThreadSafeQueue() { disablePush(); }
+  /// call to `disableQueue` is missing.
+  ~ThreadSafeQueue() { disableQueue(); }
 
   /// Blocks until another thread pushes an element via push() which is
   /// hen returned or signalLastElementWasPushed() is called resulting in an
@@ -91,12 +88,12 @@ class ThreadSafeQueue {
   std::optional pop() {
     std::unique_lock lock{mutex_};
     pushNotification_.wait(lock, [this] {
-      return !queue_.empty() || lastElementPushed_ || pushedException_;
+      return !queue_.empty() || queueDisabled_ || pushedException_;
     });
     if (pushedException_) {
       std::rethrow_exception(pushedException_);
     }
-    if (lastElementPushed_ && queue_.empty()) {
+    if (queueDisabled_ && queue_.empty()) {
       return {};
     }
     std::optional value = std::move(queue_.front());
@@ -138,7 +135,7 @@ class OrderedThreadSafeQueue {
 
   // Push the `value` to the queue that is associated with the `index`. This
   // call blocks, until `push` has been called for all indices in `[0, ...,
-  // index - 1]` or until `disablePush` was called. The remaining behavior is
+  // index - 1]` or until `disableQueue` was called. The remaining behavior is
   // equal to `ThreadSafeQueue::push`.
   bool push(size_t index, T value) {
     std::unique_lock lock{mutex_};
@@ -165,11 +162,8 @@ class OrderedThreadSafeQueue {
   }
 
   // See `ThreadSafeQueue` for details.
-  void signalLastElementWasPushed() { queue_.signalLastElementWasPushed(); }
-
-  // See `ThreadSafeQueue` for details.
-  void disablePush() {
-    queue_.disablePush();
+  void disableQueue() {
+    queue_.disableQueue();
     std::unique_lock lock{mutex_};
     pushWasDisabled_ = true;
     lock.unlock();
@@ -177,7 +171,7 @@ class OrderedThreadSafeQueue {
   }
 
   // See `ThreadSafeQueue` for details.
-  ~OrderedThreadSafeQueue() { disablePush(); }
+  ~OrderedThreadSafeQueue() { disableQueue(); }
 
   // See `ThreadSafeQueue` for details. All the returned values will be in
   // ascending consecutive order wrt the index with which they were pushed.
diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 753c9ace91..4e72a10d93 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -56,7 +56,7 @@ TEST(ThreadSafeQueue, BufferSizeIsRespected) {
       while (numPushed < numValues) {
         push(numPushed++);
       }
-      queue.signalLastElementWasPushed();
+      queue.disableQueue();
     });
 
     size_t numPopped = 0;
@@ -83,7 +83,7 @@ TEST(ThreadSafeQueue, ReturnValueOfPush) {
     // called.
     EXPECT_TRUE(push(0));
     EXPECT_EQ(queue.pop(), 0);
-    queue.disablePush();
+    queue.disableQueue();
     EXPECT_FALSE(push(1));
   };
   runWithBothQueueTypes(runTest);
@@ -105,7 +105,7 @@ TEST(ThreadSafeQueue, Concurrency) {
       }
       numThreadsDone++;
       if (numThreadsDone == numThreads) {
-        queue.signalLastElementWasPushed();
+        queue.disableQueue();
       }
     };
 
@@ -189,7 +189,7 @@ TEST(ThreadSafeQueue, PushException) {
     try {
       // The usual check as always, but at some point `pop` will throw, because
       // exceptions were pushed to the queue.
-      while (auto opt = queue.pop()) {
+      while (queue.pop()) {
         ++numPopped;
         EXPECT_LE(numPushed, numPopped + queueSize + 1 + numThreads);
       }
@@ -230,7 +230,7 @@ TEST(ThreadSafeQueue, DisablePush) {
 
       // Disable the push, make the consumers finish.
       if (numPopped == 400) {
-        queue.disablePush();
+        queue.disableQueue();
         break;
       }
     }

From 3e2a00f0c0e62eb14321fd3fcd16e88659e6b5f2 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 27 Jun 2023 11:06:37 +0200
Subject: [PATCH 069/150] Make it possible to only get the result from an
 operation if it can be read from the cache.

---
 src/engine/Operation.cpp | 17 +++++++++++++----
 src/engine/Operation.h   |  3 ++-
 test/OperationTest.cpp   | 36 +++++++++++++++++++++++++++++++++++-
 3 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/src/engine/Operation.cpp b/src/engine/Operation.cpp
index 0d35aedca3..16a02ec1b1 100644
--- a/src/engine/Operation.cpp
+++ b/src/engine/Operation.cpp
@@ -2,7 +2,7 @@
 // Chair of Algorithms and Data Structures.
 // Author: Johannes Kalmbach  (johannes.kalmbach@gmail.com)
 
-#include "./Operation.h"
+#include "engine/Operation.h"
 
 #include "engine/QueryExecutionTree.h"
 #include "util/OnDestructionDontThrowDuringStackUnwinding.h"
@@ -58,9 +58,9 @@ void Operation::recursivelySetTimeoutTimer(
   }
 }
 
-// Get the result for the subtree rooted at this element. Use existing results
-// if they are already available, otherwise trigger computation.
-shared_ptr Operation::getResult(bool isRoot) {
+// ________________________________________________________________________
+shared_ptr Operation::getResult(bool isRoot,
+                                                   bool onlyReadFromCache) {
   ad_utility::Timer timer{ad_utility::Timer::Started};
 
   if (isRoot) {
@@ -154,6 +154,15 @@ shared_ptr Operation::getResult(bool isRoot) {
       return CacheValue{std::move(result), getRuntimeInfo()};
     };
 
+    if (onlyReadFromCache) {
+      AD_CORRECTNESS_CHECK(!pinResult);
+      auto optionalResult = cache.getIfContained(cacheKey);
+      if (!optionalResult.has_value()) {
+        return nullptr;
+      }
+      updateRuntimeInformationOnSuccess(optionalResult.value(), timer.msecs());
+      return optionalResult.value()._resultPointer->resultTable();
+    }
     auto result = (pinResult) ? cache.computeOncePinned(cacheKey, computeLambda)
                               : cache.computeOnce(cacheKey, computeLambda);
 
diff --git a/src/engine/Operation.h b/src/engine/Operation.h
index 2c93a9e2fb..0a824d4487 100644
--- a/src/engine/Operation.h
+++ b/src/engine/Operation.h
@@ -119,7 +119,8 @@ class Operation {
   // Get the result for the subtree rooted at this element.
   // Use existing results if they are already available, otherwise
   // trigger computation.
-  shared_ptr getResult(bool isRoot = false);
+  shared_ptr getResult(bool isRoot = false,
+                                          bool onlyReadFromCache = false);
 
   // Use the same timeout timer for all children of an operation (= query plan
   // rooted at that operation). As soon as one child times out, the whole
diff --git a/test/OperationTest.cpp b/test/OperationTest.cpp
index 2dd4166e97..da1e6cd818 100644
--- a/test/OperationTest.cpp
+++ b/test/OperationTest.cpp
@@ -7,8 +7,11 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+using namespace ad_utility::testing;
+
+// ________________________________________________
 TEST(OperationTest, limitIsRepresentedInCacheKey) {
-  NeutralElementOperation n{ad_utility::testing::getQec()};
+  NeutralElementOperation n{getQec()};
   EXPECT_THAT(n.asString(), testing::Not(testing::HasSubstr("LIMIT 20")));
   LimitOffsetClause l;
   l._limit = 20;
@@ -20,3 +23,34 @@ TEST(OperationTest, limitIsRepresentedInCacheKey) {
   n.setLimit(l);
   EXPECT_THAT(n.asString(), testing::HasSubstr("OFFSET 34"));
 }
+
+// ________________________________________________
+TEST(OperationTest, getResultOnlyCached) {
+  auto qec = getQec();
+  qec->getQueryTreeCache().clearAll();
+  NeutralElementOperation n{qec};
+  // The second `true` means "only read the result if it was cached".
+  // We have just cleared the cache, and so this should return false.
+  EXPECT_EQ(n.getResult(true, true), nullptr);
+  EXPECT_EQ(n.getRuntimeInfo().status_, RuntimeInformation::Status::notStarted);
+  // Nothing has been stored in the cache by this call.
+  EXPECT_EQ(qec->getQueryTreeCache().numNonPinnedEntries(), 0);
+  EXPECT_EQ(qec->getQueryTreeCache().numPinnedEntries(), 0);
+
+  // This "ordinary" call to `getResult` also stores the result in the cache.
+  NeutralElementOperation n2{qec};
+  auto result = n2.getResult();
+  EXPECT_NE(result, nullptr);
+  EXPECT_EQ(n2.getRuntimeInfo().status_, RuntimeInformation::Status::completed);
+  EXPECT_EQ(n2.getRuntimeInfo().cacheStatus_,
+            ad_utility::CacheStatus::computed);
+  EXPECT_EQ(qec->getQueryTreeCache().numNonPinnedEntries(), 1);
+  EXPECT_EQ(qec->getQueryTreeCache().numPinnedEntries(), 0);
+
+  // When we now request to only return the result if it is cached, we should
+  // get exactly the same `shared_ptr` as with the previous call.
+  NeutralElementOperation n3{qec};
+  EXPECT_EQ(n3.getResult(true, true), result);
+  EXPECT_EQ(n3.getRuntimeInfo().cacheStatus_,
+            ad_utility::CacheStatus::cachedNotPinned);
+}

From 64b6652fe7928da91a0ba21fdd590403513d9d5f Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 27 Jun 2023 11:19:24 +0200
Subject: [PATCH 070/150] Add comments.

---
 src/engine/Operation.cpp |  2 ++
 src/engine/Operation.h   | 17 ++++++++++++++---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/engine/Operation.cpp b/src/engine/Operation.cpp
index 16a02ec1b1..79893c9db3 100644
--- a/src/engine/Operation.cpp
+++ b/src/engine/Operation.cpp
@@ -154,6 +154,8 @@ shared_ptr Operation::getResult(bool isRoot,
       return CacheValue{std::move(result), getRuntimeInfo()};
     };
 
+    // Handle the case that we only want the result if it was contained in the
+    // cache, and `nullptr` else.
     if (onlyReadFromCache) {
       AD_CORRECTNESS_CHECK(!pinResult);
       auto optionalResult = cache.getIfContained(cacheKey);
diff --git a/src/engine/Operation.h b/src/engine/Operation.h
index 0a824d4487..5202bfa2c6 100644
--- a/src/engine/Operation.h
+++ b/src/engine/Operation.h
@@ -116,9 +116,20 @@ class Operation {
     return _runtimeInfoWholeQuery;
   }
 
-  // Get the result for the subtree rooted at this element.
-  // Use existing results if they are already available, otherwise
-  // trigger computation.
+  /**
+   * @brief Get the result for the subtree rooted at this element. Use existing
+   * results if they are already available, otherwise trigger computation.
+   * Always returns a non-null pointer, except for when `onlyReadFromCache` is
+   * true (see below).
+   * @param isRoot Has be set to `true` iff this is the root operation of a
+   * complete query to obtain the expected behavior wrt cache pinning and
+   * runtime information in error cases.
+   * @param onlyReadFromCache If set to true the result is only returned if it
+   * can be read from the cache without any computation. If the result is not in
+   * the cache, `nullptr` will be returned.
+   * @return A shared pointer to the result. May only be `nullptr` if
+   * `onlyReadFromCache` is true.
+   */
   shared_ptr getResult(bool isRoot = false,
                                           bool onlyReadFromCache = false);
 

From 54e444493839724b1d6c8eebce49a7a4540ab4ce Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 27 Jun 2023 11:25:21 +0200
Subject: [PATCH 071/150] Fix the build.

---
 src/util/AsyncStream.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/util/AsyncStream.h b/src/util/AsyncStream.h
index 0f803dac3e..803c833595 100644
--- a/src/util/AsyncStream.h
+++ b/src/util/AsyncStream.h
@@ -40,14 +40,14 @@ cppcoro::generator runStreamAsync(
     } catch (...) {
       exception = std::current_exception();
     }
-    queue.signalLastElementWasPushed();
+    queue.disableQueue();
   }};
 
   // Only rethrow an exception from the `thread` if no exception occured in this
   // thread to avoid crashes because of multiple active exceptions.
   auto cleanup = ad_utility::makeOnDestructionDontThrowDuringStackUnwinding(
       [&queue, &thread, &exception] {
-        queue.disablePush();
+        queue.disableQueue();
         thread.join();
         // This exception will only be thrown once all the values that were
         // pushed to the queue before the exception occurred.

From 50741d04f39c699904d09536fdb56c008c827081 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 27 Jun 2023 11:40:46 +0200
Subject: [PATCH 072/150] Hopefully fix a bug on mac, and addressed some code
 smells.

---
 src/engine/Join.cpp                      | 12 +++++-------
 src/engine/Join.h                        |  4 ++--
 src/index/CompressedRelation.cpp         | 24 +++++++++++-------------
 src/index/Permutation.h                  |  8 ++++----
 src/util/JoinAlgorithms/JoinAlgorithms.h |  2 --
 5 files changed, 22 insertions(+), 28 deletions(-)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index e4da7c9a3f..1aed5fb283 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -630,13 +630,13 @@ void Join::addCombinedRowToIdTable(const ROW_A& rowA, const ROW_B& rowB,
 }
 
 // ______________________________________________________________________________________________________
-void Join::computeResultForTwoIndexScans(IdTable* resultPtr) {
+void Join::computeResultForTwoIndexScans(IdTable* resultPtr) const {
   AD_CORRECTNESS_CHECK(_left->getType() == QueryExecutionTree::SCAN &&
                        _right->getType() == QueryExecutionTree::SCAN);
   auto& result = *resultPtr;
   result.setNumColumns(getResultWidth());
 
-  auto addResultRow = [&](auto itLeft, auto itRight) {
+  auto addResultRow = [&result](auto itLeft, auto itRight) {
     const auto& l = *itLeft;
     const auto& r = *itRight;
     AD_CORRECTNESS_CHECK(l[0] == r[0]);
@@ -669,14 +669,15 @@ void Join::computeResultForIndexScanAndColumn(const IdTable& inputTable,
                                               ColumnIndex joinColumnIndexTable,
                                               const IndexScan& scan,
                                               ColumnIndex joinColumnIndexScan,
-                                              IdTable* resultPtr) {
+                                              IdTable* resultPtr) const {
   auto& result = *resultPtr;
   result.setNumColumns(getResultWidth());
 
   AD_CORRECTNESS_CHECK(joinColumnIndexScan == 0);
 
   auto joinColumn = inputTable.getColumn(joinColumnIndexTable);
-  auto addResultRow = [&, beg = joinColumn.data()](auto itLeft, auto itRight) {
+  auto addResultRow = [&result, &inputTable, &joinColumnIndexScan,
+                       beg = joinColumn.data()](auto itLeft, auto itRight) {
     const auto& l = *(inputTable.begin() + (&(*itLeft) - beg));
     const auto& r = *itRight;
     result.emplace_back();
@@ -714,9 +715,6 @@ void Join::computeResultForIndexScanAndColumn(const IdTable& inputTable,
       IndexScan::lazyScanForJoinOfColumnWithScan(joinColumn, scan);
 
   auto rightProjection = [](const auto& row) { return row[0]; };
-  // LOG(WARN) << "num blocks in first: " << std::ranges::distance(leftBlocks)
-  // << std::endl; LOG(WARN) << "num blocks in second: " <<
-  // std::ranges::distance(rightBlocks) << std::endl;
   ad_utility::zipperJoinForBlocksWithoutUndef(
       std::span{&joinColumn, 1}, rightBlocks, lessThan, addResultRow,
       std::identity{}, rightProjection);
diff --git a/src/engine/Join.h b/src/engine/Join.h
index 52332629cb..329676b199 100644
--- a/src/engine/Join.h
+++ b/src/engine/Join.h
@@ -134,12 +134,12 @@ class Join : public Operation {
 
   ResultTable computeResultForJoinWithFullScanDummy();
 
-  void computeResultForTwoIndexScans(IdTable* tablePtr);
+  void computeResultForTwoIndexScans(IdTable* tablePtr) const;
   void computeResultForIndexScanAndColumn(const IdTable& inputTable,
                                           ColumnIndex joinColumnIndexTable,
                                           const IndexScan& scan,
                                           ColumnIndex joinColumnIndexScan,
-                                          IdTable* resultPtr);
+                                          IdTable* resultPtr) const;
 
   using ScanMethodType = std::function;
 
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 0b2fcee121..7360d8aaa1 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -148,13 +148,12 @@ CompressedRelationReader::asyncParallelBlockGenerator(
     co_return;
   }
   // Note: It is important to define the `threads` before the `queue`. That way
-  // the joining destructor of the threads will see that the queue is finished and join.
+  // the joining destructor of the threads will see that the queue is finished
+  // and join.
+  std::mutex fileMutex;
   std::vector threads;
   ad_utility::data_structures::OrderedThreadSafeQueue queue{
       5};
-  // TODO We can configure whether we want to allow async access to
-  // the file.
-  std::mutex fileMutex;
   size_t blockIndex = 0;
   auto readAndDecompressBlock = [&]() {
     while (true) {
@@ -170,9 +169,8 @@ CompressedRelationReader::asyncParallelBlockGenerator(
       ++blockIndex;
       bool isLastBlock = beginBlock == endBlock;
       lock.unlock();
-      bool success = queue.push(
-          myIndex, decompressBlock(compressedBuffer, block.numRows_));
-      if (!success) {
+      if (!queue.push(myIndex,
+                      decompressBlock(compressedBuffer, block.numRows_))) {
         return;
       }
       if (isLastBlock) {
@@ -319,7 +317,7 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   }
 
   for (auto& block : asyncParallelBlockGenerator(beginBlock, endBlock, file,
-                                                 std::vector{1ul})) {
+                                                 std::vector{1UL})) {
     co_yield block;
   }
 
@@ -408,8 +406,7 @@ std::vector CompressedRelationReader::getBlocksForJoin(
 
   // The following check shouldn't be too expensive as there are only few
   // blocks.
-  auto unique = std::unique(result.begin(), result.end());
-  AD_CORRECTNESS_CHECK(unique == result.end());
+  AD_CORRECTNESS_CHECK(std::ranges::unique(result).begin() == result.end());
   return result;
 }
 
@@ -452,7 +449,7 @@ CompressedRelationReader::getBlocksForJoin(
   [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef(
       idRanges1, idRanges2, blockLessThanBlock, addRow, noop, noop);
   for (auto& vec : result) {
-    vec.erase(std::unique(vec.begin(), vec.end()), vec.end());
+    vec.erase(std::ranges::unique(vec).begin(), vec.end());
   }
   return result;
 }
@@ -489,7 +486,8 @@ void CompressedRelationReader::scan(
 
   // The first and the last block might be incomplete, compute
   // and store the partial results from them.
-  std::optional firstBlockResult, lastBlockResult;
+  std::optional firstBlockResult;
+  std::optional lastBlockResult;
   size_t totalResultSize = 0;
   if (beginBlock < endBlock) {
     firstBlockResult = readIncompleteBlock(*beginBlock);
@@ -534,7 +532,7 @@ void CompressedRelationReader::scan(
       // Read the block serially, only read the second column.
       AD_CORRECTNESS_CHECK(block.offsetsAndCompressedSize_.size() == 2);
       CompressedBlock compressedBuffer =
-          readCompressedBlockFromFile(block, file, std::vector{1ul});
+          readCompressedBlockFromFile(block, file, std::vector{1UL});
 
       // A lambda that owns the compressed block decompresses it to the
       // correct position in the result. It may safely be run in parallel
diff --git a/src/index/Permutation.h b/src/index/Permutation.h
index 02d7febfc7..5d59867297 100644
--- a/src/index/Permutation.h
+++ b/src/index/Permutation.h
@@ -54,10 +54,10 @@ class Permutation {
     }
 
     auto metadata = meta_.getMetaData(col0Id);
-    return MetaDataAndBlocks{
-        meta_.getMetaData(col0Id),
-        reader_.getBlocksFromMetadata(metadata, col1Id, meta_.blockData()),
-        col1Id};
+    return MetaDataAndBlocks{meta_.getMetaData(col0Id),
+                             CompressedRelationReader::getBlocksFromMetadata(
+                                 metadata, col1Id, meta_.blockData()),
+                             col1Id};
   }
 
   cppcoro::generator lazyScan(
diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index 8a61cda2e6..72e2155cc1 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -113,8 +113,6 @@ template <
     const FindSmallerUndefRangesLeft& findSmallerUndefRangesLeft,
     const FindSmallerUndefRangesRight& findSmallerUndefRangesRight,
     ElFromFirstNotFoundAction elFromFirstNotFoundAction = {}) {
-  // using Iterator = std::ranges::iterator_t;
-
   // If this is not an OPTIONAL join or a MINUS we can apply several
   // optimizations, so we store this information.
   static constexpr bool hasNotFoundAction =

From cf94ba463db1a5d6a3ca02663dd7afeb87a65804 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 27 Jun 2023 12:47:11 +0200
Subject: [PATCH 073/150] Make the tests more robust.

---
 test/ThreadSafeQueueTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 4e72a10d93..3a630cc4f7 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -191,7 +191,7 @@ TEST(ThreadSafeQueue, PushException) {
       // exceptions were pushed to the queue.
       while (queue.pop()) {
         ++numPopped;
-        EXPECT_LE(numPushed, numPopped + queueSize + 1 + numThreads);
+        EXPECT_LE(numPushed, numPopped + queueSize + 1 + 2 * numThreads);
       }
       FAIL() << "Should have thrown" << std::endl;
     } catch (const IntegerException& i) {

From 66ff69eeccf8bc583bd222208b5f0ae297ec5328 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 27 Jun 2023 16:17:53 +0200
Subject: [PATCH 074/150] Do not use lazy scans when the full result of the
 scans is already cached.

---
 src/engine/Join.cpp | 61 ++++++++++++++++++++++++++++++++++-----------
 src/engine/Join.h   |  1 +
 2 files changed, 47 insertions(+), 15 deletions(-)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 1aed5fb283..4c3b2e4144 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -113,16 +113,31 @@ ResultTable Join::computeResult() {
     return computeResultForJoinWithFullScanDummy();
   }
 
+  auto leftResIfCached = _left->getRootOperation()->getResult(false, true);
+  auto rightResIfCached = _right->getRootOperation()->getResult(false, true);
+
   if (_left->getType() == QueryExecutionTree::SCAN &&
       _right->getType() == QueryExecutionTree::SCAN) {
-    computeResultForTwoIndexScans(&idTable);
-    // TODO When we add triples to the
-    // index, the vocabularies of index scans will not necessarily be empty and
-    // we need a mechanism to still retrieve them when using the lazy scan.
-    return {std::move(idTable), resultSortedOn(), LocalVocab{}};
+    if (rightResIfCached && !leftResIfCached) {
+      computeResultForIndexScanAndColumn(
+          rightResIfCached->idTable(), _rightJoinCol,
+          dynamic_cast(*_left->getRootOperation()),
+          _leftJoinCol, &idTable);
+      // TODO This still has the wrong column order
+      return {std::move(idTable), resultSortedOn(), LocalVocab{}};
+
+    } else if (!leftResIfCached) {
+      computeResultForTwoIndexScans(&idTable);
+      // TODO When we add triples to the
+      // index, the vocabularies of index scans will not necessarily be empty
+      // and we need a mechanism to still retrieve them when using the lazy
+      // scan.
+      return {std::move(idTable), resultSortedOn(), LocalVocab{}};
+    }
   }
 
-  shared_ptr leftRes = _left->getResult();
+  shared_ptr leftRes =
+      leftResIfCached ? leftResIfCached : _left->getResult();
   if (leftRes->size() == 0) {
     _right->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
     // TODO When we add triples to the
@@ -133,8 +148,8 @@ ResultTable Join::computeResult() {
 
   // Note: If only one of the children is a scan, then we have made sure in the
   // constructor that it is the right child.
-  if (_right->getType() == QueryExecutionTree::SCAN) {
-    computeResultForIndexScanAndColumn(
+  if (_right->getType() == QueryExecutionTree::SCAN && !rightResIfCached) {
+    computeResultForIndexScanAndColumn(
         leftRes->idTable(), _leftJoinCol,
         dynamic_cast(*_right->getRootOperation()),
         _rightJoinCol, &idTable);
@@ -142,7 +157,8 @@ ResultTable Join::computeResult() {
             leftRes->getSharedLocalVocab()};
   }
 
-  shared_ptr rightRes = _right->getResult();
+  shared_ptr rightRes =
+      rightResIfCached ? rightResIfCached : _right->getResult();
   join(leftRes->idTable(), _leftJoinCol, rightRes->idTable(), _rightJoinCol,
        &idTable);
 
@@ -665,6 +681,7 @@ void Join::computeResultForTwoIndexScans(IdTable* resultPtr) const {
 }
 
 // ______________________________________________________________________________________________________
+template 
 void Join::computeResultForIndexScanAndColumn(const IdTable& inputTable,
                                               ColumnIndex joinColumnIndexTable,
                                               const IndexScan& scan,
@@ -683,16 +700,30 @@ void Join::computeResultForIndexScanAndColumn(const IdTable& inputTable,
     result.emplace_back();
     IdTable::row_reference lastRow = result.back();
     size_t nextIndex = 0;
-    for (size_t i = 0; i < inputTable.numColumns(); ++i) {
-      lastRow[nextIndex] = l[i];
-      ++nextIndex;
-    }
 
-    for (size_t i = 0; i < r.size(); ++i) {
-      if (i != joinColumnIndexScan) {
+    if constexpr (firstIsRight) {
+      for (size_t i = 0; i < r.size(); ++i) {
         lastRow[nextIndex] = r[i];
         ++nextIndex;
       }
+      for (size_t i = 0; i < inputTable.numColumns(); ++i) {
+        if (i != joinColumnIndexScan) {
+          lastRow[nextIndex] = l[i];
+          ++nextIndex;
+        }
+      }
+    } else {
+      for (size_t i = 0; i < inputTable.numColumns(); ++i) {
+        lastRow[nextIndex] = l[i];
+        ++nextIndex;
+      }
+
+      for (size_t i = 0; i < r.size(); ++i) {
+        if (i != joinColumnIndexScan) {
+          lastRow[nextIndex] = r[i];
+          ++nextIndex;
+        }
+      }
     }
   };
 
diff --git a/src/engine/Join.h b/src/engine/Join.h
index 329676b199..4a3284ef0a 100644
--- a/src/engine/Join.h
+++ b/src/engine/Join.h
@@ -135,6 +135,7 @@ class Join : public Operation {
   ResultTable computeResultForJoinWithFullScanDummy();
 
   void computeResultForTwoIndexScans(IdTable* tablePtr) const;
+  template 
   void computeResultForIndexScanAndColumn(const IdTable& inputTable,
                                           ColumnIndex joinColumnIndexTable,
                                           const IndexScan& scan,

From ccff07b5d002146f06607198c462743a3973234c Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 27 Jun 2023 16:55:43 +0200
Subject: [PATCH 075/150] Add some initial tests for the so far untested stuff.

---
 test/CompressedRelationsTest.cpp | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp
index 75b8e3aadf..a305c0413a 100644
--- a/test/CompressedRelationsTest.cpp
+++ b/test/CompressedRelationsTest.cpp
@@ -296,3 +296,25 @@ TEST(CompressedRelationMetadata, GettersAndSetters) {
   m.numRows_ = 43;
   ASSERT_EQ(43, m.numRows_);
 }
+
+TEST(CompressedRelationReader, getBlocksForJoin) {
+  std::vector joinColumn{V(1), V(3), V(17), V(29)};
+
+  CompressedBlockMetadata block1{
+      {}, 0, {V(16), V(0), V(0)}, {V(38), V(4), V(12)}};
+  CompressedBlockMetadata block2{
+      {}, 0, {V(39), V(0), V(0)}, {V(42), V(4), V(12)}};
+  CompressedBlockMetadata block3{
+      {}, 0, {V(42), V(4), V(13)}, {V(42), V(6), V(9)}};
+
+  CompressedRelationMetadata relation;
+  relation.col0Id_ = V(42);
+
+  std::vector blocks{block1, block2, block3};
+  CompressedRelationReader::MetadataAndBlocks metadataAndBlocks{
+      relation, blocks, std::nullopt};
+
+  auto result =
+      CompressedRelationReader::getBlocksForJoin(joinColumn, metadataAndBlocks);
+  ASSERT_THAT(result, ::testing::ElementsAre(block2));
+}

From a494db21bfacd43f70b810eeaca2b3069320c3be Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 27 Jun 2023 19:27:46 +0200
Subject: [PATCH 076/150] Some initial unit tests.

---
 src/index/CompressedRelation.h   |  2 +-
 test/CompressedRelationsTest.cpp | 28 +++++++++++++++++++++++-----
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index be2feb7a32..fd3720c24c 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -273,7 +273,7 @@ class CompressedRelationReader {
 
   struct MetadataAndBlocks {
     const CompressedRelationMetadata relationMetadata_;
-    std::span blockMetadata_;
+    const std::span blockMetadata_;
     std::optional col1Id_;
   };
   static std::vector getBlocksForJoin(
diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp
index a305c0413a..65e4c48081 100644
--- a/test/CompressedRelationsTest.cpp
+++ b/test/CompressedRelationsTest.cpp
@@ -298,8 +298,6 @@ TEST(CompressedRelationMetadata, GettersAndSetters) {
 }
 
 TEST(CompressedRelationReader, getBlocksForJoin) {
-  std::vector joinColumn{V(1), V(3), V(17), V(29)};
-
   CompressedBlockMetadata block1{
       {}, 0, {V(16), V(0), V(0)}, {V(38), V(4), V(12)}};
   CompressedBlockMetadata block2{
@@ -307,6 +305,7 @@ TEST(CompressedRelationReader, getBlocksForJoin) {
   CompressedBlockMetadata block3{
       {}, 0, {V(42), V(4), V(13)}, {V(42), V(6), V(9)}};
 
+  // We are only interested in blocks with a col0 of `42`.
   CompressedRelationMetadata relation;
   relation.col0Id_ = V(42);
 
@@ -314,7 +313,26 @@ TEST(CompressedRelationReader, getBlocksForJoin) {
   CompressedRelationReader::MetadataAndBlocks metadataAndBlocks{
       relation, blocks, std::nullopt};
 
-  auto result =
-      CompressedRelationReader::getBlocksForJoin(joinColumn, metadataAndBlocks);
-  ASSERT_THAT(result, ::testing::ElementsAre(block2));
+  auto test = [&metadataAndBlocks](
+                  const std::vector& joinColumn,
+                  const std::vector expectedBlocks,
+                  source_location l = source_location::current()) {
+    auto t = generateLocationTrace(l);
+    auto result = CompressedRelationReader::getBlocksForJoin(joinColumn,
+                                                             metadataAndBlocks);
+    EXPECT_THAT(result, ::testing::ElementsAreArray(expectedBlocks));
+  };
+
+  // Tests for a fixed col0Id, so the join is on the middle column.
+  test({V(1), V(3), V(17), V(29)}, {block2});
+  test({V(2), V(3), V(4), V(5)}, {block2, block3});
+  test({V(4)}, {block2, block3});
+  test({V(6)}, {block3});
+
+  // Test with a fixed col1Id. We now join on the last column, the first column
+  // is fixed (42), and the second column is also fixed (4).
+  metadataAndBlocks.col1Id_ = V(4);
+  test({V(11), V(27), V(30)}, {block2, block3});
+  test({V(12)}, {block2});
+  test({V(13)}, {block3});
 }

From 536de7044f76d41633491da27d8788c4c9829ec4 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 27 Jun 2023 20:08:53 +0200
Subject: [PATCH 077/150] Last changes from a review.

---
 src/util/AsyncStream.h       |  4 ++--
 src/util/ThreadSafeQueue.h   | 16 ++++++++--------
 test/ThreadSafeQueueTest.cpp |  8 ++++----
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/util/AsyncStream.h b/src/util/AsyncStream.h
index 803c833595..e1dac1e84d 100644
--- a/src/util/AsyncStream.h
+++ b/src/util/AsyncStream.h
@@ -40,14 +40,14 @@ cppcoro::generator runStreamAsync(
     } catch (...) {
       exception = std::current_exception();
     }
-    queue.disableQueue();
+    queue.finish();
   }};
 
   // Only rethrow an exception from the `thread` if no exception occured in this
   // thread to avoid crashes because of multiple active exceptions.
   auto cleanup = ad_utility::makeOnDestructionDontThrowDuringStackUnwinding(
       [&queue, &thread, &exception] {
-        queue.disableQueue();
+        queue.finish();
         thread.join();
         // This exception will only be thrown once all the values that were
         // pushed to the queue before the exception occurred.
diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index fe9ec1f42a..17fcc6faf4 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -33,7 +33,7 @@ class ThreadSafeQueue {
   const ThreadSafeQueue& operator=(ThreadSafeQueue&&) = delete;
 
   /// Push an element into the queue. Block until there is free space in the
-  /// queue or until disableQueue() was called. Return false if disableQueue()
+  /// queue or until disableQueue() was called. Return false if finish()
   /// was called. In this case the current element element and all future
   /// elements are not added to the queue.
   bool push(T value) {
@@ -64,11 +64,11 @@ class ThreadSafeQueue {
   // After calling this function, all calls to `push` will return `false` and no
   // further elements will be added to the queue. Calls to `pop` will yield the
   // elements that were already stored in the queue before the call to
-  // `disableQueue`, after those were drained, `pop` will return `nullopt`. This
+  // `finish`, after those were drained, `pop` will return `nullopt`. This
   // function can be called from the producing/pushing threads to signal that
   // all elements have been pushed, or from the consumers to signal that they
   // will not pop further elements from the queue.
-  void disableQueue() {
+  void finish() {
     std::unique_lock lock{mutex_};
     queueDisabled_ = true;
     lock.unlock();
@@ -76,11 +76,11 @@ class ThreadSafeQueue {
     popNotification_.notify_all();
   }
 
-  /// Always call `disableQueue` on destruction. This makes sure that worker
+  /// Always call `finish` on destruction. This makes sure that worker
   /// threads that pop from the queue always see std::nullopt, even if the
   /// threads that push to the queue exit via an exception or if the explicit
-  /// call to `disableQueue` is missing.
-  ~ThreadSafeQueue() { disableQueue(); }
+  /// call to `finish` is missing.
+  ~ThreadSafeQueue() { finish(); }
 
   /// Blocks until another thread pushes an element via push() which is
   /// hen returned or signalLastElementWasPushed() is called resulting in an
@@ -135,7 +135,7 @@ class OrderedThreadSafeQueue {
 
   // Push the `value` to the queue that is associated with the `index`. This
   // call blocks, until `push` has been called for all indices in `[0, ...,
-  // index - 1]` or until `disableQueue` was called. The remaining behavior is
+  // index - 1]` or until `finish` was called. The remaining behavior is
   // equal to `ThreadSafeQueue::push`.
   bool push(size_t index, T value) {
     std::unique_lock lock{mutex_};
@@ -163,7 +163,7 @@ class OrderedThreadSafeQueue {
 
   // See `ThreadSafeQueue` for details.
   void disableQueue() {
-    queue_.disableQueue();
+    queue_.finish();
     std::unique_lock lock{mutex_};
     pushWasDisabled_ = true;
     lock.unlock();
diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 3a630cc4f7..8106ebe972 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -56,7 +56,7 @@ TEST(ThreadSafeQueue, BufferSizeIsRespected) {
       while (numPushed < numValues) {
         push(numPushed++);
       }
-      queue.disableQueue();
+      queue.finish();
     });
 
     size_t numPopped = 0;
@@ -83,7 +83,7 @@ TEST(ThreadSafeQueue, ReturnValueOfPush) {
     // called.
     EXPECT_TRUE(push(0));
     EXPECT_EQ(queue.pop(), 0);
-    queue.disableQueue();
+    queue.finish();
     EXPECT_FALSE(push(1));
   };
   runWithBothQueueTypes(runTest);
@@ -105,7 +105,7 @@ TEST(ThreadSafeQueue, Concurrency) {
       }
       numThreadsDone++;
       if (numThreadsDone == numThreads) {
-        queue.disableQueue();
+        queue.finish();
       }
     };
 
@@ -230,7 +230,7 @@ TEST(ThreadSafeQueue, DisablePush) {
 
       // Disable the push, make the consumers finish.
       if (numPopped == 400) {
-        queue.disableQueue();
+        queue.finish();
         break;
       }
     }

From 97ba0f9538814881caf7b7a125019f698faea370 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 27 Jun 2023 20:12:30 +0200
Subject: [PATCH 078/150] Last rename to finish.

---
 src/util/ThreadSafeQueue.h | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index 17fcc6faf4..bf584ce6da 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -20,7 +20,7 @@ class ThreadSafeQueue {
   std::mutex mutex_;
   std::condition_variable pushNotification_;
   std::condition_variable popNotification_;
-  bool queueDisabled_ = false;
+  bool finish_ = false;
   size_t maxSize_;
 
  public:
@@ -33,14 +33,14 @@ class ThreadSafeQueue {
   const ThreadSafeQueue& operator=(ThreadSafeQueue&&) = delete;
 
   /// Push an element into the queue. Block until there is free space in the
-  /// queue or until disableQueue() was called. Return false if finish()
+  /// queue or until finish() was called. Return false if finish()
   /// was called. In this case the current element element and all future
   /// elements are not added to the queue.
   bool push(T value) {
     std::unique_lock lock{mutex_};
     popNotification_.wait(
-        lock, [this] { return queue_.size() < maxSize_ || queueDisabled_; });
-    if (queueDisabled_) {
+        lock, [this] { return queue_.size() < maxSize_ || finish_; });
+    if (finish_) {
       return false;
     }
     queue_.push(std::move(value));
@@ -55,7 +55,7 @@ class ThreadSafeQueue {
   void pushException(std::exception_ptr exception) {
     std::unique_lock lock{mutex_};
     pushedException_ = std::move(exception);
-    queueDisabled_ = true;
+    finish_ = true;
     lock.unlock();
     pushNotification_.notify_all();
     popNotification_.notify_all();
@@ -70,7 +70,7 @@ class ThreadSafeQueue {
   // will not pop further elements from the queue.
   void finish() {
     std::unique_lock lock{mutex_};
-    queueDisabled_ = true;
+    finish_ = true;
     lock.unlock();
     pushNotification_.notify_all();
     popNotification_.notify_all();
@@ -88,12 +88,12 @@ class ThreadSafeQueue {
   std::optional pop() {
     std::unique_lock lock{mutex_};
     pushNotification_.wait(lock, [this] {
-      return !queue_.empty() || queueDisabled_ || pushedException_;
+      return !queue_.empty() || finish_ || pushedException_;
     });
     if (pushedException_) {
       std::rethrow_exception(pushedException_);
     }
-    if (queueDisabled_ && queue_.empty()) {
+    if (finish_ && queue_.empty()) {
       return {};
     }
     std::optional value = std::move(queue_.front());
@@ -120,7 +120,7 @@ class OrderedThreadSafeQueue {
   std::condition_variable cv_;
   ThreadSafeQueue queue_;
   size_t nextIndex_ = 0;
-  bool pushWasDisabled_ = false;
+  bool finish_ = false;
 
  public:
   // Construct from the maximal queue size (see `ThreadSafeQueue` for details).
@@ -139,10 +139,8 @@ class OrderedThreadSafeQueue {
   // equal to `ThreadSafeQueue::push`.
   bool push(size_t index, T value) {
     std::unique_lock lock{mutex_};
-    cv_.wait(lock, [this, index]() {
-      return index == nextIndex_ || pushWasDisabled_;
-    });
-    if (pushWasDisabled_) {
+    cv_.wait(lock, [this, index]() { return index == nextIndex_ || finish_; });
+    if (finish_) {
       return false;
     }
     ++nextIndex_;
@@ -156,22 +154,22 @@ class OrderedThreadSafeQueue {
   void pushException(std::exception_ptr exception) {
     std::unique_lock l{mutex_};
     queue_.pushException(std::move(exception));
-    pushWasDisabled_ = true;
+    finish_ = true;
     l.unlock();
     cv_.notify_all();
   }
 
   // See `ThreadSafeQueue` for details.
-  void disableQueue() {
+  void finish() {
     queue_.finish();
     std::unique_lock lock{mutex_};
-    pushWasDisabled_ = true;
+    finish_ = true;
     lock.unlock();
     cv_.notify_all();
   }
 
   // See `ThreadSafeQueue` for details.
-  ~OrderedThreadSafeQueue() { disableQueue(); }
+  ~OrderedThreadSafeQueue() { finish(); }
 
   // See `ThreadSafeQueue` for details. All the returned values will be in
   // ascending consecutive order wrt the index with which they were pushed.

From 05a1ee1013a20199fd9db4b96781a71f10793685 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 28 Jun 2023 10:27:43 +0200
Subject: [PATCH 079/150] Fixed some code smells and removed some code
 duplication.

---
 src/index/CompressedRelation.cpp         | 131 ++++++++---------------
 src/index/CompressedRelation.h           |   4 +-
 src/util/JoinAlgorithms/JoinAlgorithms.h |   8 +-
 3 files changed, 52 insertions(+), 91 deletions(-)

diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 7360d8aaa1..6ca500c464 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -65,29 +65,16 @@ void CompressedRelationReader::scan(
   // Set up a lambda, that reads this block and decompresses it to
   // the result.
   auto readIncompleteBlock = [&](const auto& block) mutable {
-    // A block is uniquely identified by its start position in the file.
-    auto cacheKey = block.offsetsAndCompressedSize_.at(0).offsetInFile_;
-    auto uncompressedBuffer = blockCache_
-                                  .computeOnce(cacheKey,
-                                               [&]() {
-                                                 return readAndDecompressBlock(
-                                                     block, file, std::nullopt);
-                                               })
-                                  ._resultPointer;
-
-    // Extract the part of the block that actually belongs to the relation
-    auto numElements = metadata.numRows_;
-    AD_CORRECTNESS_CHECK(uncompressedBuffer->numColumns() ==
-                         metadata.numColumns());
-    for (size_t i = 0; i < uncompressedBuffer->numColumns(); ++i) {
-      const auto& inputCol = uncompressedBuffer->getColumn(i);
-      auto begin = inputCol.begin() + metadata.offsetInBlock_;
+    auto trimmedBlock =
+        readPossiblyIncompleteBlock(metadata, std::nullopt, file, block);
+    for (size_t i = 0; i < trimmedBlock.numColumns(); ++i) {
+      const auto& inputCol = trimmedBlock.getColumn(i);
       auto resultColumn = result->getColumn(i);
-      AD_CORRECTNESS_CHECK(numElements <= spaceLeft);
-      std::copy(begin, begin + numElements, resultColumn.begin());
+      AD_CORRECTNESS_CHECK(inputCol.size() <= resultColumn.size());
+      std::ranges::copy(inputCol, resultColumn.begin());
     }
-    rowIndexOfNextBlock += numElements;
-    spaceLeft -= numElements;
+    rowIndexOfNextBlock += trimmedBlock.size();
+    spaceLeft -= trimmedBlock.size();
   };
 
   // Read the first block if it is incomplete
@@ -174,7 +161,7 @@ CompressedRelationReader::asyncParallelBlockGenerator(
         return;
       }
       if (isLastBlock) {
-        queue.signalLastElementWasPushed();
+        queue.finish();
       }
     }
   };
@@ -223,38 +210,10 @@ cppcoro::generator CompressedRelationReader::lazyScan(
                          std::numeric_limits::max());
   }
 
-  // We have at most one block that is incomplete and thus requires trimming.
-  // Set up a lambda, that reads this block and decompresses it to
-  // the result.
-  auto readIncompleteBlock = [&](const auto& block) {
-    // A block is uniquely identified by its start position in the file.
-    auto cacheKey = block.offsetsAndCompressedSize_.at(0).offsetInFile_;
-    auto uncompressedBuffer = blockCache_
-                                  .computeOnce(cacheKey,
-                                               [&]() {
-                                                 return readAndDecompressBlock(
-                                                     block, file, std::nullopt);
-                                               })
-                                  ._resultPointer;
-
-    // Extract the part of the block that actually belongs to the relation
-    auto numElements = metadata.numRows_;
-    AD_CORRECTNESS_CHECK(uncompressedBuffer->numColumns() ==
-                         metadata.numColumns());
-    IdTable result(uncompressedBuffer->numColumns(), allocator_);
-    result.resize(numElements);
-    for (size_t i = 0; i < uncompressedBuffer->numColumns(); ++i) {
-      const auto& inputCol = uncompressedBuffer->getColumn(i);
-      auto begin = inputCol.begin() + metadata.offsetInBlock_;
-      decltype(auto) resultColumn = result.getColumn(i);
-      std::copy(begin, begin + numElements, resultColumn.begin());
-    }
-    return result;
-  };
-
   // Read the first block if it is incomplete
   if (firstBlockIsIncomplete) {
-    co_yield readIncompleteBlock(*beginBlock);
+    co_yield readPossiblyIncompleteBlock(metadata, std::nullopt, file,
+                                         *beginBlock);
     ++beginBlock;
     if (timer) {
       timer->wlock()->checkTimeoutAndThrow("IndexScan :");
@@ -269,6 +228,7 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   // TODO Timeout checks.
 }
 
+// _____________________________________________________________________________
 cppcoro::generator CompressedRelationReader::lazyScan(
     CompressedRelationMetadata metadata, Id col1Id,
     std::vector blockMetadata, ad_utility::File& file,
@@ -290,43 +250,29 @@ cppcoro::generator CompressedRelationReader::lazyScan(
     AD_CORRECTNESS_CHECK(endBlock - beginBlock <= 1);
   }
 
-  // The first and the last block might be incomplete, compute
-  // and store the partial results from them.
-  std::optional firstBlockResult;
-  std::optional lastBlockResult;
-  if (beginBlock < endBlock) {
-    firstBlockResult =
-        readPossiblyIncompleteBlock(metadata, col1Id, file, *beginBlock);
-    ++beginBlock;
+  auto getIncompleteBlock = [&](auto it) {
+    auto result = readPossiblyIncompleteBlock(metadata, col1Id, file, *it);
+    result.setColumnSubset(std::array{1});
     if (timer) {
       timer->wlock()->checkTimeoutAndThrow("IndexScan: ");
     }
-  }
-  if (beginBlock < endBlock) {
-    lastBlockResult =
-        readPossiblyIncompleteBlock(metadata, col1Id, file, *(endBlock - 1));
-    endBlock--;
-    if (timer) {
-      timer->wlock()->checkTimeoutAndThrow("IndexScan: ");
-    }
-  }
+    return result;
+  };
 
-  if (firstBlockResult.has_value()) {
-    firstBlockResult.value().setColumnSubset(std::array{1});
-    co_yield std::move(firstBlockResult.value());
+  if (beginBlock < endBlock) {
+    co_yield getIncompleteBlock(beginBlock);
+    ++beginBlock;
   }
 
-  for (auto& block : asyncParallelBlockGenerator(beginBlock, endBlock, file,
-                                                 std::vector{1UL})) {
-    co_yield block;
+  if (beginBlock < endBlock) {
+    for (auto& block : asyncParallelBlockGenerator(beginBlock, endBlock - 1,
+                                                   file, std::vector{1UL})) {
+      co_yield block;
+    }
   }
 
-  if (timer) {
-    timer->wlock()->checkTimeoutAndThrow();
-  }
-  if (lastBlockResult.has_value()) {
-    lastBlockResult.value().setColumnSubset(std::array{1});
-    co_yield lastBlockResult.value();
+  if (beginBlock < endBlock) {
+    co_yield getIncompleteBlock(endBlock - 1);
   }
 }
 
@@ -569,11 +515,19 @@ void CompressedRelationReader::scan(
 
 // _____________________________________________________________________________
 DecompressedBlock CompressedRelationReader::readPossiblyIncompleteBlock(
-    const CompressedRelationMetadata& relationMetadata, Id col1Id,
-    ad_utility::File& file,
+    const CompressedRelationMetadata& relationMetadata,
+    std::optional col1Id, ad_utility::File& file,
     const CompressedBlockMetadata& blockMetadata) const {
+  // A block is uniquely identified by its start position in the file.
+  auto cacheKey = blockMetadata.offsetsAndCompressedSize_.at(0).offsetInFile_;
   DecompressedBlock block =
-      readAndDecompressBlock(blockMetadata, file, std::nullopt);
+      blockCache_
+          .computeOnce(cacheKey,
+                       [&]() {
+                         return readAndDecompressBlock(blockMetadata, file,
+                                                       std::nullopt);
+                       })
+          ._resultPointer->clone();
   AD_CORRECTNESS_CHECK(block.numColumns() == 2);
   const auto& col1Column = block.getColumn(0);
   const auto& col2Column = block.getColumn(1);
@@ -589,7 +543,14 @@ DecompressedBlock CompressedRelationReader::readPossiblyIncompleteBlock(
   }
   auto end = containedInOnlyOneBlock ? begin + relationMetadata.numRows_
                                      : col1Column.end();
-  auto subBlock = std::ranges::equal_range(begin, end, col1Id);
+
+  auto subBlock = [&]() {
+    if (col1Id.has_value()) {
+      return std::ranges::equal_range(begin, end, col1Id.value());
+    } else {
+      return std::ranges::subrange{begin, end};
+    }
+  }();
   auto numResults = subBlock.size();
   block.erase(block.begin(),
               block.begin() + (subBlock.begin() - col1Column.begin()));
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index fd3720c24c..ed31f2c836 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -383,8 +383,8 @@ class CompressedRelationReader {
   // the blocks that actually store triples from the given `relationMetadata`'s
   // relation, else the behavior is undefined.
   DecompressedBlock readPossiblyIncompleteBlock(
-      const CompressedRelationMetadata& relationMetadata, Id col1Id,
-      ad_utility::File& file,
+      const CompressedRelationMetadata& relationMetadata,
+      std::optional col1Id, ad_utility::File& file,
       const CompressedBlockMetadata& blockMetadata) const;
 };
 
diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index 72e2155cc1..e7bbb3a8ae 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -670,10 +670,10 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
       ad_utility::isSimilar>);
   // Iterators for the two inputs. These iterators work on blocks.
-  auto it1 = leftBlocks.begin();
-  auto end1 = leftBlocks.end();
-  auto it2 = rightBlocks.begin();
-  auto end2 = rightBlocks.end();
+  auto it1 = std::begin(leftBlocks);
+  auto end1 = std::end(leftBlocks);
+  auto it2 = std::begin(rightBlocks);
+  auto end2 = std::end(rightBlocks);
 
   // Create an equality comparison from the `lessThan` predicate.
   auto eq = [&lessThan](const auto& el1, const auto& el2) {

From f983e79e8ac26e30d72c0d74ea1660840f4d64dc Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 28 Jun 2023 11:43:54 +0200
Subject: [PATCH 080/150] Added comments and several refactorings. Also log the
 time for the retrieval of the blocks.

---
 src/engine/Join.cpp              | 18 ++++++-
 src/engine/Join.h                |  4 +-
 src/index/CompressedRelation.cpp | 82 +++++++++++++++++++-------------
 src/index/CompressedRelation.h   | 77 +++++++++++++++++++++---------
 4 files changed, 121 insertions(+), 60 deletions(-)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 4c3b2e4144..af35fb783c 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -646,7 +646,7 @@ void Join::addCombinedRowToIdTable(const ROW_A& rowA, const ROW_B& rowB,
 }
 
 // ______________________________________________________________________________________________________
-void Join::computeResultForTwoIndexScans(IdTable* resultPtr) const {
+void Join::computeResultForTwoIndexScans(IdTable* resultPtr) {
   AD_CORRECTNESS_CHECK(_left->getType() == QueryExecutionTree::SCAN &&
                        _right->getType() == QueryExecutionTree::SCAN);
   auto& result = *resultPtr;
@@ -672,12 +672,17 @@ void Join::computeResultForTwoIndexScans(IdTable* resultPtr) const {
 
   auto lessThan = [](const auto& a, const auto& b) { return a[0] < b[0]; };
 
+  ad_utility::Timer timer{ad_utility::timer::Timer::InitialStatus::Started};
   auto [leftBlocks, rightBlocks] = IndexScan::lazyScanForJoinOfTwoScans(
       dynamic_cast(*_left->getRootOperation()),
       dynamic_cast(*_right->getRootOperation()));
+  getRuntimeInfo().addDetail("time-for-filtering-blocks", timer.msecs());
 
   ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks, lessThan,
                                               addResultRow);
+
+  _left->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
+  _right->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
 }
 
 // ______________________________________________________________________________________________________
@@ -686,7 +691,7 @@ void Join::computeResultForIndexScanAndColumn(const IdTable& inputTable,
                                               ColumnIndex joinColumnIndexTable,
                                               const IndexScan& scan,
                                               ColumnIndex joinColumnIndexScan,
-                                              IdTable* resultPtr) const {
+                                              IdTable* resultPtr) {
   auto& result = *resultPtr;
   result.setNumColumns(getResultWidth());
 
@@ -742,11 +747,20 @@ void Join::computeResultForIndexScanAndColumn(const IdTable& inputTable,
     }
   };
 
+  ad_utility::Timer timer{ad_utility::timer::Timer::InitialStatus::Started};
   auto rightBlocks =
       IndexScan::lazyScanForJoinOfColumnWithScan(joinColumn, scan);
 
+  getRuntimeInfo().addDetail("time-for-filtering-blocks", timer.msecs());
+
   auto rightProjection = [](const auto& row) { return row[0]; };
   ad_utility::zipperJoinForBlocksWithoutUndef(
       std::span{&joinColumn, 1}, rightBlocks, lessThan, addResultRow,
       std::identity{}, rightProjection);
+
+  if (firstIsRight) {
+    _left->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
+  } else {
+    _right->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
+  }
 }
diff --git a/src/engine/Join.h b/src/engine/Join.h
index 4a3284ef0a..92b30c7e30 100644
--- a/src/engine/Join.h
+++ b/src/engine/Join.h
@@ -134,13 +134,13 @@ class Join : public Operation {
 
   ResultTable computeResultForJoinWithFullScanDummy();
 
-  void computeResultForTwoIndexScans(IdTable* tablePtr) const;
+  void computeResultForTwoIndexScans(IdTable* resultPtr);
   template 
   void computeResultForIndexScanAndColumn(const IdTable& inputTable,
                                           ColumnIndex joinColumnIndexTable,
                                           const IndexScan& scan,
                                           ColumnIndex joinColumnIndexScan,
-                                          IdTable* resultPtr) const;
+                                          IdTable* resultPtr);
 
   using ScanMethodType = std::function;
 
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 6ca500c464..739b7228c2 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -276,35 +276,43 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   }
 }
 
-// TODO Comment those helpers. Should we register them in the header as
-// private static functions?
+// Some internal helper functions for the `getBlocksForJoin` functions below.
 namespace {
-auto getJoinColumnRangeValue = [](CompressedBlockMetadata::PermutedTriple block,
-                                  Id col0Id, std::optional col1Id) {
+// If the combination of `col0Id, col1Id` is strictly smaller than the `triple`
+// return the smallest possible ID (which is the undefined ID which never is
+// contained in a triple). If the combination is stictly larger than the triple,
+// the greatest possible ID (which is greater than all valid IDs) is returned.
+// Else, the `col2Id` is returned if  a `col1Id` is specified, otherwise the
+// `col1Id`.
+Id getJoinColumnRangeValue(CompressedBlockMetadata::PermutedTriple triple,
+                           Id col0Id, std::optional col1Id) {
   auto minId = Id::makeUndefined();
   auto maxId = Id::fromBits(std::numeric_limits::max());
-  if (block.col0Id_ < col0Id) {
+  if (triple.col0Id_ < col0Id) {
     return minId;
   }
-  if (block.col0Id_ > col0Id) {
+  if (triple.col0Id_ > col0Id) {
     return maxId;
   }
   if (!col1Id.has_value()) {
-    return block.col1Id_;
+    return triple.col1Id_;
   }
 
-  if (block.col1Id_ < col1Id.value()) {
+  if (triple.col1Id_ < col1Id.value()) {
     return minId;
   }
-  if (block.col1Id_ > col1Id.value()) {
+  if (triple.col1Id_ > col1Id.value()) {
     return maxId;
   }
-  return block.col2Id_;
-};
+  return triple.col2Id_;
+}
 
 using IdPair = std::pair;
-auto blocksToIdRanges = [](std::span blocks,
-                           Id col0Id, std::optional col1Id) {
+// Convert each of the `blocks` to an `IdPair` by calling
+// `getJoinColumnRangeValue` for the first and last triple of the block.
+std::vector blocksToIdRanges(
+    std::span blocks, Id col0Id,
+    std::optional col1Id) {
   std::vector result;
   for (const auto& block : blocks) {
     result.emplace_back(
@@ -312,7 +320,7 @@ auto blocksToIdRanges = [](std::span blocks,
         getJoinColumnRangeValue(block.lastTriple_, col0Id, col1Id));
   }
   return result;
-};
+}
 }  // namespace
 
 // _____________________________________________________________________________
@@ -320,33 +328,35 @@ std::vector CompressedRelationReader::getBlocksForJoin(
     std::span joinColum, const MetadataAndBlocks& scanMetadata) {
   // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_
   const auto& [relationMetadata, blockMetadata, col1Id] = scanMetadata;
-  auto relevantBlocks =
-      getBlocksFromMetadata(relationMetadata, std::nullopt, blockMetadata);
+  auto relevantBlocks = getBlocksFromMetadata(scanMetadata);
 
   auto idRanges =
       blocksToIdRanges(relevantBlocks, relationMetadata.col0Id_, col1Id);
 
+  // We need symmetric comparisons between `Id` and `IdPair` as well as between
+  // Ids.
   auto idLessThanBlock = [](Id id, const IdPair& block) {
     return id < block.first;
   };
-
   auto blockLessThanId = [](const IdPair& block, Id id) {
     return block.second < id;
   };
-
   auto idLessThanId = [](Id id, Id id2) { return id < id2; };
 
   auto lessThan = ad_utility::OverloadCallOperator{
       idLessThanBlock, blockLessThanId, idLessThanId};
 
+  // When we have found a matching block, we have to convert it back.
   std::vector result;
   auto addRow = [&result, begBlocks = relevantBlocks.begin(),
-                 begRanges = idRanges.begin()]([[maybe_unused]] auto it1,
-                                               auto it2) {
-    result.push_back(*(begBlocks + (it2 - begRanges)));
+                 begRanges = idRanges.begin()]([[maybe_unused]] auto idIterator,
+                                               auto blockIterator) {
+    result.push_back(*(begBlocks + (blockIterator - begRanges)));
   };
 
   auto noop = ad_utility::noop;
+
+  // Actually perform the join.
   [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef(
       joinColum, idRanges, lessThan, addRow, noop, noop);
 
@@ -361,18 +371,9 @@ std::array, 2>
 CompressedRelationReader::getBlocksForJoin(
     const MetadataAndBlocks& scanMetadata1,
     const MetadataAndBlocks& scanMetadata2) {
-  // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_
-  struct KeyLhs {
-    Id col0FirstId_;
-    Id col0LastId_;
-  };
-
-  auto relevantBlocks1 =
-      getBlocksFromMetadata(scanMetadata1.relationMetadata_, std::nullopt,
-                            scanMetadata1.blockMetadata_);
-  auto relevantBlocks2 =
-      getBlocksFromMetadata(scanMetadata2.relationMetadata_, std::nullopt,
-                            scanMetadata2.blockMetadata_);
+  // Get the
+  auto relevantBlocks1 = getBlocksFromMetadata(scanMetadata1);
+  auto relevantBlocks2 = getBlocksFromMetadata(scanMetadata2);
 
   auto blockLessThanBlock = [](const auto& block1, const auto& block2) {
     return block1.second < block2.first;
@@ -385,6 +386,8 @@ CompressedRelationReader::getBlocksForJoin(
   auto idRanges2 =
       blocksToIdRanges(relevantBlocks2, scanMetadata2.relationMetadata_.col0Id_,
                        scanMetadata2.col1Id_);
+  // When a result is found, we have to convert back from the `IdRanges` to the
+  // corresponding blocks.
   auto addRow = [&result, &relevantBlocks1, &relevantBlocks2, &idRanges1,
                  &idRanges2]([[maybe_unused]] auto it1, auto it2) {
     result[0].push_back(*(relevantBlocks1.begin() + (it1 - idRanges1.begin())));
@@ -394,6 +397,10 @@ CompressedRelationReader::getBlocksForJoin(
   auto noop = ad_utility::noop;
   [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef(
       idRanges1, idRanges2, blockLessThanBlock, addRow, noop, noop);
+
+  // There might be duplicates in the blocks that we have to eliminate.
+  // TODO We should be able to eliminate those directly in the
+  // `zipperJoinWithUndef` routine.
   for (auto& vec : result) {
     vec.erase(std::ranges::unique(vec).begin(), vec.end());
   }
@@ -544,6 +551,7 @@ DecompressedBlock CompressedRelationReader::readPossiblyIncompleteBlock(
   auto end = containedInOnlyOneBlock ? begin + relationMetadata.numRows_
                                      : col1Column.end();
 
+  // If the `col1Id` was specified, we additionally have to filter by it.
   auto subBlock = [&]() {
     if (col1Id.has_value()) {
       return std::ranges::equal_range(begin, end, col1Id.value());
@@ -843,3 +851,11 @@ CompressedRelationReader::getBlocksFromMetadata(
   AD_CORRECTNESS_CHECK(col0IdHasExclusiveBlocks || result.size() <= 1);
   return result;
 }
+
+// _____________________________________________________________________________
+std::span
+CompressedRelationReader::getBlocksFromMetadata(
+    const MetadataAndBlocks& metadata) {
+  return getBlocksFromMetadata(metadata.relationMetadata_, metadata.col1Id_,
+                               metadata.blockMetadata_);
+}
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index ed31f2c836..fc3fa8017a 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -231,6 +231,15 @@ class CompressedRelationReader {
  public:
   using Allocator = ad_utility::AllocatorWithLimit;
 
+  // Combine the metadata of a single relation with a subset of its blocks and
+  // possibly a `col1Id` for additional filtering. This is used as the input to
+  // several functions below that take such an input.
+  struct MetadataAndBlocks {
+    const CompressedRelationMetadata relationMetadata_;
+    const std::span blockMetadata_;
+    std::optional col1Id_;
+  };
+
  private:
   // This cache stores a small number of decompressed blocks. Its current
   // purpose is to make the e2e-tests run fast. They contain many SPARQL queries
@@ -268,35 +277,34 @@ class CompressedRelationReader {
             ad_utility::File& file, IdTable* result,
             ad_utility::SharedConcurrentTimeoutTimer timer) const;
 
-  // Get all the blocks that can contain an Id from the `joinColumn`.
-  // TODO Include a timeout check.
-
-  struct MetadataAndBlocks {
-    const CompressedRelationMetadata relationMetadata_;
-    const std::span blockMetadata_;
-    std::optional col1Id_;
-  };
-  static std::vector getBlocksForJoin(
-      std::span joinColum, const MetadataAndBlocks& scanMetadata);
-
-  static std::array, 2> getBlocksForJoin(
-      const MetadataAndBlocks& scanMetadata1,
-      const MetadataAndBlocks& scanMetadata2);
-
+  // Similar to `scan` (directly above), but the result of the scan is lazily
+  // computed and returned as a generator of the single blocks that are scanned.
+  // The blocks are guaranteed to be in order.
   cppcoro::generator lazyScan(
       CompressedRelationMetadata metadata,
       std::vector blockMetadata,
       ad_utility::File& file,
       ad_utility::SharedConcurrentTimeoutTimer timer) const;
 
-  cppcoro::generator lazyScan(
-      CompressedRelationMetadata metadata, Id col1Id,
-      std::vector blockMetadata,
-      ad_utility::File& file,
-      ad_utility::SharedConcurrentTimeoutTimer timer) const;
-  cppcoro::generator asyncParallelBlockGenerator(
-      auto beginBlock, auto endBlock, ad_utility::File& file,
-      std::optional> columnIndices) const;
+  // Get the blocks (an ordered subset of the blocks that are passed in via the
+  // `blockIterator`) where the `col1Id` can theoretically match one of the
+  // elements in the `idIterator` (The col0Id is fixed and specified by the
+  // `blockIterator`). The join column of the scan is the first column that is
+  // not fixed by the `blockIterator`, so the middle column (col1) in case the
+  // `blockIterator` doesn't contain a `col1Id`, or the last column (col2) else.
+  static std::vector getBlocksForJoin(
+      std::span idIterator, const MetadataAndBlocks& blockIterator);
+
+  // For each of `scanMetadata1, scanMetadata2` get the blocks (an ordered
+  // subset of the blocks in the `scanMetadata` that might contain matching
+  // elements in the following scenario: The result of `scanMetadata1` is joined
+  // with the result of `scanMetadata2`. For each of the inputs the join column
+  // is the first column that is not fixed by the metadata, so the middle column
+  // (col1) in case the `scanMetadata` doesn't contain a `col1Id`, or the last
+  // column (col2) else.
+  static std::array, 2> getBlocksForJoin(
+      const MetadataAndBlocks& scanMetadata1,
+      const MetadataAndBlocks& scanMetadata2);
 
   /**
    * @brief For a permutation XYZ, retrieve all Z for given X and Y.
@@ -319,6 +327,15 @@ class CompressedRelationReader {
             ad_utility::File& file, IdTable* result,
             ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
+  // Similar to `scan` (directly above), but the result of the scan is lazily
+  // computed and returned as a generator of the single blocks that are scanned.
+  // The blocks are guaranteed to be in order.
+  cppcoro::generator lazyScan(
+      CompressedRelationMetadata metadata, Id col1Id,
+      std::vector blockMetadata,
+      ad_utility::File& file,
+      ad_utility::SharedConcurrentTimeoutTimer timer) const;
+
   // Only get the size of the result for a given permutation XYZ for a given X
   // and Y. This can be done by scanning one or two blocks. Note: The overload
   // of this function where only the X is given is not needed, as the size of
@@ -337,6 +354,11 @@ class CompressedRelationReader {
       const CompressedRelationMetadata& metadata, std::optional col1Id,
       std::span blockMetadata);
 
+  // The same function, but specify the arguments as the `MetadataAndBlocks`
+  // struct.
+  static std::span getBlocksFromMetadata(
+      const MetadataAndBlocks& metadataAndBlocks);
+
  private:
   // Read the block that is identified by the `blockMetaData` from the `file`.
   // If `columnIndices` is `nullopt`, then all columns of the block are read,
@@ -386,6 +408,15 @@ class CompressedRelationReader {
       const CompressedRelationMetadata& relationMetadata,
       std::optional col1Id, ad_utility::File& file,
       const CompressedBlockMetadata& blockMetadata) const;
+
+  // Yield all the blocks in the range `[beginBlock, endBlock)`. If the
+  // `columnIndices` are set, that only the specified columns from the blocks
+  // are yielded, else the complete blocks are yielded. The blocks are yielded
+  // in the correct order, but asynchronously read and decompressed using
+  // multiple worker threads.
+  cppcoro::generator asyncParallelBlockGenerator(
+      auto beginBlock, auto endBlock, ad_utility::File& file,
+      std::optional> columnIndices) const;
 };
 
 #endif  // QLEVER_COMPRESSEDRELATION_H

From 07891b8abeb6a1624c7100c9183460f66596df63 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 28 Jun 2023 17:18:38 +0200
Subject: [PATCH 081/150] Some refactorings in the CompresseddRelation.cpp file
 that reduce the amount of code duplication.

---
 src/index/CompressedRelation.cpp | 94 ++++++++++++--------------------
 test/CompressedRelationsTest.cpp |  5 +-
 2 files changed, 38 insertions(+), 61 deletions(-)

diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 739b7228c2..30faab3f80 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -29,7 +29,6 @@ void CompressedRelationReader::scan(
       getBlocksFromMetadata(metadata, std::nullopt, blockMetadata);
   auto beginBlock = relevantBlocks.begin();
   auto endBlock = relevantBlocks.end();
-  Id col0Id = metadata.col0Id_;
   // The total size of the result is now known.
   result->resize(metadata.getNofElements());
 
@@ -41,26 +40,6 @@ void CompressedRelationReader::scan(
   // in the result (only needed for checking of invariants).
   size_t spaceLeft = result->size();
 
-  // The first block might contain entries that are not part of our
-  // actual scan result.
-  bool firstBlockIsIncomplete =
-      beginBlock < endBlock && (beginBlock->firstTriple_.col0Id_ < col0Id ||
-                                beginBlock->lastTriple_.col0Id_ > col0Id);
-  auto lastBlock = endBlock - 1;
-
-  bool lastBlockIsIncomplete =
-      beginBlock < lastBlock && (lastBlock->firstTriple_.col0Id_ < col0Id ||
-                                 lastBlock->lastTriple_.col0Id_ > col0Id);
-
-  // Invariant: A relation spans multiple blocks exclusively or several
-  // entities are stored completely in the same Block.
-  AD_CORRECTNESS_CHECK(!firstBlockIsIncomplete || (beginBlock == lastBlock));
-  AD_CORRECTNESS_CHECK(!lastBlockIsIncomplete);
-  if (firstBlockIsIncomplete) {
-    AD_CORRECTNESS_CHECK(metadata.offsetInBlock_ !=
-                         std::numeric_limits::max());
-  }
-
   // We have at most one block that is incomplete and thus requires trimming.
   // Set up a lambda, that reads this block and decompresses it to
   // the result.
@@ -77,13 +56,11 @@ void CompressedRelationReader::scan(
     spaceLeft -= trimmedBlock.size();
   };
 
-  // Read the first block if it is incomplete
-  if (firstBlockIsIncomplete) {
-    readIncompleteBlock(*beginBlock);
-    ++beginBlock;
-    if (timer) {
-      timer->wlock()->checkTimeoutAndThrow("IndexScan :");
-    }
+  // Read the first block (it might be incomplete)
+  readIncompleteBlock(*beginBlock);
+  ++beginBlock;
+  if (timer) {
+    timer->wlock()->checkTimeoutAndThrow("IndexScan :");
   }
 
   // Read all the other (complete!) blocks in parallel
@@ -127,6 +104,7 @@ void CompressedRelationReader::scan(
   }
 }
 
+// ____________________________________________________________________________
 cppcoro::generator
 CompressedRelationReader::asyncParallelBlockGenerator(
     auto beginBlock, auto endBlock, ad_utility::File& file,
@@ -189,35 +167,13 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   if (beginBlock == endBlock) {
     co_return;
   }
-  Id col0Id = metadata.col0Id_;
-  // The first block might contain entries that are not part of our
-  // actual scan result.
-  bool firstBlockIsIncomplete =
-      beginBlock < endBlock && (beginBlock->firstTriple_.col0Id_ < col0Id ||
-                                beginBlock->lastTriple_.col0Id_ > col0Id);
-  auto lastBlock = endBlock - 1;
-
-  bool lastBlockIsIncomplete =
-      beginBlock < lastBlock && (lastBlock->firstTriple_.col0Id_ < col0Id ||
-                                 lastBlock->lastTriple_.col0Id_ > col0Id);
-
-  // Invariant: A relation spans multiple blocks exclusively or several
-  // entities are stored completely in the same Block.
-  AD_CORRECTNESS_CHECK(!firstBlockIsIncomplete || (beginBlock == lastBlock));
-  AD_CORRECTNESS_CHECK(!lastBlockIsIncomplete);
-  if (firstBlockIsIncomplete) {
-    AD_CORRECTNESS_CHECK(metadata.offsetInBlock_ !=
-                         std::numeric_limits::max());
-  }
-
-  // Read the first block if it is incomplete
-  if (firstBlockIsIncomplete) {
-    co_yield readPossiblyIncompleteBlock(metadata, std::nullopt, file,
-                                         *beginBlock);
-    ++beginBlock;
-    if (timer) {
-      timer->wlock()->checkTimeoutAndThrow("IndexScan :");
-    }
+
+  // Read the first block, it might be incomplete
+  co_yield readPossiblyIncompleteBlock(metadata, std::nullopt, file,
+                                       *beginBlock);
+  ++beginBlock;
+  if (timer) {
+    timer->wlock()->checkTimeoutAndThrow("IndexScan :");
   }
 
   for (auto& block :
@@ -371,7 +327,6 @@ std::array, 2>
 CompressedRelationReader::getBlocksForJoin(
     const MetadataAndBlocks& scanMetadata1,
     const MetadataAndBlocks& scanMetadata2) {
-  // Get the
   auto relevantBlocks1 = getBlocksFromMetadata(scanMetadata1);
   auto relevantBlocks2 = getBlocksFromMetadata(scanMetadata2);
 
@@ -849,6 +804,29 @@ CompressedRelationReader::getBlocksFromMetadata(
       metadata.offsetInBlock_ == std::numeric_limits::max();
   // `result` might also be empty if no block was found at all.
   AD_CORRECTNESS_CHECK(col0IdHasExclusiveBlocks || result.size() <= 1);
+
+  if (!result.empty()) {
+    // Check some invariants of the splitting of the relations.
+    bool firstIncomplete = result.front().firstTriple_.col0Id_ < col0Id ||
+                           result.front().lastTriple_.col0Id_ > col0Id;
+
+    auto lastBlock = result.end() - 1;
+
+    bool lastIncomplete = result.begin() < lastBlock &&
+                          (lastBlock->firstTriple_.col0Id_ < col0Id ||
+                           lastBlock->lastTriple_.col0Id_ > col0Id);
+
+    // Invariant: A relation spans multiple blocks exclusively or several
+    // entities are stored completely in the same Block.
+    if (!col1Id.has_value()) {
+      AD_CORRECTNESS_CHECK(!firstIncomplete || (result.begin() == lastBlock));
+      AD_CORRECTNESS_CHECK(!lastIncomplete);
+    }
+    if (firstIncomplete) {
+      AD_CORRECTNESS_CHECK(!col0IdHasExclusiveBlocks);
+    }
+    // AD_CORRECTNESS_CHECK(firstIncomplete || !col0IdHasExclusiveBlocks);
+  }
   return result;
 }
 
diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp
index 65e4c48081..70274cda88 100644
--- a/test/CompressedRelationsTest.cpp
+++ b/test/CompressedRelationsTest.cpp
@@ -6,7 +6,6 @@
 
 #include "./IndexTestHelpers.h"
 #include "index/CompressedRelation.h"
-#include "index/Permutation.h"
 #include "util/GTestHelpers.h"
 #include "util/Serializer/ByteBufferSerializer.h"
 #include "util/SourceLocation.h"
@@ -301,7 +300,7 @@ TEST(CompressedRelationReader, getBlocksForJoin) {
   CompressedBlockMetadata block1{
       {}, 0, {V(16), V(0), V(0)}, {V(38), V(4), V(12)}};
   CompressedBlockMetadata block2{
-      {}, 0, {V(39), V(0), V(0)}, {V(42), V(4), V(12)}};
+      {}, 0, {V(42), V(3), V(0)}, {V(42), V(4), V(12)}};
   CompressedBlockMetadata block3{
       {}, 0, {V(42), V(4), V(13)}, {V(42), V(6), V(9)}};
 
@@ -315,7 +314,7 @@ TEST(CompressedRelationReader, getBlocksForJoin) {
 
   auto test = [&metadataAndBlocks](
                   const std::vector& joinColumn,
-                  const std::vector expectedBlocks,
+                  const std::vector& expectedBlocks,
                   source_location l = source_location::current()) {
     auto t = generateLocationTrace(l);
     auto result = CompressedRelationReader::getBlocksForJoin(joinColumn,

From c3675661889de894d479decc32fde86c124c62da Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 28 Jun 2023 20:02:12 +0200
Subject: [PATCH 082/150] Several further refactorings.

---
 src/engine/IndexScan.cpp  | 13 ++++-----
 src/engine/IndexScan.h    |  6 ++---
 src/index/Permutation.cpp | 35 ++++++++++++++++++++++++
 src/index/Permutation.h   | 57 +++++++++++++++++----------------------
 4 files changed, 70 insertions(+), 41 deletions(-)

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index 264bb9d431..8875ab731d 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -279,9 +279,10 @@ std::array IndexScan::getPermutedTriple()
           triple[permutation[2]]};
 }
 
+// ___________________________________________________________________________
 // TODO include a timeout timer.
-cppcoro::generator IndexScan::getLazyScan(const IndexScan& s,
-                                                   const auto& blocks) {
+cppcoro::generator IndexScan::getLazyScan(
+    const IndexScan& s, std::vector blocks) {
   const IndexImpl& index = s.getIndex().getImpl();
   Id col0Id = s.getPermutedTriple()[0]->toValueId(index.getVocab()).value();
   std::optional col1Id;
@@ -289,16 +290,16 @@ cppcoro::generator IndexScan::getLazyScan(const IndexScan& s,
     col1Id = s.getPermutedTriple()[1]->toValueId(index.getVocab()).value();
   }
   if (!col1Id.has_value()) {
-    return index.getPermutation(s.permutation()).lazyScan(col0Id, blocks);
+    return index.getPermutation(s.permutation()).lazyScan(col0Id, std::move(blocks));
   } else {
     return index.getPermutation(s.permutation())
-        .lazyScan(col0Id, col1Id.value(), blocks);
+        .lazyScan(col0Id, col1Id.value(), std::move(blocks));
   }
 };
 
 // ________________________________________________________________
-auto IndexScan::getMetadataForScan(const IndexScan& s)
-    -> std::optional {
+std::optional IndexScan::getMetadataForScan(const IndexScan& s)
+     {
   auto permutedTriple = s.getPermutedTriple();
   const IndexImpl& index = s.getIndex().getImpl();
   std::optional optId = permutedTriple[0]->toValueId(index.getVocab());
diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h
index 4f16db8c75..3932e2bbcc 100644
--- a/src/engine/IndexScan.h
+++ b/src/engine/IndexScan.h
@@ -49,10 +49,10 @@ class IndexScan : public Operation {
       const IndexScan& s1, const IndexScan& s2);
   static cppcoro::generator lazyScanForJoinOfColumnWithScan(
       std::span joinColumn, const IndexScan& s);
-  static cppcoro::generator getLazyScan(const IndexScan& s,
-                                                 const auto& blocks);
+  static cppcoro::generator getLazyScan(
+      const IndexScan& s, std::vector blocks);
   static auto getMetadataForScan(const IndexScan& s)
-      -> std::optional;
+      -> std::optional;
 
  private:
   // TODO Make the `getSizeEstimateBeforeLimit()` function `const` for
diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp
index 66abb6d4a4..0cd80be994 100644
--- a/src/index/Permutation.cpp
+++ b/src/index/Permutation.cpp
@@ -114,3 +114,38 @@ std::string_view Permutation::toString(Permutation::Enum permutation) {
   }
   AD_FAIL();
 }
+
+// _____________________________________________________________________
+std::optional Permutation::getMetadataAndBlocks(
+    Id col0Id, std::optional col1Id) const {
+  if (!meta_.col0IdExists(col0Id)) {
+    return std::nullopt;
+  }
+
+  auto metadata = meta_.getMetaData(col0Id);
+  return MetadataAndBlocks{meta_.getMetaData(col0Id),
+                           CompressedRelationReader::getBlocksFromMetadata(
+                               metadata, col1Id, meta_.blockData()),
+                           col1Id};
+}
+
+// _____________________________________________________________________
+cppcoro::generator Permutation::lazyScan(
+    Id col0Id, std::vector blocks,
+    ad_utility::SharedConcurrentTimeoutTimer timer) const {
+  if (!meta_.col0IdExists(col0Id)) {
+    return {};
+  }
+  return reader_.lazyScan(meta_.getMetaData(col0Id), std::move(blocks), file_, timer);
+}
+
+// _____________________________________________________________________
+cppcoro::generator Permutation::lazyScan(
+    Id col0Id, Id col1Id, const std::vector& blocks,
+    ad_utility::SharedConcurrentTimeoutTimer timer) const {
+  if (!meta_.col0IdExists(col0Id)) {
+    return {};
+  }
+  return reader_.lazyScan(meta_.getMetaData(col0Id), col1Id, blocks, file_,
+                          timer);
+}
diff --git a/src/index/Permutation.h b/src/index/Permutation.h
index 5d59867297..8f3f2caaaa 100644
--- a/src/index/Permutation.h
+++ b/src/index/Permutation.h
@@ -45,39 +45,7 @@ class Permutation {
   // everything that has to be done when reading an index from disk
   void loadFromDisk(const std::string& onDiskBase);
 
-  using MetaDataAndBlocks = CompressedRelationReader::MetadataAndBlocks;
 
-  std::optional getMetadataAndBlocks(
-      Id col0Id, std::optional col1Id) const {
-    if (!meta_.col0IdExists(col0Id)) {
-      return std::nullopt;
-    }
-
-    auto metadata = meta_.getMetaData(col0Id);
-    return MetaDataAndBlocks{meta_.getMetaData(col0Id),
-                             CompressedRelationReader::getBlocksFromMetadata(
-                                 metadata, col1Id, meta_.blockData()),
-                             col1Id};
-  }
-
-  cppcoro::generator lazyScan(
-      Id col0Id, const std::vector& blocks,
-      ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const {
-    if (!meta_.col0IdExists(col0Id)) {
-      return {};
-    }
-    return reader_.lazyScan(meta_.getMetaData(col0Id), blocks, file_, timer);
-  }
-
-  cppcoro::generator lazyScan(
-      Id col0Id, Id col1Id, const std::vector& blocks,
-      ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const {
-    if (!meta_.col0IdExists(col0Id)) {
-      return {};
-    }
-    return reader_.lazyScan(meta_.getMetaData(col0Id), col1Id, blocks, file_,
-                            timer);
-  }
   /// For a given ID for the first column, retrieve all IDs of the second and
   /// third column, and store them in `result`. This is just a thin wrapper
   /// around `CompressedRelationMetaData::scan`.
@@ -90,6 +58,31 @@ class Permutation {
   void scan(Id col0Id, Id col1Id, IdTable* result,
             ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
+  // Typedef to propagate the `MetadataAndblocks` type.
+  using MetadataAndBlocks = CompressedRelationReader::MetadataAndBlocks;
+
+
+  // The overloaded function `lazyScan` is similar to `scan` (see above) with the following differences:
+  // - The result is returned as a lazy generator of blocks.
+  // - The block metadata must be passed in manually. It can be obtained via the `getMetadataAndBlocks` function below
+  //   and then be prefiltered. The blocks must be passed in in ascending order and must only contain blocks that contain
+  //   the given `col0Id` (combined with the `col1Id` for the second overload), else the behavior is undefined.
+  // TODO We should only communicate these interface via the `MetadataAndBlocks` class and make this a
+  // strong class that always maintains its invariants.
+  cppcoro::generator lazyScan(
+      Id col0Id, std::vector blocks,
+      ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
+
+  cppcoro::generator lazyScan(
+      Id col0Id, Id col1Id, const std::vector& blocks,
+      ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
+
+  // Return the relation metadata for the relation specified by the `col0Id` along with the metadata of all the
+  // blocks that contain this relation (also prefiltered by the `col1Id` if specified). If the `col0Id` does not exist
+  // in this permutation, `nullopt` is returned.
+  std::optional getMetadataAndBlocks(
+      Id col0Id, std::optional col1Id) const;
+
   /// Similar to the previous `scan` function, but only get the size of the
   /// result
   size_t getResultSizeOfScan(Id col0Id, Id col1Id) const;

From a6d8d54ce601bf0af087b3d84cbed52480e8e349 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 28 Jun 2023 20:17:19 +0200
Subject: [PATCH 083/150] Added comments etc.

---
 src/engine/IndexScan.cpp  |  8 +++++---
 src/engine/IndexScan.h    | 19 +++++++++++++++----
 src/engine/Join.h         | 10 +++++++++-
 src/index/Permutation.cpp |  3 ++-
 src/index/Permutation.h   | 24 ++++++++++++++----------
 5 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index 8875ab731d..b3c9fc12c0 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -290,7 +290,8 @@ cppcoro::generator IndexScan::getLazyScan(
     col1Id = s.getPermutedTriple()[1]->toValueId(index.getVocab()).value();
   }
   if (!col1Id.has_value()) {
-    return index.getPermutation(s.permutation()).lazyScan(col0Id, std::move(blocks));
+    return index.getPermutation(s.permutation())
+        .lazyScan(col0Id, std::move(blocks));
   } else {
     return index.getPermutation(s.permutation())
         .lazyScan(col0Id, col1Id.value(), std::move(blocks));
@@ -298,8 +299,8 @@ cppcoro::generator IndexScan::getLazyScan(
 };
 
 // ________________________________________________________________
-std::optional IndexScan::getMetadataForScan(const IndexScan& s)
-     {
+std::optional IndexScan::getMetadataForScan(
+    const IndexScan& s) {
   auto permutedTriple = s.getPermutedTriple();
   const IndexImpl& index = s.getIndex().getImpl();
   std::optional optId = permutedTriple[0]->toValueId(index.getVocab());
@@ -334,6 +335,7 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans(
 // ________________________________________________________________
 cppcoro::generator IndexScan::lazyScanForJoinOfColumnWithScan(
     std::span joinColumn, const IndexScan& s) {
+  AD_EXPENSIVE_CHECK(std::ranges::is_sorted(joinColumn));
   AD_CONTRACT_CHECK(s.numVariables_ < 3);
 
   auto metaBlocks1 = getMetadataForScan(s);
diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h
index 3932e2bbcc..a46f997c73 100644
--- a/src/engine/IndexScan.h
+++ b/src/engine/IndexScan.h
@@ -45,14 +45,19 @@ class IndexScan : public Operation {
   // can be read from the Metadata.
   size_t getExactSize() const { return sizeEstimate_; }
 
+  // Return two generators that lazily yield the results of `s1` and `s2` in
+  // blocks, but only the blocks that can theoretically contain matching rows
+  // when performing a join on the first variable of `s1` with the first
+  // variable of `s2`.
   static std::array, 2> lazyScanForJoinOfTwoScans(
       const IndexScan& s1, const IndexScan& s2);
+
+  // Return a generator that lazily yields the result of `s` in blocks, but only
+  // the blocks that can theoretically contain matching rows when performing a
+  // join between the first variable of `s`  with the `joinColumn`. Requires
+  // that the `joinColumn` is sorted, else the behavior is undefined.
   static cppcoro::generator lazyScanForJoinOfColumnWithScan(
       std::span joinColumn, const IndexScan& s);
-  static cppcoro::generator getLazyScan(
-      const IndexScan& s, std::vector blocks);
-  static auto getMetadataForScan(const IndexScan& s)
-      -> std::optional;
 
  private:
   // TODO Make the `getSizeEstimateBeforeLimit()` function `const` for
@@ -98,4 +103,10 @@ class IndexScan : public Operation {
   // `permutation_`. For example if `permutation_ == PSO` then the result is
   // {&predicate_, &subject_, &object_}
   std::array getPermutedTriple() const;
+
+  //  Helper functions for the public `getLazyScanFor...` functions (see above).
+  static cppcoro::generator getLazyScan(
+      const IndexScan& s, std::vector blocks);
+  static std::optional getMetadataForScan(
+      const IndexScan& s);
 };
diff --git a/src/engine/Join.h b/src/engine/Join.h
index 92b30c7e30..0ad87b9bdb 100644
--- a/src/engine/Join.h
+++ b/src/engine/Join.h
@@ -134,8 +134,16 @@ class Join : public Operation {
 
   ResultTable computeResultForJoinWithFullScanDummy();
 
+  // A special implementation that is called when both children are
+  // `IndexScan`s. Uses the lazy scans to only retrieve the subset of the
+  // `IndexScan`s that is actually needed without fully materializing them.
   void computeResultForTwoIndexScans(IdTable* resultPtr);
-  template 
+
+  // A special implementation that is called when one of the children is an
+  // `IndexScan`. The argument `scanIsLeft` determines whether the `IndexScan`
+  // is the left or the right child of this `Join`. This needs to be known to
+  // determine the correct order of the columns in the result.
+  template 
   void computeResultForIndexScanAndColumn(const IdTable& inputTable,
                                           ColumnIndex joinColumnIndexTable,
                                           const IndexScan& scan,
diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp
index 0cd80be994..c917bd793d 100644
--- a/src/index/Permutation.cpp
+++ b/src/index/Permutation.cpp
@@ -136,7 +136,8 @@ cppcoro::generator Permutation::lazyScan(
   if (!meta_.col0IdExists(col0Id)) {
     return {};
   }
-  return reader_.lazyScan(meta_.getMetaData(col0Id), std::move(blocks), file_, timer);
+  return reader_.lazyScan(meta_.getMetaData(col0Id), std::move(blocks), file_,
+                          timer);
 }
 
 // _____________________________________________________________________
diff --git a/src/index/Permutation.h b/src/index/Permutation.h
index 8f3f2caaaa..1e1c537ee6 100644
--- a/src/index/Permutation.h
+++ b/src/index/Permutation.h
@@ -45,7 +45,6 @@ class Permutation {
   // everything that has to be done when reading an index from disk
   void loadFromDisk(const std::string& onDiskBase);
 
-
   /// For a given ID for the first column, retrieve all IDs of the second and
   /// third column, and store them in `result`. This is just a thin wrapper
   /// around `CompressedRelationMetaData::scan`.
@@ -61,14 +60,18 @@ class Permutation {
   // Typedef to propagate the `MetadataAndblocks` type.
   using MetadataAndBlocks = CompressedRelationReader::MetadataAndBlocks;
 
-
-  // The overloaded function `lazyScan` is similar to `scan` (see above) with the following differences:
+  // The overloaded function `lazyScan` is similar to `scan` (see above) with
+  // the following differences:
   // - The result is returned as a lazy generator of blocks.
-  // - The block metadata must be passed in manually. It can be obtained via the `getMetadataAndBlocks` function below
-  //   and then be prefiltered. The blocks must be passed in in ascending order and must only contain blocks that contain
-  //   the given `col0Id` (combined with the `col1Id` for the second overload), else the behavior is undefined.
-  // TODO We should only communicate these interface via the `MetadataAndBlocks` class and make this a
-  // strong class that always maintains its invariants.
+  // - The block metadata must be passed in manually. It can be obtained via the
+  // `getMetadataAndBlocks` function below
+  //   and then be prefiltered. The blocks must be passed in in ascending order
+  //   and must only contain blocks that contain the given `col0Id` (combined
+  //   with the `col1Id` for the second overload), else the behavior is
+  //   undefined.
+  // TODO We should only communicate these interface via the
+  // `MetadataAndBlocks` class and make this a strong class that always
+  // maintains its invariants.
   cppcoro::generator lazyScan(
       Id col0Id, std::vector blocks,
       ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
@@ -77,8 +80,9 @@ class Permutation {
       Id col0Id, Id col1Id, const std::vector& blocks,
       ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
-  // Return the relation metadata for the relation specified by the `col0Id` along with the metadata of all the
-  // blocks that contain this relation (also prefiltered by the `col1Id` if specified). If the `col0Id` does not exist
+  // Return the relation metadata for the relation specified by the `col0Id`
+  // along with the metadata of all the blocks that contain this relation (also
+  // prefiltered by the `col1Id` if specified). If the `col0Id` does not exist
   // in this permutation, `nullopt` is returned.
   std::optional getMetadataAndBlocks(
       Id col0Id, std::optional col1Id) const;

From 213cf8b38f21fc2cf79638e45f27f16974e5cac6 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 28 Jun 2023 20:19:14 +0200
Subject: [PATCH 084/150] Not much done yet.

---
 src/util/Generator.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/util/Generator.h b/src/util/Generator.h
index 15b46c79cf..4d4dc24837 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -19,6 +19,11 @@ namespace cppcoro {
 template 
 class generator;
 
+struct AddDetail {
+  std::string key_;
+  nlohmann::json value_;
+};
+
 namespace detail {
 template 
 class generator_promise {

From e5f0e4cab98d49bccde233547ab6d68a85b9c11d Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Thu, 29 Jun 2023 13:51:26 +0200
Subject: [PATCH 085/150] This doesn't work, find the bug.

---
 src/engine/IndexScan.cpp         |  11 +-
 src/engine/Join.cpp              |   2 +-
 src/global/Constants.h           |   5 +-
 src/index/CompressedRelation.cpp | 259 ++++++++++++++-----------------
 src/index/CompressedRelation.h   |  57 +++----
 src/index/Index.cpp              |  22 +--
 src/index/Index.h                |  35 +----
 src/index/IndexImpl.cpp          |  47 ++----
 src/index/IndexImpl.h            |  41 ++---
 src/index/Permutation.cpp        |  46 +++---
 src/index/Permutation.h          |  18 +--
 src/index/Permutations.h         | 146 -----------------
 src/index/TriplesView.h          |   2 +-
 test/CompressedRelationsTest.cpp |  18 +--
 test/IndexTest.cpp               |   5 +-
 test/TriplesViewTest.cpp         |   3 +-
 16 files changed, 225 insertions(+), 492 deletions(-)
 delete mode 100644 src/index/Permutations.h

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index b3c9fc12c0..bf6ea2609e 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -120,7 +120,7 @@ ResultTable IndexScan::computeResult() {
   const auto& index = _executionContext->getIndex();
   const auto permutedTriple = getPermutedTriple();
   if (numVariables_ == 2) {
-    index.scan(*permutedTriple[0], &idTable, permutation_, _timeoutTimer);
+    index.scan(*permutedTriple[0], std::nullopt, &idTable, permutation_, _timeoutTimer);
   } else if (numVariables_ == 1) {
     index.scan(*permutedTriple[0], *permutedTriple[1], &idTable, permutation_,
                _timeoutTimer);
@@ -289,13 +289,8 @@ cppcoro::generator IndexScan::getLazyScan(
   if (s.numVariables_ == 1) {
     col1Id = s.getPermutedTriple()[1]->toValueId(index.getVocab()).value();
   }
-  if (!col1Id.has_value()) {
-    return index.getPermutation(s.permutation())
-        .lazyScan(col0Id, std::move(blocks));
-  } else {
-    return index.getPermutation(s.permutation())
-        .lazyScan(col0Id, col1Id.value(), std::move(blocks));
-  }
+  return index.getPermutation(s.permutation())
+      .lazyScan(col0Id, col1Id, std::move(blocks));
 };
 
 // ________________________________________________________________
diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index af35fb783c..9aa062cb30 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -268,7 +268,7 @@ Join::ScanMethodType Join::getScanMethod(
   const auto& idx = _executionContext->getIndex();
   const auto scanLambda = [&idx](const Permutation::Enum perm) {
     return
-        [&idx, perm](Id id, IdTable* idTable) { idx.scan(id, idTable, perm); };
+        [&idx, perm](Id id, IdTable* idTable) { idx.scan(id, std::nullopt, idTable, perm); };
   };
   AD_CORRECTNESS_CHECK(scan.getResultWidth() == 3);
   return scanLambda(scan.permutation());
diff --git a/src/global/Constants.h b/src/global/Constants.h
index 92892c8787..ae0681e6a7 100644
--- a/src/global/Constants.h
+++ b/src/global/Constants.h
@@ -180,7 +180,10 @@ inline auto& RuntimeParameters() {
       // timeout exception.
       Double<"sort-estimate-cancellation-factor">{3.0},
       SizeT<"cache-max-num-entries">{1000}, SizeT<"cache-max-size-gb">{30},
-      SizeT<"cache-max-size-gb-single-entry">{5}};
+      SizeT<"cache-max-size-gb-single-entry">{5},
+      SizeT<"lazy-index-scan-queue-size">{5},
+      SizeT<"lazy-index-scan-num-threads">{10}
+  };
   return params;
 }
 
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 30faab3f80..aaeb5f5c27 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -18,19 +18,19 @@
 using namespace std::chrono_literals;
 
 // ____________________________________________________________________________
-void CompressedRelationReader::scan(
+IdTable CompressedRelationReader::scan(
     const CompressedRelationMetadata& metadata,
     std::span blockMetadata,
-    ad_utility::File& file, IdTable* result,
-    ad_utility::SharedConcurrentTimeoutTimer timer) const {
-  AD_CONTRACT_CHECK(result->numColumns() == NumColumns);
+    ad_utility::File& file, const TimeoutTimer& timer) const {
+
+  IdTable result(NumColumns, allocator_);
 
   auto relevantBlocks =
       getBlocksFromMetadata(metadata, std::nullopt, blockMetadata);
   auto beginBlock = relevantBlocks.begin();
   auto endBlock = relevantBlocks.end();
   // The total size of the result is now known.
-  result->resize(metadata.getNofElements());
+  result.resize(metadata.getNofElements());
 
   // The position in the result to which the next block is being
   // decompressed.
@@ -38,7 +38,7 @@ void CompressedRelationReader::scan(
 
   // The number of rows for which we still have space
   // in the result (only needed for checking of invariants).
-  size_t spaceLeft = result->size();
+  size_t spaceLeft = result.size();
 
   // We have at most one block that is incomplete and thus requires trimming.
   // Set up a lambda, that reads this block and decompresses it to
@@ -48,7 +48,7 @@ void CompressedRelationReader::scan(
         readPossiblyIncompleteBlock(metadata, std::nullopt, file, block);
     for (size_t i = 0; i < trimmedBlock.numColumns(); ++i) {
       const auto& inputCol = trimmedBlock.getColumn(i);
-      auto resultColumn = result->getColumn(i);
+      auto resultColumn = result.getColumn(i);
       AD_CORRECTNESS_CHECK(inputCol.size() <= resultColumn.size());
       std::ranges::copy(inputCol, resultColumn.begin());
     }
@@ -56,12 +56,10 @@ void CompressedRelationReader::scan(
     spaceLeft -= trimmedBlock.size();
   };
 
-  // Read the first block (it might be incomplete)
+  // Read the first block (it might be incomplete).
   readIncompleteBlock(*beginBlock);
   ++beginBlock;
-  if (timer) {
-    timer->wlock()->checkTimeoutAndThrow("IndexScan :");
-  }
+  checkTimeout(timer);
 
   // Read all the other (complete!) blocks in parallel
   if (beginBlock < endBlock) {
@@ -83,7 +81,7 @@ void CompressedRelationReader::scan(
           ad_utility::TimeBlockAndLog tbl{"Decompressing a block"};
 
           decompressBlockToExistingIdTable(compressedBuffer, block.numRows_,
-                                           *result, rowIndexOfNextBlock);
+                                           result, rowIndexOfNextBlock);
         };
 
         // The `decompressLambda` can now run in parallel
@@ -102,55 +100,66 @@ void CompressedRelationReader::scan(
       AD_CORRECTNESS_CHECK(spaceLeft == 0);
     }  // End of omp parallel region, all the decompression was handled now.
   }
+  return result;
 }
 
 // ____________________________________________________________________________
 cppcoro::generator
 CompressedRelationReader::asyncParallelBlockGenerator(
     auto beginBlock, auto endBlock, ad_utility::File& file,
-    std::optional> columnIndices) const {
+    std::optional> columnIndices,
+    const TimeoutTimer& timer) const {
   if (beginBlock == endBlock) {
     co_return;
   }
   // Note: It is important to define the `threads` before the `queue`. That way
   // the joining destructor of the threads will see that the queue is finished
   // and join.
-  std::mutex fileMutex;
+  std::mutex blockIteratorMutex;
   std::vector threads;
+  const size_t queueSize = RuntimeParameters().get<"lazy-index-scan-queue-size">();
   ad_utility::data_structures::OrderedThreadSafeQueue queue{
-      5};
-  size_t blockIndex = 0;
+      queueSize};
+  auto blockIterator = beginBlock;
   auto readAndDecompressBlock = [&]() {
-    while (true) {
-      std::unique_lock lock{fileMutex};
-      if (beginBlock == endBlock) {
-        return;
-      }
-      auto block = *beginBlock;
-      CompressedBlock compressedBuffer =
-          readCompressedBlockFromFile(block, file, columnIndices);
-      ++beginBlock;
-      size_t myIndex = blockIndex;
-      ++blockIndex;
-      bool isLastBlock = beginBlock == endBlock;
-      lock.unlock();
-      if (!queue.push(myIndex,
-                      decompressBlock(compressedBuffer, block.numRows_))) {
-        return;
-      }
-      if (isLastBlock) {
-        queue.finish();
+    try {
+      while (true) {
+        std::unique_lock lock{blockIteratorMutex};
+        if (blockIterator == endBlock) {
+          return;
+        }
+        auto block = *blockIterator;
+        auto myIndex = static_cast(blockIterator - beginBlock);
+        ++blockIterator;
+        bool isLastBlock = blockIterator == endBlock;
+        // Note: the reading of the block could also happen without holding the lock. We still perform it inside the
+        // lock to avoid contention of the file. On a fast SSD we could possibly change this, but this has to be
+        // investigated.
+        CompressedBlock compressedBlock =
+            readCompressedBlockFromFile(block, file, columnIndices);
+        lock.unlock();
+        bool pushWasSuccessful = queue.push(
+            myIndex, decompressBlock(compressedBlock, block.numRows_));
+        checkTimeout(timer);
+        if (!pushWasSuccessful) {
+          return;
+        }
+        if (isLastBlock) {
+          queue.finish();
+        }
       }
+    } catch (...) {
+      queue.pushException(std::current_exception());
     }
   };
 
-  // TODO get the number of optimal threads from the operating
-  // system.
-  for (size_t j = 0; j < 10; ++j) {
+  const size_t numThreads = RuntimeParameters().get<"lazy-index-scan-num-threads">();
+  for ([[maybe_unused]] auto j : std::views::iota(0u,numThreads)) {
     threads.emplace_back(readAndDecompressBlock);
   }
 
   while (auto opt = queue.pop()) {
+    checkTimeout(timer);
     co_yield opt.value();
   }
 }
@@ -159,11 +168,11 @@ CompressedRelationReader::asyncParallelBlockGenerator(
 cppcoro::generator CompressedRelationReader::lazyScan(
     CompressedRelationMetadata metadata,
     std::vector blockMetadata, ad_utility::File& file,
-    ad_utility::SharedConcurrentTimeoutTimer timer) const {
+    const TimeoutTimer& timer) const {
   auto relevantBlocks =
       getBlocksFromMetadata(metadata, std::nullopt, blockMetadata);
-  auto beginBlock = relevantBlocks.begin();
-  auto endBlock = relevantBlocks.end();
+  const auto beginBlock = relevantBlocks.begin();
+  const auto endBlock = relevantBlocks.end();
   if (beginBlock == endBlock) {
     co_return;
   }
@@ -171,24 +180,19 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   // Read the first block, it might be incomplete
   co_yield readPossiblyIncompleteBlock(metadata, std::nullopt, file,
                                        *beginBlock);
-  ++beginBlock;
-  if (timer) {
-    timer->wlock()->checkTimeoutAndThrow("IndexScan :");
-  }
+  checkTimeout(timer);
 
-  for (auto& block :
-       asyncParallelBlockGenerator(beginBlock, endBlock, file, std::nullopt)) {
+  for (auto& block : asyncParallelBlockGenerator(
+           beginBlock + 1, endBlock, file, std::nullopt, timer)) {
     co_yield block;
   }
-
-  // TODO Timeout checks.
 }
 
 // _____________________________________________________________________________
 cppcoro::generator CompressedRelationReader::lazyScan(
     CompressedRelationMetadata metadata, Id col1Id,
     std::vector blockMetadata, ad_utility::File& file,
-    ad_utility::SharedConcurrentTimeoutTimer timer) const {
+    const TimeoutTimer& timer) const {
   auto relevantBlocks = getBlocksFromMetadata(metadata, col1Id, blockMetadata);
   auto beginBlock = relevantBlocks.begin();
   auto endBlock = relevantBlocks.end();
@@ -209,25 +213,20 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   auto getIncompleteBlock = [&](auto it) {
     auto result = readPossiblyIncompleteBlock(metadata, col1Id, file, *it);
     result.setColumnSubset(std::array{1});
-    if (timer) {
-      timer->wlock()->checkTimeoutAndThrow("IndexScan: ");
-    }
+    checkTimeout(timer);
     return result;
   };
 
   if (beginBlock < endBlock) {
     co_yield getIncompleteBlock(beginBlock);
-    ++beginBlock;
   }
 
-  if (beginBlock < endBlock) {
-    for (auto& block : asyncParallelBlockGenerator(beginBlock, endBlock - 1,
-                                                   file, std::vector{1UL})) {
+  if (beginBlock + 1 < endBlock) {
+    for (auto& block :
+         asyncParallelBlockGenerator(beginBlock + 1, endBlock - 1, file,
+                                     std::vector{1UL}, timer)) {
       co_yield block;
     }
-  }
-
-  if (beginBlock < endBlock) {
     co_yield getIncompleteBlock(endBlock - 1);
   }
 }
@@ -240,81 +239,62 @@ namespace {
 // the greatest possible ID (which is greater than all valid IDs) is returned.
 // Else, the `col2Id` is returned if  a `col1Id` is specified, otherwise the
 // `col1Id`.
-Id getJoinColumnRangeValue(CompressedBlockMetadata::PermutedTriple triple,
-                           Id col0Id, std::optional col1Id) {
-  auto minId = Id::makeUndefined();
-  auto maxId = Id::fromBits(std::numeric_limits::max());
-  if (triple.col0Id_ < col0Id) {
-    return minId;
-  }
-  if (triple.col0Id_ > col0Id) {
-    return maxId;
+auto  getRelevantIdFromTriple(CompressedBlockMetadata::PermutedTriple triple, const CompressedRelationReader::MetadataAndBlocks& metadataAndBlocks
+                           ) {
+  auto idForNonMatchingBlock = [](Id fromTriple, Id key)-> std::optional {
+    if (fromTriple < key) {
+      return Id::min();
+    }
+    if (fromTriple > key) {
+      return Id::max();
+    }
+    return std::nullopt;
+  };
+
+  if (auto optId = idForNonMatchingBlock(triple.col0Id_, metadataAndBlocks.relationMetadata_.col0Id_)) {
+    return optId.value();
   }
-  if (!col1Id.has_value()) {
+  if (!metadataAndBlocks.col1Id_.has_value()) {
     return triple.col1Id_;
   }
 
-  if (triple.col1Id_ < col1Id.value()) {
-    return minId;
-  }
-  if (triple.col1Id_ > col1Id.value()) {
-    return maxId;
-  }
-  return triple.col2Id_;
-}
+  return idForNonMatchingBlock(triple.col1Id_, metadataAndBlocks.col1Id_.value()).value_or(triple.col2Id_);
 
-using IdPair = std::pair;
-// Convert each of the `blocks` to an `IdPair` by calling
-// `getJoinColumnRangeValue` for the first and last triple of the block.
-std::vector blocksToIdRanges(
-    std::span blocks, Id col0Id,
-    std::optional col1Id) {
-  std::vector result;
-  for (const auto& block : blocks) {
-    result.emplace_back(
-        getJoinColumnRangeValue(block.firstTriple_, col0Id, col1Id),
-        getJoinColumnRangeValue(block.lastTriple_, col0Id, col1Id));
-  }
-  return result;
+  return triple.col2Id_;
 }
 }  // namespace
 
 // _____________________________________________________________________________
 std::vector CompressedRelationReader::getBlocksForJoin(
-    std::span joinColum, const MetadataAndBlocks& scanMetadata) {
+    std::span joinColum, const MetadataAndBlocks& metadataAndBlocks) {
   // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_
-  const auto& [relationMetadata, blockMetadata, col1Id] = scanMetadata;
-  auto relevantBlocks = getBlocksFromMetadata(scanMetadata);
+  auto relevantBlocks = getBlocksFromMetadata(metadataAndBlocks);
 
-  auto idRanges =
-      blocksToIdRanges(relevantBlocks, relationMetadata.col0Id_, col1Id);
-
-  // We need symmetric comparisons between `Id` and `IdPair` as well as between
+  // We need symmetric comparisons between `Id` and `IdPair`. as well as between
   // Ids.
-  auto idLessThanBlock = [](Id id, const IdPair& block) {
-    return id < block.first;
+  auto idLessThanBlock = [&metadataAndBlocks](Id id, const CompressedBlockMetadata& block) {
+    return id < getRelevantIdFromTriple(block.firstTriple_, metadataAndBlocks);
   };
-  auto blockLessThanId = [](const IdPair& block, Id id) {
-    return block.second < id;
+
+  auto blockLessThanId = [&metadataAndBlocks](const CompressedBlockMetadata& block, Id id) {
+    return getRelevantIdFromTriple(block.lastTriple_, metadataAndBlocks) < id;
   };
-  auto idLessThanId = [](Id id, Id id2) { return id < id2; };
 
   auto lessThan = ad_utility::OverloadCallOperator{
-      idLessThanBlock, blockLessThanId, idLessThanId};
+      idLessThanBlock, blockLessThanId};
 
   // When we have found a matching block, we have to convert it back.
   std::vector result;
-  auto addRow = [&result, begBlocks = relevantBlocks.begin(),
-                 begRanges = idRanges.begin()]([[maybe_unused]] auto idIterator,
+  auto addRow = [&result]([[maybe_unused]] auto idIterator,
                                                auto blockIterator) {
-    result.push_back(*(begBlocks + (blockIterator - begRanges)));
+    result.push_back(*blockIterator);
   };
 
   auto noop = ad_utility::noop;
 
   // Actually perform the join.
   [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef(
-      joinColum, idRanges, lessThan, addRow, noop, noop);
+      joinColum, relevantBlocks, lessThan, addRow, noop, noop);
 
   // The following check shouldn't be too expensive as there are only few
   // blocks.
@@ -325,37 +305,31 @@ std::vector CompressedRelationReader::getBlocksForJoin(
 // _____________________________________________________________________________
 std::array, 2>
 CompressedRelationReader::getBlocksForJoin(
-    const MetadataAndBlocks& scanMetadata1,
-    const MetadataAndBlocks& scanMetadata2) {
-  auto relevantBlocks1 = getBlocksFromMetadata(scanMetadata1);
-  auto relevantBlocks2 = getBlocksFromMetadata(scanMetadata2);
+    const MetadataAndBlocks& metadataAndBlocks1,
+    const MetadataAndBlocks& metadataAndBlocks2) {
+  auto relevantBlocks1 = getBlocksFromMetadata(metadataAndBlocks1);
+  auto relevantBlocks2 = getBlocksFromMetadata(metadataAndBlocks2);
 
-  auto blockLessThanBlock = [](const auto& block1, const auto& block2) {
-    return block1.second < block2.first;
+  auto blockLessThanBlock = [&](const CompressedBlockMetadata& block1, const CompressedBlockMetadata& block2) {
+    return getRelevantIdFromTriple(block1.lastTriple_, metadataAndBlocks1) < getRelevantIdFromTriple(block2.firstTriple_, metadataAndBlocks2);
   };
 
   std::array, 2> result;
-  auto idRanges1 =
-      blocksToIdRanges(relevantBlocks1, scanMetadata1.relationMetadata_.col0Id_,
-                       scanMetadata1.col1Id_);
-  auto idRanges2 =
-      blocksToIdRanges(relevantBlocks2, scanMetadata2.relationMetadata_.col0Id_,
-                       scanMetadata2.col1Id_);
   // When a result is found, we have to convert back from the `IdRanges` to the
   // corresponding blocks.
-  auto addRow = [&result, &relevantBlocks1, &relevantBlocks2, &idRanges1,
-                 &idRanges2]([[maybe_unused]] auto it1, auto it2) {
-    result[0].push_back(*(relevantBlocks1.begin() + (it1 - idRanges1.begin())));
-    result[1].push_back(*(relevantBlocks2.begin() + (it2 - idRanges2.begin())));
+  auto addRow = [&result](auto it1, auto it2) {
+    result[0].push_back(*it1);
+    result[1].push_back(*it2);
   };
 
   auto noop = ad_utility::noop;
   [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef(
-      idRanges1, idRanges2, blockLessThanBlock, addRow, noop, noop);
+      relevantBlocks1, relevantBlocks2, blockLessThanBlock, addRow, noop, noop);
 
-  // There might be duplicates in the blocks that we have to eliminate.
-  // TODO We should be able to eliminate those directly in the
-  // `zipperJoinWithUndef` routine.
+  // There might be duplicates in the blocks that we have to eliminate. We could in theory eliminate them directly
+  // in the `zipperJoinWithUndef` routine more efficiently, but this would make this already very complex method even
+  // harder to read. The joining of the blocks doesn't seem to be a significant time factor, so we leave it like this
+  // for now.
   for (auto& vec : result) {
     vec.erase(std::ranges::unique(vec).begin(), vec.end());
   }
@@ -363,11 +337,12 @@ CompressedRelationReader::getBlocksForJoin(
 }
 
 // _____________________________________________________________________________
-void CompressedRelationReader::scan(
+IdTable CompressedRelationReader::scan(
     const CompressedRelationMetadata& metadata, Id col1Id,
     const vector& blocks, ad_utility::File& file,
-    IdTable* result, ad_utility::SharedConcurrentTimeoutTimer timer) const {
-  AD_CONTRACT_CHECK(result->numColumns() == 1);
+    const TimeoutTimer& timer) const {
+
+  IdTable result(1, allocator_);
 
   // Get all the blocks  that possibly might contain our pair of col0Id and
   // col1Id
@@ -401,17 +376,13 @@ void CompressedRelationReader::scan(
     firstBlockResult = readIncompleteBlock(*beginBlock);
     totalResultSize += firstBlockResult.value().size();
     ++beginBlock;
-    if (timer) {
-      timer->wlock()->checkTimeoutAndThrow("IndexScan: ");
-    }
+    checkTimeout(timer);
   }
   if (beginBlock < endBlock) {
     lastBlockResult = readIncompleteBlock(*(endBlock - 1));
     totalResultSize += lastBlockResult.value().size();
     endBlock--;
-    if (timer) {
-      timer->wlock()->checkTimeoutAndThrow("IndexScan: ");
-    }
+    checkTimeout(timer);
   }
 
   // Determine the total size of the result.
@@ -420,13 +391,13 @@ void CompressedRelationReader::scan(
                                      [](const auto& count, const auto& block) {
                                        return count + block.numRows_;
                                      });
-  result->resize(totalResultSize);
+  result.resize(totalResultSize);
 
   size_t rowIndexOfNextBlockStart = 0;
   // Insert the first block into the result;
   if (firstBlockResult.has_value()) {
     std::ranges::copy(firstBlockResult.value().getColumn(1),
-                      result->getColumn(0).data());
+                      result.getColumn(0).data());
     rowIndexOfNextBlockStart = firstBlockResult.value().numRows();
   }
 
@@ -444,13 +415,13 @@ void CompressedRelationReader::scan(
 
       // A lambda that owns the compressed block decompresses it to the
       // correct position in the result. It may safely be run in parallel
-      auto decompressLambda = [rowIndexOfNextBlockStart, &block, result,
+      auto decompressLambda = [rowIndexOfNextBlockStart, &block, &result,
                                compressedBuffer =
                                    std::move(compressedBuffer)]() mutable {
         ad_utility::TimeBlockAndLog tbl{"Decompression a block"};
 
         decompressBlockToExistingIdTable(compressedBuffer, block.numRows_,
-                                         *result, rowIndexOfNextBlockStart);
+                                         result, rowIndexOfNextBlockStart);
       };
 
       // Register an OpenMP task that performs the decompression of this
@@ -469,10 +440,11 @@ void CompressedRelationReader::scan(
   // Add the last block.
   if (lastBlockResult.has_value()) {
     std::ranges::copy(lastBlockResult.value().getColumn(1),
-                      result->getColumn(0).data() + rowIndexOfNextBlockStart);
+                      result.getColumn(0).data() + rowIndexOfNextBlockStart);
     rowIndexOfNextBlockStart += lastBlockResult.value().size();
   }
-  AD_CORRECTNESS_CHECK(rowIndexOfNextBlockStart == result->size());
+  AD_CORRECTNESS_CHECK(rowIndexOfNextBlockStart == result.size());
+  return result;
 }
 
 // _____________________________________________________________________________
@@ -825,7 +797,6 @@ CompressedRelationReader::getBlocksFromMetadata(
     if (firstIncomplete) {
       AD_CORRECTNESS_CHECK(!col0IdHasExclusiveBlocks);
     }
-    // AD_CORRECTNESS_CHECK(firstIncomplete || !col0IdHasExclusiveBlocks);
   }
   return result;
 }
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index fc3fa8017a..276687909d 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -230,10 +230,11 @@ class CompressedRelationWriter {
 class CompressedRelationReader {
  public:
   using Allocator = ad_utility::AllocatorWithLimit;
+  using TimeoutTimer = ad_utility::SharedConcurrentTimeoutTimer;
 
-  // Combine the metadata of a single relation with a subset of its blocks and
-  // possibly a `col1Id` for additional filtering. This is used as the input to
-  // several functions below that take such an input.
+  // The metadata of a single relation together with a subset of its
+  // blocks and possibly a `col1Id` for additional filtering. This is used as
+  // the input to several functions below that take such an input.
   struct MetadataAndBlocks {
     const CompressedRelationMetadata relationMetadata_;
     const std::span blockMetadata_;
@@ -264,18 +265,16 @@ class CompressedRelationReader {
    * @param blockMetadata The metadata of the on-disk blocks for the given
    * permutation.
    * @param file The file in which the permutation is stored.
-   * @param result The ID table to which we write the result. It must have
-   * exactly two columns.
    * @param timer If specified (!= nullptr) a `TimeoutException` will be thrown
    *          if the timer runs out during the exeuction of this function.
    *
    * The arguments `metadata`, `blocks`, and `file` must all be obtained from
    * The same `CompressedRelationWriter` (see below).
    */
-  void scan(const CompressedRelationMetadata& metadata,
+  IdTable scan(const CompressedRelationMetadata& metadata,
             std::span blockMetadata,
-            ad_utility::File& file, IdTable* result,
-            ad_utility::SharedConcurrentTimeoutTimer timer) const;
+            ad_utility::File& file,
+            const TimeoutTimer& timer) const;
 
   // Similar to `scan` (directly above), but the result of the scan is lazily
   // computed and returned as a generator of the single blocks that are scanned.
@@ -283,28 +282,27 @@ class CompressedRelationReader {
   cppcoro::generator lazyScan(
       CompressedRelationMetadata metadata,
       std::vector blockMetadata,
-      ad_utility::File& file,
-      ad_utility::SharedConcurrentTimeoutTimer timer) const;
+      ad_utility::File& file, const TimeoutTimer& timer) const;
 
   // Get the blocks (an ordered subset of the blocks that are passed in via the
-  // `blockIterator`) where the `col1Id` can theoretically match one of the
+  // `metadataAndBlocks`) where the `col1Id` can theoretically match one of the
   // elements in the `idIterator` (The col0Id is fixed and specified by the
-  // `blockIterator`). The join column of the scan is the first column that is
-  // not fixed by the `blockIterator`, so the middle column (col1) in case the
-  // `blockIterator` doesn't contain a `col1Id`, or the last column (col2) else.
+  // `metadataAndBlocks`). The join column of the scan is the first column that is
+  // not fixed by the `metadataAndBlocks`, so the middle column (col1) in case the
+  // `metadataAndBlocks` doesn't contain a `col1Id`, or the last column (col2) else.
   static std::vector getBlocksForJoin(
-      std::span idIterator, const MetadataAndBlocks& blockIterator);
+      std::span idIterator, const MetadataAndBlocks& metadataAndBlocks);
 
-  // For each of `scanMetadata1, scanMetadata2` get the blocks (an ordered
+  // For each of `metadataAndBlocks1, metadataAndBlocks2` get the blocks (an ordered
   // subset of the blocks in the `scanMetadata` that might contain matching
-  // elements in the following scenario: The result of `scanMetadata1` is joined
-  // with the result of `scanMetadata2`. For each of the inputs the join column
+  // elements in the following scenario: The result of `metadataAndBlocks1` is joined
+  // with the result of `metadataAndBlocks2`. For each of the inputs the join column
   // is the first column that is not fixed by the metadata, so the middle column
   // (col1) in case the `scanMetadata` doesn't contain a `col1Id`, or the last
   // column (col2) else.
   static std::array, 2> getBlocksForJoin(
-      const MetadataAndBlocks& scanMetadata1,
-      const MetadataAndBlocks& scanMetadata2);
+      const MetadataAndBlocks& metadataAndBlocks1,
+      const MetadataAndBlocks& metadataAndBlocks2);
 
   /**
    * @brief For a permutation XYZ, retrieve all Z for given X and Y.
@@ -322,10 +320,10 @@ class CompressedRelationReader {
    * The arguments `metadata`, `blocks`, and `file` must all be obtained from
    * The same `CompressedRelationWriter` (see below).
    */
-  void scan(const CompressedRelationMetadata& metadata, Id col1Id,
+  IdTable scan(const CompressedRelationMetadata& metadata, Id col1Id,
             const vector& blocks,
-            ad_utility::File& file, IdTable* result,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
+            ad_utility::File& file,
+            const TimeoutTimer& timer = nullptr) const;
 
   // Similar to `scan` (directly above), but the result of the scan is lazily
   // computed and returned as a generator of the single blocks that are scanned.
@@ -333,8 +331,7 @@ class CompressedRelationReader {
   cppcoro::generator lazyScan(
       CompressedRelationMetadata metadata, Id col1Id,
       std::vector blockMetadata,
-      ad_utility::File& file,
-      ad_utility::SharedConcurrentTimeoutTimer timer) const;
+      ad_utility::File& file, const TimeoutTimer& timer) const;
 
   // Only get the size of the result for a given permutation XYZ for a given X
   // and Y. This can be done by scanning one or two blocks. Note: The overload
@@ -416,7 +413,15 @@ class CompressedRelationReader {
   // multiple worker threads.
   cppcoro::generator asyncParallelBlockGenerator(
       auto beginBlock, auto endBlock, ad_utility::File& file,
-      std::optional> columnIndices) const;
+      std::optional> columnIndices,
+      const TimeoutTimer& timer) const;
+
+  // A helper function to abstract away the timeout check:
+  static void checkTimeout(const ad_utility::SharedConcurrentTimeoutTimer& timer) {
+    if (timer) {
+      timer->wlock()->checkTimeoutAndThrow("IndexScan :");
+    }
+  }
 };
 
 #endif  // QLEVER_COMPRESSEDRELATION_H
diff --git a/src/index/Index.cpp b/src/index/Index.cpp
index 672a42405a..dcede376ab 100644
--- a/src/index/Index.cpp
+++ b/src/index/Index.cpp
@@ -346,26 +346,20 @@ vector Index::getMultiplicities(const TripleComponent& key,
   return pimpl_->getMultiplicities(key, p);
 }
 
-// _____________________________________________________
-void Index::scan(Id key, IdTable* result, Permutation::Enum p,
-                 ad_utility::SharedConcurrentTimeoutTimer timer) const {
-  return pimpl_->scan(key, result, p, std::move(timer));
-}
-
-// _____________________________________________________
-void Index::scan(const TripleComponent& key, IdTable* result,
-                 const Permutation::Enum& p,
-                 ad_utility::SharedConcurrentTimeoutTimer timer) const {
-  return pimpl_->scan(key, result, p, std::move(timer));
-}
-
 // _____________________________________________________
 void Index::scan(const TripleComponent& col0String,
-                 const TripleComponent& col1String, IdTable* result,
+                 std::optional> col1String, IdTable* result,
                  Permutation::Enum p,
                  ad_utility::SharedConcurrentTimeoutTimer timer) const {
   return pimpl_->scan(col0String, col1String, result, p, std::move(timer));
 }
+// _____________________________________________________________________________
+void Index::scan(Id col0Id,
+          std::optional col1Id, IdTable* result,
+          Permutation::Enum p,
+          ad_utility::SharedConcurrentTimeoutTimer timer) const {
+  return pimpl_->scan(col0Id, col1Id, result, p, std::move(timer));
+}
 
 // _____________________________________________________
 size_t Index::getResultSizeOfScan(const TripleComponent& col0String,
diff --git a/src/index/Index.h b/src/index/Index.h
index 6a32fa7bb9..4693b6ac8b 100644
--- a/src/index/Index.h
+++ b/src/index/Index.h
@@ -265,33 +265,6 @@ class Index {
   // ___________________________________________________________________
   vector getMultiplicities(Permutation::Enum p) const;
 
-  /**
-   * @brief Perform a scan for one key i.e. retrieve all YZ from the XYZ
-   * permutation for a specific key value of X
-   * @tparam Permutation The permutations Index::POS()... have different types
-   * @param key The key (in Id space) for which to search, e.g. fixed value for
-   * O in OSP permutation.
-   * @param result The Id table to which we will write. Must have 2 columns.
-   * @param p The Permutation::Enum to use (in particularly POS(), SOP,...
-   * members of Index class).
-   */
-  void scan(Id key, IdTable* result, Permutation::Enum p,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
-
-  /**
-   * @brief Perform a scan for one key i.e. retrieve all YZ from the XYZ
-   * permutation for a specific key value of X
-   * @tparam Permutation The permutations Index::POS()... have different types
-   * @param key The key (as a raw string that is yet to be transformed to index
-   * space) for which to search, e.g. fixed value for O in OSP permutation.
-   * @param result The Id table to which we will write. Must have 2 columns.
-   * @param p The Permutation::Enum to use (in particularly POS(), SOP,...
-   * members of Index class).
-   */
-  void scan(const TripleComponent& key, IdTable* result,
-            const Permutation::Enum& p,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
-
   /**
    * @brief Perform a scan for two keys i.e. retrieve all Z from the XYZ
    * permutation for specific key values of X and Y.
@@ -308,7 +281,13 @@ class Index {
    */
   // _____________________________________________________________________________
   void scan(const TripleComponent& col0String,
-            const TripleComponent& col1String, IdTable* result,
+            std::optional> col1String, IdTable* result,
+            Permutation::Enum p,
+            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
+
+  // _____________________________________________________________________________
+  void scan(Id col0Id,
+            std::optional col1Id, IdTable* result,
             Permutation::Enum p,
             ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp
index 397fb0f312..7428f5bca5 100644
--- a/src/index/IndexImpl.cpp
+++ b/src/index/IndexImpl.cpp
@@ -1281,45 +1281,24 @@ vector IndexImpl::getMultiplicities(
   return {m[p.keyOrder_[0]], m[p.keyOrder_[1]], m[p.keyOrder_[2]]};
 }
 
-// ___________________________________________________________________
-void IndexImpl::scan(Id key, IdTable* result, const Permutation::Enum& p,
-                     ad_utility::SharedConcurrentTimeoutTimer timer) const {
-  getPermutation(p).scan(key, result, std::move(timer));
-}
-
-// ___________________________________________________________________
-void IndexImpl::scan(const TripleComponent& key, IdTable* result,
-                     Permutation::Enum permutation,
-                     ad_utility::SharedConcurrentTimeoutTimer timer) const {
-  const auto& p = getPermutation(permutation);
-  LOG(DEBUG) << "Performing " << p.readableName_
-             << " scan for full list for: " << key << "\n";
-
-  if (std::optional id = key.toValueId(getVocab()); id.has_value()) {
-    LOG(TRACE) << "Successfully got key ID.\n";
-    scan(id.value(), result, permutation, std::move(timer));
-  }
-  LOG(DEBUG) << "Scan done, got " << result->size() << " elements.\n";
-}
-
 // _____________________________________________________________________________
-void IndexImpl::scan(const TripleComponent& col0String,
-                     const TripleComponent& col1String, IdTable* result,
-                     const Permutation::Enum& permutation,
+void IndexImpl::scan(
+    const TripleComponent& col0String,
+    std::optional> col1String, IdTable* result, const Permutation::Enum& permutation,
                      ad_utility::SharedConcurrentTimeoutTimer timer) const {
   std::optional col0Id = col0String.toValueId(getVocab());
-  std::optional col1Id = col1String.toValueId(getVocab());
-  const auto& p = getPermutation(permutation);
-  if (!col0Id.has_value() || !col1Id.has_value()) {
-    LOG(DEBUG) << "Key " << col0String << " or key " << col1String
-               << " were not found in the vocabulary \n";
+  std::optional col1Id = col1String.has_value() ? col1String.value().get().toValueId(getVocab()) : std::nullopt;
+  if (!col0Id.has_value() || (col1String.has_value() && !col1Id.has_value())) {
     return;
   }
-
-  LOG(DEBUG) << "Performing " << p.readableName_ << "  scan of relation "
-             << col0String << " with fixed subject: " << col1String << "...\n";
-
-  p.scan(col0Id.value(), col1Id.value(), result, timer);
+  return scan(col0Id.value(), col1Id, result, permutation,  timer);
+}
+// _____________________________________________________________________________
+void IndexImpl::scan(Id col0Id,
+          std::optional col1Id, IdTable* result,
+          Permutation::Enum p,
+          ad_utility::SharedConcurrentTimeoutTimer timer) const {
+  getPermutation(p).scan(col0Id, col1Id, result, timer);
 }
 
 // _____________________________________________________________________________
diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h
index 99fc0d4a0e..54891ae666 100644
--- a/src/index/IndexImpl.h
+++ b/src/index/IndexImpl.h
@@ -397,35 +397,6 @@ class IndexImpl {
   // ___________________________________________________________________
   vector getMultiplicities(Permutation::Enum permutation) const;
 
-  /**
-   * @brief Perform a scan for one key i.e. retrieve all YZ from the XYZ
-   * permutation for a specific key value of X
-   * @tparam Permutation The permutations IndexImpl::POS()... have different
-   * types
-   * @param key The key (in Id space) for which to search, e.g. fixed value for
-   * O in OSP permutation.
-   * @param result The Id table to which we will write. Must have 2 columns.
-   * @param p The Permutation::Enum to use (in particularly POS(), SOP,...
-   * members of IndexImpl class).
-   */
-  void scan(Id key, IdTable* result, const Permutation::Enum& p,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
-
-  /**
-   * @brief Perform a scan for one key i.e. retrieve all YZ from the XYZ
-   * permutation for a specific key value of X
-   * @tparam Permutation The permutations IndexImpl::POS()... have different
-   * types
-   * @param key The key (as a raw string that is yet to be transformed to index
-   * space) for which to search, e.g. fixed value for O in OSP permutation.
-   * @param result The Id table to which we will write. Must have 2 columns.
-   * @param p The Permutation::Enum to use (in particularly POS(), SOP,...
-   * members of IndexImpl class).
-   */
-  void scan(const TripleComponent& key, IdTable* result,
-            Permutation::Enum permutation,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
-
   /**
    * @brief Perform a scan for two keys i.e. retrieve all Z from the XYZ
    * permutation for specific key values of X and Y.
@@ -440,9 +411,15 @@ class IndexImpl {
    * members of IndexImpl class).
    */
   // _____________________________________________________________________________
-  void scan(const TripleComponent& col0String,
-            const TripleComponent& col1String, IdTable* result,
-            const Permutation::Enum& permutation,
+  void scan(
+      const TripleComponent& col0String,
+      std::optional> col1String, IdTable* result, const Permutation::Enum& permutation,
+            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
+
+  // _____________________________________________________________________________
+  void scan(Id col0Id,
+            std::optional col1Id, IdTable* result,
+            Permutation::Enum p,
             ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
   // _____________________________________________________________________________
diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp
index c917bd793d..d0bf9acae1 100644
--- a/src/index/Permutation.cpp
+++ b/src/index/Permutation.cpp
@@ -38,30 +38,25 @@ void Permutation::loadFromDisk(const std::string& onDiskBase) {
 }
 
 // _____________________________________________________________________
-void Permutation::scan(Id col0Id, IdTable* result,
-                       ad_utility::SharedConcurrentTimeoutTimer timer) const {
+void Permutation::scan(Id col0Id, std::optional col1Id, IdTable* result,
+                       const TimeoutTimer& timer) const {
   if (!isLoaded_) {
     throw std::runtime_error("This query requires the permutation " +
                              readableName_ + ", which was not loaded");
   }
-  if (!meta_.col0IdExists(col0Id)) {
-    return;
-  }
-  const auto& metaData = meta_.getMetaData(col0Id);
-  return reader_.scan(metaData, meta_.blockData(), file_, result,
-                      std::move(timer));
-}
 
-// _____________________________________________________________________
-void Permutation::scan(Id col0Id, Id col1Id, IdTable* result,
-                       ad_utility::SharedConcurrentTimeoutTimer timer) const {
   if (!meta_.col0IdExists(col0Id)) {
     return;
   }
   const auto& metaData = meta_.getMetaData(col0Id);
 
-  return reader_.scan(metaData, col1Id, meta_.blockData(), file_, result,
-                      timer);
+  if (col1Id.has_value()) {
+    *result = reader_.scan(metaData, col1Id.value(), meta_.blockData(), file_,
+                        timer);
+  } else {
+    *result =  reader_.scan(metaData, meta_.blockData(), file_,
+                        timer);
+  }
 }
 
 // _____________________________________________________________________
@@ -131,22 +126,17 @@ std::optional Permutation::getMetadataAndBlocks(
 
 // _____________________________________________________________________
 cppcoro::generator Permutation::lazyScan(
-    Id col0Id, std::vector blocks,
-    ad_utility::SharedConcurrentTimeoutTimer timer) const {
+    Id col0Id, std::optional col1Id,
+    const std::vector& blocks,
+    const TimeoutTimer& timer) const {
   if (!meta_.col0IdExists(col0Id)) {
     return {};
   }
-  return reader_.lazyScan(meta_.getMetaData(col0Id), std::move(blocks), file_,
-                          timer);
-}
-
-// _____________________________________________________________________
-cppcoro::generator Permutation::lazyScan(
-    Id col0Id, Id col1Id, const std::vector& blocks,
-    ad_utility::SharedConcurrentTimeoutTimer timer) const {
-  if (!meta_.col0IdExists(col0Id)) {
-    return {};
+  if (col1Id.has_value()) {
+    return reader_.lazyScan(meta_.getMetaData(col0Id), col1Id.value(), blocks, file_,
+                            timer);
+  } else {
+    return reader_.lazyScan(meta_.getMetaData(col0Id), std::move(blocks), file_,
+                            timer);
   }
-  return reader_.lazyScan(meta_.getMetaData(col0Id), col1Id, blocks, file_,
-                          timer);
 }
diff --git a/src/index/Permutation.h b/src/index/Permutation.h
index 1e1c537ee6..7ac595ccd9 100644
--- a/src/index/Permutation.h
+++ b/src/index/Permutation.h
@@ -32,6 +32,7 @@ class Permutation {
 
   using MetaData = IndexMetaDataMmapView;
   using Allocator = ad_utility::AllocatorWithLimit;
+  using TimeoutTimer = ad_utility::SharedConcurrentTimeoutTimer;
   // Convert a permutation to the corresponding string, etc. `PSO` is converted
   // to "PSO".
   static std::string_view toString(Enum permutation);
@@ -48,14 +49,12 @@ class Permutation {
   /// For a given ID for the first column, retrieve all IDs of the second and
   /// third column, and store them in `result`. This is just a thin wrapper
   /// around `CompressedRelationMetaData::scan`.
-  void scan(Id col0Id, IdTable* result,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
-
+  /// TODO unify the comments.
   /// For given IDs for the first and second column, retrieve all IDs of the
   /// third column, and store them in `result`. This is just a thin wrapper
   /// around `CompressedRelationMetaData::scan`.
-  void scan(Id col0Id, Id col1Id, IdTable* result,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
+  void scan(Id col0Id, std::optional col1Id, IdTable* result,
+            const TimeoutTimer& timer = nullptr) const;
 
   // Typedef to propagate the `MetadataAndblocks` type.
   using MetadataAndBlocks = CompressedRelationReader::MetadataAndBlocks;
@@ -73,12 +72,9 @@ class Permutation {
   // `MetadataAndBlocks` class and make this a strong class that always
   // maintains its invariants.
   cppcoro::generator lazyScan(
-      Id col0Id, std::vector blocks,
-      ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
-
-  cppcoro::generator lazyScan(
-      Id col0Id, Id col1Id, const std::vector& blocks,
-      ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
+      Id col0Id, std::optional col1Id,
+      const std::vector& blocks,
+      const TimeoutTimer& timer = nullptr) const;
 
   // Return the relation metadata for the relation specified by the `col0Id`
   // along with the metadata of all the blocks that contain this relation (also
diff --git a/src/index/Permutations.h b/src/index/Permutations.h
deleted file mode 100644
index 593ef7fcdf..0000000000
--- a/src/index/Permutations.h
+++ /dev/null
@@ -1,146 +0,0 @@
-// Copyright 2018, University of Freiburg,
-// Chair of Algorithms and Data Structures.
-// Author: Johannes Kalmbach (johannes.kalmbach@gmail.com)
-#pragma once
-
-#include 
-#include 
-
-#include "global/Constants.h"
-#include "index/IndexMetaData.h"
-#include "util/File.h"
-#include "util/Log.h"
-
-// Helper class to store static properties of the different permutations to
-// avoid code duplication. The first template parameter is a search functor for
-// STXXL.
-class Permutation {
- public:
-  /// Identifiers for the six possible permutations.
-  enum struct Enum { PSO, POS, SPO, SOP, OPS, OSP };
-  // Unfortunately there is a bug in GCC that doesn't allow use to simply use
-  // `using enum`.
-  static constexpr auto PSO = Enum::PSO;
-  static constexpr auto POS = Enum::POS;
-  static constexpr auto SPO = Enum::SPO;
-  static constexpr auto SOP = Enum::SOP;
-  static constexpr auto OPS = Enum::OPS;
-  static constexpr auto OSP = Enum::OSP;
-  using MetaData = IndexMetaDataMmapView;
-  Permutation(string name, string suffix, array order)
-      : _readableName(std::move(name)),
-        _fileSuffix(std::move(suffix)),
-        _keyOrder(order) {}
-
-  // everything that has to be done when reading an index from disk
-  void loadFromDisk(const std::string& onDiskBase) {
-    if constexpr (MetaData::_isMmapBased) {
-      _meta.setup(onDiskBase + ".index" + _fileSuffix + MMAP_FILE_SUFFIX,
-                  ad_utility::ReuseTag(), ad_utility::AccessPattern::Random);
-    }
-    auto filename = string(onDiskBase + ".index" + _fileSuffix);
-    try {
-      _file.open(filename, "r");
-    } catch (const std::runtime_error& e) {
-      AD_THROW("Could not open the index file " + filename +
-               " for reading. Please check that you have read access to "
-               "this file. If it does not exist, your index is broken.");
-    }
-    _meta.readFromFile(&_file);
-    LOG(INFO) << "Registered " << _readableName
-              << " permutation: " << _meta.statistics() << std::endl;
-    _isLoaded = true;
-  }
-
-  /// For a given ID for the first column, retrieve all IDs of the second and
-  /// third column, and store them in `result`. This is just a thin wrapper
-  /// around `CompressedRelationMetaData::scan`.
-  template 
-  void scan(Id col0Id, IdTableImpl* result,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const {
-    if (!_isLoaded) {
-      throw std::runtime_error("This query requires the permutation " +
-                               _readableName + ", which was not loaded");
-    }
-    if (!_meta.col0IdExists(col0Id)) {
-      return;
-    }
-    const auto& metaData = _meta.getMetaData(col0Id);
-    return _reader.scan(metaData, _meta.blockData(), _file, result,
-                        std::move(timer));
-  }
-  /// For given IDs for the first and second column, retrieve all IDs of the
-  /// third column, and store them in `result`. This is just a thin wrapper
-  /// around `CompressedRelationMetaData::scan`.
-  template 
-  void scan(Id col0Id, Id col1Id, IdTableImpl* result,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const {
-    if (!_meta.col0IdExists(col0Id)) {
-      return;
-    }
-    const auto& metaData = _meta.getMetaData(col0Id);
-
-    return _reader.scan(metaData, col1Id, _meta.blockData(), _file, result,
-                        timer);
-  }
-
-  struct MetaDataAndBlocks {
-    const CompressedRelationMetadata relationMetadata_;
-    std::span blockMetadata_;
-  };
-
-  std::optional getMetadataAndBlocks(
-      Id col0Id, std::optional col1Id) const {
-    if (!_meta.col0IdExists(col0Id)) {
-      return std::nullopt;
-    }
-
-    auto metadata = _meta.getMetaData(col0Id);
-    return MetaDataAndBlocks{
-        _meta.getMetaData(col0Id),
-        _reader.getBlocksFromMetadata(metadata, col1Id, _meta.blockData())};
-  }
-
-  cppcoro::generator lazyScan(
-      Id col0Id, const std::vector& blocks,
-      ad_utility::AllocatorWithLimit allocator,
-      ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const {
-    if (!_meta.col0IdExists(col0Id)) {
-      return {};
-    }
-    return _reader.lazyScan(_meta.getMetaData(col0Id), blocks, _file,
-                            std::move(allocator), timer);
-  }
-
-  cppcoro::generator lazyScan(
-      Id col0Id, Id col1Id, const std::vector& blocks,
-      ad_utility::AllocatorWithLimit allocator,
-      ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const {
-    if (!_meta.col0IdExists(col0Id)) {
-      return {};
-    }
-    return _reader.lazyScan(_meta.getMetaData(col0Id), col1Id, blocks, _file,
-                            std::move(allocator), timer);
-  }
-
-  // _______________________________________________________
-  void setKbName(const string& name) { _meta.setName(name); }
-
-  // for Log output, e.g. "POS"
-  const std::string _readableName;
-  // e.g. ".pos"
-  const std::string _fileSuffix;
-  // order of the 3 keys S(0), P(1), and O(2) for which this permutation is
-  // sorted. Needed for the createPermutation function in the Index class
-  // e.g. {1, 0, 2} for PsO
-  const array _keyOrder;
-
-  const MetaData& metaData() const { return _meta; }
-  MetaData _meta;
-
-  mutable ad_utility::File _file;
-
-  CompressedRelationReader _reader;
-
-  bool _isLoaded = false;
-};
diff --git a/src/index/TriplesView.h b/src/index/TriplesView.h
index 5b1edb4ff6..8f88b5f08b 100644
--- a/src/index/TriplesView.h
+++ b/src/index/TriplesView.h
@@ -79,7 +79,7 @@ cppcoro::generator> TriplesView(
       col1And2.clear();
       Id id = it.getId();
       // TODO We could also pass a timeout pointer here.
-      permutation.scan(id, &col1And2);
+      permutation.scan(id, std::nullopt, &col1And2);
       for (const auto& row : col1And2) {
         std::array triple{id, row[0], row[1]};
         if (isTripleIgnored(triple)) [[unlikely]] {
diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp
index 70274cda88..cb08db112e 100644
--- a/test/CompressedRelationsTest.cpp
+++ b/test/CompressedRelationsTest.cpp
@@ -125,13 +125,7 @@ void testCompressedRelations(const std::vector& inputs,
     ASSERT_FLOAT_EQ(m.numRows_ / static_cast(i + 1),
                     m.multiplicityCol1_);
     // Scan for all distinct `col0` and check that we get the expected result.
-    IdTable table{2, ad_utility::testing::makeAllocator()};
-    reader.scan(metaData[i], blocks, file, &table, timer);
-    {
-      IdTable wrongNumCols{3, ad_utility::testing::makeAllocator()};
-      ASSERT_THROW(reader.scan(metaData[i], blocks, file, &wrongNumCols, timer),
-                   ad_utility::Exception);
-    }
+    IdTable table = reader.scan(metaData[i], blocks, file, timer);
     const auto& col1And2 = inputs[i].col1And2_;
     checkThatTablesAreEqual(col1And2, table);
 
@@ -149,11 +143,11 @@ void testCompressedRelations(const std::vector& inputs,
     std::vector> col3;
 
     auto scanAndCheck = [&]() {
-      IdTable tableWidthOne{1, ad_utility::testing::makeAllocator()};
       auto size =
           reader.getResultSizeOfScan(metaData[i], V(lastCol1Id), blocks, file);
-      reader.scan(metaData[i], V(lastCol1Id), blocks, file, &tableWidthOne,
+      IdTable tableWidthOne = reader.scan(metaData[i], V(lastCol1Id), blocks, file,
                   timer);
+      ASSERT_EQ(tableWidthOne.numColumns(), 1);
       EXPECT_EQ(size, tableWidthOne.numRows());
       checkThatTablesAreEqual(col3, tableWidthOne);
       tableWidthOne.clear();
@@ -162,12 +156,6 @@ void testCompressedRelations(const std::vector& inputs,
         tableWidthOne.insertAtEnd(block.begin(), block.end());
       }
       checkThatTablesAreEqual(col3, tableWidthOne);
-      {
-        IdTable wrongNumCols{2, ad_utility::testing::makeAllocator()};
-        ASSERT_THROW(reader.scan(metaData[i], V(lastCol1Id), blocks, file,
-                                 &wrongNumCols, timer),
-                     ad_utility::Exception);
-      }
     };
     for (size_t j = 0; j < col1And2.size(); ++j) {
       if (col1And2[j][0] == lastCol1Id) {
diff --git a/test/IndexTest.cpp b/test/IndexTest.cpp
index 3f25f6d9db..ea08f8547b 100644
--- a/test/IndexTest.cpp
+++ b/test/IndexTest.cpp
@@ -32,7 +32,8 @@ auto makeTestScanWidthOne = [](const IndexImpl& index) {
                       ad_utility::source_location::current()) {
     auto t = generateLocationTrace(l);
     IdTable result(1, makeAllocator());
-    index.scan(c0, c1, &result, permutation);
+    TripleComponent c1Tc{c1};
+    index.scan(c0, std::cref(c1Tc), &result, permutation);
     ASSERT_EQ(result, makeIdTableFromVector(expected));
   };
 };
@@ -48,7 +49,7 @@ auto makeTestScanWidthTwo = [](const IndexImpl& index) {
                       ad_utility::source_location::current()) {
     auto t = generateLocationTrace(l);
     IdTable wol(2, makeAllocator());
-    index.scan(c0, &wol, permutation);
+    index.scan(c0, std::nullopt, &wol, permutation);
     ASSERT_EQ(wol, makeIdTableFromVector(expected));
   };
 };
diff --git a/test/TriplesViewTest.cpp b/test/TriplesViewTest.cpp
index f959180521..49c43c0ee0 100644
--- a/test/TriplesViewTest.cpp
+++ b/test/TriplesViewTest.cpp
@@ -15,7 +15,8 @@ auto V = ad_utility::testing::VocabId;
 // This struct mocks the structure of the actual `Permutation::Enum` types used
 // in QLever for testing the `TriplesView`.
 struct DummyPermutation {
-  void scan(Id col0Id, auto* result) const {
+  void scan(Id col0Id, std::optional col1Id,  auto* result) const {
+    AD_CORRECTNESS_CHECK(!col1Id.has_value());
     result->reserve(col0Id.getVocabIndex().get());
     for (size_t i = 0; i < col0Id.getVocabIndex().get(); ++i) {
       result->push_back(std::array{V((i + 1) * col0Id.getVocabIndex().get()),

From 322423c7c5cd6a59117ee48f2d2a046a92f24b96 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Thu, 29 Jun 2023 17:01:26 +0200
Subject: [PATCH 086/150] Several refactorings that fortunately work again.

---
 src/engine/IndexScan.cpp         |  3 +-
 src/engine/Join.cpp              |  5 +-
 src/global/Constants.h           |  6 +--
 src/index/CompressedRelation.cpp | 82 ++++++++++++++++++++------------
 src/index/CompressedRelation.h   | 47 ++++++++++--------
 src/index/Index.cpp              | 16 +++----
 src/index/Index.h                | 12 ++---
 src/index/IndexImpl.cpp          | 20 ++++----
 src/index/IndexImpl.h            |  8 ++--
 src/index/Permutation.cpp        | 16 +++----
 src/index/Permutation.h          |  4 +-
 src/index/TriplesView.h          |  7 +--
 test/CompressedRelationsTest.cpp |  4 +-
 test/TriplesViewTest.cpp         | 10 ++--
 14 files changed, 135 insertions(+), 105 deletions(-)

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index bf6ea2609e..ceebf03a7f 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -120,7 +120,8 @@ ResultTable IndexScan::computeResult() {
   const auto& index = _executionContext->getIndex();
   const auto permutedTriple = getPermutedTriple();
   if (numVariables_ == 2) {
-    index.scan(*permutedTriple[0], std::nullopt, &idTable, permutation_, _timeoutTimer);
+    index.scan(*permutedTriple[0], std::nullopt, &idTable, permutation_,
+               _timeoutTimer);
   } else if (numVariables_ == 1) {
     index.scan(*permutedTriple[0], *permutedTriple[1], &idTable, permutation_,
                _timeoutTimer);
diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 9aa062cb30..042e76ea71 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -267,8 +267,9 @@ Join::ScanMethodType Join::getScanMethod(
   // during its lifetime
   const auto& idx = _executionContext->getIndex();
   const auto scanLambda = [&idx](const Permutation::Enum perm) {
-    return
-        [&idx, perm](Id id, IdTable* idTable) { idx.scan(id, std::nullopt, idTable, perm); };
+    return [&idx, perm](Id id, IdTable* idTable) {
+      idx.scan(id, std::nullopt, idTable, perm);
+    };
   };
   AD_CORRECTNESS_CHECK(scan.getResultWidth() == 3);
   return scanLambda(scan.permutation());
diff --git a/src/global/Constants.h b/src/global/Constants.h
index ae0681e6a7..02ff7cc70d 100644
--- a/src/global/Constants.h
+++ b/src/global/Constants.h
@@ -179,11 +179,11 @@ inline auto& RuntimeParameters() {
       // factor than the remaining time, then the sort is canceled with a
       // timeout exception.
       Double<"sort-estimate-cancellation-factor">{3.0},
-      SizeT<"cache-max-num-entries">{1000}, SizeT<"cache-max-size-gb">{30},
+      SizeT<"cache-max-num-entries">{1000},
+      SizeT<"cache-max-size-gb">{30},
       SizeT<"cache-max-size-gb-single-entry">{5},
       SizeT<"lazy-index-scan-queue-size">{5},
-      SizeT<"lazy-index-scan-num-threads">{10}
-  };
+      SizeT<"lazy-index-scan-num-threads">{10}};
   return params;
 }
 
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index aaeb5f5c27..09d274cfbe 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -22,7 +22,6 @@ IdTable CompressedRelationReader::scan(
     const CompressedRelationMetadata& metadata,
     std::span blockMetadata,
     ad_utility::File& file, const TimeoutTimer& timer) const {
-
   IdTable result(NumColumns, allocator_);
 
   auto relevantBlocks =
@@ -117,7 +116,8 @@ CompressedRelationReader::asyncParallelBlockGenerator(
   // and join.
   std::mutex blockIteratorMutex;
   std::vector threads;
-  const size_t queueSize = RuntimeParameters().get<"lazy-index-scan-queue-size">();
+  const size_t queueSize =
+      RuntimeParameters().get<"lazy-index-scan-queue-size">();
   ad_utility::data_structures::OrderedThreadSafeQueue queue{
       queueSize};
   auto blockIterator = beginBlock;
@@ -132,8 +132,9 @@ CompressedRelationReader::asyncParallelBlockGenerator(
         auto myIndex = static_cast(blockIterator - beginBlock);
         ++blockIterator;
         bool isLastBlock = blockIterator == endBlock;
-        // Note: the reading of the block could also happen without holding the lock. We still perform it inside the
-        // lock to avoid contention of the file. On a fast SSD we could possibly change this, but this has to be
+        // Note: the reading of the block could also happen without holding the
+        // lock. We still perform it inside the lock to avoid contention of the
+        // file. On a fast SSD we could possibly change this, but this has to be
         // investigated.
         CompressedBlock compressedBlock =
             readCompressedBlockFromFile(block, file, columnIndices);
@@ -153,8 +154,9 @@ CompressedRelationReader::asyncParallelBlockGenerator(
     }
   };
 
-  const size_t numThreads = RuntimeParameters().get<"lazy-index-scan-num-threads">();
-  for ([[maybe_unused]] auto j : std::views::iota(0u,numThreads)) {
+  const size_t numThreads =
+      RuntimeParameters().get<"lazy-index-scan-num-threads">();
+  for ([[maybe_unused]] auto j : std::views::iota(0u, numThreads)) {
     threads.emplace_back(readAndDecompressBlock);
   }
 
@@ -168,11 +170,12 @@ CompressedRelationReader::asyncParallelBlockGenerator(
 cppcoro::generator CompressedRelationReader::lazyScan(
     CompressedRelationMetadata metadata,
     std::vector blockMetadata, ad_utility::File& file,
-    const TimeoutTimer& timer) const {
+    TimeoutTimer timer) const {
   auto relevantBlocks =
       getBlocksFromMetadata(metadata, std::nullopt, blockMetadata);
   const auto beginBlock = relevantBlocks.begin();
   const auto endBlock = relevantBlocks.end();
+
   if (beginBlock == endBlock) {
     co_return;
   }
@@ -182,8 +185,8 @@ cppcoro::generator CompressedRelationReader::lazyScan(
                                        *beginBlock);
   checkTimeout(timer);
 
-  for (auto& block : asyncParallelBlockGenerator(
-           beginBlock + 1, endBlock, file, std::nullopt, timer)) {
+  for (auto& block : asyncParallelBlockGenerator(beginBlock + 1, endBlock, file,
+                                                 std::nullopt, timer)) {
     co_yield block;
   }
 }
@@ -192,7 +195,7 @@ cppcoro::generator CompressedRelationReader::lazyScan(
 cppcoro::generator CompressedRelationReader::lazyScan(
     CompressedRelationMetadata metadata, Id col1Id,
     std::vector blockMetadata, ad_utility::File& file,
-    const TimeoutTimer& timer) const {
+    TimeoutTimer timer) const {
   auto relevantBlocks = getBlocksFromMetadata(metadata, col1Id, blockMetadata);
   auto beginBlock = relevantBlocks.begin();
   auto endBlock = relevantBlocks.end();
@@ -222,9 +225,8 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   }
 
   if (beginBlock + 1 < endBlock) {
-    for (auto& block :
-         asyncParallelBlockGenerator(beginBlock + 1, endBlock - 1, file,
-                                     std::vector{1UL}, timer)) {
+    for (auto& block : asyncParallelBlockGenerator(
+             beginBlock + 1, endBlock - 1, file, std::vector{1UL}, timer)) {
       co_yield block;
     }
     co_yield getIncompleteBlock(endBlock - 1);
@@ -239,9 +241,10 @@ namespace {
 // the greatest possible ID (which is greater than all valid IDs) is returned.
 // Else, the `col2Id` is returned if  a `col1Id` is specified, otherwise the
 // `col1Id`.
-auto  getRelevantIdFromTriple(CompressedBlockMetadata::PermutedTriple triple, const CompressedRelationReader::MetadataAndBlocks& metadataAndBlocks
-                           ) {
-  auto idForNonMatchingBlock = [](Id fromTriple, Id key)-> std::optional {
+auto getRelevantIdFromTriple(
+    CompressedBlockMetadata::PermutedTriple triple,
+    const CompressedRelationReader::MetadataAndBlocks& metadataAndBlocks) {
+  auto idForNonMatchingBlock = [](Id fromTriple, Id key) -> std::optional {
     if (fromTriple < key) {
       return Id::min();
     }
@@ -251,14 +254,17 @@ auto  getRelevantIdFromTriple(CompressedBlockMetadata::PermutedTriple triple, co
     return std::nullopt;
   };
 
-  if (auto optId = idForNonMatchingBlock(triple.col0Id_, metadataAndBlocks.relationMetadata_.col0Id_)) {
+  if (auto optId = idForNonMatchingBlock(
+          triple.col0Id_, metadataAndBlocks.relationMetadata_.col0Id_)) {
     return optId.value();
   }
   if (!metadataAndBlocks.col1Id_.has_value()) {
     return triple.col1Id_;
   }
 
-  return idForNonMatchingBlock(triple.col1Id_, metadataAndBlocks.col1Id_.value()).value_or(triple.col2Id_);
+  return idForNonMatchingBlock(triple.col1Id_,
+                               metadataAndBlocks.col1Id_.value())
+      .value_or(triple.col2Id_);
 
   return triple.col2Id_;
 }
@@ -272,21 +278,23 @@ std::vector CompressedRelationReader::getBlocksForJoin(
 
   // We need symmetric comparisons between `Id` and `IdPair`. as well as between
   // Ids.
-  auto idLessThanBlock = [&metadataAndBlocks](Id id, const CompressedBlockMetadata& block) {
+  auto idLessThanBlock = [&metadataAndBlocks](
+                             Id id, const CompressedBlockMetadata& block) {
     return id < getRelevantIdFromTriple(block.firstTriple_, metadataAndBlocks);
   };
 
-  auto blockLessThanId = [&metadataAndBlocks](const CompressedBlockMetadata& block, Id id) {
+  auto blockLessThanId = [&metadataAndBlocks](
+                             const CompressedBlockMetadata& block, Id id) {
     return getRelevantIdFromTriple(block.lastTriple_, metadataAndBlocks) < id;
   };
 
-  auto lessThan = ad_utility::OverloadCallOperator{
-      idLessThanBlock, blockLessThanId};
+  auto lessThan =
+      ad_utility::OverloadCallOperator{idLessThanBlock, blockLessThanId};
 
   // When we have found a matching block, we have to convert it back.
   std::vector result;
   auto addRow = [&result]([[maybe_unused]] auto idIterator,
-                                               auto blockIterator) {
+                          auto blockIterator) {
     result.push_back(*blockIterator);
   };
 
@@ -310,8 +318,22 @@ CompressedRelationReader::getBlocksForJoin(
   auto relevantBlocks1 = getBlocksFromMetadata(metadataAndBlocks1);
   auto relevantBlocks2 = getBlocksFromMetadata(metadataAndBlocks2);
 
-  auto blockLessThanBlock = [&](const CompressedBlockMetadata& block1, const CompressedBlockMetadata& block2) {
-    return getRelevantIdFromTriple(block1.lastTriple_, metadataAndBlocks1) < getRelevantIdFromTriple(block2.firstTriple_, metadataAndBlocks2);
+  auto metadataForBlock =
+      [&](const CompressedBlockMetadata& block) -> decltype(auto) {
+    if (relevantBlocks1.data() <= &block &&
+        &block < relevantBlocks1.data() + relevantBlocks1.size()) {
+      return metadataAndBlocks1;
+    } else {
+      return metadataAndBlocks2;
+    }
+  };
+
+  auto blockLessThanBlock = [&](const CompressedBlockMetadata& block1,
+                                const CompressedBlockMetadata& block2) {
+    return getRelevantIdFromTriple(block1.lastTriple_,
+                                   metadataForBlock(block1)) <
+           getRelevantIdFromTriple(block2.firstTriple_,
+                                   metadataForBlock(block2));
   };
 
   std::array, 2> result;
@@ -326,10 +348,11 @@ CompressedRelationReader::getBlocksForJoin(
   [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef(
       relevantBlocks1, relevantBlocks2, blockLessThanBlock, addRow, noop, noop);
 
-  // There might be duplicates in the blocks that we have to eliminate. We could in theory eliminate them directly
-  // in the `zipperJoinWithUndef` routine more efficiently, but this would make this already very complex method even
-  // harder to read. The joining of the blocks doesn't seem to be a significant time factor, so we leave it like this
-  // for now.
+  // There might be duplicates in the blocks that we have to eliminate. We could
+  // in theory eliminate them directly in the `zipperJoinWithUndef` routine more
+  // efficiently, but this would make this already very complex method even
+  // harder to read. The joining of the blocks doesn't seem to be a significant
+  // time factor, so we leave it like this for now.
   for (auto& vec : result) {
     vec.erase(std::ranges::unique(vec).begin(), vec.end());
   }
@@ -341,7 +364,6 @@ IdTable CompressedRelationReader::scan(
     const CompressedRelationMetadata& metadata, Id col1Id,
     const vector& blocks, ad_utility::File& file,
     const TimeoutTimer& timer) const {
-
   IdTable result(1, allocator_);
 
   // Get all the blocks  that possibly might contain our pair of col0Id and
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index 276687909d..347f684bd3 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -272,9 +272,8 @@ class CompressedRelationReader {
    * The same `CompressedRelationWriter` (see below).
    */
   IdTable scan(const CompressedRelationMetadata& metadata,
-            std::span blockMetadata,
-            ad_utility::File& file,
-            const TimeoutTimer& timer) const;
+               std::span blockMetadata,
+               ad_utility::File& file, const TimeoutTimer& timer) const;
 
   // Similar to `scan` (directly above), but the result of the scan is lazily
   // computed and returned as a generator of the single blocks that are scanned.
@@ -282,24 +281,26 @@ class CompressedRelationReader {
   cppcoro::generator lazyScan(
       CompressedRelationMetadata metadata,
       std::vector blockMetadata,
-      ad_utility::File& file, const TimeoutTimer& timer) const;
+      ad_utility::File& file, TimeoutTimer timer) const;
 
   // Get the blocks (an ordered subset of the blocks that are passed in via the
   // `metadataAndBlocks`) where the `col1Id` can theoretically match one of the
   // elements in the `idIterator` (The col0Id is fixed and specified by the
-  // `metadataAndBlocks`). The join column of the scan is the first column that is
-  // not fixed by the `metadataAndBlocks`, so the middle column (col1) in case the
-  // `metadataAndBlocks` doesn't contain a `col1Id`, or the last column (col2) else.
+  // `metadataAndBlocks`). The join column of the scan is the first column that
+  // is not fixed by the `metadataAndBlocks`, so the middle column (col1) in
+  // case the `metadataAndBlocks` doesn't contain a `col1Id`, or the last column
+  // (col2) else.
   static std::vector getBlocksForJoin(
-      std::span idIterator, const MetadataAndBlocks& metadataAndBlocks);
-
-  // For each of `metadataAndBlocks1, metadataAndBlocks2` get the blocks (an ordered
-  // subset of the blocks in the `scanMetadata` that might contain matching
-  // elements in the following scenario: The result of `metadataAndBlocks1` is joined
-  // with the result of `metadataAndBlocks2`. For each of the inputs the join column
-  // is the first column that is not fixed by the metadata, so the middle column
-  // (col1) in case the `scanMetadata` doesn't contain a `col1Id`, or the last
-  // column (col2) else.
+      std::span idIterator,
+      const MetadataAndBlocks& metadataAndBlocks);
+
+  // For each of `metadataAndBlocks1, metadataAndBlocks2` get the blocks (an
+  // ordered subset of the blocks in the `scanMetadata` that might contain
+  // matching elements in the following scenario: The result of
+  // `metadataAndBlocks1` is joined with the result of `metadataAndBlocks2`. For
+  // each of the inputs the join column is the first column that is not fixed by
+  // the metadata, so the middle column (col1) in case the `scanMetadata`
+  // doesn't contain a `col1Id`, or the last column (col2) else.
   static std::array, 2> getBlocksForJoin(
       const MetadataAndBlocks& metadataAndBlocks1,
       const MetadataAndBlocks& metadataAndBlocks2);
@@ -321,9 +322,9 @@ class CompressedRelationReader {
    * The same `CompressedRelationWriter` (see below).
    */
   IdTable scan(const CompressedRelationMetadata& metadata, Id col1Id,
-            const vector& blocks,
-            ad_utility::File& file,
-            const TimeoutTimer& timer = nullptr) const;
+               const vector& blocks,
+               ad_utility::File& file,
+               const TimeoutTimer& timer = nullptr) const;
 
   // Similar to `scan` (directly above), but the result of the scan is lazily
   // computed and returned as a generator of the single blocks that are scanned.
@@ -331,7 +332,7 @@ class CompressedRelationReader {
   cppcoro::generator lazyScan(
       CompressedRelationMetadata metadata, Id col1Id,
       std::vector blockMetadata,
-      ad_utility::File& file, const TimeoutTimer& timer) const;
+      ad_utility::File& file, TimeoutTimer timer) const;
 
   // Only get the size of the result for a given permutation XYZ for a given X
   // and Y. This can be done by scanning one or two blocks. Note: The overload
@@ -356,6 +357,9 @@ class CompressedRelationReader {
   static std::span getBlocksFromMetadata(
       const MetadataAndBlocks& metadataAndBlocks);
 
+  // Get access to the underlying allocator
+  const Allocator& allocator() const { return allocator_; }
+
  private:
   // Read the block that is identified by the `blockMetaData` from the `file`.
   // If `columnIndices` is `nullopt`, then all columns of the block are read,
@@ -417,7 +421,8 @@ class CompressedRelationReader {
       const TimeoutTimer& timer) const;
 
   // A helper function to abstract away the timeout check:
-  static void checkTimeout(const ad_utility::SharedConcurrentTimeoutTimer& timer) {
+  static void checkTimeout(
+      const ad_utility::SharedConcurrentTimeoutTimer& timer) {
     if (timer) {
       timer->wlock()->checkTimeoutAndThrow("IndexScan :");
     }
diff --git a/src/index/Index.cpp b/src/index/Index.cpp
index dcede376ab..ae062babb8 100644
--- a/src/index/Index.cpp
+++ b/src/index/Index.cpp
@@ -347,17 +347,17 @@ vector Index::getMultiplicities(const TripleComponent& key,
 }
 
 // _____________________________________________________
-void Index::scan(const TripleComponent& col0String,
-                 std::optional> col1String, IdTable* result,
-                 Permutation::Enum p,
-                 ad_utility::SharedConcurrentTimeoutTimer timer) const {
+void Index::scan(
+    const TripleComponent& col0String,
+    std::optional> col1String,
+    IdTable* result, Permutation::Enum p,
+    ad_utility::SharedConcurrentTimeoutTimer timer) const {
   return pimpl_->scan(col0String, col1String, result, p, std::move(timer));
 }
 // _____________________________________________________________________________
-void Index::scan(Id col0Id,
-          std::optional col1Id, IdTable* result,
-          Permutation::Enum p,
-          ad_utility::SharedConcurrentTimeoutTimer timer) const {
+void Index::scan(Id col0Id, std::optional col1Id, IdTable* result,
+                 Permutation::Enum p,
+                 ad_utility::SharedConcurrentTimeoutTimer timer) const {
   return pimpl_->scan(col0Id, col1Id, result, p, std::move(timer));
 }
 
diff --git a/src/index/Index.h b/src/index/Index.h
index 4693b6ac8b..d86409781b 100644
--- a/src/index/Index.h
+++ b/src/index/Index.h
@@ -280,14 +280,14 @@ class Index {
    * members of Index class).
    */
   // _____________________________________________________________________________
-  void scan(const TripleComponent& col0String,
-            std::optional> col1String, IdTable* result,
-            Permutation::Enum p,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
+  void scan(
+      const TripleComponent& col0String,
+      std::optional> col1String,
+      IdTable* result, Permutation::Enum p,
+      ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
   // _____________________________________________________________________________
-  void scan(Id col0Id,
-            std::optional col1Id, IdTable* result,
+  void scan(Id col0Id, std::optional col1Id, IdTable* result,
             Permutation::Enum p,
             ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp
index 7428f5bca5..83f081d518 100644
--- a/src/index/IndexImpl.cpp
+++ b/src/index/IndexImpl.cpp
@@ -1284,21 +1284,23 @@ vector IndexImpl::getMultiplicities(
 // _____________________________________________________________________________
 void IndexImpl::scan(
     const TripleComponent& col0String,
-    std::optional> col1String, IdTable* result, const Permutation::Enum& permutation,
-                     ad_utility::SharedConcurrentTimeoutTimer timer) const {
+    std::optional> col1String,
+    IdTable* result, const Permutation::Enum& permutation,
+    ad_utility::SharedConcurrentTimeoutTimer timer) const {
   std::optional col0Id = col0String.toValueId(getVocab());
-  std::optional col1Id = col1String.has_value() ? col1String.value().get().toValueId(getVocab()) : std::nullopt;
+  std::optional col1Id =
+      col1String.has_value() ? col1String.value().get().toValueId(getVocab())
+                             : std::nullopt;
   if (!col0Id.has_value() || (col1String.has_value() && !col1Id.has_value())) {
     return;
   }
-  return scan(col0Id.value(), col1Id, result, permutation,  timer);
+  return scan(col0Id.value(), col1Id, result, permutation, timer);
 }
 // _____________________________________________________________________________
-void IndexImpl::scan(Id col0Id,
-          std::optional col1Id, IdTable* result,
-          Permutation::Enum p,
-          ad_utility::SharedConcurrentTimeoutTimer timer) const {
-  getPermutation(p).scan(col0Id, col1Id, result, timer);
+void IndexImpl::scan(Id col0Id, std::optional col1Id, IdTable* result,
+                     Permutation::Enum p,
+                     ad_utility::SharedConcurrentTimeoutTimer timer) const {
+  *result = getPermutation(p).scan(col0Id, col1Id, timer);
 }
 
 // _____________________________________________________________________________
diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h
index 54891ae666..119bd926ec 100644
--- a/src/index/IndexImpl.h
+++ b/src/index/IndexImpl.h
@@ -413,12 +413,12 @@ class IndexImpl {
   // _____________________________________________________________________________
   void scan(
       const TripleComponent& col0String,
-      std::optional> col1String, IdTable* result, const Permutation::Enum& permutation,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
+      std::optional> col1String,
+      IdTable* result, const Permutation::Enum& permutation,
+      ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
   // _____________________________________________________________________________
-  void scan(Id col0Id,
-            std::optional col1Id, IdTable* result,
+  void scan(Id col0Id, std::optional col1Id, IdTable* result,
             Permutation::Enum p,
             ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp
index d0bf9acae1..be8b3859f5 100644
--- a/src/index/Permutation.cpp
+++ b/src/index/Permutation.cpp
@@ -38,24 +38,24 @@ void Permutation::loadFromDisk(const std::string& onDiskBase) {
 }
 
 // _____________________________________________________________________
-void Permutation::scan(Id col0Id, std::optional col1Id, IdTable* result,
-                       const TimeoutTimer& timer) const {
+IdTable Permutation::scan(Id col0Id, std::optional col1Id,
+                          const TimeoutTimer& timer) const {
   if (!isLoaded_) {
     throw std::runtime_error("This query requires the permutation " +
                              readableName_ + ", which was not loaded");
   }
 
   if (!meta_.col0IdExists(col0Id)) {
-    return;
+    size_t numColumns = col1Id.has_value() ? 1 : 2;
+    return IdTable{numColumns, reader_.allocator()};
   }
   const auto& metaData = meta_.getMetaData(col0Id);
 
   if (col1Id.has_value()) {
-    *result = reader_.scan(metaData, col1Id.value(), meta_.blockData(), file_,
+    return reader_.scan(metaData, col1Id.value(), meta_.blockData(), file_,
                         timer);
   } else {
-    *result =  reader_.scan(metaData, meta_.blockData(), file_,
-                        timer);
+    return reader_.scan(metaData, meta_.blockData(), file_, timer);
   }
 }
 
@@ -133,8 +133,8 @@ cppcoro::generator Permutation::lazyScan(
     return {};
   }
   if (col1Id.has_value()) {
-    return reader_.lazyScan(meta_.getMetaData(col0Id), col1Id.value(), blocks, file_,
-                            timer);
+    return reader_.lazyScan(meta_.getMetaData(col0Id), col1Id.value(), blocks,
+                            file_, timer);
   } else {
     return reader_.lazyScan(meta_.getMetaData(col0Id), std::move(blocks), file_,
                             timer);
diff --git a/src/index/Permutation.h b/src/index/Permutation.h
index 7ac595ccd9..23c1f6bc81 100644
--- a/src/index/Permutation.h
+++ b/src/index/Permutation.h
@@ -53,8 +53,8 @@ class Permutation {
   /// For given IDs for the first and second column, retrieve all IDs of the
   /// third column, and store them in `result`. This is just a thin wrapper
   /// around `CompressedRelationMetaData::scan`.
-  void scan(Id col0Id, std::optional col1Id, IdTable* result,
-            const TimeoutTimer& timer = nullptr) const;
+  IdTable scan(Id col0Id, std::optional col1Id,
+               const TimeoutTimer& timer = nullptr) const;
 
   // Typedef to propagate the `MetadataAndblocks` type.
   using MetadataAndBlocks = CompressedRelationReader::MetadataAndBlocks;
diff --git a/src/index/TriplesView.h b/src/index/TriplesView.h
index 8f88b5f08b..6bd7a07572 100644
--- a/src/index/TriplesView.h
+++ b/src/index/TriplesView.h
@@ -71,15 +71,12 @@ cppcoro::generator> TriplesView(
   // specified memory limit.
   // TODO Implement the scanning of large relations lazily and in
   // blocks, making the limit here unnecessary.
-  using Tuple = std::array;
-  auto tupleAllocator = allocator.as();
-  IdTable col1And2{2, allocator};
   for (auto& [begin, end] : allowedRanges) {
     for (auto it = begin; it != end; ++it) {
-      col1And2.clear();
       Id id = it.getId();
       // TODO We could also pass a timeout pointer here.
-      permutation.scan(id, std::nullopt, &col1And2);
+      IdTable col1And2 = permutation.scan(id, std::nullopt);
+      AD_CORRECTNESS_CHECK(col1And2.numColumns() == 2);
       for (const auto& row : col1And2) {
         std::array triple{id, row[0], row[1]};
         if (isTripleIgnored(triple)) [[unlikely]] {
diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp
index cb08db112e..3c5e8b2a84 100644
--- a/test/CompressedRelationsTest.cpp
+++ b/test/CompressedRelationsTest.cpp
@@ -145,8 +145,8 @@ void testCompressedRelations(const std::vector& inputs,
     auto scanAndCheck = [&]() {
       auto size =
           reader.getResultSizeOfScan(metaData[i], V(lastCol1Id), blocks, file);
-      IdTable tableWidthOne = reader.scan(metaData[i], V(lastCol1Id), blocks, file,
-                  timer);
+      IdTable tableWidthOne =
+          reader.scan(metaData[i], V(lastCol1Id), blocks, file, timer);
       ASSERT_EQ(tableWidthOne.numColumns(), 1);
       EXPECT_EQ(size, tableWidthOne.numRows());
       checkThatTablesAreEqual(col3, tableWidthOne);
diff --git a/test/TriplesViewTest.cpp b/test/TriplesViewTest.cpp
index 49c43c0ee0..bf20b70a43 100644
--- a/test/TriplesViewTest.cpp
+++ b/test/TriplesViewTest.cpp
@@ -15,13 +15,15 @@ auto V = ad_utility::testing::VocabId;
 // This struct mocks the structure of the actual `Permutation::Enum` types used
 // in QLever for testing the `TriplesView`.
 struct DummyPermutation {
-  void scan(Id col0Id, std::optional col1Id,  auto* result) const {
+  IdTable scan(Id col0Id, std::optional col1Id) const {
+    IdTable result(2, ad_utility::makeUnlimitedAllocator());
     AD_CORRECTNESS_CHECK(!col1Id.has_value());
-    result->reserve(col0Id.getVocabIndex().get());
+    result.reserve(col0Id.getVocabIndex().get());
     for (size_t i = 0; i < col0Id.getVocabIndex().get(); ++i) {
-      result->push_back(std::array{V((i + 1) * col0Id.getVocabIndex().get()),
-                                   V((i + 2) * col0Id.getVocabIndex().get())});
+      result.push_back(std::array{V((i + 1) * col0Id.getVocabIndex().get()),
+                                  V((i + 2) * col0Id.getVocabIndex().get())});
     }
+    return result;
   }
 
   struct Metadata {

From 207fff0ea2c8432dc77be3b2b5569dd923952ebf Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Thu, 29 Jun 2023 18:20:51 +0200
Subject: [PATCH 087/150] Almost all scans return IdTable now.

---
 src/engine/IndexScan.cpp |  5 +++--
 src/engine/Join.cpp      | 10 ++++------
 src/engine/Join.h        |  2 +-
 src/index/Index.cpp      | 11 ++++++-----
 src/index/Index.h        |  6 +++---
 src/index/IndexImpl.cpp  | 13 +++++++------
 src/index/IndexImpl.h    |  6 +++---
 test/IndexTest.cpp       |  6 ++----
 8 files changed, 29 insertions(+), 30 deletions(-)

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index ceebf03a7f..5037b8e424 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -120,15 +120,16 @@ ResultTable IndexScan::computeResult() {
   const auto& index = _executionContext->getIndex();
   const auto permutedTriple = getPermutedTriple();
   if (numVariables_ == 2) {
-    index.scan(*permutedTriple[0], std::nullopt, &idTable, permutation_,
+    idTable = index.scan(*permutedTriple[0], std::nullopt, permutation_,
                _timeoutTimer);
   } else if (numVariables_ == 1) {
-    index.scan(*permutedTriple[0], *permutedTriple[1], &idTable, permutation_,
+    idTable = index.scan(*permutedTriple[0], *permutedTriple[1], permutation_,
                _timeoutTimer);
   } else {
     AD_CORRECTNESS_CHECK(numVariables_ == 3);
     computeFullScan(&idTable, permutation_);
   }
+  AD_CORRECTNESS_CHECK(idTable.numColumns() == numVariables_);
   LOG(DEBUG) << "IndexScan result computation done.\n";
 
   return {std::move(idTable), resultSortedOn(), LocalVocab{}};
diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 042e76ea71..ea2b9a18df 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -267,8 +267,8 @@ Join::ScanMethodType Join::getScanMethod(
   // during its lifetime
   const auto& idx = _executionContext->getIndex();
   const auto scanLambda = [&idx](const Permutation::Enum perm) {
-    return [&idx, perm](Id id, IdTable* idTable) {
-      idx.scan(id, std::nullopt, idTable, perm);
+    return [&idx, perm](Id id){
+      return idx.scan(id, std::nullopt, perm);
     };
   };
   AD_CORRECTNESS_CHECK(scan.getResultWidth() == 3);
@@ -297,8 +297,7 @@ void Join::doComputeJoinWithFullScanDummyRight(const IdTable& ndr,
       // The scan is a relatively expensive disk operation, so we can afford to
       // check for timeouts before each call.
       checkTimeout();
-      IdTable jr(2, _executionContext->getAllocator());
-      scan(currentJoinId, &jr);
+      IdTable jr = scan(currentJoinId);
       LOG(TRACE) << "Got #items: " << jr.size() << endl;
       // Build the cross product.
       appendCrossProduct(joinItemFrom, joinItemEnd, jr.cbegin(), jr.cend(),
@@ -311,8 +310,7 @@ void Join::doComputeJoinWithFullScanDummyRight(const IdTable& ndr,
   }
   // Do the scan for the final element.
   LOG(TRACE) << "Inner scan with ID: " << currentJoinId << endl;
-  IdTable jr(2, _executionContext->getAllocator());
-  scan(currentJoinId, &jr);
+  IdTable jr = scan(currentJoinId);
   LOG(TRACE) << "Got #items: " << jr.size() << endl;
   // Build the cross product.
   appendCrossProduct(joinItemFrom, joinItemEnd, jr.cbegin(), jr.cend(), res);
diff --git a/src/engine/Join.h b/src/engine/Join.h
index 0ad87b9bdb..cd547dc53a 100644
--- a/src/engine/Join.h
+++ b/src/engine/Join.h
@@ -150,7 +150,7 @@ class Join : public Operation {
                                           ColumnIndex joinColumnIndexScan,
                                           IdTable* resultPtr);
 
-  using ScanMethodType = std::function;
+  using ScanMethodType = std::function;
 
   ScanMethodType getScanMethod(
       std::shared_ptr fullScanDummyTree) const;
diff --git a/src/index/Index.cpp b/src/index/Index.cpp
index ae062babb8..ddf32140ad 100644
--- a/src/index/Index.cpp
+++ b/src/index/Index.cpp
@@ -347,18 +347,19 @@ vector Index::getMultiplicities(const TripleComponent& key,
 }
 
 // _____________________________________________________
-void Index::scan(
+IdTable Index::scan(
     const TripleComponent& col0String,
     std::optional> col1String,
-    IdTable* result, Permutation::Enum p,
+    Permutation::Enum p,
     ad_utility::SharedConcurrentTimeoutTimer timer) const {
-  return pimpl_->scan(col0String, col1String, result, p, std::move(timer));
+  return pimpl_->scan(col0String, col1String, p, std::move(timer));
 }
+
 // _____________________________________________________________________________
-void Index::scan(Id col0Id, std::optional col1Id, IdTable* result,
+IdTable Index::scan(Id col0Id, std::optional col1Id,
                  Permutation::Enum p,
                  ad_utility::SharedConcurrentTimeoutTimer timer) const {
-  return pimpl_->scan(col0Id, col1Id, result, p, std::move(timer));
+  return pimpl_->scan(col0Id, col1Id, p, std::move(timer));
 }
 
 // _____________________________________________________
diff --git a/src/index/Index.h b/src/index/Index.h
index d86409781b..e0d77d2081 100644
--- a/src/index/Index.h
+++ b/src/index/Index.h
@@ -280,14 +280,14 @@ class Index {
    * members of Index class).
    */
   // _____________________________________________________________________________
-  void scan(
+  IdTable scan(
       const TripleComponent& col0String,
       std::optional> col1String,
-      IdTable* result, Permutation::Enum p,
+      Permutation::Enum p,
       ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
   // _____________________________________________________________________________
-  void scan(Id col0Id, std::optional col1Id, IdTable* result,
+  IdTable scan(Id col0Id, std::optional col1Id,
             Permutation::Enum p,
             ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp
index 83f081d518..ececff6d2f 100644
--- a/src/index/IndexImpl.cpp
+++ b/src/index/IndexImpl.cpp
@@ -1282,25 +1282,26 @@ vector IndexImpl::getMultiplicities(
 }
 
 // _____________________________________________________________________________
-void IndexImpl::scan(
+IdTable IndexImpl::scan(
     const TripleComponent& col0String,
     std::optional> col1String,
-    IdTable* result, const Permutation::Enum& permutation,
+    const Permutation::Enum& permutation,
     ad_utility::SharedConcurrentTimeoutTimer timer) const {
   std::optional col0Id = col0String.toValueId(getVocab());
   std::optional col1Id =
       col1String.has_value() ? col1String.value().get().toValueId(getVocab())
                              : std::nullopt;
   if (!col0Id.has_value() || (col1String.has_value() && !col1Id.has_value())) {
-    return;
+    size_t numColumns = col1String.has_value() ? 1 : 2;
+    return IdTable{numColumns, allocator_};
   }
-  return scan(col0Id.value(), col1Id, result, permutation, timer);
+  return scan(col0Id.value(), col1Id, permutation, timer);
 }
 // _____________________________________________________________________________
-void IndexImpl::scan(Id col0Id, std::optional col1Id, IdTable* result,
+IdTable IndexImpl::scan(Id col0Id, std::optional col1Id,
                      Permutation::Enum p,
                      ad_utility::SharedConcurrentTimeoutTimer timer) const {
-  *result = getPermutation(p).scan(col0Id, col1Id, timer);
+  return getPermutation(p).scan(col0Id, col1Id, timer);
 }
 
 // _____________________________________________________________________________
diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h
index 119bd926ec..adf026dce3 100644
--- a/src/index/IndexImpl.h
+++ b/src/index/IndexImpl.h
@@ -411,14 +411,14 @@ class IndexImpl {
    * members of IndexImpl class).
    */
   // _____________________________________________________________________________
-  void scan(
+  IdTable scan(
       const TripleComponent& col0String,
       std::optional> col1String,
-      IdTable* result, const Permutation::Enum& permutation,
+      const Permutation::Enum& permutation,
       ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
   // _____________________________________________________________________________
-  void scan(Id col0Id, std::optional col1Id, IdTable* result,
+  IdTable scan(Id col0Id, std::optional col1Id,
             Permutation::Enum p,
             ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
diff --git a/test/IndexTest.cpp b/test/IndexTest.cpp
index ea08f8547b..8474d7adbd 100644
--- a/test/IndexTest.cpp
+++ b/test/IndexTest.cpp
@@ -31,9 +31,8 @@ auto makeTestScanWidthOne = [](const IndexImpl& index) {
                   ad_utility::source_location l =
                       ad_utility::source_location::current()) {
     auto t = generateLocationTrace(l);
-    IdTable result(1, makeAllocator());
     TripleComponent c1Tc{c1};
-    index.scan(c0, std::cref(c1Tc), &result, permutation);
+    IdTable result = index.scan(c0, std::cref(c1Tc), permutation);
     ASSERT_EQ(result, makeIdTableFromVector(expected));
   };
 };
@@ -48,8 +47,7 @@ auto makeTestScanWidthTwo = [](const IndexImpl& index) {
                   ad_utility::source_location l =
                       ad_utility::source_location::current()) {
     auto t = generateLocationTrace(l);
-    IdTable wol(2, makeAllocator());
-    index.scan(c0, std::nullopt, &wol, permutation);
+    IdTable wol = index.scan(c0, std::nullopt, permutation);
     ASSERT_EQ(wol, makeIdTableFromVector(expected));
   };
 };

From 6eb0b6810afb29af5ed6e0028b2bb171d92e46ea Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Thu, 29 Jun 2023 20:30:06 +0200
Subject: [PATCH 088/150] Changes from Hannah with review.

---
 src/engine/IndexScan.cpp                 |   4 +-
 src/engine/Join.cpp                      |   4 +-
 src/index/CompressedRelation.cpp         |  32 ++++---
 src/index/Index.cpp                      |   8 +-
 src/index/Index.h                        |   5 +-
 src/index/IndexImpl.cpp                  |   4 +-
 src/index/IndexImpl.h                    |   5 +-
 src/util/JoinAlgorithms/JoinAlgorithms.h | 111 +++++++++++++----------
 8 files changed, 97 insertions(+), 76 deletions(-)

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index 5037b8e424..cdcb8df074 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -121,10 +121,10 @@ ResultTable IndexScan::computeResult() {
   const auto permutedTriple = getPermutedTriple();
   if (numVariables_ == 2) {
     idTable = index.scan(*permutedTriple[0], std::nullopt, permutation_,
-               _timeoutTimer);
+                         _timeoutTimer);
   } else if (numVariables_ == 1) {
     idTable = index.scan(*permutedTriple[0], *permutedTriple[1], permutation_,
-               _timeoutTimer);
+                         _timeoutTimer);
   } else {
     AD_CORRECTNESS_CHECK(numVariables_ == 3);
     computeFullScan(&idTable, permutation_);
diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index ea2b9a18df..0f6d0a6f72 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -267,9 +267,7 @@ Join::ScanMethodType Join::getScanMethod(
   // during its lifetime
   const auto& idx = _executionContext->getIndex();
   const auto scanLambda = [&idx](const Permutation::Enum perm) {
-    return [&idx, perm](Id id){
-      return idx.scan(id, std::nullopt, perm);
-    };
+    return [&idx, perm](Id id) { return idx.scan(id, std::nullopt, perm); };
   };
   AD_CORRECTNESS_CHECK(scan.getResultWidth() == 3);
   return scanLambda(scan.permutation());
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 09d274cfbe..5d3e1c16c0 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -22,7 +22,7 @@ IdTable CompressedRelationReader::scan(
     const CompressedRelationMetadata& metadata,
     std::span blockMetadata,
     ad_utility::File& file, const TimeoutTimer& timer) const {
-  IdTable result(NumColumns, allocator_);
+  IdTable result(2, allocator_);
 
   auto relevantBlocks =
       getBlocksFromMetadata(metadata, std::nullopt, blockMetadata);
@@ -126,9 +126,18 @@ CompressedRelationReader::asyncParallelBlockGenerator(
       while (true) {
         std::unique_lock lock{blockIteratorMutex};
         if (blockIterator == endBlock) {
+          // Note: We cannot call `queue.finish()` here, as the last block might
+          // not yet be pushed because it is handled by another thread.
           return;
         }
+        // Note: taking a copy here is probably not necessary (the lifetime of
+        // all the blocks is long enough, so a `const&` would suffice), but the
+        // copy is cheap and makes the code more robust.
         auto block = *blockIterator;
+        // Note: The order of the following three lines is important: The index
+        // of the current block depends on the current value of `blockIterator`,
+        // but we have to increment `blockIterator` to determine whether this
+        // was the last block.
         auto myIndex = static_cast(blockIterator - beginBlock);
         ++blockIterator;
         bool isLastBlock = blockIterator == endBlock;
@@ -145,6 +154,9 @@ CompressedRelationReader::asyncParallelBlockGenerator(
         if (!pushWasSuccessful) {
           return;
         }
+        // Note: Only the thread that actually pushes the last block knows when
+        // it is safe to call `finish` to signal that all blocks have been
+        // succesfully pushed to the queue.
         if (isLastBlock) {
           queue.finish();
         }
@@ -233,14 +245,14 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   }
 }
 
-// Some internal helper functions for the `getBlocksForJoin` functions below.
 namespace {
-// If the combination of `col0Id, col1Id` is strictly smaller than the `triple`
-// return the smallest possible ID (which is the undefined ID which never is
-// contained in a triple). If the combination is stictly larger than the triple,
-// the greatest possible ID (which is greater than all valid IDs) is returned.
-// Else, the `col2Id` is returned if  a `col1Id` is specified, otherwise the
-// `col1Id`.
+// An internal helper function for the `getBlocksForJoin` functions below.
+// Get the ID from the `triple` that pertains to the join column (the col2 if
+// the col1 is specified, else the col1). There are two special cases: If the
+// triple doesn't match the `col0` and (if specified) the `col1` then a sentinel
+// value is returned that is `Id::min` if the triple is lower than all matching
+// triples, and `Id::max` if is higher. That way we can consistently compare a
+// single ID from a join column with a complete triple from a block.
 auto getRelevantIdFromTriple(
     CompressedBlockMetadata::PermutedTriple triple,
     const CompressedRelationReader::MetadataAndBlocks& metadataAndBlocks) {
@@ -265,15 +277,13 @@ auto getRelevantIdFromTriple(
   return idForNonMatchingBlock(triple.col1Id_,
                                metadataAndBlocks.col1Id_.value())
       .value_or(triple.col2Id_);
-
-  return triple.col2Id_;
 }
 }  // namespace
 
 // _____________________________________________________________________________
 std::vector CompressedRelationReader::getBlocksForJoin(
     std::span joinColum, const MetadataAndBlocks& metadataAndBlocks) {
-  // get all the blocks where col0FirstId_ <= col0Id <= col0LastId_
+  // Get all the blocks where `col0FirstId_ <= col0Id <= col0LastId_`.
   auto relevantBlocks = getBlocksFromMetadata(metadataAndBlocks);
 
   // We need symmetric comparisons between `Id` and `IdPair`. as well as between
diff --git a/src/index/Index.cpp b/src/index/Index.cpp
index ddf32140ad..dde33d2a80 100644
--- a/src/index/Index.cpp
+++ b/src/index/Index.cpp
@@ -350,15 +350,13 @@ vector Index::getMultiplicities(const TripleComponent& key,
 IdTable Index::scan(
     const TripleComponent& col0String,
     std::optional> col1String,
-    Permutation::Enum p,
-    ad_utility::SharedConcurrentTimeoutTimer timer) const {
+    Permutation::Enum p, ad_utility::SharedConcurrentTimeoutTimer timer) const {
   return pimpl_->scan(col0String, col1String, p, std::move(timer));
 }
 
 // _____________________________________________________________________________
-IdTable Index::scan(Id col0Id, std::optional col1Id,
-                 Permutation::Enum p,
-                 ad_utility::SharedConcurrentTimeoutTimer timer) const {
+IdTable Index::scan(Id col0Id, std::optional col1Id, Permutation::Enum p,
+                    ad_utility::SharedConcurrentTimeoutTimer timer) const {
   return pimpl_->scan(col0Id, col1Id, p, std::move(timer));
 }
 
diff --git a/src/index/Index.h b/src/index/Index.h
index e0d77d2081..2dece39636 100644
--- a/src/index/Index.h
+++ b/src/index/Index.h
@@ -287,9 +287,8 @@ class Index {
       ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
   // _____________________________________________________________________________
-  IdTable scan(Id col0Id, std::optional col1Id,
-            Permutation::Enum p,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
+  IdTable scan(Id col0Id, std::optional col1Id, Permutation::Enum p,
+               ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
   // Similar to the previous overload of `scan`, but only get the exact size of
   // the scan result.
diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp
index ececff6d2f..61005802b7 100644
--- a/src/index/IndexImpl.cpp
+++ b/src/index/IndexImpl.cpp
@@ -1299,8 +1299,8 @@ IdTable IndexImpl::scan(
 }
 // _____________________________________________________________________________
 IdTable IndexImpl::scan(Id col0Id, std::optional col1Id,
-                     Permutation::Enum p,
-                     ad_utility::SharedConcurrentTimeoutTimer timer) const {
+                        Permutation::Enum p,
+                        ad_utility::SharedConcurrentTimeoutTimer timer) const {
   return getPermutation(p).scan(col0Id, col1Id, timer);
 }
 
diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h
index adf026dce3..cb61447998 100644
--- a/src/index/IndexImpl.h
+++ b/src/index/IndexImpl.h
@@ -418,9 +418,8 @@ class IndexImpl {
       ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
   // _____________________________________________________________________________
-  IdTable scan(Id col0Id, std::optional col1Id,
-            Permutation::Enum p,
-            ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
+  IdTable scan(Id col0Id, std::optional col1Id, Permutation::Enum p,
+               ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
   // _____________________________________________________________________________
   size_t getResultSizeOfScan(const TripleComponent& col0,
diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index e7bbb3a8ae..7b10c80a61 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -93,7 +93,9 @@ concept BinaryIteratorFunction =
  * (`addDuplicatesFromLeft = true`) it would be `AC, AD, BC, BD`. Note that this
  * currently only works for the exact matches and has no effects on the merging
  * with undefined values. This is useful when there are no undefined values and
- * we are only interested in the unique results from `right`.
+ * we are only interested in the unique results from `right`. This is used in
+ * `CompressedRelations.cpp` where we intersect a column of IDs with a set of
+ * block metadata and are only interested in the matching blocks.
  * @return 0 if the result is sorted, > 0 if the result is not sorted. `Sorted`
  * means that all the calls to `compatibleRowAction` were ordered wrt
  * `lessThan`. A result being out of order can happen if two rows with UNDEF
@@ -563,11 +565,11 @@ namespace detail {
 using Range = std::pair;
 
 // Store a contiguous random-access range (e.g. `std::vector`or `std::span`,
-// together with a pair of indices
-// `[beginIndex, endIndex)` that denote a contiguous subrange of the container.
-// Note that this approach is more robust than storing iterators or subranges
-// directly instead of indices, because many containers have their iterators
-// invalidated when they are being moved (e.g. `std::string` or `IdTable`).
+// together with a pair of indices `[beginIndex, endIndex)` that denote a
+// contiguous subrange of the range. Note that this approach is more robust
+// than storing iterators or subranges directly instead of indices, because many
+// containers have their iterators invalidated when they are being moved (e.g.
+// `std::string` or `IdTable`).
 template 
 class BlockAndSubrange {
  private:
@@ -578,8 +580,8 @@ class BlockAndSubrange {
   // The reference type of the underlying container.
   using reference = std::iterator_traits::reference;
 
-  // Construct from a container object, the initial subrange will represent the
-  // whole container.
+  // Construct from a container object, where the initial subrange will
+  // represent the whole container.
   explicit BlockAndSubrange(Block block)
       : block_{std::move(block)}, subrange_{0, block_.size()} {}
 
@@ -623,15 +625,16 @@ class BlockAndSubrange {
 /**
  * @brief Perform a zipper/merge join between two sorted inputs that are given
  * as blocks of inputs, e.g. `std::vector>` or
- * `ad_utility::generator>`. The blocks can be specified via a
+ * `ad_utility::generator>`. The blocks can be specified via an
  * input range (e.g. a generator), but each single block must be a random access
- * range (e.g. vector). The inputs will be moved from. The space complexity is
+ * range (e.g. vector). The blocks will be moved from. The space complexity is
  * `O(size of result)`. The result is only correct if none of the inputs
  * contains UNDEF values.
- * @param leftBlocks The left input to the join algorithm. Typically a range of
- * blocks of rows of IDs (e.g.`std::vector` or
- * `ad_utility::generator`).
- * @param rightBlocks The right input to the join algorithm.
+ * @param leftBlocks The left input range to the join algorithm. We use this for
+ * a range of blocks of rows of IDs (e.g.`std::vector` or
+ * `ad_utility::generator`). Each element will be moved from.
+ * @param rightBlocks The right input range to the join algorithm. Each element
+ * will be moved from.
  * @param lessThan This function is called with one element from one of the
  * blocks of `leftBlocks` and `rightBlocks` each and must return `true` if the
  * first argument comes before the second one. The concatenation of all blocks
@@ -692,31 +695,41 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
   };
 
   // Read the minimal number of unread blocks from `leftBlocks` into
-  // `sameBlocksLeft` and from `rightBlocks` into `sameBlocksRight` until the
-  // following conditions hold:
-  // * at least one block is contained in `sameBlocksLeft` and `sameBlocksRight`
-  // each.
-  // * assume that the last element from sameBlocksLeft[0] is less than or equal
-  // to the last element of sameBlocksLeft[1] (otherwise the condition is
-  // switched)
-  // * Then all the blocks that contain elements less than or equal to this
-  // minimum are read into the respective buffers. For example the following
-  // scenario might occur:
-  //   sameBlocksLeft:  [0-3], [3-3], [3-5]
-  //   sameBlocksRight: [0-3], [3-7]
-  // The consequence of these postconditions is that we can always join at least
-  // on block from each input in the next step and can correctly handle the case
-  // that there are duplicates of the last element of the block in the adjacent
-  // blocks. If either of `sameBlocksLeft` or `sameBlocksRight` are empty after
-  // calling this function, then there are no more blocks to join and the
-  // algorithm can terminate.
+  // `sameBlocksLeft` and from `rightBlocks` into `sameBlocksRight` s.t. at
+  // least one of these blocks can be fully processed. For example consider the
+  // inputs:
+  //   leftBlocks:  [0-3], [3-3], [3-5], ...
+  //   rightBlocks: [0-3], [3-7], ...
+  // All of these five blocks have to be processed at once to fully process at
+  // least one block. Afterwards we have fully processed all blocks except for
+  // the [3-7] block which has to stay in `sameBlocksRight` before the next call
+  // to `fillBuffer`. To ensure this, all the following conditions must hold. 0.
+  // All blocks that were previously read into `sameBlocksLeft/Right` but have
+  // not yet been fully processed are still stored in those buffers. This
+  // precondition is enforced by the `joinBuffers` lambda below.
+  // 1. At least one block is contained in `sameBlocksLeft` and
+  // `sameBlocksRight` each.
+  // 2. Consider the minimum of the last element in `sameBlocksLeft[0]` and the
+  // last element of `sameBlocksRight[0]` after condition 1 is fulfilled. All
+  // blocks that contain elements equal to this minimum are read into the
+  // respective buffers. No blocks that don't fulfill this condition are read
+  // into the buffers.
+  //
+  // The only exception to these conditions can happen if we are at the end of
+  // one of the inputs. In that case either of `sameBlocksLeft` or
+  // `sameBlocksRight` is empty after calling this function. Then we have
+  // finished processing all blocks and can finish the overall algorithm.
   auto fillBuffer = [&]() {
     AD_CORRECTNESS_CHECK(sameBlocksLeft.size() <= 1);
     AD_CORRECTNESS_CHECK(sameBlocksRight.size() <= 1);
 
+    // If the `targetBuffer` is empty, read the next nonempty block from `[it,
+    // end)` if there is one.
     auto fillWithAtLeastOne = [&lessThan](auto& targetBuffer, auto& it,
                                           const auto& end) {
-      (void)lessThan;  // only needed when expensive checks are enabled.
+      // `lessThan` is only needed when compiling with expensive checks enabled,
+      // so we suppress the warning about `lessThan` being unused.
+      (void)lessThan;
       while (targetBuffer.empty() && it != end) {
         if (!it->empty()) {
           AD_EXPENSIVE_CHECK(std::ranges::is_sorted(*it, lessThan));
@@ -729,9 +742,11 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     fillWithAtLeastOne(sameBlocksRight, it2, end2);
 
     if (sameBlocksLeft.empty() || sameBlocksRight.empty()) {
+      // One of the inputs was exhausted, we are done.
       return;
     }
 
+    // Add the remaining blocks such that condition 2 from above is fulfilled.
     auto fillEqualToMinimum = [minEl = getMinEl(), &lessThan, &eq](
                                   auto& targetBuffer, auto& it,
                                   const auto& end) {
@@ -762,10 +777,10 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
   };
 
   // Join the first block in `sameBlocksLeft` with the first block in
-  // `sameBlocksRight`, but leave the last element in each block untouched (it
-  // might have matches in subsequent blocks). The fully joined parts of the
-  // block are then removed from `sameBlocksLeft/Right`, as they are not needed
-  // anymore.
+  // `sameBlocksRight`, but ignore all elements that >= min(lastL, lastR) where
+  // `lastL` is the last element of `sameBlocksLeft[0]`, lastR similarly. The
+  // fully joined parts of the block are then removed from
+  // `sameBlocksLeft/Right`, as they are not needed anymore.
   auto joinAndRemoveBeginning = [&]() {
     // Get the first blocks.
     AD_CORRECTNESS_CHECK(!sameBlocksLeft.empty());
@@ -787,21 +802,20 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     sameBlocksRight.at(0).setSubrange(itR, r.end());
   };
 
-  // Cleanup the `blocks` (either `sameBlocksLeft` or `sameBlocksRight`, by
-  // removing blocks and parts of blocks, s.t. only elements `>=
-  // lastHandledElement` remain. This function expects that all but the last
-  // block can completely be deleted.
+  // Remove all elements from blocks (either `sameBlocksLeft` or
+  // `sameBlocksRight`) s.t. only elements `> lastProcessedElement` remain. This
+  // effectively deletes all blocks completely, except maybe the last one.
   auto removeAllButUnjoined = [lessThan](
                                   Blocks& blocks,
-                                  ProjectedEl lastHandledElement) {
+                                  ProjectedEl lastProcessedElement) {
     // Erase all but the last block.
     AD_CORRECTNESS_CHECK(!blocks.empty());
     blocks.erase(blocks.begin(), blocks.end() - 1);
 
-    // Delete the part from the last block that is `<= lastHandledElement`.
+    // Delete the part from the last block that is `<= lastProcessedElement`.
     decltype(auto) remainingBlock = blocks.at(0).subrange();
-    auto beginningOfUnjoined =
-        std::ranges::upper_bound(remainingBlock, lastHandledElement, lessThan);
+    auto beginningOfUnjoined = std::ranges::upper_bound(
+        remainingBlock, lastProcessedElement, lessThan);
     remainingBlock =
         std::ranges::subrange{beginningOfUnjoined, remainingBlock.end()};
     // If the last block also was already handled completely, delete it (this
@@ -817,9 +831,12 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
   auto joinBuffers = [&]() {
     // Join the beginning of the first blocks and remove it from the input.
     joinAndRemoveBeginning();
-    // Prepare the subranges of the loaded blocks that are equal to the
-    // last element that we can safely join.
+
     ProjectedEl minEl = getMinEl();
+    // Return a vector of subranges of all elements in `input` that are equal to
+    // the last element that we can safely join (this is the `minEl`).
+    // Effectively these are all the blocks completely except maybe for the last
+    // one, of which we can only get a prefix.
     auto pushRelevantSubranges = [&minEl, &lessThan](const auto& input) {
       using Subrange = decltype(std::as_const(input.front()).subrange());
       std::vector result;

From 0c906a66111a1f52feac6e90f51bb45c8597df2b Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 30 Jun 2023 11:24:12 +0200
Subject: [PATCH 089/150] Several additional improvementts. Currently chasing
 doesn the UB in the new clang version.

---
 src/engine/IndexScan.cpp                 |  8 ++---
 src/index/Index.h                        |  8 ++---
 src/index/IndexImpl.h                    | 13 --------
 src/index/Permutation.cpp                | 16 ++++++---
 src/index/Permutation.h                  | 17 ++++------
 src/index/PermutationExporterMain.cpp    |  2 +-
 src/index/TriplesView.h                  | 29 ++++++++--------
 src/util/JoinAlgorithms/JoinAlgorithms.h | 42 ++++++++++++------------
 test/TriplesViewTest.cpp                 | 18 ++++++----
 9 files changed, 72 insertions(+), 81 deletions(-)

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index cdcb8df074..14b0998b4e 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -259,9 +259,8 @@ void IndexScan::computeFullScan(IdTable* result,
   size_t i = 0;
   const auto& permutationImpl =
       getExecutionContext()->getIndex().getImpl().getPermutation(permutation);
-  auto triplesView =
-      TriplesView(permutationImpl, getExecutionContext()->getAllocator(),
-                  ignoredRanges, isTripleIgnored);
+  auto triplesView = TriplesView(permutationImpl, ignoredRanges,
+                                 isTripleIgnored, _timeoutTimer);
   for (const auto& triple : triplesView) {
     if (i >= resultSize) {
       break;
@@ -282,7 +281,6 @@ std::array IndexScan::getPermutedTriple()
 }
 
 // ___________________________________________________________________________
-// TODO include a timeout timer.
 cppcoro::generator IndexScan::getLazyScan(
     const IndexScan& s, std::vector blocks) {
   const IndexImpl& index = s.getIndex().getImpl();
@@ -292,7 +290,7 @@ cppcoro::generator IndexScan::getLazyScan(
     col1Id = s.getPermutedTriple()[1]->toValueId(index.getVocab()).value();
   }
   return index.getPermutation(s.permutation())
-      .lazyScan(col0Id, col1Id, std::move(blocks));
+      .lazyScan(col0Id, col1Id, std::move(blocks), s._timeoutTimer);
 };
 
 // ________________________________________________________________
diff --git a/src/index/Index.h b/src/index/Index.h
index 2dece39636..9582bfbaf8 100644
--- a/src/index/Index.h
+++ b/src/index/Index.h
@@ -266,8 +266,9 @@ class Index {
   vector getMultiplicities(Permutation::Enum p) const;
 
   /**
-   * @brief Perform a scan for two keys i.e. retrieve all Z from the XYZ
-   * permutation for specific key values of X and Y.
+   * @brief Perform a scan for one or two keys i.e. retrieve all YZ from the XYZ
+   * permutation for specific key values of X if `col1String` is `nullopt`, and
+   * all Z for the given XY if `col1String` is specified.
    * @tparam Permutation The permutations Index::POS()... have different types
    * @param col0String The first key (as a raw string that is yet to be
    * transformed to index space) for which to search, e.g. fixed value for O in
@@ -279,14 +280,13 @@ class Index {
    * @param p The Permutation::Enum to use (in particularly POS(), SOP,...
    * members of Index class).
    */
-  // _____________________________________________________________________________
   IdTable scan(
       const TripleComponent& col0String,
       std::optional> col1String,
       Permutation::Enum p,
       ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
-  // _____________________________________________________________________________
+  // Similar to the overload of `scan` above, but the keys are specified as IDs.
   IdTable scan(Id col0Id, std::optional col1Id, Permutation::Enum p,
                ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) const;
 
diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h
index cb61447998..3e6d53d43e 100644
--- a/src/index/IndexImpl.h
+++ b/src/index/IndexImpl.h
@@ -397,19 +397,6 @@ class IndexImpl {
   // ___________________________________________________________________
   vector getMultiplicities(Permutation::Enum permutation) const;
 
-  /**
-   * @brief Perform a scan for two keys i.e. retrieve all Z from the XYZ
-   * permutation for specific key values of X and Y.
-   * @param col0String The first key (as a raw string that is yet to be
-   * transformed to index space) for which to search, e.g. fixed value for O in
-   * OSP permutation.
-   * @param col1String The second key (as a raw string that is yet to be
-   * transformed to index space) for which to search, e.g. fixed value for S in
-   * OSP permutation.
-   * @param result The Id table to which we will write. Must have 2 columns.
-   * @param p The Permutation::Enum to use (in particularly POS(), SOP,...
-   * members of IndexImpl class).
-   */
   // _____________________________________________________________________________
   IdTable scan(
       const TripleComponent& col0String,
diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp
index be8b3859f5..128f183b7d 100644
--- a/src/index/Permutation.cpp
+++ b/src/index/Permutation.cpp
@@ -127,16 +127,22 @@ std::optional Permutation::getMetadataAndBlocks(
 // _____________________________________________________________________
 cppcoro::generator Permutation::lazyScan(
     Id col0Id, std::optional col1Id,
-    const std::vector& blocks,
+    std::optional> blocks,
     const TimeoutTimer& timer) const {
   if (!meta_.col0IdExists(col0Id)) {
     return {};
   }
+  auto relationMetadata = meta_.getMetaData(col0Id);
+  if (!blocks.has_value()) {
+    auto blockSpan = CompressedRelationReader::getBlocksFromMetadata(
+        relationMetadata, col1Id, meta_.blockData());
+    blocks = std::vector(blockSpan.begin(), blockSpan.end());
+  }
   if (col1Id.has_value()) {
-    return reader_.lazyScan(meta_.getMetaData(col0Id), col1Id.value(), blocks,
-                            file_, timer);
+    return reader_.lazyScan(meta_.getMetaData(col0Id), col1Id.value(),
+                            std::move(blocks.value()), file_, timer);
   } else {
-    return reader_.lazyScan(meta_.getMetaData(col0Id), std::move(blocks), file_,
-                            timer);
+    return reader_.lazyScan(meta_.getMetaData(col0Id),
+                            std::move(blocks.value()), file_, timer);
   }
 }
diff --git a/src/index/Permutation.h b/src/index/Permutation.h
index 23c1f6bc81..6be8aebbec 100644
--- a/src/index/Permutation.h
+++ b/src/index/Permutation.h
@@ -46,34 +46,31 @@ class Permutation {
   // everything that has to be done when reading an index from disk
   void loadFromDisk(const std::string& onDiskBase);
 
-  /// For a given ID for the first column, retrieve all IDs of the second and
-  /// third column, and store them in `result`. This is just a thin wrapper
-  /// around `CompressedRelationMetaData::scan`.
-  /// TODO unify the comments.
-  /// For given IDs for the first and second column, retrieve all IDs of the
-  /// third column, and store them in `result`. This is just a thin wrapper
-  /// around `CompressedRelationMetaData::scan`.
+  /// For a given ID for the col0, retrieve all IDs of the col1 and col2.
+  /// If `col1Id` is specified, only the col2 is returned for triples that
+  /// additionally have the specified col1. .This is just a thin wrapper around
+  /// `CompressedRelationMetaData::scan`.
   IdTable scan(Id col0Id, std::optional col1Id,
                const TimeoutTimer& timer = nullptr) const;
 
   // Typedef to propagate the `MetadataAndblocks` type.
   using MetadataAndBlocks = CompressedRelationReader::MetadataAndBlocks;
 
-  // The overloaded function `lazyScan` is similar to `scan` (see above) with
+  // The function `lazyScan` is similar to `scan` (see above) with
   // the following differences:
   // - The result is returned as a lazy generator of blocks.
   // - The block metadata must be passed in manually. It can be obtained via the
   // `getMetadataAndBlocks` function below
   //   and then be prefiltered. The blocks must be passed in in ascending order
   //   and must only contain blocks that contain the given `col0Id` (combined
-  //   with the `col1Id` for the second overload), else the behavior is
+  //   with the `col1Id` if specified), else the behavior is
   //   undefined.
   // TODO We should only communicate these interface via the
   // `MetadataAndBlocks` class and make this a strong class that always
   // maintains its invariants.
   cppcoro::generator lazyScan(
       Id col0Id, std::optional col1Id,
-      const std::vector& blocks,
+      std::optional> blocks,
       const TimeoutTimer& timer = nullptr) const;
 
   // Return the relation metadata for the relation specified by the `col0Id`
diff --git a/src/index/PermutationExporterMain.cpp b/src/index/PermutationExporterMain.cpp
index a522afa2fe..e945f24628 100644
--- a/src/index/PermutationExporterMain.cpp
+++ b/src/index/PermutationExporterMain.cpp
@@ -12,7 +12,7 @@ void dumpToStdout(const Permutation& permutation) {
   ad_utility::AllocatorWithLimit allocator{
       ad_utility::makeAllocationMemoryLeftThreadsafeObject(
           std::numeric_limits::max())};
-  auto triples = TriplesView(permutation, allocator);
+  auto triples = TriplesView(permutation);
   size_t i = 0;
   for (auto triple : triples) {
     std::cout << triple[0] << " " << triple[1] << " " << triple[2] << std::endl;
diff --git a/src/index/TriplesView.h b/src/index/TriplesView.h
index 6bd7a07572..d4f536ce39 100644
--- a/src/index/TriplesView.h
+++ b/src/index/TriplesView.h
@@ -32,9 +32,9 @@ inline auto alwaysReturnFalse = [](auto&&...) { return false; };
  */
 template 
 cppcoro::generator> TriplesView(
-    const auto& permutation, ad_utility::AllocatorWithLimit allocator,
-    detail::IgnoredRelations ignoredRanges = {},
-    IsTripleIgnored isTripleIgnored = IsTripleIgnored{}) {
+    const auto& permutation, detail::IgnoredRelations ignoredRanges = {},
+    IsTripleIgnored isTripleIgnored = IsTripleIgnored{},
+    ad_utility::SharedConcurrentTimeoutTimer timer = nullptr) {
   std::sort(ignoredRanges.begin(), ignoredRanges.end());
 
   const auto& metaData = permutation.meta_.data();
@@ -66,23 +66,20 @@ cppcoro::generator> TriplesView(
     allowedRanges.emplace_back(beginOfAllowed, endOfAllowed);
   }
 
-  // Currently complete relations are yielded at once. This might take a lot of
-  // space for certain predicates in the Pxx permutations, so we respect the
-  // specified memory limit.
-  // TODO Implement the scanning of large relations lazily and in
-  // blocks, making the limit here unnecessary.
   for (auto& [begin, end] : allowedRanges) {
     for (auto it = begin; it != end; ++it) {
       Id id = it.getId();
-      // TODO We could also pass a timeout pointer here.
-      IdTable col1And2 = permutation.scan(id, std::nullopt);
-      AD_CORRECTNESS_CHECK(col1And2.numColumns() == 2);
-      for (const auto& row : col1And2) {
-        std::array triple{id, row[0], row[1]};
-        if (isTripleIgnored(triple)) [[unlikely]] {
-          continue;
+      auto blockGenerator = permutation.lazyScan(id, std::nullopt, std::nullopt,
+                                                 std::move(timer));
+      for (const IdTable& col1And2 : blockGenerator) {
+        AD_CORRECTNESS_CHECK(col1And2.numColumns() == 2);
+        for (const auto& row : col1And2) {
+          std::array triple{id, row[0], row[1]};
+          if (isTripleIgnored(triple)) [[unlikely]] {
+            continue;
+          }
+          co_yield triple;
         }
-        co_yield triple;
       }
     }
   }
diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index 7b10c80a61..9ced4b3b3b 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -631,9 +631,9 @@ class BlockAndSubrange {
  * `O(size of result)`. The result is only correct if none of the inputs
  * contains UNDEF values.
  * @param leftBlocks The left input range to the join algorithm. We use this for
- * a range of blocks of rows of IDs (e.g.`std::vector` or
- * `ad_utility::generator`). Each element will be moved from.
- * @param rightBlocks The right input range to the join algorithm. Each element
+ * blocks of rows of IDs (e.g.`std::vector` or
+ * `ad_utility::generator`). Each block will be moved from.
+ * @param rightBlocks The right input range to the join algorithm. Each block
  * will be moved from.
  * @param lessThan This function is called with one element from one of the
  * blocks of `leftBlocks` and `rightBlocks` each and must return `true` if the
@@ -700,20 +700,20 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
   // inputs:
   //   leftBlocks:  [0-3], [3-3], [3-5], ...
   //   rightBlocks: [0-3], [3-7], ...
-  // All of these five blocks have to be processed at once to fully process at
-  // least one block. Afterwards we have fully processed all blocks except for
-  // the [3-7] block which has to stay in `sameBlocksRight` before the next call
-  // to `fillBuffer`. To ensure this, all the following conditions must hold. 0.
-  // All blocks that were previously read into `sameBlocksLeft/Right` but have
-  // not yet been fully processed are still stored in those buffers. This
+  // All of these five blocks have to be processed at once in ordere to be able
+  // to fully process at least one block. Afterwards we have fully processed all
+  // blocks except for the [3-7] block, which has to stay in `sameBlocksRight`
+  // before the next call to `fillBuffer`. To ensure this, all the following
+  // conditions must hold.
+  // 1. All blocks that were previously read into `sameBlocksLeft/Right` but
+  // have not yet been fully processed are still stored in those buffers. This
   // precondition is enforced by the `joinBuffers` lambda below.
-  // 1. At least one block is contained in `sameBlocksLeft` and
+  // 2. At least one block is contained in `sameBlocksLeft` and
   // `sameBlocksRight` each.
-  // 2. Consider the minimum of the last element in `sameBlocksLeft[0]` and the
-  // last element of `sameBlocksRight[0]` after condition 1 is fulfilled. All
+  // 3. Consider the minimum of the last element in `sameBlocksLeft[0]` and the
+  // last element of `sameBlocksRight[0]` after condition 2 is fulfilled. All
   // blocks that contain elements equal to this minimum are read into the
-  // respective buffers. No blocks that don't fulfill this condition are read
-  // into the buffers.
+  // respective buffers. Only blocks that fulfill this condition are read.
   //
   // The only exception to these conditions can happen if we are at the end of
   // one of the inputs. In that case either of `sameBlocksLeft` or
@@ -746,7 +746,7 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
       return;
     }
 
-    // Add the remaining blocks such that condition 2 from above is fulfilled.
+    // Add the remaining blocks such that condition 3 from above is fulfilled.
     auto fillEqualToMinimum = [minEl = getMinEl(), &lessThan, &eq](
                                   auto& targetBuffer, auto& it,
                                   const auto& end) {
@@ -778,8 +778,8 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
 
   // Join the first block in `sameBlocksLeft` with the first block in
   // `sameBlocksRight`, but ignore all elements that >= min(lastL, lastR) where
-  // `lastL` is the last element of `sameBlocksLeft[0]`, lastR similarly. The
-  // fully joined parts of the block are then removed from
+  // `lastL` is the last element of `sameBlocksLeft[0]`, and `lastR`
+  // analogously. The fully joined parts of the block are then removed from
   // `sameBlocksLeft/Right`, as they are not needed anymore.
   auto joinAndRemoveBeginning = [&]() {
     // Get the first blocks.
@@ -802,9 +802,9 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     sameBlocksRight.at(0).setSubrange(itR, r.end());
   };
 
-  // Remove all elements from blocks (either `sameBlocksLeft` or
+  // Remove all elements from `blocks` (either `sameBlocksLeft` or
   // `sameBlocksRight`) s.t. only elements `> lastProcessedElement` remain. This
-  // effectively deletes all blocks completely, except maybe the last one.
+  // effectively removes all blocks completely, except maybe the last one.
   auto removeAllButUnjoined = [lessThan](
                                   Blocks& blocks,
                                   ProjectedEl lastProcessedElement) {
@@ -835,8 +835,8 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     ProjectedEl minEl = getMinEl();
     // Return a vector of subranges of all elements in `input` that are equal to
     // the last element that we can safely join (this is the `minEl`).
-    // Effectively these are all the blocks completely except maybe for the last
-    // one, of which we can only get a prefix.
+    // Effectively, these subranges cover all the blocks completely except maybe
+    // the last one, which might contain elements `> minEl` at the end.
     auto pushRelevantSubranges = [&minEl, &lessThan](const auto& input) {
       using Subrange = decltype(std::as_const(input.front()).subrange());
       std::vector result;
diff --git a/test/TriplesViewTest.cpp b/test/TriplesViewTest.cpp
index bf20b70a43..b29315bf55 100644
--- a/test/TriplesViewTest.cpp
+++ b/test/TriplesViewTest.cpp
@@ -8,7 +8,6 @@
 #include "./util/IdTestHelpers.h"
 #include "index/TriplesView.h"
 
-using ad_utility::testing::makeAllocator;
 namespace {
 auto V = ad_utility::testing::VocabId;
 
@@ -26,6 +25,15 @@ struct DummyPermutation {
     return result;
   }
 
+  cppcoro::generator lazyScan(
+      Id col0Id, std::optional col1Id,
+      std::optional> blocks,
+      const auto&) const {
+    AD_CORRECTNESS_CHECK(!blocks.has_value());
+    auto table = scan(col0Id, col1Id);
+    co_yield table;
+  }
+
   struct Metadata {
     struct Data {
       std::vector _ids{V(1), V(3), V(5), V(7), V(8), V(10), V(13)};
@@ -59,7 +67,7 @@ std::vector> expectedResult() {
 
 TEST(TriplesView, AllTriples) {
   std::vector> result;
-  for (auto triple : TriplesView(DummyPermutation{}, makeAllocator())) {
+  for (auto triple : TriplesView(DummyPermutation{})) {
     result.push_back(triple);
   }
   ASSERT_EQ(result, expectedResult());
@@ -74,8 +82,7 @@ TEST(TriplesView, IgnoreRanges) {
   });
   std::vector> ignoredRanges{
       {V(0), V(4)}, {V(7), V(8)}, {V(13), V(87593)}};
-  for (auto triple :
-       TriplesView(DummyPermutation{}, makeAllocator(), ignoredRanges)) {
+  for (auto triple : TriplesView(DummyPermutation{}, ignoredRanges)) {
     result.push_back(triple);
   }
   ASSERT_EQ(result, expected);
@@ -88,8 +95,7 @@ TEST(TriplesView, IgnoreTriples) {
     return triple[1].getVocabIndex().get() % 2 == 0;
   };
   std::erase_if(expected, isTripleIgnored);
-  for (auto triple :
-       TriplesView(DummyPermutation{}, makeAllocator(), {}, isTripleIgnored)) {
+  for (auto triple : TriplesView(DummyPermutation{}, {}, isTripleIgnored)) {
     result.push_back(triple);
   }
   ASSERT_EQ(result, expected);

From 5fa348c905dafc1955ede7c27fd94f318c3df596 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 30 Jun 2023 13:12:28 +0200
Subject: [PATCH 090/150] Changes from a round of reviews with Hannah.

---
 src/engine/IndexScan.cpp                 | 10 +++---
 src/engine/IndexScan.h                   |  8 ++---
 src/engine/Join.cpp                      | 45 ++++++++++++------------
 src/engine/Join.h                        | 10 +++---
 src/index/Permutation.h                  | 19 +++++-----
 src/util/JoinAlgorithms/JoinAlgorithms.h |  2 +-
 6 files changed, 48 insertions(+), 46 deletions(-)

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index 14b0998b4e..94055168f7 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -298,16 +298,16 @@ std::optional IndexScan::getMetadataForScan(
     const IndexScan& s) {
   auto permutedTriple = s.getPermutedTriple();
   const IndexImpl& index = s.getIndex().getImpl();
-  std::optional optId = permutedTriple[0]->toValueId(index.getVocab());
-  std::optional optId2 =
+  std::optional col0Id = permutedTriple[0]->toValueId(index.getVocab());
+  std::optional col1Id =
       s.numVariables_ == 2 ? std::nullopt
                            : permutedTriple[1]->toValueId(index.getVocab());
-  if (!optId.has_value() || (!optId2.has_value() && s.numVariables_ == 1)) {
+  if (!col0Id.has_value() || (!col1Id.has_value() && s.numVariables_ == 1)) {
     return std::nullopt;
   }
 
   return index.getPermutation(s.permutation())
-      .getMetadataAndBlocks(optId.value(), optId2);
+      .getMetadataAndBlocks(col0Id.value(), col1Id);
 };
 
 // ________________________________________________________________
@@ -331,7 +331,7 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans(
 cppcoro::generator IndexScan::lazyScanForJoinOfColumnWithScan(
     std::span joinColumn, const IndexScan& s) {
   AD_EXPENSIVE_CHECK(std::ranges::is_sorted(joinColumn));
-  AD_CONTRACT_CHECK(s.numVariables_ < 3);
+  AD_CONTRACT_CHECK(s.numVariables_ == 1 || s.numVariables_ == 2);
 
   auto metaBlocks1 = getMetadataForScan(s);
 
diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h
index a46f997c73..f9815932a2 100644
--- a/src/engine/IndexScan.h
+++ b/src/engine/IndexScan.h
@@ -47,15 +47,15 @@ class IndexScan : public Operation {
 
   // Return two generators that lazily yield the results of `s1` and `s2` in
   // blocks, but only the blocks that can theoretically contain matching rows
-  // when performing a join on the first variable of `s1` with the first
-  // variable of `s2`.
+  // when performing a join on the first column of the result of `s1` with the
+  // first column of the result of `s2`.
   static std::array, 2> lazyScanForJoinOfTwoScans(
       const IndexScan& s1, const IndexScan& s2);
 
   // Return a generator that lazily yields the result of `s` in blocks, but only
   // the blocks that can theoretically contain matching rows when performing a
-  // join between the first variable of `s`  with the `joinColumn`. Requires
-  // that the `joinColumn` is sorted, else the behavior is undefined.
+  // join between the first column of the result of `s`  with the `joinColumn`.
+  // Requires that the `joinColumn` is sorted, else the behavior is undefined.
   static cppcoro::generator lazyScanForJoinOfColumnWithScan(
       std::span joinColumn, const IndexScan& s);
 
diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 0f6d0a6f72..e5e265243b 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -31,7 +31,8 @@ Join::Join(QueryExecutionContext* qec, std::shared_ptr t1,
   t1 = QueryExecutionTree::createSortedTree(std::move(t1), {t1JoinCol});
   t2 = QueryExecutionTree::createSortedTree(std::move(t2), {t2JoinCol});
 
-  // Make sure subtrees are ordered so that identical queries can be identified.
+  // Make sure that the subtrees are ordered so that identical queries can be
+  // identified.
   auto swapChildren = [&]() {
     std::swap(t1, t2);
     std::swap(t1JoinCol, t2JoinCol);
@@ -119,11 +120,10 @@ ResultTable Join::computeResult() {
   if (_left->getType() == QueryExecutionTree::SCAN &&
       _right->getType() == QueryExecutionTree::SCAN) {
     if (rightResIfCached && !leftResIfCached) {
-      computeResultForIndexScanAndColumn(
+      computeResultForIndexScanAndIdTable(
           rightResIfCached->idTable(), _rightJoinCol,
           dynamic_cast(*_left->getRootOperation()),
           _leftJoinCol, &idTable);
-      // TODO This still has the wrong column order
       return {std::move(idTable), resultSortedOn(), LocalVocab{}};
 
     } else if (!leftResIfCached) {
@@ -149,7 +149,7 @@ ResultTable Join::computeResult() {
   // Note: If only one of the children is a scan, then we have made sure in the
   // constructor that it is the right child.
   if (_right->getType() == QueryExecutionTree::SCAN && !rightResIfCached) {
-    computeResultForIndexScanAndColumn(
+    computeResultForIndexScanAndIdTable(
         leftRes->idTable(), _leftJoinCol,
         dynamic_cast(*_right->getRootOperation()),
         _rightJoinCol, &idTable);
@@ -283,6 +283,7 @@ void Join::doComputeJoinWithFullScanDummyRight(const IdTable& ndr,
   const ScanMethodType scan = getScanMethod(_right);
   // Iterate through non-dummy.
   Id currentJoinId = ndr(0, _leftJoinCol);
+  // TODO This can be simplified using `std::views::chunk_by`.
   auto joinItemFrom = ndr.begin();
   auto joinItemEnd = ndr.begin();
   for (size_t i = 0; i < ndr.size(); ++i) {
@@ -684,45 +685,45 @@ void Join::computeResultForTwoIndexScans(IdTable* resultPtr) {
 
 // ______________________________________________________________________________________________________
 template 
-void Join::computeResultForIndexScanAndColumn(const IdTable& inputTable,
-                                              ColumnIndex joinColumnIndexTable,
-                                              const IndexScan& scan,
-                                              ColumnIndex joinColumnIndexScan,
-                                              IdTable* resultPtr) {
+void Join::computeResultForIndexScanAndIdTable(
+    const IdTable& idTable, ColumnIndex joinColumnIndexIdTable,
+    const IndexScan& scan, ColumnIndex joinColumnIndexScan,
+    IdTable* resultPtr) {
   auto& result = *resultPtr;
   result.setNumColumns(getResultWidth());
 
   AD_CORRECTNESS_CHECK(joinColumnIndexScan == 0);
 
-  auto joinColumn = inputTable.getColumn(joinColumnIndexTable);
-  auto addResultRow = [&result, &inputTable, &joinColumnIndexScan,
-                       beg = joinColumn.data()](auto itLeft, auto itRight) {
-    const auto& l = *(inputTable.begin() + (&(*itLeft) - beg));
-    const auto& r = *itRight;
+  auto joinColumn = idTable.getColumn(joinColumnIndexIdTable);
+  auto addResultRow = [&result, &idTable, &joinColumnIndexScan,
+                       beg = joinColumn.data()](auto itIdTable,
+                                                auto itIndexScan) {
+    const auto& idTableRow = *(idTable.begin() + (&(*itIdTable) - beg));
+    const auto& indexScanRow = *itIndexScan;
     result.emplace_back();
     IdTable::row_reference lastRow = result.back();
     size_t nextIndex = 0;
 
     if constexpr (firstIsRight) {
-      for (size_t i = 0; i < r.size(); ++i) {
-        lastRow[nextIndex] = r[i];
+      for (size_t i = 0; i < indexScanRow.size(); ++i) {
+        lastRow[nextIndex] = indexScanRow[i];
         ++nextIndex;
       }
-      for (size_t i = 0; i < inputTable.numColumns(); ++i) {
+      for (size_t i = 0; i < idTable.numColumns(); ++i) {
         if (i != joinColumnIndexScan) {
-          lastRow[nextIndex] = l[i];
+          lastRow[nextIndex] = idTableRow[i];
           ++nextIndex;
         }
       }
     } else {
-      for (size_t i = 0; i < inputTable.numColumns(); ++i) {
-        lastRow[nextIndex] = l[i];
+      for (size_t i = 0; i < idTable.numColumns(); ++i) {
+        lastRow[nextIndex] = idTableRow[i];
         ++nextIndex;
       }
 
-      for (size_t i = 0; i < r.size(); ++i) {
+      for (size_t i = 0; i < indexScanRow.size(); ++i) {
         if (i != joinColumnIndexScan) {
-          lastRow[nextIndex] = r[i];
+          lastRow[nextIndex] = indexScanRow[i];
           ++nextIndex;
         }
       }
diff --git a/src/engine/Join.h b/src/engine/Join.h
index cd547dc53a..c0eaed871b 100644
--- a/src/engine/Join.h
+++ b/src/engine/Join.h
@@ -144,11 +144,11 @@ class Join : public Operation {
   // is the left or the right child of this `Join`. This needs to be known to
   // determine the correct order of the columns in the result.
   template 
-  void computeResultForIndexScanAndColumn(const IdTable& inputTable,
-                                          ColumnIndex joinColumnIndexTable,
-                                          const IndexScan& scan,
-                                          ColumnIndex joinColumnIndexScan,
-                                          IdTable* resultPtr);
+  void computeResultForIndexScanAndIdTable(const IdTable& itIdTable,
+                                           ColumnIndex itIndexScan,
+                                           const IndexScan& scan,
+                                           ColumnIndex joinColumnIndexScan,
+                                           IdTable* resultPtr);
 
   using ScanMethodType = std::function;
 
diff --git a/src/index/Permutation.h b/src/index/Permutation.h
index 6be8aebbec..7b0fd339fb 100644
--- a/src/index/Permutation.h
+++ b/src/index/Permutation.h
@@ -33,6 +33,7 @@ class Permutation {
   using MetaData = IndexMetaDataMmapView;
   using Allocator = ad_utility::AllocatorWithLimit;
   using TimeoutTimer = ad_utility::SharedConcurrentTimeoutTimer;
+
   // Convert a permutation to the corresponding string, etc. `PSO` is converted
   // to "PSO".
   static std::string_view toString(Enum permutation);
@@ -46,10 +47,10 @@ class Permutation {
   // everything that has to be done when reading an index from disk
   void loadFromDisk(const std::string& onDiskBase);
 
-  /// For a given ID for the col0, retrieve all IDs of the col1 and col2.
-  /// If `col1Id` is specified, only the col2 is returned for triples that
-  /// additionally have the specified col1. .This is just a thin wrapper around
-  /// `CompressedRelationMetaData::scan`.
+  // For a given ID for the col0, retrieve all IDs of the col1 and col2.
+  // If `col1Id` is specified, only the col2 is returned for triples that
+  // additionally have the specified col1. .This is just a thin wrapper around
+  // `CompressedRelationMetaData::scan`.
   IdTable scan(Id col0Id, std::optional col1Id,
                const TimeoutTimer& timer = nullptr) const;
 
@@ -59,13 +60,13 @@ class Permutation {
   // The function `lazyScan` is similar to `scan` (see above) with
   // the following differences:
   // - The result is returned as a lazy generator of blocks.
-  // - The block metadata must be passed in manually. It can be obtained via the
+  // - The block metadata must be given manually. It can be obtained via the
   // `getMetadataAndBlocks` function below
-  //   and then be prefiltered. The blocks must be passed in in ascending order
+  //   and then be prefiltered. The blocks must be given in ascending order
   //   and must only contain blocks that contain the given `col0Id` (combined
   //   with the `col1Id` if specified), else the behavior is
   //   undefined.
-  // TODO We should only communicate these interface via the
+  // TODO We should only communicate this interface via the
   // `MetadataAndBlocks` class and make this a strong class that always
   // maintains its invariants.
   cppcoro::generator lazyScan(
@@ -73,8 +74,8 @@ class Permutation {
       std::optional> blocks,
       const TimeoutTimer& timer = nullptr) const;
 
-  // Return the relation metadata for the relation specified by the `col0Id`
-  // along with the metadata of all the blocks that contain this relation (also
+  // Return the metadata for the relation specified by the `col0Id`
+  // along with the metadata for all the blocks that contain this relation (also
   // prefiltered by the `col1Id` if specified). If the `col0Id` does not exist
   // in this permutation, `nullopt` is returned.
   std::optional getMetadataAndBlocks(
diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index 9ced4b3b3b..f04e74831f 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -700,7 +700,7 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
   // inputs:
   //   leftBlocks:  [0-3], [3-3], [3-5], ...
   //   rightBlocks: [0-3], [3-7], ...
-  // All of these five blocks have to be processed at once in ordere to be able
+  // All of these five blocks have to be processed at once in order to be able
   // to fully process at least one block. Afterwards we have fully processed all
   // blocks except for the [3-7] block, which has to stay in `sameBlocksRight`
   // before the next call to `fillBuffer`. To ensure this, all the following

From c6104a54a0abad030d1066e3a9e4db3ece036d93 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 30 Jun 2023 15:06:34 +0200
Subject: [PATCH 091/150] Extend the `generator` class by the functionality to
 add details.

---
 src/util/Generator.h   | 28 ++++++++++++++++++++++++++--
 test/CMakeLists.txt    |  2 ++
 test/GeneratorTest.cpp | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 63 insertions(+), 2 deletions(-)
 create mode 100644 test/GeneratorTest.cpp

diff --git a/src/util/Generator.h b/src/util/Generator.h
index 4d4dc24837..ae412bceec 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -1,5 +1,5 @@
 ///////////////////////////////////////////////////////////////////////////////
-// Copyright (c) Lewis Baker
+// Copyright (c) Lewis Baker, Johannes Kalmbach (functionality to add details).
 // Licenced under MIT license. See LICENSE.txt for details.
 ///////////////////////////////////////////////////////////////////////////////
 #ifndef CPPCORO_GENERATOR_HPP_INCLUDED
@@ -13,12 +13,16 @@
 
 // Coroutines are still experimental in clang libcpp, therefore adapt the
 // appropriate namespaces by including the convenience header.
-#include "./Coroutines.h"
+#include "nlohmann/json.hpp"
+#include "util/Coroutines.h"
 
 namespace cppcoro {
 template 
 class generator;
 
+// This struct can be `co_await`ed inside a `generator` to add a value to a
+// dictionary, that can then be accessed from outside via the `details()`
+// function of the generator object. For an example see `GeneratorTest.cpp`.
 struct AddDetail {
   std::string key_;
   nlohmann::json value_;
@@ -35,6 +39,17 @@ class generator_promise {
   using reference_type = std::conditional_t, T, T&>;
   using pointer_type = std::remove_reference_t*;
 
+  struct DetailAwaiter {
+    generator_promise& promise_;
+    AddDetail detail_;
+    bool await_ready() {
+      promise_.details()[detail_.key_] = detail_.value_;
+      return true;
+    }
+    bool await_suspend(std::coroutine_handle<>) noexcept { return false; }
+    void await_resume() noexcept {}
+  };
+
   generator_promise() = default;
 
   generator get_return_object() noexcept;
@@ -72,9 +87,16 @@ class generator_promise {
     }
   }
 
+  DetailAwaiter await_transform(AddDetail detail) {
+    return {*this, std::move(detail)};
+  }
+
+  nlohmann::json& details() { return m_details; }
+
  private:
   pointer_type m_value;
   std::exception_ptr m_exception;
+  nlohmann::json m_details;
 };
 
 struct generator_sentinel {};
@@ -184,6 +206,8 @@ class [[nodiscard]] generator {
     std::swap(m_coroutine, other.m_coroutine);
   }
 
+  const nlohmann::json& details() { return m_coroutine.promise().details(); }
+
  private:
   friend class detail::generator_promise;
 
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 55ae0711b7..840e692395 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -331,3 +331,5 @@ addLinkAndDiscoverTest(CtreHelpersTest)
 addLinkAndDiscoverTest(ComparisonWithNanTest)
 
 addLinkAndDiscoverTest(ThreadSafeQueueTest)
+
+addLinkAndDiscoverTest(GeneratorTest)
diff --git a/test/GeneratorTest.cpp b/test/GeneratorTest.cpp
new file mode 100644
index 0000000000..cf6cca7771
--- /dev/null
+++ b/test/GeneratorTest.cpp
@@ -0,0 +1,35 @@
+//  Copyright 2023, University of Freiburg,
+//                  Chair of Algorithms and Data Structures.
+//  Author: Johannes Kalmbach 
+
+#include 
+
+#include "util/Generator.h"
+
+// A simple generator that first yields three numbers and then adds a detail
+// value, that we can then extract after iterating over it.
+cppcoro::generator simpleGen(double detailValue) {
+  co_await cppcoro::AddDetail{"started", true};
+  co_yield 1;
+  co_yield 42;
+  co_yield 43;
+  co_await cppcoro::AddDetail{"detail", detailValue};
+};
+
+// Test the behavior of the `simpleGen` above
+TEST(Generator, details) {
+  auto gen = simpleGen(17.3);
+  int result{};
+  // The first detail is only added after the call to `begin()` in the for loop
+  // below
+  ASSERT_TRUE(gen.details().empty());
+  for (int i : gen) {
+    result += i;
+    // The detail is only
+    ASSERT_EQ(gen.details().size(), 1);
+    ASSERT_TRUE(gen.details()["started"]);
+  }
+  ASSERT_EQ(result, 86);
+  ASSERT_EQ(gen.details().size(), 2);
+  ASSERT_EQ(gen.details()["detail"], 17.3);
+}

From dcbae5408b552f114f4a24910aea4e7dafd8fb3b Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 30 Jun 2023 16:25:19 +0200
Subject: [PATCH 092/150] Add additional information.

---
 src/engine/Join.cpp                      |  9 +++---
 src/index/CompressedRelation.cpp         | 40 +++++++++++++++++++-----
 src/index/CompressedRelation.h           |  4 +--
 src/util/Generator.h                     | 21 ++++++++-----
 src/util/JoinAlgorithms/JoinAlgorithms.h | 19 ++++++-----
 test/GeneratorTest.cpp                   |  4 +--
 6 files changed, 67 insertions(+), 30 deletions(-)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index e5e265243b..4af601862c 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -756,9 +756,8 @@ void Join::computeResultForIndexScanAndIdTable(
       std::span{&joinColumn, 1}, rightBlocks, lessThan, addResultRow,
       std::identity{}, rightProjection);
 
-  if (firstIsRight) {
-    _left->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
-  } else {
-    _right->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
-  }
+  auto& scanTree = firstIsRight ? _left : _right;
+  scanTree->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
+  scanTree->getRootOperation()->getRuntimeInfo().details_.update(
+      rightBlocks.details());
 }
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 5d3e1c16c0..093bc6f845 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -172,10 +172,14 @@ CompressedRelationReader::asyncParallelBlockGenerator(
     threads.emplace_back(readAndDecompressBlock);
   }
 
+  ad_utility::Timer popTimer{ad_utility::timer::Timer::InitialStatus::Started};
   while (auto opt = queue.pop()) {
+    popTimer.stop();
     checkTimeout(timer);
     co_yield opt.value();
+    popTimer.cont();
   }
+  co_await cppcoro::AddDetail{"blocking-time-block-reading", popTimer.msecs()};
 }
 
 // _____________________________________________________________________________
@@ -188,7 +192,12 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   const auto beginBlock = relevantBlocks.begin();
   const auto endBlock = relevantBlocks.end();
 
+  size_t numBlocks = 0;
+  size_t numElements = 0;
+
   if (beginBlock == endBlock) {
+    co_await cppcoro::AddDetail{"num-blocks", numBlocks};
+    co_await cppcoro::AddDetail{"num-elements", numElements};
     co_return;
   }
 
@@ -197,10 +206,16 @@ cppcoro::generator CompressedRelationReader::lazyScan(
                                        *beginBlock);
   checkTimeout(timer);
 
-  for (auto& block : asyncParallelBlockGenerator(beginBlock + 1, endBlock, file,
-                                                 std::nullopt, timer)) {
+  auto blockGenerator = asyncParallelBlockGenerator(beginBlock + 1, endBlock,
+                                                    file, std::nullopt, timer);
+  for (auto& block : blockGenerator) {
+    numElements += block.numRows();
+    ++numBlocks;
     co_yield block;
   }
+  co_await cppcoro::AddDetail{blockGenerator.details()};
+  co_await cppcoro::AddDetail{"num-blocks", numBlocks};
+  co_await cppcoro::AddDetail{"num-elements", numElements};
 }
 
 // _____________________________________________________________________________
@@ -282,7 +297,8 @@ auto getRelevantIdFromTriple(
 
 // _____________________________________________________________________________
 std::vector CompressedRelationReader::getBlocksForJoin(
-    std::span joinColum, const MetadataAndBlocks& metadataAndBlocks) {
+    std::span joinColumn,
+    const MetadataAndBlocks& metadataAndBlocks) {
   // Get all the blocks where `col0FirstId_ <= col0Id <= col0LastId_`.
   auto relevantBlocks = getBlocksFromMetadata(metadataAndBlocks);
 
@@ -303,16 +319,26 @@ std::vector CompressedRelationReader::getBlocksForJoin(
 
   // When we have found a matching block, we have to convert it back.
   std::vector result;
-  auto addRow = [&result]([[maybe_unused]] auto idIterator,
-                          auto blockIterator) {
+  auto addRowZipper = [&result]([[maybe_unused]] auto idIterator,
+                                auto blockIterator) {
+    result.push_back(*blockIterator);
+  };
+  auto addRowGallop = [&result](auto blockIterator,
+                                [[maybe_unused]] auto idIterator) {
     result.push_back(*blockIterator);
   };
 
   auto noop = ad_utility::noop;
 
   // Actually perform the join.
-  [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef(
-      joinColum, relevantBlocks, lessThan, addRow, noop, noop);
+  if (joinColumn.size() / relevantBlocks.size() > GALLOP_THRESHOLD) {
+    ad_utility::gallopingJoin(relevantBlocks, joinColumn, lessThan,
+                                     addRowGallop);
+  } else {
+    [[maybe_unused]] auto numOutOfOrder =
+        ad_utility::zipperJoinWithUndef(
+            joinColumn, relevantBlocks, lessThan, addRowZipper, noop, noop);
+  }
 
   // The following check shouldn't be too expensive as there are only few
   // blocks.
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index 347f684bd3..de0a5a7313 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -285,13 +285,13 @@ class CompressedRelationReader {
 
   // Get the blocks (an ordered subset of the blocks that are passed in via the
   // `metadataAndBlocks`) where the `col1Id` can theoretically match one of the
-  // elements in the `idIterator` (The col0Id is fixed and specified by the
+  // elements in the `joinColumn` (The col0Id is fixed and specified by the
   // `metadataAndBlocks`). The join column of the scan is the first column that
   // is not fixed by the `metadataAndBlocks`, so the middle column (col1) in
   // case the `metadataAndBlocks` doesn't contain a `col1Id`, or the last column
   // (col2) else.
   static std::vector getBlocksForJoin(
-      std::span idIterator,
+      std::span joinColumn,
       const MetadataAndBlocks& metadataAndBlocks);
 
   // For each of `metadataAndBlocks1, metadataAndBlocks2` get the blocks (an
diff --git a/src/util/Generator.h b/src/util/Generator.h
index ae412bceec..df6056476c 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -13,8 +13,9 @@
 
 // Coroutines are still experimental in clang libcpp, therefore adapt the
 // appropriate namespaces by including the convenience header.
-#include "nlohmann/json.hpp"
 #include "util/Coroutines.h"
+#include "util/HashMap.h"
+#include "util/json.h"
 
 namespace cppcoro {
 template 
@@ -23,9 +24,13 @@ class generator;
 // This struct can be `co_await`ed inside a `generator` to add a value to a
 // dictionary, that can then be accessed from outside via the `details()`
 // function of the generator object. For an example see `GeneratorTest.cpp`.
+using Details = ad_utility::HashMap;
 struct AddDetail {
-  std::string key_;
-  nlohmann::json value_;
+  Details details_;
+  AddDetail(std::string key, nlohmann::json value) {
+    details_[std::move(key)] = value;
+  }
+  AddDetail(Details details) : details_{std::move(details)} {}
 };
 
 namespace detail {
@@ -43,7 +48,9 @@ class generator_promise {
     generator_promise& promise_;
     AddDetail detail_;
     bool await_ready() {
-      promise_.details()[detail_.key_] = detail_.value_;
+      for (auto& [key, value] : detail_.details_) {
+        promise_.details()[std::move(key)] = std::move(value);
+      }
       return true;
     }
     bool await_suspend(std::coroutine_handle<>) noexcept { return false; }
@@ -91,12 +98,12 @@ class generator_promise {
     return {*this, std::move(detail)};
   }
 
-  nlohmann::json& details() { return m_details; }
+  Details& details() { return m_details; }
 
  private:
   pointer_type m_value;
   std::exception_ptr m_exception;
-  nlohmann::json m_details;
+  Details m_details;
 };
 
 struct generator_sentinel {};
@@ -206,7 +213,7 @@ class [[nodiscard]] generator {
     std::swap(m_coroutine, other.m_coroutine);
   }
 
-  const nlohmann::json& details() { return m_coroutine.promise().details(); }
+  const Details& details() { return m_coroutine.promise().details(); }
 
  private:
   friend class detail::generator_promise;
diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index f04e74831f..1b031f6033 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -333,12 +333,13 @@ template <
  * implement very efficient OPTIONAL or MINUS if neither of the inputs contains
  * UNDEF values, and if the left operand is much smaller.
  */
-template 
 void gallopingJoin(
-    const Range& smaller, const Range& larger,
-    BinaryRangePredicate auto const& lessThan,
-    BinaryIteratorFunction auto const& action,
+    const RangeSmaller& smaller, const RangeLarger& larger,
+    auto const& lessThan, auto const& action,
     ElementFromSmallerNotFoundAction elementFromSmallerNotFoundAction = {}) {
   auto itSmall = std::begin(smaller);
   auto endSmall = std::end(smaller);
@@ -403,9 +404,13 @@ void gallopingJoin(
         itLarge, endLarge, [&](const auto& row) { return eq(row, *itSmall); });
 
     for (; itSmall != endSameSmall; ++itSmall) {
-      for (auto innerItLarge = itLarge; innerItLarge != endSameLarge;
-           ++innerItLarge) {
-        action(itSmall, innerItLarge);
+      if constexpr (!addDuplicatesFromLarge) {
+        action(itSmall, itLarge);
+      } else {
+        for (auto innerItLarge = itLarge; innerItLarge != endSameLarge;
+             ++innerItLarge) {
+          action(itSmall, innerItLarge);
+        }
       }
     }
     itSmall = endSameSmall;
diff --git a/test/GeneratorTest.cpp b/test/GeneratorTest.cpp
index cf6cca7771..e349fcf759 100644
--- a/test/GeneratorTest.cpp
+++ b/test/GeneratorTest.cpp
@@ -27,9 +27,9 @@ TEST(Generator, details) {
     result += i;
     // The detail is only
     ASSERT_EQ(gen.details().size(), 1);
-    ASSERT_TRUE(gen.details()["started"]);
+    ASSERT_TRUE(gen.details().at("started"));
   }
   ASSERT_EQ(result, 86);
   ASSERT_EQ(gen.details().size(), 2);
-  ASSERT_EQ(gen.details()["detail"], 17.3);
+  ASSERT_EQ(gen.details().at("detail"), 17.3);
 }

From 0a975119af8d854bae29cbb29eb34d1ca7d62d9d Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 30 Jun 2023 16:37:35 +0200
Subject: [PATCH 093/150] Add additional details and size information.

---
 src/engine/Join.cpp | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 4af601862c..03cb9191fc 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -681,6 +681,18 @@ void Join::computeResultForTwoIndexScans(IdTable* resultPtr) {
 
   _left->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
   _right->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
+  _right->getRootOperation()->getRuntimeInfo().details_.update(
+      rightBlocks.details());
+  _left->getRootOperation()->getRuntimeInfo().details_.update(
+      leftBlocks.details());
+  if (rightBlocks.details().contains("num-elements")) {
+    _right->getRootOperation()->getRuntimeInfo().numRows_ =
+        static_cast(rightBlocks.details().at("num-elements"));
+  }
+  if (leftBlocks.details().contains("num-elements")) {
+    _left->getRootOperation()->getRuntimeInfo().numRows_ =
+        static_cast(leftBlocks.details().at("num-elements"));
+  }
 }
 
 // ______________________________________________________________________________________________________
@@ -760,4 +772,8 @@ void Join::computeResultForIndexScanAndIdTable(
   scanTree->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
   scanTree->getRootOperation()->getRuntimeInfo().details_.update(
       rightBlocks.details());
+  if (rightBlocks.details().contains("num-elements")) {
+    scanTree->getRootOperation()->getRuntimeInfo().numRows_ =
+        static_cast(rightBlocks.details().at("num-elements"));
+  }
 }

From 0e8d2bb3e00fcf5703d18a31ba1c844d25dbbb19 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 3 Jul 2023 15:19:31 +0200
Subject: [PATCH 094/150] Changed the interface and the bugs.

---
 benchmark/infrastructure/BenchmarkToJson.cpp |  2 +-
 src/engine/ExportQueryExecutionTrees.h       |  2 +-
 src/engine/Server.h                          |  2 +-
 src/util/Generator.h                         | 79 ++++++++++----------
 test/GeneratorTest.cpp                       | 24 +++---
 5 files changed, 57 insertions(+), 52 deletions(-)

diff --git a/benchmark/infrastructure/BenchmarkToJson.cpp b/benchmark/infrastructure/BenchmarkToJson.cpp
index e971e29752..f0122adf5d 100644
--- a/benchmark/infrastructure/BenchmarkToJson.cpp
+++ b/benchmark/infrastructure/BenchmarkToJson.cpp
@@ -9,7 +9,7 @@
 #include 
 
 #include "../benchmark/infrastructure/Benchmark.h"
-#include "nlohmann/json.hpp"
+#include "util/json.h"
 
 namespace ad_benchmark {
 // ___________________________________________________________________________
diff --git a/src/engine/ExportQueryExecutionTrees.h b/src/engine/ExportQueryExecutionTrees.h
index c07c9fd501..f73b568261 100644
--- a/src/engine/ExportQueryExecutionTrees.h
+++ b/src/engine/ExportQueryExecutionTrees.h
@@ -5,9 +5,9 @@
 #include 
 
 #include "engine/QueryExecutionTree.h"
-#include "nlohmann/json.hpp"
 #include "parser/data/LimitOffsetClause.h"
 #include "util/http/MediaTypes.h"
+#include "util/json.h"
 
 #pragma once
 
diff --git a/src/engine/Server.h b/src/engine/Server.h
index 5455353074..10aa906c86 100644
--- a/src/engine/Server.h
+++ b/src/engine/Server.h
@@ -14,13 +14,13 @@
 #include "engine/QueryExecutionTree.h"
 #include "engine/SortPerformanceEstimator.h"
 #include "index/Index.h"
-#include "nlohmann/json.hpp"
 #include "parser/ParseException.h"
 #include "parser/SparqlParser.h"
 #include "util/AllocatorWithLimit.h"
 #include "util/Timer.h"
 #include "util/http/HttpServer.h"
 #include "util/http/streamable_body.h"
+#include "util/json.h"
 
 using nlohmann::json;
 using std::string;
diff --git a/src/util/Generator.h b/src/util/Generator.h
index ae412bceec..43d922163d 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -13,23 +13,21 @@
 
 // Coroutines are still experimental in clang libcpp, therefore adapt the
 // appropriate namespaces by including the convenience header.
-#include "nlohmann/json.hpp"
 #include "util/Coroutines.h"
+#include "util/TypeTraits.h"
 
 namespace cppcoro {
-template 
+template 
 class generator;
 
-// This struct can be `co_await`ed inside a `generator` to add a value to a
-// dictionary, that can then be accessed from outside via the `details()`
-// function of the generator object. For an example see `GeneratorTest.cpp`.
-struct AddDetail {
-  std::string key_;
-  nlohmann::json value_;
-};
+// This struct can be `co_await`ed inside a `generator` to obtain a reference to
+// the details object ( the value of which is a template parameter to the
+// generator). For an example see `GeneratorTest.cpp`.
+struct GetDetails {};
+static constexpr GetDetails getDetails;
 
 namespace detail {
-template 
+template 
 class generator_promise {
  public:
   // Even if the generator only yields `const` values, the `value_type`
@@ -39,20 +37,9 @@ class generator_promise {
   using reference_type = std::conditional_t, T, T&>;
   using pointer_type = std::remove_reference_t*;
 
-  struct DetailAwaiter {
-    generator_promise& promise_;
-    AddDetail detail_;
-    bool await_ready() {
-      promise_.details()[detail_.key_] = detail_.value_;
-      return true;
-    }
-    bool await_suspend(std::coroutine_handle<>) noexcept { return false; }
-    void await_resume() noexcept {}
-  };
-
   generator_promise() = default;
 
-  generator get_return_object() noexcept;
+  generator get_return_object() noexcept;
 
   constexpr std::suspend_always initial_suspend() const noexcept { return {}; }
   constexpr std::suspend_always final_suspend() const noexcept { return {}; }
@@ -87,32 +74,43 @@ class generator_promise {
     }
   }
 
-  DetailAwaiter await_transform(AddDetail detail) {
-    return {*this, std::move(detail)};
+  // The machinery to expose the stored `Details` via
+  // `co_await cppcoro::getDetails`.
+  struct DetailAwaiter {
+    generator_promise& promise_;
+    bool await_ready() { return true; }
+    bool await_suspend(std::coroutine_handle<>) noexcept { return false; }
+    Details& await_resume() noexcept { return promise_.details(); }
+  };
+
+  DetailAwaiter await_transform(
+      [[maybe_unused]] ad_utility::SimilarTo auto&& detail) {
+    return {*this};
   }
 
-  nlohmann::json& details() { return m_details; }
+  Details& details() { return m_details; }
 
  private:
   pointer_type m_value;
   std::exception_ptr m_exception;
-  nlohmann::json m_details;
+  Details m_details{};
 };
 
 struct generator_sentinel {};
 
-template 
+template 
 class generator_iterator {
-  using coroutine_handle = std::coroutine_handle>;
+  using promise_type = generator_promise;
+  using coroutine_handle = std::coroutine_handle;
 
  public:
   using iterator_category = std::input_iterator_tag;
   // What type should we use for counting elements of a potentially infinite
   // sequence?
   using difference_type = std::ptrdiff_t;
-  using value_type = typename generator_promise::value_type;
-  using reference = typename generator_promise::reference_type;
-  using pointer = typename generator_promise::pointer_type;
+  using value_type = typename promise_type::value_type;
+  using reference = typename promise_type::reference_type;
+  using pointer = typename promise_type::pointer_type;
 
   // Iterator needs to be default-constructible to satisfy the Range concept.
   generator_iterator() noexcept : m_coroutine(nullptr) {}
@@ -161,11 +159,11 @@ class generator_iterator {
 };
 }  // namespace detail
 
-template 
+template 
 class [[nodiscard]] generator {
  public:
-  using promise_type = detail::generator_promise;
-  using iterator = detail::generator_iterator;
+  using promise_type = detail::generator_promise;
+  using iterator = detail::generator_iterator;
   using value_type = typename iterator::value_type;
 
   generator() noexcept : m_coroutine(nullptr) {}
@@ -206,10 +204,10 @@ class [[nodiscard]] generator {
     std::swap(m_coroutine, other.m_coroutine);
   }
 
-  const nlohmann::json& details() { return m_coroutine.promise().details(); }
+  const Details& details() { return m_coroutine.promise().details(); }
 
  private:
-  friend class detail::generator_promise;
+  friend class detail::generator_promise;
 
   explicit generator(std::coroutine_handle coroutine) noexcept
       : m_coroutine(coroutine) {}
@@ -223,10 +221,11 @@ void swap(generator& a, generator& b) {
 }
 
 namespace detail {
-template 
-generator generator_promise::get_return_object() noexcept {
-  using coroutine_handle = std::coroutine_handle>;
-  return generator{coroutine_handle::from_promise(*this)};
+template 
+generator
+generator_promise::get_return_object() noexcept {
+  using coroutine_handle = std::coroutine_handle>;
+  return generator{coroutine_handle::from_promise(*this)};
 }
 }  // namespace detail
 
diff --git a/test/GeneratorTest.cpp b/test/GeneratorTest.cpp
index cf6cca7771..706838773f 100644
--- a/test/GeneratorTest.cpp
+++ b/test/GeneratorTest.cpp
@@ -6,14 +6,20 @@
 
 #include "util/Generator.h"
 
+struct Details {
+  bool begin_ = false;
+  bool end_ = false;
+};
+
 // A simple generator that first yields three numbers and then adds a detail
 // value, that we can then extract after iterating over it.
-cppcoro::generator simpleGen(double detailValue) {
-  co_await cppcoro::AddDetail{"started", true};
+cppcoro::generator simpleGen(double detailValue) {
+  auto& details = co_await cppcoro::getDetails;
+  details.begin_ = true;
   co_yield 1;
   co_yield 42;
   co_yield 43;
-  co_await cppcoro::AddDetail{"detail", detailValue};
+  details.end_ = true;
 };
 
 // Test the behavior of the `simpleGen` above
@@ -22,14 +28,14 @@ TEST(Generator, details) {
   int result{};
   // The first detail is only added after the call to `begin()` in the for loop
   // below
-  ASSERT_TRUE(gen.details().empty());
+  ASSERT_FALSE(gen.details().begin_);
+  ASSERT_FALSE(gen.details().end_);
   for (int i : gen) {
     result += i;
-    // The detail is only
-    ASSERT_EQ(gen.details().size(), 1);
-    ASSERT_TRUE(gen.details()["started"]);
+    ASSERT_TRUE(gen.details().begin_);
+    ASSERT_FALSE(gen.details().end_);
   }
   ASSERT_EQ(result, 86);
-  ASSERT_EQ(gen.details().size(), 2);
-  ASSERT_EQ(gen.details()["detail"], 17.3);
+  ASSERT_FALSE(gen.details().begin_);
+  ASSERT_TRUE(gen.details().end_);
 }

From 8a866530194930049264dcac8b627e678bf2ee5c Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 3 Jul 2023 16:20:37 +0200
Subject: [PATCH 095/150] Started to do stuff for perfomannce, currently not
 yet done.

---
 src/util/JoinAlgorithms/JoinAlgorithms.h | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index 1b031f6033..f667d3a021 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -609,6 +609,13 @@ class BlockAndSubrange {
                                  block_.begin() + subrange_.second};
   }
 
+  const auto& fullBlock() const {
+    return block_;
+  }
+  auto& fullBlock() {
+    return block_;
+  }
+
   // Specify the subrange by using two iterators `begin` and `end`. The
   // iterators must be valid iterators that point into the container, this is
   // checked by an assertion. The only legal way to obtain such iterators is to
@@ -797,10 +804,16 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     ProjectedEl minEl = getMinEl();
     auto itL = std::ranges::lower_bound(l, minEl, lessThan);
     auto itR = std::ranges::lower_bound(r, minEl, lessThan);
+
+    std::vector> matchingIndices;
+    auto addRowIndex = [&matchingIndices, begL = sameBlocksLeft.at(0).fullBlock().begin(), begR = sameBlocksRight.at(0).fullBlock().begin()](auto itFromL, auto itFromR) {
+      matchingIndices.emplace_back(itFromL - begL, itFromR - begR);
+    };
+
     [[maybe_unused]] auto res =
         zipperJoinWithUndef(std::ranges::subrange{l.begin(), itL},
                             std::ranges::subrange{r.begin(), itR}, lessThan,
-                            compatibleRowAction, noop, noop);
+                            addRowIndex, noop, noop);
 
     // Remove the joined elements.
     sameBlocksLeft.at(0).setSubrange(itL, l.end());

From 6aaa445650fe8e99cebf31664b8c0b386b191e77 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 3 Jul 2023 16:23:22 +0200
Subject: [PATCH 096/150] Fix the unit tests.

---
 test/GeneratorTest.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/GeneratorTest.cpp b/test/GeneratorTest.cpp
index 706838773f..56b347d6d2 100644
--- a/test/GeneratorTest.cpp
+++ b/test/GeneratorTest.cpp
@@ -13,7 +13,7 @@ struct Details {
 
 // A simple generator that first yields three numbers and then adds a detail
 // value, that we can then extract after iterating over it.
-cppcoro::generator simpleGen(double detailValue) {
+cppcoro::generator simpleGen() {
   auto& details = co_await cppcoro::getDetails;
   details.begin_ = true;
   co_yield 1;
@@ -24,10 +24,10 @@ cppcoro::generator simpleGen(double detailValue) {
 
 // Test the behavior of the `simpleGen` above
 TEST(Generator, details) {
-  auto gen = simpleGen(17.3);
+  auto gen = simpleGen();
   int result{};
-  // The first detail is only added after the call to `begin()` in the for loop
-  // below
+  // `details().begin_` is only set to true after the call to `begin()` in the
+  // for loop below
   ASSERT_FALSE(gen.details().begin_);
   ASSERT_FALSE(gen.details().end_);
   for (int i : gen) {
@@ -36,6 +36,6 @@ TEST(Generator, details) {
     ASSERT_FALSE(gen.details().end_);
   }
   ASSERT_EQ(result, 86);
-  ASSERT_FALSE(gen.details().begin_);
+  ASSERT_TRUE(gen.details().begin_);
   ASSERT_TRUE(gen.details().end_);
 }

From e45d06465c31f2a2e4e16560755c036f1a74dff3 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 3 Jul 2023 19:34:40 +0200
Subject: [PATCH 097/150] Still in the middle of anything.

---
 src/engine/AddCombinedRowToTable.h       | 10 ++++
 src/util/JoinAlgorithms/JoinAlgorithms.h | 60 ++++++++++++++----------
 2 files changed, 46 insertions(+), 24 deletions(-)

diff --git a/src/engine/AddCombinedRowToTable.h b/src/engine/AddCombinedRowToTable.h
index 1dbe65c686..dadcc4da8c 100644
--- a/src/engine/AddCombinedRowToTable.h
+++ b/src/engine/AddCombinedRowToTable.h
@@ -93,6 +93,16 @@ class AddCombinedRowToIdTable {
     }
   }
 
+  void addRows(const std::vector>& indices) {
+    flush();
+    indexBuffer_.reserve(indices.size());
+    // TODO This is rather ineffective.
+    for (auto [l, r] : indices) {
+      indexBuffer_.push_back(TargetIndexAndRowIndices{nextIndex_, {l, r}});
+    }
+    flush();
+  }
+
   // The next free row in the output will be created from
   // `inputLeft_[rowIndexA]`. The columns from `inputRight_` will all be set to
   // UNDEF
diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index f667d3a021..b02c918a3e 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -578,7 +578,7 @@ using Range = std::pair;
 template 
 class BlockAndSubrange {
  private:
-  Block block_;
+  std::shared_ptr block_;
   Range subrange_;
 
  public:
@@ -588,32 +588,36 @@ class BlockAndSubrange {
   // Construct from a container object, where the initial subrange will
   // represent the whole container.
   explicit BlockAndSubrange(Block block)
-      : block_{std::move(block)}, subrange_{0, block_.size()} {}
+      : block_{std::make_shared(std::move(block))}, subrange_{0, block_->size()} {}
 
   // Return a reference to the last element of the currently specified subrange.
   reference back() {
-    AD_CORRECTNESS_CHECK(subrange_.second - 1 < block_.size());
-    return block_[subrange_.second - 1];
+    AD_CORRECTNESS_CHECK(subrange_.second - 1 < block_->size());
+    return (*block_)[subrange_.second - 1];
   }
 
   // Return the currently specified subrange as a `std::ranges::subrange`
   // object.
   auto subrange() {
-    return std::ranges::subrange{block_.begin() + subrange_.first,
-                                 block_.begin() + subrange_.second};
+    return std::ranges::subrange{fullBlock().begin() + subrange_.first,
+                                 fullBlock().begin() + subrange_.second};
   }
 
   // The const overload of the `subrange` method (see above).
   auto subrange() const {
-    return std::ranges::subrange{block_.begin() + subrange_.first,
-                                 block_.begin() + subrange_.second};
+    return std::ranges::subrange{fullBlock().begin() + subrange_.first,
+                                 fullBlock().begin() + subrange_.second};
+  }
+
+  Range getIndices() {
+    return subrange_;
   }
 
   const auto& fullBlock() const {
-    return block_;
+    return *block_;
   }
   auto& fullBlock() {
-    return block_;
+    return *block_;
   }
 
   // Specify the subrange by using two iterators `begin` and `end`. The
@@ -623,13 +627,13 @@ class BlockAndSubrange {
   // subrange.
   void setSubrange(auto begin, auto end) {
     auto checkIt = [this](const auto& it) {
-      AD_CONTRACT_CHECK(block_.begin() <= it && it <= block_.end());
+      AD_CONTRACT_CHECK(block_.begin() <= it && it <= fullBlock().end());
     };
     checkIt(begin);
     checkIt(end);
     AD_CONTRACT_CHECK(begin <= end);
-    subrange_.first = begin - block_.begin();
-    subrange_.second = end - block_.begin();
+    subrange_.first = begin - fullBlock().begin();
+    subrange_.second = end - fullBlock().begin();
   }
 };
 }  // namespace detail
@@ -697,8 +701,11 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
 
   // In these buffers we will store blocks that all contain the same elements
   // and thus their cartesian products match.
-  std::vector> sameBlocksLeft;
-  std::vector> sameBlocksRight;
+  using LeftBlockVec = std::vector>;
+  using RightBlockVec = std::vector>;
+  LeftBlockVec sameBlocksLeft;
+  using RightBlockVec = std::vector>;
+  RightBlockVec sameBlocksRight;
 
   auto getMinEl = [&leftProjection, &rightProjection, &sameBlocksLeft,
                    &sameBlocksRight, &lessThan]() -> ProjectedEl {
@@ -779,6 +786,8 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     // TODO use `std::views::cartesian_product`.
     for (const auto& lBlock : blocksLeft) {
       for (const auto& rBlock : blocksRight) {
+        compatibleRowAction.setInput(lBlock.fullBlock(), rBlock.fullBlock());
+
         for (const auto& lEl : lBlock) {
           for (const auto& rEl : rBlock) {
             compatibleRowAction(&lEl, &rEl);
@@ -799,6 +808,8 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     AD_CORRECTNESS_CHECK(!sameBlocksRight.empty());
     decltype(auto) l = sameBlocksLeft.at(0).subrange();
     decltype(auto) r = sameBlocksRight.at(0).subrange();
+    auto& fullBlockLeft = sameBlocksLeft.at(0).fullBlock();
+    auto& fullBlockRight = sameBlocksRight.at(0).fullBlock();
 
     // Compute the range that is safe to join and perform the join.
     ProjectedEl minEl = getMinEl();
@@ -806,7 +817,7 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     auto itR = std::ranges::lower_bound(r, minEl, lessThan);
 
     std::vector> matchingIndices;
-    auto addRowIndex = [&matchingIndices, begL = sameBlocksLeft.at(0).fullBlock().begin(), begR = sameBlocksRight.at(0).fullBlock().begin()](auto itFromL, auto itFromR) {
+    auto addRowIndex = [&matchingIndices, begL = fullBlockLeft.begin(), begR = fullBlockRight.begin()](auto itFromL, auto itFromR) {
       matchingIndices.emplace_back(itFromL - begL, itFromR - begR);
     };
 
@@ -815,6 +826,8 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
                             std::ranges::subrange{r.begin(), itR}, lessThan,
                             addRowIndex, noop, noop);
 
+    compatibleRowAction(fullBlockLeft, fullBlockRight, matchingIndices);
+
     // Remove the joined elements.
     sameBlocksLeft.at(0).setSubrange(itL, l.end());
     sameBlocksRight.at(0).setSubrange(itR, r.end());
@@ -856,15 +869,14 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     // Effectively, these subranges cover all the blocks completely except maybe
     // the last one, which might contain elements `> minEl` at the end.
     auto pushRelevantSubranges = [&minEl, &lessThan](const auto& input) {
-      using Subrange = decltype(std::as_const(input.front()).subrange());
-      std::vector result;
-      for (size_t i = 0; i < input.size() - 1; ++i) {
-        result.push_back(input[i].subrange());
-      }
-      if (!input.empty()) {
-        result.push_back(
-            std::ranges::equal_range(input.back().subrange(), minEl, lessThan));
+      using Subrange = std::decay_t;
+      auto result = input;
+      if (result.empty()) {
+        return result;
       }
+      auto& last = result.back();
+      auto range = std::ranges::equal_range(last.subrange(), minEl, lessThan);
+      last.setSubrange(range.begin(), range.end());
       return result;
     };
     auto l = pushRelevantSubranges(sameBlocksLeft);

From 34d4a2184eea518ccbf1eb506612ab7b26d7fb75 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 4 Jul 2023 08:50:30 +0200
Subject: [PATCH 098/150] Small changes from a review with Hannahh.

---
 src/util/Generator.h   | 2 +-
 test/GeneratorTest.cpp | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/util/Generator.h b/src/util/Generator.h
index 43d922163d..23b2cb6780 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -21,7 +21,7 @@ template 
 class generator;
 
 // This struct can be `co_await`ed inside a `generator` to obtain a reference to
-// the details object ( the value of which is a template parameter to the
+// the details object (the value of which is a template parameter to the
 // generator). For an example see `GeneratorTest.cpp`.
 struct GetDetails {};
 static constexpr GetDetails getDetails;
diff --git a/test/GeneratorTest.cpp b/test/GeneratorTest.cpp
index 56b347d6d2..862f077b29 100644
--- a/test/GeneratorTest.cpp
+++ b/test/GeneratorTest.cpp
@@ -14,7 +14,7 @@ struct Details {
 // A simple generator that first yields three numbers and then adds a detail
 // value, that we can then extract after iterating over it.
 cppcoro::generator simpleGen() {
-  auto& details = co_await cppcoro::getDetails;
+  Details& details = co_await cppcoro::getDetails;
   details.begin_ = true;
   co_yield 1;
   co_yield 42;
@@ -22,7 +22,7 @@ cppcoro::generator simpleGen() {
   details.end_ = true;
 };
 
-// Test the behavior of the `simpleGen` above
+// Test the behavior of the `simpleGen` above.
 TEST(Generator, details) {
   auto gen = simpleGen();
   int result{};

From 461c92cd6aeb66dfaab7a56bfd7ac369b2bf7c9b Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 4 Jul 2023 19:58:00 +0200
Subject: [PATCH 099/150] Several performance improvements. This has yet to be
 cleaned up.

---
 src/engine/AddCombinedRowToTable.h       |  21 ++--
 src/engine/IndexScan.cpp                 |   8 ++
 src/engine/Join.cpp                      | 149 +++++++++++++----------
 src/engine/Sort.cpp                      |   2 +
 src/engine/idTable/IdTable.h             |   8 +-
 src/global/Constants.h                   |   2 +-
 src/index/CompressedRelation.cpp         |  57 +++++++--
 src/util/JoinAlgorithms/JoinAlgorithms.h |  44 +++----
 test/JoinAlgorithmsTest.cpp              |  39 ++++--
 9 files changed, 214 insertions(+), 116 deletions(-)

diff --git a/src/engine/AddCombinedRowToTable.h b/src/engine/AddCombinedRowToTable.h
index dadcc4da8c..a26916b617 100644
--- a/src/engine/AddCombinedRowToTable.h
+++ b/src/engine/AddCombinedRowToTable.h
@@ -68,12 +68,14 @@ class AddCombinedRowToIdTable {
         inputRight_{input2},
         resultTable_{std::move(output)},
         bufferSize_{bufferSize} {
+    /*
     AD_CORRECTNESS_CHECK(resultTable_.numColumns() == input1.numColumns() +
                                                           input2.numColumns() -
                                                           numJoinColumns);
     AD_CORRECTNESS_CHECK(input1.numColumns() >= numJoinColumns &&
                          input2.numColumns() >= numJoinColumns);
     AD_CORRECTNESS_CHECK(resultTable_.empty());
+     */
   }
 
   // Return the number of UNDEF values per column.
@@ -93,14 +95,19 @@ class AddCombinedRowToIdTable {
     }
   }
 
-  void addRows(const std::vector>& indices) {
-    flush();
-    indexBuffer_.reserve(indices.size());
-    // TODO This is rather ineffective.
-    for (auto [l, r] : indices) {
-      indexBuffer_.push_back(TargetIndexAndRowIndices{nextIndex_, {l, r}});
+  void setInput(const auto& left, const auto& right) {
+    auto toView = [](const T& table) {
+      if constexpr (requires { table.template asStaticView<0>(); }) {
+        return table.template asStaticView<0>();
+      } else {
+        return table;
+      }
+    };
+    if (nextIndex_ != 0) {
+      flush();
     }
-    flush();
+    inputLeft_ = toView(left);
+    inputRight_ = toView(right);
   }
 
   // The next free row in the output will be created from
diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index 94055168f7..e9c346f706 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -325,6 +325,14 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans(
       metaBlocks1.value(), metaBlocks2.value());
 
   return {getLazyScan(s1, blocks1), getLazyScan(s2, blocks2)};
+  /*
+  std::vector
+  bl1(metaBlocks1.value().blockMetadata_.begin(),metaBlocks1.value().blockMetadata_.end());
+  std::vector
+  bl2(metaBlocks2.value().blockMetadata_.begin(),metaBlocks2.value().blockMetadata_.end());
+
+  return {getLazyScan(s1,bl1), getLazyScan(s2, bl2)};
+   */
 }
 
 // ________________________________________________________________
diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 03cb9191fc..20b28e9abf 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -643,41 +643,65 @@ void Join::addCombinedRowToIdTable(const ROW_A& rowA, const ROW_B& rowB,
   }
 }
 
+template 
+struct IdTableAndFirstCol {
+  Table table_;
+  using iterator = std::decay_t;
+  IdTableAndFirstCol(Table t) : table_{std::move(t)} {}
+  auto begin() { return table_.getColumn(0).begin(); }
+  auto end() { return table_.getColumn(0).end(); }
+
+  decltype(auto) col() { return table_.getColumn(0); }
+  decltype(auto) col() const { return table_.getColumn(0); }
+
+  bool empty() { return col().empty(); }
+
+  const Id& operator[](size_t idx) { return col()[idx]; }
+  const Id& operator[](size_t idx) const { return col()[idx]; }
+
+  size_t size() const { return col().size(); }
+
+  template 
+  auto asStaticView() const {
+    return table_.template asStaticView();
+  }
+};
+
+cppcoro::generator> fasterGenerator(
+    cppcoro::generator gen) {
+  for (auto& table : gen) {
+    IdTableAndFirstCol t{std::move(table)};
+    co_yield t;
+  }
+  co_await cppcoro::AddDetail{gen.details()};
+}
+
 // ______________________________________________________________________________________________________
 void Join::computeResultForTwoIndexScans(IdTable* resultPtr) {
   AD_CORRECTNESS_CHECK(_left->getType() == QueryExecutionTree::SCAN &&
                        _right->getType() == QueryExecutionTree::SCAN);
+  // TODO Assert that the join column index is 0 in both inputs;
   auto& result = *resultPtr;
   result.setNumColumns(getResultWidth());
+  IdTable dummy{getExecutionContext()->getAllocator()};
+  ad_utility::AddCombinedRowToIdTable rowAdder{
+      1, dummy.asStaticView<0>(), dummy.asStaticView<0>(), std::move(result)};
 
-  auto addResultRow = [&result](auto itLeft, auto itRight) {
-    const auto& l = *itLeft;
-    const auto& r = *itRight;
-    AD_CORRECTNESS_CHECK(l[0] == r[0]);
-    result.emplace_back();
-    IdTable::row_reference lastRow = result.back();
-    lastRow[0] = l[0];
-    size_t nextIndex = 1;
-    for (size_t i = 1; i < l.size(); ++i) {
-      lastRow[nextIndex] = l[i];
-      ++nextIndex;
-    }
-    for (size_t i = 1; i < r.size(); ++i) {
-      lastRow[nextIndex] = r[i];
-      ++nextIndex;
-    }
-  };
-
-  auto lessThan = [](const auto& a, const auto& b) { return a[0] < b[0]; };
+  // TODO We can also only pass in the joinColumns.
+  // auto lessThan = [](const auto& a, const auto& b) { return a[0] < b[0]; };
 
   ad_utility::Timer timer{ad_utility::timer::Timer::InitialStatus::Started};
-  auto [leftBlocks, rightBlocks] = IndexScan::lazyScanForJoinOfTwoScans(
-      dynamic_cast(*_left->getRootOperation()),
-      dynamic_cast(*_right->getRootOperation()));
+  auto [leftBlocksInternal, rightBlocksInternal] =
+      IndexScan::lazyScanForJoinOfTwoScans(
+          dynamic_cast(*_left->getRootOperation()),
+          dynamic_cast(*_right->getRootOperation()));
   getRuntimeInfo().addDetail("time-for-filtering-blocks", timer.msecs());
 
-  ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks, lessThan,
-                                              addResultRow);
+  auto leftBlocks = fasterGenerator(std::move(leftBlocksInternal));
+  auto rightBlocks = fasterGenerator(std::move(rightBlocksInternal));
+
+  ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks,
+                                              std::less{}, rowAdder);
 
   _left->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
   _right->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
@@ -693,6 +717,7 @@ void Join::computeResultForTwoIndexScans(IdTable* resultPtr) {
     _left->getRootOperation()->getRuntimeInfo().numRows_ =
         static_cast(leftBlocks.details().at("num-elements"));
   }
+  result = std::move(rowAdder).resultTable();
 }
 
 // ______________________________________________________________________________________________________
@@ -704,43 +729,31 @@ void Join::computeResultForIndexScanAndIdTable(
   auto& result = *resultPtr;
   result.setNumColumns(getResultWidth());
 
+  auto jcLeft = firstIsRight ? joinColumnIndexScan : joinColumnIndexIdTable;
+  auto jcRight = firstIsRight ? joinColumnIndexIdTable : joinColumnIndexScan;
+  size_t numColsLeft =
+      firstIsRight ? scan.getResultWidth() : idTable.numColumns();
+  size_t numColsRight =
+      firstIsRight ? idTable.numColumns() : scan.getResultWidth();
+
+  auto joinColMap = ad_utility::JoinColumnMapping{
+      {{jcLeft, jcRight}}, numColsLeft, numColsRight};
+  IdTable dummy{getExecutionContext()->getAllocator()};
+  ad_utility::AddCombinedRowToIdTable rowAdder{
+      1, dummy.asStaticView<0>(), dummy.asStaticView<0>(), std::move(result)};
+
   AD_CORRECTNESS_CHECK(joinColumnIndexScan == 0);
 
-  auto joinColumn = idTable.getColumn(joinColumnIndexIdTable);
-  auto addResultRow = [&result, &idTable, &joinColumnIndexScan,
-                       beg = joinColumn.data()](auto itIdTable,
-                                                auto itIndexScan) {
-    const auto& idTableRow = *(idTable.begin() + (&(*itIdTable) - beg));
-    const auto& indexScanRow = *itIndexScan;
-    result.emplace_back();
-    IdTable::row_reference lastRow = result.back();
-    size_t nextIndex = 0;
-
-    if constexpr (firstIsRight) {
-      for (size_t i = 0; i < indexScanRow.size(); ++i) {
-        lastRow[nextIndex] = indexScanRow[i];
-        ++nextIndex;
-      }
-      for (size_t i = 0; i < idTable.numColumns(); ++i) {
-        if (i != joinColumnIndexScan) {
-          lastRow[nextIndex] = idTableRow[i];
-          ++nextIndex;
-        }
-      }
+  auto permutation = [&]() {
+    if (firstIsRight) {
+      return IdTableAndFirstCol{
+          idTable.asColumnSubsetView(joinColMap.permutationRight())};
     } else {
-      for (size_t i = 0; i < idTable.numColumns(); ++i) {
-        lastRow[nextIndex] = idTableRow[i];
-        ++nextIndex;
-      }
-
-      for (size_t i = 0; i < indexScanRow.size(); ++i) {
-        if (i != joinColumnIndexScan) {
-          lastRow[nextIndex] = indexScanRow[i];
-          ++nextIndex;
-        }
-      }
+      return IdTableAndFirstCol{
+          idTable.asColumnSubsetView(joinColMap.permutationLeft())};
     }
-  };
+  }();
+  // auto joinColumn = permutation.getColumn(0);
 
   auto lessThan = [](const A& a, const B& b) {
     static constexpr bool aIsId = ad_utility::isSimilar;
@@ -758,15 +771,25 @@ void Join::computeResultForIndexScanAndIdTable(
   };
 
   ad_utility::Timer timer{ad_utility::timer::Timer::InitialStatus::Started};
-  auto rightBlocks =
-      IndexScan::lazyScanForJoinOfColumnWithScan(joinColumn, scan);
+  auto rightBlocksInternal =
+      IndexScan::lazyScanForJoinOfColumnWithScan(permutation.col(), scan);
+  auto rightBlocks = fasterGenerator(std::move(rightBlocksInternal));
 
   getRuntimeInfo().addDetail("time-for-filtering-blocks", timer.msecs());
 
-  auto rightProjection = [](const auto& row) { return row[0]; };
-  ad_utility::zipperJoinForBlocksWithoutUndef(
-      std::span{&joinColumn, 1}, rightBlocks, lessThan, addResultRow,
-      std::identity{}, rightProjection);
+  auto projection = std::identity{};
+  // TODO Only pass in the joinColumns.
+  if (firstIsRight) {
+    ad_utility::zipperJoinForBlocksWithoutUndef(
+        rightBlocks, std::span{&permutation, 1}, std::less{}, rowAdder,
+        projection, projection);
+  } else {
+    ad_utility::zipperJoinForBlocksWithoutUndef(
+        std::span{&permutation, 1}, rightBlocks, std::less{}, rowAdder,
+        projection, projection);
+  }
+  result = std::move(rowAdder).resultTable();
+  result.setColumnSubset(joinColMap.permutationResult());
 
   auto& scanTree = firstIsRight ? _left : _right;
   scanTree->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
diff --git a/src/engine/Sort.cpp b/src/engine/Sort.cpp
index b08c2dbd71..327bdfd7eb 100644
--- a/src/engine/Sort.cpp
+++ b/src/engine/Sort.cpp
@@ -75,7 +75,9 @@ ResultTable Sort::computeResult() {
   }
 
   LOG(DEBUG) << "Sort result computation..." << endl;
+  ad_utility::Timer t{ad_utility::timer::Timer::InitialStatus::Started};
   IdTable idTable = subRes->idTable().clone();
+  getRuntimeInfo().addDetail("time-cloning", t.msecs());
   Engine::sort(idTable, sortColumnIndices_);
 
   LOG(DEBUG) << "Sort result computation done." << endl;
diff --git a/src/engine/idTable/IdTable.h b/src/engine/idTable/IdTable.h
index 0b12f84557..2718a27c6c 100644
--- a/src/engine/idTable/IdTable.h
+++ b/src/engine/idTable/IdTable.h
@@ -599,12 +599,14 @@ class IdTable {
   // TODO We should probably change the names of all those
   // typedefs (`iterator` as well as `row_type` etc.) to `PascalCase` for
   // consistency.
-  using iterator = ad_utility::IteratorForAccessOperator<
-      IdTable, IteratorHelper,
-      ad_utility::IsConst::False, row_type, row_reference>;
   using const_iterator = ad_utility::IteratorForAccessOperator<
       IdTable, IteratorHelper,
       ad_utility::IsConst::True, row_type, const_row_reference>;
+  using iterator = std::conditional_t<
+      isView, const_iterator,
+      ad_utility::IteratorForAccessOperator<
+          IdTable, IteratorHelper,
+          ad_utility::IsConst::False, row_type, row_reference>>;
 
   // The usual overloads of `begin()` and `end()` for const and mutable
   // `IdTable`s.
diff --git a/src/global/Constants.h b/src/global/Constants.h
index 02ff7cc70d..3634f34581 100644
--- a/src/global/Constants.h
+++ b/src/global/Constants.h
@@ -182,7 +182,7 @@ inline auto& RuntimeParameters() {
       SizeT<"cache-max-num-entries">{1000},
       SizeT<"cache-max-size-gb">{30},
       SizeT<"cache-max-size-gb-single-entry">{5},
-      SizeT<"lazy-index-scan-queue-size">{5},
+      SizeT<"lazy-index-scan-queue-size">{20},
       SizeT<"lazy-index-scan-num-threads">{10}};
   return params;
 }
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 093bc6f845..fbff78b546 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -145,9 +145,9 @@ CompressedRelationReader::asyncParallelBlockGenerator(
         // lock. We still perform it inside the lock to avoid contention of the
         // file. On a fast SSD we could possibly change this, but this has to be
         // investigated.
+        lock.unlock();
         CompressedBlock compressedBlock =
             readCompressedBlockFromFile(block, file, columnIndices);
-        lock.unlock();
         bool pushWasSuccessful = queue.push(
             myIndex, decompressBlock(compressedBlock, block.numRows_));
         checkTimeout(timer);
@@ -195,6 +195,9 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   size_t numBlocks = 0;
   size_t numElements = 0;
 
+  co_await cppcoro::AddDetail{"num-blocks-before-yielding",
+                              endBlock - beginBlock};
+
   if (beginBlock == endBlock) {
     co_await cppcoro::AddDetail{"num-blocks", numBlocks};
     co_await cppcoro::AddDetail{"num-elements", numElements};
@@ -202,8 +205,11 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   }
 
   // Read the first block, it might be incomplete
-  co_yield readPossiblyIncompleteBlock(metadata, std::nullopt, file,
-                                       *beginBlock);
+  auto firstBlock =
+      readPossiblyIncompleteBlock(metadata, std::nullopt, file, *beginBlock);
+  ++numBlocks;
+  numElements += firstBlock.numRows();
+  co_yield firstBlock;
   checkTimeout(timer);
 
   auto blockGenerator = asyncParallelBlockGenerator(beginBlock + 1, endBlock,
@@ -330,6 +336,21 @@ std::vector CompressedRelationReader::getBlocksForJoin(
 
   auto noop = ad_utility::noop;
 
+  for (const auto& block : relevantBlocks) {
+    auto rng =
+        std::equal_range(joinColumn.begin(), joinColumn.end(), block, lessThan);
+    if (rng.first != rng.second) {
+      result.push_back(block);
+    }
+  }
+  /*
+  for (const auto& block : relevantBlocks2) {
+    if (!std::ranges::equal_range(relevantBlocks1, block, blockLessThanBlock)
+        .empty()) {
+      result[1].push_back(block);
+    }
+  }
+
   // Actually perform the join.
   if (joinColumn.size() / relevantBlocks.size() > GALLOP_THRESHOLD) {
     ad_utility::gallopingJoin(relevantBlocks, joinColumn, lessThan,
@@ -339,6 +360,7 @@ std::vector CompressedRelationReader::getBlocksForJoin(
         ad_utility::zipperJoinWithUndef(
             joinColumn, relevantBlocks, lessThan, addRowZipper, noop, noop);
   }
+   */
 
   // The following check shouldn't be too expensive as there are only few
   // blocks.
@@ -373,16 +395,30 @@ CompressedRelationReader::getBlocksForJoin(
   };
 
   std::array, 2> result;
-  // When a result is found, we have to convert back from the `IdRanges` to the
-  // corresponding blocks.
-  auto addRow = [&result](auto it1, auto it2) {
-    result[0].push_back(*it1);
-    result[1].push_back(*it2);
-  };
 
+  AD_CONTRACT_CHECK(
+      std::ranges::is_sorted(relevantBlocks1, blockLessThanBlock));
+  AD_CONTRACT_CHECK(
+      std::ranges::is_sorted(relevantBlocks2, blockLessThanBlock));
+
+  /*
   auto noop = ad_utility::noop;
   [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef(
       relevantBlocks1, relevantBlocks2, blockLessThanBlock, addRow, noop, noop);
+      */
+
+  for (const auto& block : relevantBlocks1) {
+    if (!std::ranges::equal_range(relevantBlocks2, block, blockLessThanBlock)
+             .empty()) {
+      result[0].push_back(block);
+    }
+  }
+  for (const auto& block : relevantBlocks2) {
+    if (!std::ranges::equal_range(relevantBlocks1, block, blockLessThanBlock)
+             .empty()) {
+      result[1].push_back(block);
+    }
+  }
 
   // There might be duplicates in the blocks that we have to eliminate. We could
   // in theory eliminate them directly in the `zipperJoinWithUndef` routine more
@@ -390,6 +426,9 @@ CompressedRelationReader::getBlocksForJoin(
   // harder to read. The joining of the blocks doesn't seem to be a significant
   // time factor, so we leave it like this for now.
   for (auto& vec : result) {
+    std::ranges::sort(vec, {}, [](const CompressedBlockMetadata& b) {
+      return b.offsetsAndCompressedSize_.at(0).offsetInFile_;
+    });
     vec.erase(std::ranges::unique(vec).begin(), vec.end());
   }
   return result;
diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index b02c918a3e..cc96b564d6 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -583,12 +583,15 @@ class BlockAndSubrange {
 
  public:
   // The reference type of the underlying container.
-  using reference = std::iterator_traits::reference;
+  // using reference = std::iterator_traits::reference;
+  using reference = std::iterator_traits::value_type;
 
   // Construct from a container object, where the initial subrange will
   // represent the whole container.
   explicit BlockAndSubrange(Block block)
-      : block_{std::make_shared(std::move(block))}, subrange_{0, block_->size()} {}
+      : block_{std::make_shared(std::move(block))},
+        subrange_{0, block_->size()} {}
 
   // Return a reference to the last element of the currently specified subrange.
   reference back() {
@@ -609,16 +612,10 @@ class BlockAndSubrange {
                                  fullBlock().begin() + subrange_.second};
   }
 
-  Range getIndices() {
-    return subrange_;
-  }
+  Range getIndices() const { return subrange_; }
 
-  const auto& fullBlock() const {
-    return *block_;
-  }
-  auto& fullBlock() {
-    return *block_;
-  }
+  const auto& fullBlock() const { return *block_; }
+  auto& fullBlock() { return *block_; }
 
   // Specify the subrange by using two iterators `begin` and `end`. The
   // iterators must be valid iterators that point into the container, this is
@@ -627,7 +624,7 @@ class BlockAndSubrange {
   // subrange.
   void setSubrange(auto begin, auto end) {
     auto checkIt = [this](const auto& it) {
-      AD_CONTRACT_CHECK(block_.begin() <= it && it <= fullBlock().end());
+      AD_CONTRACT_CHECK(fullBlock().begin() <= it && it <= fullBlock().end());
     };
     checkIt(begin);
     checkIt(end);
@@ -668,7 +665,7 @@ template >;
   using RightBlockVec = std::vector>;
+  // TODO The sameBlocksLeft/Right can possibly become very large.
+  // They should respect the memory limit.
   LeftBlockVec sameBlocksLeft;
   using RightBlockVec = std::vector>;
   RightBlockVec sameBlocksRight;
@@ -788,9 +787,11 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
       for (const auto& rBlock : blocksRight) {
         compatibleRowAction.setInput(lBlock.fullBlock(), rBlock.fullBlock());
 
-        for (const auto& lEl : lBlock) {
-          for (const auto& rEl : rBlock) {
-            compatibleRowAction(&lEl, &rEl);
+        for (size_t i : std::views::iota(lBlock.getIndices().first,
+                                         lBlock.getIndices().second)) {
+          for (size_t j : std::views::iota(rBlock.getIndices().first,
+                                           rBlock.getIndices().second)) {
+            compatibleRowAction.addRow(i, j);
           }
         }
       }
@@ -816,9 +817,11 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     auto itL = std::ranges::lower_bound(l, minEl, lessThan);
     auto itR = std::ranges::lower_bound(r, minEl, lessThan);
 
-    std::vector> matchingIndices;
-    auto addRowIndex = [&matchingIndices, begL = fullBlockLeft.begin(), begR = fullBlockRight.begin()](auto itFromL, auto itFromR) {
-      matchingIndices.emplace_back(itFromL - begL, itFromR - begR);
+    compatibleRowAction.setInput(fullBlockLeft, fullBlockRight);
+    auto addRowIndex = [begL = fullBlockLeft.begin(),
+                        begR = fullBlockRight.begin(),
+                        &compatibleRowAction](auto itFromL, auto itFromR) {
+      compatibleRowAction.addRow(itFromL - begL, itFromR - begR);
     };
 
     [[maybe_unused]] auto res =
@@ -826,8 +829,6 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
                             std::ranges::subrange{r.begin(), itR}, lessThan,
                             addRowIndex, noop, noop);
 
-    compatibleRowAction(fullBlockLeft, fullBlockRight, matchingIndices);
-
     // Remove the joined elements.
     sameBlocksLeft.at(0).setSubrange(itL, l.end());
     sameBlocksRight.at(0).setSubrange(itR, r.end());
@@ -869,7 +870,6 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     // Effectively, these subranges cover all the blocks completely except maybe
     // the last one, which might contain elements `> minEl` at the end.
     auto pushRelevantSubranges = [&minEl, &lessThan](const auto& input) {
-      using Subrange = std::decay_t;
       auto result = input;
       if (result.empty()) {
         return result;
diff --git a/test/JoinAlgorithmsTest.cpp b/test/JoinAlgorithmsTest.cpp
index 78509a3087..85f4e4c8a2 100644
--- a/test/JoinAlgorithmsTest.cpp
+++ b/test/JoinAlgorithmsTest.cpp
@@ -17,14 +17,26 @@ using Block = std::vector>;
 using NestedBlock = std::vector;
 using JoinResult = std::vector>;
 
-auto makeRowAdder(JoinResult& target) {
-  // `it1, it2` must be (const) iterators to a `Block`.
-  return [&target](auto it1, auto it2) {
-    auto [x1, x2] = *it1;
-    auto [y1, y2] = *it2;
+struct RowAdder {
+  const Block* left_{};
+  const Block* right_{};
+  JoinResult* target_{};
+
+  void setInput(const Block& left, const Block& right) {
+    left_ = &left;
+    right_ = &right;
+  }
+
+  void addRow(size_t leftIndex, size_t rightIndex) {
+    auto [x1, x2] = (*left_)[leftIndex];
+    auto [y1, y2] = (*right_)[rightIndex];
     AD_CONTRACT_CHECK(x1 == y1);
-    target.push_back(std::array{x1, x2, y2});
-  };
+    target_->push_back(std::array{x1, x2, y2});
+  }
+};
+
+auto makeRowAdder(JoinResult& target) {
+  return RowAdder{nullptr, nullptr, &target};
 }
 
 using ad_utility::source_location;
@@ -37,7 +49,8 @@ void testJoin(const NestedBlock& a, const NestedBlock& b, JoinResult expected,
   auto trace = generateLocationTrace(l);
   JoinResult result;
   auto compare = [](auto l, auto r) { return l[0] < r[0]; };
-  zipperJoinForBlocksWithoutUndef(a, b, compare, makeRowAdder(result));
+  auto adder = makeRowAdder(result);
+  zipperJoinForBlocksWithoutUndef(a, b, compare, adder);
   // The result must be sorted on the first column
   EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first));
   // The exact order of the elements with the same first column is not important
@@ -47,9 +60,13 @@ void testJoin(const NestedBlock& a, const NestedBlock& b, JoinResult expected,
   for (auto& [x, y, z] : expected) {
     std::swap(y, z);
   }
-  zipperJoinForBlocksWithoutUndef(b, a, compare, makeRowAdder(result));
-  EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first));
-  EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expected));
+  {
+    auto adder = makeRowAdder(result);
+    zipperJoinForBlocksWithoutUndef(b, a, compare, adder);
+    EXPECT_TRUE(
+        std::ranges::is_sorted(result, std::less<>{}, ad_utility::first));
+    EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expected));
+  }
 }
 }  // namespace
 

From 5d51c249c8e82fac9ead347c6e78c95e09d850df Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 10:45:23 +0200
Subject: [PATCH 100/150] Improved some infrastructure and added missing
 runtime information.

---
 src/engine/AddCombinedRowToTable.h          |  67 +++++++----
 src/engine/Join.cpp                         | 124 +++++++-------------
 src/engine/Join.h                           |  11 +-
 src/index/CompressedRelation.cpp            |  84 ++++---------
 src/util/JoinAlgorithms/JoinColumnMapping.h |  43 +++++++
 5 files changed, 159 insertions(+), 170 deletions(-)

diff --git a/src/engine/AddCombinedRowToTable.h b/src/engine/AddCombinedRowToTable.h
index a26916b617..ba7d6afe5e 100644
--- a/src/engine/AddCombinedRowToTable.h
+++ b/src/engine/AddCombinedRowToTable.h
@@ -19,8 +19,8 @@ namespace ad_utility {
 class AddCombinedRowToIdTable {
   std::vector numUndefinedPerColumn_;
   size_t numJoinColumns_;
-  IdTableView<0> inputLeft_;
-  IdTableView<0> inputRight_;
+  std::optional> inputLeft_;
+  std::optional> inputRight_;
   IdTable resultTable_;
 
   // This struct stores the information, which row indices from the input are
@@ -57,25 +57,24 @@ class AddCombinedRowToIdTable {
 
  public:
   // Construct from the number of join columns, the two inputs, and the output.
-  // The `bufferSize` can be configured for testing.
+  // The `bufferSize` can be configured for testing. If the inputs are
+  // `std::nullopt`, this means that the inputs have to be set to an explicit
+  // call to `setInput` before adding rows. This is used for the lazy join
+  // operations (see Join.cpp) where the input changes over time.
   explicit AddCombinedRowToIdTable(size_t numJoinColumns,
-                                   const IdTableView<0>& input1,
-                                   const IdTableView<0>& input2, IdTable output,
-                                   size_t bufferSize = 100'000)
+                                   std::optional> input1,
+                                   std::optional> input2,
+                                   IdTable output, size_t bufferSize = 100'000)
       : numUndefinedPerColumn_(output.numColumns()),
         numJoinColumns_{numJoinColumns},
-        inputLeft_{input1},
-        inputRight_{input2},
+        inputLeft_{std::move(input1)},
+        inputRight_{std::move(input2)},
         resultTable_{std::move(output)},
         bufferSize_{bufferSize} {
-    /*
-    AD_CORRECTNESS_CHECK(resultTable_.numColumns() == input1.numColumns() +
-                                                          input2.numColumns() -
-                                                          numJoinColumns);
-    AD_CORRECTNESS_CHECK(input1.numColumns() >= numJoinColumns &&
-                         input2.numColumns() >= numJoinColumns);
+    if (inputLeft_.has_value() && inputRight_.has_value()) {
+      checkNumColumns();
+    }
     AD_CORRECTNESS_CHECK(resultTable_.empty());
-     */
   }
 
   // Return the number of UNDEF values per column.
@@ -87,6 +86,7 @@ class AddCombinedRowToIdTable {
   // The next free row in the output will be created from
   // `inputLeft_[rowIndexA]` and `inputRight_[rowIndexB]`.
   void addRow(size_t rowIndexA, size_t rowIndexB) {
+    AD_EXPENSIVE_CHECK(inputLeft_.has_value() && inputRight_.has_value());
     indexBuffer_.push_back(
         TargetIndexAndRowIndices{nextIndex_, {rowIndexA, rowIndexB}});
     ++nextIndex_;
@@ -95,7 +95,11 @@ class AddCombinedRowToIdTable {
     }
   }
 
-  void setInput(const auto& left, const auto& right) {
+  // Set or reset the input. All following calls to `addRow` then refer to
+  // indices in the new input. Before resetting, `flush()` is called, so all the
+  // rows from the previous inputs get materialized before deleting the old
+  // inputs.
+  void setInput(const auto& inputLeft, const auto& inputRight) {
     auto toView = [](const T& table) {
       if constexpr (requires { table.template asStaticView<0>(); }) {
         return table.template asStaticView<0>();
@@ -104,16 +108,19 @@ class AddCombinedRowToIdTable {
       }
     };
     if (nextIndex_ != 0) {
+      AD_CORRECTNESS_CHECK(inputLeft_.has_value() && inputRight_.has_value());
       flush();
     }
-    inputLeft_ = toView(left);
-    inputRight_ = toView(right);
+    inputLeft_ = toView(inputLeft);
+    inputRight_ = toView(inputRight);
+    checkNumColumns();
   }
 
   // The next free row in the output will be created from
   // `inputLeft_[rowIndexA]`. The columns from `inputRight_` will all be set to
   // UNDEF
   void addOptionalRow(size_t rowIndexA) {
+    AD_EXPENSIVE_CHECK(inputLeft_.has_value() && inputRight_.has_value());
     optionalIndexBuffer_.push_back(
         TargetIndexAndRowIndex{nextIndex_, rowIndexA});
     ++nextIndex_;
@@ -142,6 +149,7 @@ class AddCombinedRowToIdTable {
   // have to call it manually after adding the last row, else the destructor
   // will throw an exception.
   void flush() {
+    AD_CONTRACT_CHECK(inputLeft_.has_value() && inputRight_.has_value());
     auto& result = resultTable_;
     size_t oldSize = result.size();
     AD_CORRECTNESS_CHECK(nextIndex_ ==
@@ -164,8 +172,8 @@ class AddCombinedRowToIdTable {
     // `nextResultColIdx`-th column of the result.
     auto writeJoinColumn = [&result, &mergeWithUndefined, oldSize, this](
                                size_t colIdx, size_t resultColIdx) {
-      const auto& colLeft = inputLeft_.getColumn(colIdx);
-      const auto& colRight = inputRight_.getColumn(colIdx);
+      const auto& colLeft = inputLeft().getColumn(colIdx);
+      const auto& colRight = inputRight().getColumn(colIdx);
       // TODO Implement prefetching.
       decltype(auto) resultCol = result.getColumn(resultColIdx);
       size_t& numUndef = numUndefinedPerColumn_.at(resultColIdx);
@@ -195,8 +203,8 @@ class AddCombinedRowToIdTable {
     // code that was very hard to read for humans.
     auto writeNonJoinColumn = [&result, oldSize, this](
                                   size_t colIdx, size_t resultColIdx) {
-      decltype(auto) col = isColFromLeft ? inputLeft_.getColumn(colIdx)
-                                         : inputRight_.getColumn(colIdx);
+      decltype(auto) col = isColFromLeft ? inputLeft().getColumn(colIdx)
+                                         : inputRight().getColumn(colIdx);
       // TODO Implement prefetching.
       decltype(auto) resultCol = result.getColumn(resultColIdx);
       size_t& numUndef = numUndefinedPerColumn_.at(resultColIdx);
@@ -234,13 +242,13 @@ class AddCombinedRowToIdTable {
     }
 
     // Then the remaining columns from the first input.
-    for (size_t col = numJoinColumns_; col < inputLeft_.numColumns(); ++col) {
+    for (size_t col = numJoinColumns_; col < inputLeft().numColumns(); ++col) {
       writeNonJoinColumn.operator()(col, nextResultColIdx);
       ++nextResultColIdx;
     }
 
     // Then the remaining columns from the second input.
-    for (size_t col = numJoinColumns_; col < inputRight_.numColumns(); col++) {
+    for (size_t col = numJoinColumns_; col < inputRight().numColumns(); col++) {
       writeNonJoinColumn.operator()(col, nextResultColIdx);
       ++nextResultColIdx;
     }
@@ -249,5 +257,16 @@ class AddCombinedRowToIdTable {
     optionalIndexBuffer_.clear();
     nextIndex_ = 0;
   }
+  const IdTableView<0>& inputLeft() const { return inputLeft_.value(); }
+
+  const IdTableView<0>& inputRight() const { return inputRight_.value(); }
+
+  void checkNumColumns() const {
+    AD_CORRECTNESS_CHECK(resultTable_.numColumns() ==
+                         inputLeft().numColumns() + inputRight().numColumns() -
+                             numJoinColumns_);
+    AD_CORRECTNESS_CHECK(inputLeft().numColumns() >= numJoinColumns_ &&
+                         inputRight().numColumns() >= numJoinColumns_);
+  }
 };
 }  // namespace ad_utility
diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 20b28e9abf..ed18d6f796 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -120,14 +120,14 @@ ResultTable Join::computeResult() {
   if (_left->getType() == QueryExecutionTree::SCAN &&
       _right->getType() == QueryExecutionTree::SCAN) {
     if (rightResIfCached && !leftResIfCached) {
-      computeResultForIndexScanAndIdTable(
+      idTable = computeResultForIndexScanAndIdTable(
           rightResIfCached->idTable(), _rightJoinCol,
           dynamic_cast(*_left->getRootOperation()),
-          _leftJoinCol, &idTable);
+          _leftJoinCol);
       return {std::move(idTable), resultSortedOn(), LocalVocab{}};
 
     } else if (!leftResIfCached) {
-      computeResultForTwoIndexScans(&idTable);
+      idTable = computeResultForTwoIndexScans();
       // TODO When we add triples to the
       // index, the vocabularies of index scans will not necessarily be empty
       // and we need a mechanism to still retrieve them when using the lazy
@@ -149,10 +149,10 @@ ResultTable Join::computeResult() {
   // Note: If only one of the children is a scan, then we have made sure in the
   // constructor that it is the right child.
   if (_right->getType() == QueryExecutionTree::SCAN && !rightResIfCached) {
-    computeResultForIndexScanAndIdTable(
+    idTable = computeResultForIndexScanAndIdTable(
         leftRes->idTable(), _leftJoinCol,
         dynamic_cast(*_right->getRootOperation()),
-        _rightJoinCol, &idTable);
+        _rightJoinCol);
     return {std::move(idTable), resultSortedOn(),
             leftRes->getSharedLocalVocab()};
   }
@@ -643,52 +643,30 @@ void Join::addCombinedRowToIdTable(const ROW_A& rowA, const ROW_B& rowB,
   }
 }
 
-template 
-struct IdTableAndFirstCol {
-  Table table_;
-  using iterator = std::decay_t;
-  IdTableAndFirstCol(Table t) : table_{std::move(t)} {}
-  auto begin() { return table_.getColumn(0).begin(); }
-  auto end() { return table_.getColumn(0).end(); }
-
-  decltype(auto) col() { return table_.getColumn(0); }
-  decltype(auto) col() const { return table_.getColumn(0); }
-
-  bool empty() { return col().empty(); }
-
-  const Id& operator[](size_t idx) { return col()[idx]; }
-  const Id& operator[](size_t idx) const { return col()[idx]; }
-
-  size_t size() const { return col().size(); }
-
-  template 
-  auto asStaticView() const {
-    return table_.template asStaticView();
-  }
-};
-
-cppcoro::generator> fasterGenerator(
+namespace {
+// Convert a `generator` for more
+// efficient access in the join columns below.
+cppcoro::generator> liftGenerator(
     cppcoro::generator gen) {
   for (auto& table : gen) {
-    IdTableAndFirstCol t{std::move(table)};
+    ad_utility::IdTableAndFirstCol t{std::move(table)};
     co_yield t;
   }
   co_await cppcoro::AddDetail{gen.details()};
 }
+}  // namespace
 
 // ______________________________________________________________________________________________________
-void Join::computeResultForTwoIndexScans(IdTable* resultPtr) {
+IdTable Join::computeResultForTwoIndexScans() {
   AD_CORRECTNESS_CHECK(_left->getType() == QueryExecutionTree::SCAN &&
                        _right->getType() == QueryExecutionTree::SCAN);
-  // TODO Assert that the join column index is 0 in both inputs;
-  auto& result = *resultPtr;
-  result.setNumColumns(getResultWidth());
-  IdTable dummy{getExecutionContext()->getAllocator()};
+  // The join column already is the first column in both inputs, so we don't
+  // have to permute the inputs and results for the `AddCombinedRowToIdTable`
+  // class to work correctly.
+  AD_CORRECTNESS_CHECK(_leftJoinCol == 0 && _rightJoinCol == 0);
   ad_utility::AddCombinedRowToIdTable rowAdder{
-      1, dummy.asStaticView<0>(), dummy.asStaticView<0>(), std::move(result)};
-
-  // TODO We can also only pass in the joinColumns.
-  // auto lessThan = [](const auto& a, const auto& b) { return a[0] < b[0]; };
+      1, std::nullopt, std::nullopt,
+      IdTable{getResultWidth(), getExecutionContext()->getAllocator()}};
 
   ad_utility::Timer timer{ad_utility::timer::Timer::InitialStatus::Started};
   auto [leftBlocksInternal, rightBlocksInternal] =
@@ -697,8 +675,8 @@ void Join::computeResultForTwoIndexScans(IdTable* resultPtr) {
           dynamic_cast(*_right->getRootOperation()));
   getRuntimeInfo().addDetail("time-for-filtering-blocks", timer.msecs());
 
-  auto leftBlocks = fasterGenerator(std::move(leftBlocksInternal));
-  auto rightBlocks = fasterGenerator(std::move(rightBlocksInternal));
+  auto leftBlocks = liftGenerator(std::move(leftBlocksInternal));
+  auto rightBlocks = liftGenerator(std::move(rightBlocksInternal));
 
   ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks,
                                               std::less{}, rowAdder);
@@ -717,69 +695,52 @@ void Join::computeResultForTwoIndexScans(IdTable* resultPtr) {
     _left->getRootOperation()->getRuntimeInfo().numRows_ =
         static_cast(leftBlocks.details().at("num-elements"));
   }
-  result = std::move(rowAdder).resultTable();
+  return std::move(rowAdder).resultTable();
 }
 
 // ______________________________________________________________________________________________________
-template 
-void Join::computeResultForIndexScanAndIdTable(
+template 
+IdTable Join::computeResultForIndexScanAndIdTable(
     const IdTable& idTable, ColumnIndex joinColumnIndexIdTable,
-    const IndexScan& scan, ColumnIndex joinColumnIndexScan,
-    IdTable* resultPtr) {
-  auto& result = *resultPtr;
-  result.setNumColumns(getResultWidth());
-
-  auto jcLeft = firstIsRight ? joinColumnIndexScan : joinColumnIndexIdTable;
-  auto jcRight = firstIsRight ? joinColumnIndexIdTable : joinColumnIndexScan;
+    const IndexScan& scan, ColumnIndex joinColumnIndexScan) {
+  // We first have to permute the columns.
+  // TODO Maybe we can reduce the complexity for the
+  // `idTableIsRightInput` switch.
+  auto jcLeft =
+      idTableIsRightInput ? joinColumnIndexScan : joinColumnIndexIdTable;
+  auto jcRight =
+      idTableIsRightInput ? joinColumnIndexIdTable : joinColumnIndexScan;
   size_t numColsLeft =
-      firstIsRight ? scan.getResultWidth() : idTable.numColumns();
+      idTableIsRightInput ? scan.getResultWidth() : idTable.numColumns();
   size_t numColsRight =
-      firstIsRight ? idTable.numColumns() : scan.getResultWidth();
+      idTableIsRightInput ? idTable.numColumns() : scan.getResultWidth();
 
   auto joinColMap = ad_utility::JoinColumnMapping{
       {{jcLeft, jcRight}}, numColsLeft, numColsRight};
-  IdTable dummy{getExecutionContext()->getAllocator()};
   ad_utility::AddCombinedRowToIdTable rowAdder{
-      1, dummy.asStaticView<0>(), dummy.asStaticView<0>(), std::move(result)};
+      1, std::nullopt, std::nullopt,
+      IdTable{getResultWidth(), getExecutionContext()->getAllocator()}};
 
   AD_CORRECTNESS_CHECK(joinColumnIndexScan == 0);
-
   auto permutation = [&]() {
-    if (firstIsRight) {
-      return IdTableAndFirstCol{
+    if (idTableIsRightInput) {
+      return ad_utility::IdTableAndFirstCol{
           idTable.asColumnSubsetView(joinColMap.permutationRight())};
     } else {
-      return IdTableAndFirstCol{
+      return ad_utility::IdTableAndFirstCol{
           idTable.asColumnSubsetView(joinColMap.permutationLeft())};
     }
   }();
-  // auto joinColumn = permutation.getColumn(0);
-
-  auto lessThan = [](const A& a, const B& b) {
-    static constexpr bool aIsId = ad_utility::isSimilar;
-    static constexpr bool bIsId = ad_utility::isSimilar;
-
-    if constexpr (aIsId && bIsId) {
-      return a < b;
-    } else if constexpr (aIsId) {
-      return a < b[0];
-    } else if constexpr (bIsId) {
-      return a[0] < b;
-    } else {
-      return a[0] < b[0];
-    }
-  };
 
   ad_utility::Timer timer{ad_utility::timer::Timer::InitialStatus::Started};
   auto rightBlocksInternal =
       IndexScan::lazyScanForJoinOfColumnWithScan(permutation.col(), scan);
-  auto rightBlocks = fasterGenerator(std::move(rightBlocksInternal));
+  auto rightBlocks = liftGenerator(std::move(rightBlocksInternal));
 
   getRuntimeInfo().addDetail("time-for-filtering-blocks", timer.msecs());
 
   auto projection = std::identity{};
-  // TODO Only pass in the joinColumns.
-  if (firstIsRight) {
+  if (idTableIsRightInput) {
     ad_utility::zipperJoinForBlocksWithoutUndef(
         rightBlocks, std::span{&permutation, 1}, std::less{}, rowAdder,
         projection, projection);
@@ -788,10 +749,10 @@ void Join::computeResultForIndexScanAndIdTable(
         std::span{&permutation, 1}, rightBlocks, std::less{}, rowAdder,
         projection, projection);
   }
-  result = std::move(rowAdder).resultTable();
+  auto result = std::move(rowAdder).resultTable();
   result.setColumnSubset(joinColMap.permutationResult());
 
-  auto& scanTree = firstIsRight ? _left : _right;
+  auto& scanTree = idTableIsRightInput ? _left : _right;
   scanTree->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
   scanTree->getRootOperation()->getRuntimeInfo().details_.update(
       rightBlocks.details());
@@ -799,4 +760,5 @@ void Join::computeResultForIndexScanAndIdTable(
     scanTree->getRootOperation()->getRuntimeInfo().numRows_ =
         static_cast(rightBlocks.details().at("num-elements"));
   }
+  return result;
 }
diff --git a/src/engine/Join.h b/src/engine/Join.h
index c0eaed871b..8eaf446357 100644
--- a/src/engine/Join.h
+++ b/src/engine/Join.h
@@ -137,18 +137,17 @@ class Join : public Operation {
   // A special implementation that is called when both children are
   // `IndexScan`s. Uses the lazy scans to only retrieve the subset of the
   // `IndexScan`s that is actually needed without fully materializing them.
-  void computeResultForTwoIndexScans(IdTable* resultPtr);
+  IdTable computeResultForTwoIndexScans();
 
   // A special implementation that is called when one of the children is an
   // `IndexScan`. The argument `scanIsLeft` determines whether the `IndexScan`
   // is the left or the right child of this `Join`. This needs to be known to
   // determine the correct order of the columns in the result.
   template 
-  void computeResultForIndexScanAndIdTable(const IdTable& itIdTable,
-                                           ColumnIndex itIndexScan,
-                                           const IndexScan& scan,
-                                           ColumnIndex joinColumnIndexScan,
-                                           IdTable* resultPtr);
+  IdTable computeResultForIndexScanAndIdTable(const IdTable& itIdTable,
+                                              ColumnIndex itIndexScan,
+                                              const IndexScan& scan,
+                                              ColumnIndex joinColumnIndexScan);
 
   using ScanMethodType = std::function;
 
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index fbff78b546..15fe0b4bd7 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -192,14 +192,11 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   const auto beginBlock = relevantBlocks.begin();
   const auto endBlock = relevantBlocks.end();
 
-  size_t numBlocks = 0;
   size_t numElements = 0;
 
-  co_await cppcoro::AddDetail{"num-blocks-before-yielding",
-                              endBlock - beginBlock};
+  co_await cppcoro::AddDetail{"num-blocks", endBlock - beginBlock};
 
   if (beginBlock == endBlock) {
-    co_await cppcoro::AddDetail{"num-blocks", numBlocks};
     co_await cppcoro::AddDetail{"num-elements", numElements};
     co_return;
   }
@@ -207,7 +204,6 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   // Read the first block, it might be incomplete
   auto firstBlock =
       readPossiblyIncompleteBlock(metadata, std::nullopt, file, *beginBlock);
-  ++numBlocks;
   numElements += firstBlock.numRows();
   co_yield firstBlock;
   checkTimeout(timer);
@@ -216,11 +212,9 @@ cppcoro::generator CompressedRelationReader::lazyScan(
                                                     file, std::nullopt, timer);
   for (auto& block : blockGenerator) {
     numElements += block.numRows();
-    ++numBlocks;
     co_yield block;
   }
   co_await cppcoro::AddDetail{blockGenerator.details()};
-  co_await cppcoro::AddDetail{"num-blocks", numBlocks};
   co_await cppcoro::AddDetail{"num-elements", numElements};
 }
 
@@ -233,7 +227,10 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   auto beginBlock = relevantBlocks.begin();
   auto endBlock = relevantBlocks.end();
 
+  co_await cppcoro::AddDetail{"num-blocks", endBlock - beginBlock};
+  size_t numElements = 0;
   if (beginBlock == endBlock) {
+    co_await cppcoro::AddDetail{"num-elements", numElements};
     co_return;
   }
 
@@ -258,12 +255,18 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   }
 
   if (beginBlock + 1 < endBlock) {
-    for (auto& block : asyncParallelBlockGenerator(
-             beginBlock + 1, endBlock - 1, file, std::vector{1UL}, timer)) {
+    auto blockGenerator = asyncParallelBlockGenerator(
+        beginBlock + 1, endBlock - 1, file, std::vector{1UL}, timer);
+    for (auto& block : blockGenerator) {
+      numElements += block.numRows();
       co_yield block;
     }
-    co_yield getIncompleteBlock(endBlock - 1);
+    co_await cppcoro::AddDetail{blockGenerator.details()};
+    auto lastBlock = getIncompleteBlock(endBlock - 1);
+    numElements += lastBlock.numRows();
+    co_yield lastBlock;
   }
+  co_await cppcoro::AddDetail{"num-elements", numElements};
 }
 
 namespace {
@@ -308,8 +311,7 @@ std::vector CompressedRelationReader::getBlocksForJoin(
   // Get all the blocks where `col0FirstId_ <= col0Id <= col0LastId_`.
   auto relevantBlocks = getBlocksFromMetadata(metadataAndBlocks);
 
-  // We need symmetric comparisons between `Id` and `IdPair`. as well as between
-  // Ids.
+  // We need symmetric comparisons between Ids and blocks.
   auto idLessThanBlock = [&metadataAndBlocks](
                              Id id, const CompressedBlockMetadata& block) {
     return id < getRelevantIdFromTriple(block.firstTriple_, metadataAndBlocks);
@@ -319,23 +321,14 @@ std::vector CompressedRelationReader::getBlocksForJoin(
                              const CompressedBlockMetadata& block, Id id) {
     return getRelevantIdFromTriple(block.lastTriple_, metadataAndBlocks) < id;
   };
-
   auto lessThan =
       ad_utility::OverloadCallOperator{idLessThanBlock, blockLessThanId};
 
-  // When we have found a matching block, we have to convert it back.
+  // Find the matching blocks by performing binary search on the `joinColumn`.
+  // Note that it is tempting to reuse the `zipperJoinWithUndef` routine, but
+  // this doesn't work because the implicit equality defined by `!lessThan(a,b)
+  // && !lessThan(b, a)` is not transitive.
   std::vector result;
-  auto addRowZipper = [&result]([[maybe_unused]] auto idIterator,
-                                auto blockIterator) {
-    result.push_back(*blockIterator);
-  };
-  auto addRowGallop = [&result](auto blockIterator,
-                                [[maybe_unused]] auto idIterator) {
-    result.push_back(*blockIterator);
-  };
-
-  auto noop = ad_utility::noop;
-
   for (const auto& block : relevantBlocks) {
     auto rng =
         std::equal_range(joinColumn.begin(), joinColumn.end(), block, lessThan);
@@ -343,25 +336,6 @@ std::vector CompressedRelationReader::getBlocksForJoin(
       result.push_back(block);
     }
   }
-  /*
-  for (const auto& block : relevantBlocks2) {
-    if (!std::ranges::equal_range(relevantBlocks1, block, blockLessThanBlock)
-        .empty()) {
-      result[1].push_back(block);
-    }
-  }
-
-  // Actually perform the join.
-  if (joinColumn.size() / relevantBlocks.size() > GALLOP_THRESHOLD) {
-    ad_utility::gallopingJoin(relevantBlocks, joinColumn, lessThan,
-                                     addRowGallop);
-  } else {
-    [[maybe_unused]] auto numOutOfOrder =
-        ad_utility::zipperJoinWithUndef(
-            joinColumn, relevantBlocks, lessThan, addRowZipper, noop, noop);
-  }
-   */
-
   // The following check shouldn't be too expensive as there are only few
   // blocks.
   AD_CORRECTNESS_CHECK(std::ranges::unique(result).begin() == result.end());
@@ -401,12 +375,10 @@ CompressedRelationReader::getBlocksForJoin(
   AD_CONTRACT_CHECK(
       std::ranges::is_sorted(relevantBlocks2, blockLessThanBlock));
 
-  /*
-  auto noop = ad_utility::noop;
-  [[maybe_unused]] auto numOutOfOrder = ad_utility::zipperJoinWithUndef(
-      relevantBlocks1, relevantBlocks2, blockLessThanBlock, addRow, noop, noop);
-      */
-
+  // Find the matching blocks on each side by performing binary search on the
+  // other side. Note that it is tempting to reuse the `zipperJoinWithUndef`
+  // routine, but this doesn't work because the implicit equality defined by
+  // `!lessThan(a,b) && !lessThan(b, a)` is not transitive.
   for (const auto& block : relevantBlocks1) {
     if (!std::ranges::equal_range(relevantBlocks2, block, blockLessThanBlock)
              .empty()) {
@@ -420,16 +392,10 @@ CompressedRelationReader::getBlocksForJoin(
     }
   }
 
-  // There might be duplicates in the blocks that we have to eliminate. We could
-  // in theory eliminate them directly in the `zipperJoinWithUndef` routine more
-  // efficiently, but this would make this already very complex method even
-  // harder to read. The joining of the blocks doesn't seem to be a significant
-  // time factor, so we leave it like this for now.
+  // The following check shouldn't be too expensive as there are only few
+  // blocks.
   for (auto& vec : result) {
-    std::ranges::sort(vec, {}, [](const CompressedBlockMetadata& b) {
-      return b.offsetsAndCompressedSize_.at(0).offsetInFile_;
-    });
-    vec.erase(std::ranges::unique(vec).begin(), vec.end());
+    AD_CORRECTNESS_CHECK(std::ranges::unique(vec).begin() == vec.end());
   }
   return result;
 }
diff --git a/src/util/JoinAlgorithms/JoinColumnMapping.h b/src/util/JoinAlgorithms/JoinColumnMapping.h
index 6b75315583..9c5c649d79 100644
--- a/src/util/JoinAlgorithms/JoinColumnMapping.h
+++ b/src/util/JoinAlgorithms/JoinColumnMapping.h
@@ -95,4 +95,47 @@ class JoinColumnMapping {
     }
   }
 };
+
+// A class that stores a complete `IdTable`, but when being treated as a range
+// via the `begin/end/operator[]` functions, then it only gives access to the
+// first column. This is very useful for the lazy join implementations
+// (currently used in `Join.cpp`, where we need very efficient access to the
+// join column for comparing rows, but also need to store the complete table to
+// be able to write the other columns of a matching row to the result.
+// This class is templated so we can use it for `IdTable` as well as for
+// `IdTableView`.
+template 
+struct IdTableAndFirstCol {
+ private:
+  Table table_;
+
+ public:
+  // Typedef needed for generic interfaces.
+  using iterator = std::decay_t;
+
+  // Construct by taking ownership of the table.
+  IdTableAndFirstCol(Table t) : table_{std::move(t)} {}
+
+  // Get access to the first column.
+  decltype(auto) col() { return table_.getColumn(0); }
+  decltype(auto) col() const { return table_.getColumn(0); }
+
+  // The following functions all refer to the same column.
+  auto begin() { return col().begin(); }
+  auto end() { return col().end(); }
+
+  bool empty() { return col().empty(); }
+
+  const Id& operator[](size_t idx) const { return col()[idx]; }
+
+  size_t size() const { return col().size(); }
+
+  // This interface is required in `Join.cpp` by the `AddCombinedRowToTable`
+  // class. Calling this function yields the same type, no matter if `Table` is
+  // `IdTable` or `IdTableView`.
+  template 
+  IdTableView asStaticView() const {
+    return table_.template asStaticView();
+  }
+};
 }  // namespace ad_utility

From 6168e3940a27b804cea7ffc3d777c572a58b6ef8 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 11:01:45 +0200
Subject: [PATCH 101/150] Added an empty details class as the default.

---
 src/util/Generator.h | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/util/Generator.h b/src/util/Generator.h
index 23b2cb6780..b05eec0fa4 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -17,17 +17,21 @@
 #include "util/TypeTraits.h"
 
 namespace cppcoro {
-template 
-class generator;
-
 // This struct can be `co_await`ed inside a `generator` to obtain a reference to
 // the details object (the value of which is a template parameter to the
 // generator). For an example see `GeneratorTest.cpp`.
 struct GetDetails {};
 static constexpr GetDetails getDetails;
 
+// This struct is used as the default of the details object for the case that
+// there are no details
+struct NoDetails {};
+
+template 
+class generator;
+
 namespace detail {
-template 
+template 
 class generator_promise {
  public:
   // Even if the generator only yields `const` values, the `value_type`
@@ -93,7 +97,8 @@ class generator_promise {
  private:
   pointer_type m_value;
   std::exception_ptr m_exception;
-  Details m_details{};
+  // If the `Details` type is empty, we don't need it to occupy any space.
+  [[no_unique_address]] Details m_details{};
 };
 
 struct generator_sentinel {};
@@ -159,7 +164,7 @@ class generator_iterator {
 };
 }  // namespace detail
 
-template 
+template 
 class [[nodiscard]] generator {
  public:
   using promise_type = detail::generator_promise;

From f6fff69ec4905fcec95fa3be8c6c70dbd8cf37ba Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 11:51:50 +0200
Subject: [PATCH 102/150] Merged in hannah's merge.

---
 benchmark/infrastructure/BenchmarkToJson.cpp |  2 +-
 src/engine/ExportQueryExecutionTrees.h       |  2 +-
 src/engine/Server.h                          |  2 +-
 src/util/Generator.h                         | 82 +++++++++-----------
 test/GeneratorTest.cpp                       | 24 +++---
 5 files changed, 55 insertions(+), 57 deletions(-)

diff --git a/benchmark/infrastructure/BenchmarkToJson.cpp b/benchmark/infrastructure/BenchmarkToJson.cpp
index e971e29752..f0122adf5d 100644
--- a/benchmark/infrastructure/BenchmarkToJson.cpp
+++ b/benchmark/infrastructure/BenchmarkToJson.cpp
@@ -9,7 +9,7 @@
 #include 
 
 #include "../benchmark/infrastructure/Benchmark.h"
-#include "nlohmann/json.hpp"
+#include "util/json.h"
 
 namespace ad_benchmark {
 // ___________________________________________________________________________
diff --git a/src/engine/ExportQueryExecutionTrees.h b/src/engine/ExportQueryExecutionTrees.h
index c07c9fd501..f73b568261 100644
--- a/src/engine/ExportQueryExecutionTrees.h
+++ b/src/engine/ExportQueryExecutionTrees.h
@@ -5,9 +5,9 @@
 #include 
 
 #include "engine/QueryExecutionTree.h"
-#include "nlohmann/json.hpp"
 #include "parser/data/LimitOffsetClause.h"
 #include "util/http/MediaTypes.h"
+#include "util/json.h"
 
 #pragma once
 
diff --git a/src/engine/Server.h b/src/engine/Server.h
index 5455353074..10aa906c86 100644
--- a/src/engine/Server.h
+++ b/src/engine/Server.h
@@ -14,13 +14,13 @@
 #include "engine/QueryExecutionTree.h"
 #include "engine/SortPerformanceEstimator.h"
 #include "index/Index.h"
-#include "nlohmann/json.hpp"
 #include "parser/ParseException.h"
 #include "parser/SparqlParser.h"
 #include "util/AllocatorWithLimit.h"
 #include "util/Timer.h"
 #include "util/http/HttpServer.h"
 #include "util/http/streamable_body.h"
+#include "util/json.h"
 
 using nlohmann::json;
 using std::string;
diff --git a/src/util/Generator.h b/src/util/Generator.h
index df6056476c..43d922163d 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -14,27 +14,20 @@
 // Coroutines are still experimental in clang libcpp, therefore adapt the
 // appropriate namespaces by including the convenience header.
 #include "util/Coroutines.h"
-#include "util/HashMap.h"
-#include "util/json.h"
+#include "util/TypeTraits.h"
 
 namespace cppcoro {
-template 
+template 
 class generator;
 
-// This struct can be `co_await`ed inside a `generator` to add a value to a
-// dictionary, that can then be accessed from outside via the `details()`
-// function of the generator object. For an example see `GeneratorTest.cpp`.
-using Details = ad_utility::HashMap;
-struct AddDetail {
-  Details details_;
-  AddDetail(std::string key, nlohmann::json value) {
-    details_[std::move(key)] = value;
-  }
-  AddDetail(Details details) : details_{std::move(details)} {}
-};
+// This struct can be `co_await`ed inside a `generator` to obtain a reference to
+// the details object ( the value of which is a template parameter to the
+// generator). For an example see `GeneratorTest.cpp`.
+struct GetDetails {};
+static constexpr GetDetails getDetails;
 
 namespace detail {
-template 
+template 
 class generator_promise {
  public:
   // Even if the generator only yields `const` values, the `value_type`
@@ -44,22 +37,9 @@ class generator_promise {
   using reference_type = std::conditional_t, T, T&>;
   using pointer_type = std::remove_reference_t*;
 
-  struct DetailAwaiter {
-    generator_promise& promise_;
-    AddDetail detail_;
-    bool await_ready() {
-      for (auto& [key, value] : detail_.details_) {
-        promise_.details()[std::move(key)] = std::move(value);
-      }
-      return true;
-    }
-    bool await_suspend(std::coroutine_handle<>) noexcept { return false; }
-    void await_resume() noexcept {}
-  };
-
   generator_promise() = default;
 
-  generator get_return_object() noexcept;
+  generator get_return_object() noexcept;
 
   constexpr std::suspend_always initial_suspend() const noexcept { return {}; }
   constexpr std::suspend_always final_suspend() const noexcept { return {}; }
@@ -94,8 +74,18 @@ class generator_promise {
     }
   }
 
-  DetailAwaiter await_transform(AddDetail detail) {
-    return {*this, std::move(detail)};
+  // The machinery to expose the stored `Details` via
+  // `co_await cppcoro::getDetails`.
+  struct DetailAwaiter {
+    generator_promise& promise_;
+    bool await_ready() { return true; }
+    bool await_suspend(std::coroutine_handle<>) noexcept { return false; }
+    Details& await_resume() noexcept { return promise_.details(); }
+  };
+
+  DetailAwaiter await_transform(
+      [[maybe_unused]] ad_utility::SimilarTo auto&& detail) {
+    return {*this};
   }
 
   Details& details() { return m_details; }
@@ -103,23 +93,24 @@ class generator_promise {
  private:
   pointer_type m_value;
   std::exception_ptr m_exception;
-  Details m_details;
+  Details m_details{};
 };
 
 struct generator_sentinel {};
 
-template 
+template 
 class generator_iterator {
-  using coroutine_handle = std::coroutine_handle>;
+  using promise_type = generator_promise;
+  using coroutine_handle = std::coroutine_handle;
 
  public:
   using iterator_category = std::input_iterator_tag;
   // What type should we use for counting elements of a potentially infinite
   // sequence?
   using difference_type = std::ptrdiff_t;
-  using value_type = typename generator_promise::value_type;
-  using reference = typename generator_promise::reference_type;
-  using pointer = typename generator_promise::pointer_type;
+  using value_type = typename promise_type::value_type;
+  using reference = typename promise_type::reference_type;
+  using pointer = typename promise_type::pointer_type;
 
   // Iterator needs to be default-constructible to satisfy the Range concept.
   generator_iterator() noexcept : m_coroutine(nullptr) {}
@@ -168,11 +159,11 @@ class generator_iterator {
 };
 }  // namespace detail
 
-template 
+template 
 class [[nodiscard]] generator {
  public:
-  using promise_type = detail::generator_promise;
-  using iterator = detail::generator_iterator;
+  using promise_type = detail::generator_promise;
+  using iterator = detail::generator_iterator;
   using value_type = typename iterator::value_type;
 
   generator() noexcept : m_coroutine(nullptr) {}
@@ -216,7 +207,7 @@ class [[nodiscard]] generator {
   const Details& details() { return m_coroutine.promise().details(); }
 
  private:
-  friend class detail::generator_promise;
+  friend class detail::generator_promise;
 
   explicit generator(std::coroutine_handle coroutine) noexcept
       : m_coroutine(coroutine) {}
@@ -230,10 +221,11 @@ void swap(generator& a, generator& b) {
 }
 
 namespace detail {
-template 
-generator generator_promise::get_return_object() noexcept {
-  using coroutine_handle = std::coroutine_handle>;
-  return generator{coroutine_handle::from_promise(*this)};
+template 
+generator
+generator_promise::get_return_object() noexcept {
+  using coroutine_handle = std::coroutine_handle>;
+  return generator{coroutine_handle::from_promise(*this)};
 }
 }  // namespace detail
 
diff --git a/test/GeneratorTest.cpp b/test/GeneratorTest.cpp
index e349fcf759..706838773f 100644
--- a/test/GeneratorTest.cpp
+++ b/test/GeneratorTest.cpp
@@ -6,14 +6,20 @@
 
 #include "util/Generator.h"
 
+struct Details {
+  bool begin_ = false;
+  bool end_ = false;
+};
+
 // A simple generator that first yields three numbers and then adds a detail
 // value, that we can then extract after iterating over it.
-cppcoro::generator simpleGen(double detailValue) {
-  co_await cppcoro::AddDetail{"started", true};
+cppcoro::generator simpleGen(double detailValue) {
+  auto& details = co_await cppcoro::getDetails;
+  details.begin_ = true;
   co_yield 1;
   co_yield 42;
   co_yield 43;
-  co_await cppcoro::AddDetail{"detail", detailValue};
+  details.end_ = true;
 };
 
 // Test the behavior of the `simpleGen` above
@@ -22,14 +28,14 @@ TEST(Generator, details) {
   int result{};
   // The first detail is only added after the call to `begin()` in the for loop
   // below
-  ASSERT_TRUE(gen.details().empty());
+  ASSERT_FALSE(gen.details().begin_);
+  ASSERT_FALSE(gen.details().end_);
   for (int i : gen) {
     result += i;
-    // The detail is only
-    ASSERT_EQ(gen.details().size(), 1);
-    ASSERT_TRUE(gen.details().at("started"));
+    ASSERT_TRUE(gen.details().begin_);
+    ASSERT_FALSE(gen.details().end_);
   }
   ASSERT_EQ(result, 86);
-  ASSERT_EQ(gen.details().size(), 2);
-  ASSERT_EQ(gen.details().at("detail"), 17.3);
+  ASSERT_FALSE(gen.details().begin_);
+  ASSERT_TRUE(gen.details().end_);
 }

From 99f219f9c93dff75d9951e660ddde9afa9d27dfc Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 3 Jul 2023 16:23:22 +0200
Subject: [PATCH 103/150] Fix the unit tests.

---
 test/GeneratorTest.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/GeneratorTest.cpp b/test/GeneratorTest.cpp
index 706838773f..56b347d6d2 100644
--- a/test/GeneratorTest.cpp
+++ b/test/GeneratorTest.cpp
@@ -13,7 +13,7 @@ struct Details {
 
 // A simple generator that first yields three numbers and then adds a detail
 // value, that we can then extract after iterating over it.
-cppcoro::generator simpleGen(double detailValue) {
+cppcoro::generator simpleGen() {
   auto& details = co_await cppcoro::getDetails;
   details.begin_ = true;
   co_yield 1;
@@ -24,10 +24,10 @@ cppcoro::generator simpleGen(double detailValue) {
 
 // Test the behavior of the `simpleGen` above
 TEST(Generator, details) {
-  auto gen = simpleGen(17.3);
+  auto gen = simpleGen();
   int result{};
-  // The first detail is only added after the call to `begin()` in the for loop
-  // below
+  // `details().begin_` is only set to true after the call to `begin()` in the
+  // for loop below
   ASSERT_FALSE(gen.details().begin_);
   ASSERT_FALSE(gen.details().end_);
   for (int i : gen) {
@@ -36,6 +36,6 @@ TEST(Generator, details) {
     ASSERT_FALSE(gen.details().end_);
   }
   ASSERT_EQ(result, 86);
-  ASSERT_FALSE(gen.details().begin_);
+  ASSERT_TRUE(gen.details().begin_);
   ASSERT_TRUE(gen.details().end_);
 }

From fc083336884fd6667ee5674b3df867ea624964ed Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 4 Jul 2023 08:50:30 +0200
Subject: [PATCH 104/150] Small changes from a review with Hannahh.

---
 src/util/Generator.h   | 2 +-
 test/GeneratorTest.cpp | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/util/Generator.h b/src/util/Generator.h
index 43d922163d..23b2cb6780 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -21,7 +21,7 @@ template 
 class generator;
 
 // This struct can be `co_await`ed inside a `generator` to obtain a reference to
-// the details object ( the value of which is a template parameter to the
+// the details object (the value of which is a template parameter to the
 // generator). For an example see `GeneratorTest.cpp`.
 struct GetDetails {};
 static constexpr GetDetails getDetails;
diff --git a/test/GeneratorTest.cpp b/test/GeneratorTest.cpp
index 56b347d6d2..862f077b29 100644
--- a/test/GeneratorTest.cpp
+++ b/test/GeneratorTest.cpp
@@ -14,7 +14,7 @@ struct Details {
 // A simple generator that first yields three numbers and then adds a detail
 // value, that we can then extract after iterating over it.
 cppcoro::generator simpleGen() {
-  auto& details = co_await cppcoro::getDetails;
+  Details& details = co_await cppcoro::getDetails;
   details.begin_ = true;
   co_yield 1;
   co_yield 42;
@@ -22,7 +22,7 @@ cppcoro::generator simpleGen() {
   details.end_ = true;
 };
 
-// Test the behavior of the `simpleGen` above
+// Test the behavior of the `simpleGen` above.
 TEST(Generator, details) {
   auto gen = simpleGen();
   int result{};

From 63ff2f13bd7ff5dff326510e25259cb79871c77c Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 11:01:45 +0200
Subject: [PATCH 105/150] Added an empty details class as the default.

---
 src/util/Generator.h | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/util/Generator.h b/src/util/Generator.h
index 23b2cb6780..b05eec0fa4 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -17,17 +17,21 @@
 #include "util/TypeTraits.h"
 
 namespace cppcoro {
-template 
-class generator;
-
 // This struct can be `co_await`ed inside a `generator` to obtain a reference to
 // the details object (the value of which is a template parameter to the
 // generator). For an example see `GeneratorTest.cpp`.
 struct GetDetails {};
 static constexpr GetDetails getDetails;
 
+// This struct is used as the default of the details object for the case that
+// there are no details
+struct NoDetails {};
+
+template 
+class generator;
+
 namespace detail {
-template 
+template 
 class generator_promise {
  public:
   // Even if the generator only yields `const` values, the `value_type`
@@ -93,7 +97,8 @@ class generator_promise {
  private:
   pointer_type m_value;
   std::exception_ptr m_exception;
-  Details m_details{};
+  // If the `Details` type is empty, we don't need it to occupy any space.
+  [[no_unique_address]] Details m_details{};
 };
 
 struct generator_sentinel {};
@@ -159,7 +164,7 @@ class generator_iterator {
 };
 }  // namespace detail
 
-template 
+template 
 class [[nodiscard]] generator {
  public:
   using promise_type = detail::generator_promise;

From a4e5db2bdd774bb6f20417b5d9dd04518c9d9360 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 11:45:32 +0200
Subject: [PATCH 106/150] Several additional cleanups. Also fixed the missing
 runtime info for singel column scans.

---
 src/engine/IndexScan.cpp         | 16 +++---------
 src/engine/IndexScan.h           |  6 ++---
 src/engine/Join.cpp              | 42 ++++++++++++++------------------
 src/index/CompressedRelation.cpp | 39 +++++++++++++++--------------
 src/index/CompressedRelation.h   | 24 +++++++++++-------
 src/index/Permutation.cpp        |  2 +-
 src/index/Permutation.h          |  6 +++--
 7 files changed, 64 insertions(+), 71 deletions(-)

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index e9c346f706..70fb0ff663 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -281,7 +281,7 @@ std::array IndexScan::getPermutedTriple()
 }
 
 // ___________________________________________________________________________
-cppcoro::generator IndexScan::getLazyScan(
+Permutation::IdTableGenerator IndexScan::getLazyScan(
     const IndexScan& s, std::vector blocks) {
   const IndexImpl& index = s.getIndex().getImpl();
   Id col0Id = s.getPermutedTriple()[0]->toValueId(index.getVocab()).value();
@@ -311,8 +311,8 @@ std::optional IndexScan::getMetadataForScan(
 };
 
 // ________________________________________________________________
-std::array, 2> IndexScan::lazyScanForJoinOfTwoScans(
-    const IndexScan& s1, const IndexScan& s2) {
+std::array
+IndexScan::lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2) {
   AD_CONTRACT_CHECK(s1.numVariables_ < 3 && s2.numVariables_ < 3);
 
   auto metaBlocks1 = getMetadataForScan(s1);
@@ -325,18 +325,10 @@ std::array, 2> IndexScan::lazyScanForJoinOfTwoScans(
       metaBlocks1.value(), metaBlocks2.value());
 
   return {getLazyScan(s1, blocks1), getLazyScan(s2, blocks2)};
-  /*
-  std::vector
-  bl1(metaBlocks1.value().blockMetadata_.begin(),metaBlocks1.value().blockMetadata_.end());
-  std::vector
-  bl2(metaBlocks2.value().blockMetadata_.begin(),metaBlocks2.value().blockMetadata_.end());
-
-  return {getLazyScan(s1,bl1), getLazyScan(s2, bl2)};
-   */
 }
 
 // ________________________________________________________________
-cppcoro::generator IndexScan::lazyScanForJoinOfColumnWithScan(
+Permutation::IdTableGenerator IndexScan::lazyScanForJoinOfColumnWithScan(
     std::span joinColumn, const IndexScan& s) {
   AD_EXPENSIVE_CHECK(std::ranges::is_sorted(joinColumn));
   AD_CONTRACT_CHECK(s.numVariables_ == 1 || s.numVariables_ == 2);
diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h
index f9815932a2..e3d8bc5879 100644
--- a/src/engine/IndexScan.h
+++ b/src/engine/IndexScan.h
@@ -49,14 +49,14 @@ class IndexScan : public Operation {
   // blocks, but only the blocks that can theoretically contain matching rows
   // when performing a join on the first column of the result of `s1` with the
   // first column of the result of `s2`.
-  static std::array, 2> lazyScanForJoinOfTwoScans(
+  static std::array lazyScanForJoinOfTwoScans(
       const IndexScan& s1, const IndexScan& s2);
 
   // Return a generator that lazily yields the result of `s` in blocks, but only
   // the blocks that can theoretically contain matching rows when performing a
   // join between the first column of the result of `s`  with the `joinColumn`.
   // Requires that the `joinColumn` is sorted, else the behavior is undefined.
-  static cppcoro::generator lazyScanForJoinOfColumnWithScan(
+  static Permutation::IdTableGenerator lazyScanForJoinOfColumnWithScan(
       std::span joinColumn, const IndexScan& s);
 
  private:
@@ -105,7 +105,7 @@ class IndexScan : public Operation {
   std::array getPermutedTriple() const;
 
   //  Helper functions for the public `getLazyScanFor...` functions (see above).
-  static cppcoro::generator getLazyScan(
+  static Permutation::IdTableGenerator getLazyScan(
       const IndexScan& s, std::vector blocks);
   static std::optional getMetadataForScan(
       const IndexScan& s);
diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index ed18d6f796..39235432bc 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -646,13 +646,24 @@ void Join::addCombinedRowToIdTable(const ROW_A& rowA, const ROW_B& rowB,
 namespace {
 // Convert a `generator` for more
 // efficient access in the join columns below.
-cppcoro::generator> liftGenerator(
-    cppcoro::generator gen) {
+cppcoro::generator,
+                   CompressedRelationReader::LazyScanMetadata>
+liftGenerator(Permutation::IdTableGenerator gen) {
   for (auto& table : gen) {
     ad_utility::IdTableAndFirstCol t{std::move(table)};
     co_yield t;
   }
-  co_await cppcoro::AddDetail{gen.details()};
+  co_await cppcoro::getDetails = gen.details();
+}
+
+void updateRuntimeInfoForLazyScan(
+    QueryExecutionTree& scanTree,
+    const CompressedRelationReader::LazyScanMetadata& metadata) {
+  scanTree.getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
+  auto& rti = scanTree.getRootOperation()->getRuntimeInfo();
+  rti.numRows_ = metadata.numElementsRead_;
+  rti.totalTime_ = static_cast(metadata.blockingTimeMs_);
+  rti.addDetail("num-blocks-read", metadata.numBlocksRead_);
 }
 }  // namespace
 
@@ -681,20 +692,9 @@ IdTable Join::computeResultForTwoIndexScans() {
   ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks,
                                               std::less{}, rowAdder);
 
-  _left->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
-  _right->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
-  _right->getRootOperation()->getRuntimeInfo().details_.update(
-      rightBlocks.details());
-  _left->getRootOperation()->getRuntimeInfo().details_.update(
-      leftBlocks.details());
-  if (rightBlocks.details().contains("num-elements")) {
-    _right->getRootOperation()->getRuntimeInfo().numRows_ =
-        static_cast(rightBlocks.details().at("num-elements"));
-  }
-  if (leftBlocks.details().contains("num-elements")) {
-    _left->getRootOperation()->getRuntimeInfo().numRows_ =
-        static_cast(leftBlocks.details().at("num-elements"));
-  }
+  updateRuntimeInfoForLazyScan(*_left, leftBlocks.details());
+  updateRuntimeInfoForLazyScan(*_right, rightBlocks.details());
+
   return std::move(rowAdder).resultTable();
 }
 
@@ -753,12 +753,6 @@ IdTable Join::computeResultForIndexScanAndIdTable(
   result.setColumnSubset(joinColMap.permutationResult());
 
   auto& scanTree = idTableIsRightInput ? _left : _right;
-  scanTree->getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
-  scanTree->getRootOperation()->getRuntimeInfo().details_.update(
-      rightBlocks.details());
-  if (rightBlocks.details().contains("num-elements")) {
-    scanTree->getRootOperation()->getRuntimeInfo().numRows_ =
-        static_cast(rightBlocks.details().at("num-elements"));
-  }
+  updateRuntimeInfoForLazyScan(*scanTree, rightBlocks.details());
   return result;
 }
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 15fe0b4bd7..236d13b04f 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -103,7 +103,7 @@ IdTable CompressedRelationReader::scan(
 }
 
 // ____________________________________________________________________________
-cppcoro::generator
+CompressedRelationReader::IdTableGenerator
 CompressedRelationReader::asyncParallelBlockGenerator(
     auto beginBlock, auto endBlock, ad_utility::File& file,
     std::optional> columnIndices,
@@ -179,11 +179,12 @@ CompressedRelationReader::asyncParallelBlockGenerator(
     co_yield opt.value();
     popTimer.cont();
   }
-  co_await cppcoro::AddDetail{"blocking-time-block-reading", popTimer.msecs()};
+  LazyScanMetadata& details = co_await cppcoro::getDetails;
+  details.blockingTimeMs_ = popTimer.msecs();
 }
 
 // _____________________________________________________________________________
-cppcoro::generator CompressedRelationReader::lazyScan(
+CompressedRelationReader::IdTableGenerator CompressedRelationReader::lazyScan(
     CompressedRelationMetadata metadata,
     std::vector blockMetadata, ad_utility::File& file,
     TimeoutTimer timer) const {
@@ -192,34 +193,31 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   const auto beginBlock = relevantBlocks.begin();
   const auto endBlock = relevantBlocks.end();
 
-  size_t numElements = 0;
-
-  co_await cppcoro::AddDetail{"num-blocks", endBlock - beginBlock};
+  LazyScanMetadata& details = co_await cppcoro::getDetails;
+  details.numBlocksRead_ = endBlock - beginBlock;
 
   if (beginBlock == endBlock) {
-    co_await cppcoro::AddDetail{"num-elements", numElements};
     co_return;
   }
 
   // Read the first block, it might be incomplete
   auto firstBlock =
       readPossiblyIncompleteBlock(metadata, std::nullopt, file, *beginBlock);
-  numElements += firstBlock.numRows();
+  details.numElementsRead_ += firstBlock.numRows();
   co_yield firstBlock;
   checkTimeout(timer);
 
   auto blockGenerator = asyncParallelBlockGenerator(beginBlock + 1, endBlock,
                                                     file, std::nullopt, timer);
   for (auto& block : blockGenerator) {
-    numElements += block.numRows();
+    details.numElementsRead_ += block.numRows();
     co_yield block;
   }
-  co_await cppcoro::AddDetail{blockGenerator.details()};
-  co_await cppcoro::AddDetail{"num-elements", numElements};
+  details.blockingTimeMs_ = blockGenerator.details().blockingTimeMs_;
 }
 
 // _____________________________________________________________________________
-cppcoro::generator CompressedRelationReader::lazyScan(
+CompressedRelationReader::IdTableGenerator CompressedRelationReader::lazyScan(
     CompressedRelationMetadata metadata, Id col1Id,
     std::vector blockMetadata, ad_utility::File& file,
     TimeoutTimer timer) const {
@@ -227,10 +225,10 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   auto beginBlock = relevantBlocks.begin();
   auto endBlock = relevantBlocks.end();
 
-  co_await cppcoro::AddDetail{"num-blocks", endBlock - beginBlock};
-  size_t numElements = 0;
+  LazyScanMetadata& details = co_await cppcoro::getDetails;
+  details.numBlocksRead_ = endBlock - beginBlock;
+
   if (beginBlock == endBlock) {
-    co_await cppcoro::AddDetail{"num-elements", numElements};
     co_return;
   }
 
@@ -251,22 +249,23 @@ cppcoro::generator CompressedRelationReader::lazyScan(
   };
 
   if (beginBlock < endBlock) {
-    co_yield getIncompleteBlock(beginBlock);
+    auto block = getIncompleteBlock(beginBlock);
+    details.numElementsRead_ += block.numRows();
+    co_yield block;
   }
 
   if (beginBlock + 1 < endBlock) {
     auto blockGenerator = asyncParallelBlockGenerator(
         beginBlock + 1, endBlock - 1, file, std::vector{1UL}, timer);
     for (auto& block : blockGenerator) {
-      numElements += block.numRows();
+      details.numElementsRead_ += block.numRows();
       co_yield block;
     }
-    co_await cppcoro::AddDetail{blockGenerator.details()};
+    details.blockingTimeMs_ = blockGenerator.details().blockingTimeMs_;
     auto lastBlock = getIncompleteBlock(endBlock - 1);
-    numElements += lastBlock.numRows();
+    details.numElementsRead_ += lastBlock.numRows();
     co_yield lastBlock;
   }
-  co_await cppcoro::AddDetail{"num-elements", numElements};
 }
 
 namespace {
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index de0a5a7313..16ab1a3556 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -241,6 +241,14 @@ class CompressedRelationReader {
     std::optional col1Id_;
   };
 
+  struct LazyScanMetadata {
+    size_t numBlocksRead_ = 0;
+    size_t numElementsRead_ = 0;
+    size_t blockingTimeMs_ = 0;
+  };
+
+  using IdTableGenerator = cppcoro::generator;
+
  private:
   // This cache stores a small number of decompressed blocks. Its current
   // purpose is to make the e2e-tests run fast. They contain many SPARQL queries
@@ -278,10 +286,9 @@ class CompressedRelationReader {
   // Similar to `scan` (directly above), but the result of the scan is lazily
   // computed and returned as a generator of the single blocks that are scanned.
   // The blocks are guaranteed to be in order.
-  cppcoro::generator lazyScan(
-      CompressedRelationMetadata metadata,
-      std::vector blockMetadata,
-      ad_utility::File& file, TimeoutTimer timer) const;
+  IdTableGenerator lazyScan(CompressedRelationMetadata metadata,
+                            std::vector blockMetadata,
+                            ad_utility::File& file, TimeoutTimer timer) const;
 
   // Get the blocks (an ordered subset of the blocks that are passed in via the
   // `metadataAndBlocks`) where the `col1Id` can theoretically match one of the
@@ -329,10 +336,9 @@ class CompressedRelationReader {
   // Similar to `scan` (directly above), but the result of the scan is lazily
   // computed and returned as a generator of the single blocks that are scanned.
   // The blocks are guaranteed to be in order.
-  cppcoro::generator lazyScan(
-      CompressedRelationMetadata metadata, Id col1Id,
-      std::vector blockMetadata,
-      ad_utility::File& file, TimeoutTimer timer) const;
+  IdTableGenerator lazyScan(CompressedRelationMetadata metadata, Id col1Id,
+                            std::vector blockMetadata,
+                            ad_utility::File& file, TimeoutTimer timer) const;
 
   // Only get the size of the result for a given permutation XYZ for a given X
   // and Y. This can be done by scanning one or two blocks. Note: The overload
@@ -415,7 +421,7 @@ class CompressedRelationReader {
   // are yielded, else the complete blocks are yielded. The blocks are yielded
   // in the correct order, but asynchronously read and decompressed using
   // multiple worker threads.
-  cppcoro::generator asyncParallelBlockGenerator(
+  IdTableGenerator asyncParallelBlockGenerator(
       auto beginBlock, auto endBlock, ad_utility::File& file,
       std::optional> columnIndices,
       const TimeoutTimer& timer) const;
diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp
index 128f183b7d..f1ad345c45 100644
--- a/src/index/Permutation.cpp
+++ b/src/index/Permutation.cpp
@@ -125,7 +125,7 @@ std::optional Permutation::getMetadataAndBlocks(
 }
 
 // _____________________________________________________________________
-cppcoro::generator Permutation::lazyScan(
+Permutation::IdTableGenerator Permutation::lazyScan(
     Id col0Id, std::optional col1Id,
     std::optional> blocks,
     const TimeoutTimer& timer) const {
diff --git a/src/index/Permutation.h b/src/index/Permutation.h
index 7b0fd339fb..547f529232 100644
--- a/src/index/Permutation.h
+++ b/src/index/Permutation.h
@@ -54,9 +54,11 @@ class Permutation {
   IdTable scan(Id col0Id, std::optional col1Id,
                const TimeoutTimer& timer = nullptr) const;
 
-  // Typedef to propagate the `MetadataAndblocks` type.
+  // Typedef to propagate the `MetadataAndblocks` and `IdTableGenerator` type.
   using MetadataAndBlocks = CompressedRelationReader::MetadataAndBlocks;
 
+  using IdTableGenerator = CompressedRelationReader::IdTableGenerator;
+
   // The function `lazyScan` is similar to `scan` (see above) with
   // the following differences:
   // - The result is returned as a lazy generator of blocks.
@@ -69,7 +71,7 @@ class Permutation {
   // TODO We should only communicate this interface via the
   // `MetadataAndBlocks` class and make this a strong class that always
   // maintains its invariants.
-  cppcoro::generator lazyScan(
+  IdTableGenerator lazyScan(
       Id col0Id, std::optional col1Id,
       std::optional> blocks,
       const TimeoutTimer& timer = nullptr) const;

From 180456aa3ff7c7cb122c306ec90b180a6f1027df Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 12:07:16 +0200
Subject: [PATCH 107/150] Better comments. Next step: Only a single lazy scan,
 and a better generator stuff interface.

---
 src/util/JoinAlgorithms/JoinAlgorithms.h | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index cc96b564d6..2f8b6976a8 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -653,11 +653,13 @@ class BlockAndSubrange {
  * first argument comes before the second one. The concatenation of all blocks
  * in `leftBlocks` must be sorted according to this function. The same
  * requirement holds for `rightBlocks`.
- * @param compatibleRowAction When an element from a block from `leftBlock` and
- * an element from a block from `rightBlock` match (because they compare equal
- * wrt the `lessThan` relation), this function is called with the two
- * *iterators* to the  elements, i.e . `compatibleRowAction(itToLeft,
- * itToRight)`
+ * @param compatibleRowAction Needs to have two member functions:
+ * `setInput(leftBlock, rightBlock)` and `addRow(size_t, size_t)`. When the
+ * `i`-th element of a `leftBlock` and the `j`-th element `rightBlock` match
+ * (because they compare equal wrt the `lessThan` relation), then first
+ * `setInput(leftBlock, rightBlock)` and then `addRow(i, j)` is called. Of
+ * course `setInput` is only called once if there are several matching pairs of
+ * elements from the same pair of blocks.
  */
 template 
Date: Wed, 5 Jul 2023 13:46:55 +0200
Subject: [PATCH 108/150] Add the possibility to register an external details
 object.

---
 src/util/Generator.h   | 22 ++++++++++++++++++++--
 test/GeneratorTest.cpp | 29 +++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/src/util/Generator.h b/src/util/Generator.h
index b05eec0fa4..d6706d93d1 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -14,6 +14,7 @@
 // Coroutines are still experimental in clang libcpp, therefore adapt the
 // appropriate namespaces by including the convenience header.
 #include "util/Coroutines.h"
+#include "util/Exception.h"
 #include "util/TypeTraits.h"
 
 namespace cppcoro {
@@ -92,13 +93,26 @@ class generator_promise {
     return {*this};
   }
 
-  Details& details() { return m_details; }
+  static constexpr bool hasDetails = !std::is_same_v;
+  Details& details() requires hasDetails {
+    return std::holds_alternative
(m_details)
+               ? std::get
(m_details)
+               : *std::get(m_details);
+  }
+
+  void setDetailsPointer(Details* pointer) {
+    AD_CONTRACT_CHECK(pointer != nullptr);
+    m_details = pointer;
+  }
 
  private:
   pointer_type m_value;
   std::exception_ptr m_exception;
+
+  using DetailStorage =
+      std::conditional_t, Details>;
   // If the `Details` type is empty, we don't need it to occupy any space.
-  [[no_unique_address]] Details m_details{};
+  [[no_unique_address]] DetailStorage m_details{};
 };
 
 struct generator_sentinel {};
@@ -211,6 +225,10 @@ class [[nodiscard]] generator {
 
   const Details& details() { return m_coroutine.promise().details(); }
 
+  void setDetailsPointer(Details* pointer) {
+    m_coroutine.promise().setDetailsPointer(pointer);
+  }
+
  private:
   friend class detail::generator_promise;
 
diff --git a/test/GeneratorTest.cpp b/test/GeneratorTest.cpp
index 862f077b29..a10afeed44 100644
--- a/test/GeneratorTest.cpp
+++ b/test/GeneratorTest.cpp
@@ -39,3 +39,32 @@ TEST(Generator, details) {
   ASSERT_TRUE(gen.details().begin_);
   ASSERT_TRUE(gen.details().end_);
 }
+
+// Test the behavior of the `simpleGen` with an explicit external details object
+TEST(Generator, externalDetails) {
+  auto gen = simpleGen();
+  Details details{};
+  ASSERT_NE(&details, &gen.details());
+  gen.setDetailsPointer(&details);
+  ASSERT_EQ(&details, &gen.details());
+  int result{};
+  // `details().begin_` is only set to true after the call to `begin()` in the
+  // for loop below
+  ASSERT_FALSE(gen.details().begin_);
+  ASSERT_FALSE(details.begin_);
+  ASSERT_FALSE(gen.details().end_);
+  ASSERT_FALSE(details.end_);
+  for (int i : gen) {
+    result += i;
+    ASSERT_TRUE(gen.details().begin_);
+    ASSERT_TRUE(details.begin_);
+    ASSERT_FALSE(gen.details().end_);
+    ASSERT_FALSE(details.end_);
+  }
+  ASSERT_EQ(result, 86);
+  ASSERT_TRUE(gen.details().begin_);
+  ASSERT_TRUE(details.begin_);
+  ASSERT_TRUE(gen.details().end_);
+  ASSERT_TRUE(details.end_);
+  ASSERT_EQ(&details, &gen.details());
+}

From 907992a3f3e96ad7a535c614aef0669b536e9c7f Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 14:21:53 +0200
Subject: [PATCH 109/150] Several smaller cleanups and fixed as possible source
 of the ubsan unhappiness.

---
 src/engine/AddCombinedRowToTable.h       | 10 ++++-
 src/index/CompressedRelation.cpp         | 54 ++++++++++++++++--------
 src/index/CompressedRelation.h           |  4 +-
 src/util/JoinAlgorithms/JoinAlgorithms.h | 18 +++++---
 test/JoinAlgorithmsTest.cpp              |  4 ++
 5 files changed, 63 insertions(+), 27 deletions(-)

diff --git a/src/engine/AddCombinedRowToTable.h b/src/engine/AddCombinedRowToTable.h
index ba7d6afe5e..4f4922dbed 100644
--- a/src/engine/AddCombinedRowToTable.h
+++ b/src/engine/AddCombinedRowToTable.h
@@ -149,11 +149,19 @@ class AddCombinedRowToIdTable {
   // have to call it manually after adding the last row, else the destructor
   // will throw an exception.
   void flush() {
-    AD_CONTRACT_CHECK(inputLeft_.has_value() && inputRight_.has_value());
     auto& result = resultTable_;
     size_t oldSize = result.size();
     AD_CORRECTNESS_CHECK(nextIndex_ ==
                          indexBuffer_.size() + optionalIndexBuffer_.size());
+    // Sometimes the left input and right input are not valid anymore, because
+    // the `IdTable`s they point to have already been destroyed. This case is
+    // okay, as long as there was a manual call to `flush` before the inputs
+    // went out of scope. However, the call to `resultTable()` will still
+    // unconditionally flush. The following check makes this behavior defined.
+    if (nextIndex_ == 0) {
+      return;
+    }
+    AD_CONTRACT_CHECK(inputLeft_.has_value() && inputRight_.has_value());
     result.resize(oldSize + nextIndex_);
 
     // Sometimes columns are combined where one value is UNDEF and the other one
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 236d13b04f..013cb9a172 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -10,6 +10,7 @@
 #include "util/ConcurrentCache.h"
 #include "util/Generator.h"
 #include "util/JoinAlgorithms/JoinAlgorithms.h"
+#include "util/OnDestructionDontThrowDuringStackUnwinding.h"
 #include "util/OverloadCallOperator.h"
 #include "util/ThreadSafeQueue.h"
 #include "util/TypeTraits.h"
@@ -43,8 +44,8 @@ IdTable CompressedRelationReader::scan(
   // Set up a lambda, that reads this block and decompresses it to
   // the result.
   auto readIncompleteBlock = [&](const auto& block) mutable {
-    auto trimmedBlock =
-        readPossiblyIncompleteBlock(metadata, std::nullopt, file, block);
+    auto trimmedBlock = readPossiblyIncompleteBlock(metadata, std::nullopt,
+                                                    file, block, std::nullopt);
     for (size_t i = 0; i < trimmedBlock.numColumns(); ++i) {
       const auto& inputCol = trimmedBlock.getColumn(i);
       auto resultColumn = result.getColumn(i);
@@ -108,6 +109,7 @@ CompressedRelationReader::asyncParallelBlockGenerator(
     auto beginBlock, auto endBlock, ad_utility::File& file,
     std::optional> columnIndices,
     const TimeoutTimer& timer) const {
+  LazyScanMetadata& details = co_await cppcoro::getDetails;
   if (beginBlock == endBlock) {
     co_return;
   }
@@ -173,13 +175,20 @@ CompressedRelationReader::asyncParallelBlockGenerator(
   }
 
   ad_utility::Timer popTimer{ad_utility::timer::Timer::InitialStatus::Started};
+  // In case the coroutine is destroyed early we still want to have this
+  // information.
+  auto setTimer = ad_utility::makeOnDestructionDontThrowDuringStackUnwinding(
+      [&details, &popTimer]() { details.blockingTimeMs_ = popTimer.msecs(); });
   while (auto opt = queue.pop()) {
     popTimer.stop();
     checkTimeout(timer);
+    ++details.numBlocksRead_;
+    details.numElementsRead_ += opt.value().numRows();
     co_yield opt.value();
     popTimer.cont();
   }
-  LazyScanMetadata& details = co_await cppcoro::getDetails;
+  // The `OnDestruction...` above might be called too late, so we manually set
+  // the timer again.
   details.blockingTimeMs_ = popTimer.msecs();
 }
 
@@ -194,26 +203,25 @@ CompressedRelationReader::IdTableGenerator CompressedRelationReader::lazyScan(
   const auto endBlock = relevantBlocks.end();
 
   LazyScanMetadata& details = co_await cppcoro::getDetails;
-  details.numBlocksRead_ = endBlock - beginBlock;
+  size_t numBlocksTotal = endBlock - beginBlock;
 
   if (beginBlock == endBlock) {
     co_return;
   }
 
   // Read the first block, it might be incomplete
-  auto firstBlock =
-      readPossiblyIncompleteBlock(metadata, std::nullopt, file, *beginBlock);
-  details.numElementsRead_ += firstBlock.numRows();
+  auto firstBlock = readPossiblyIncompleteBlock(metadata, std::nullopt, file,
+                                                *beginBlock, std::ref(details));
   co_yield firstBlock;
   checkTimeout(timer);
 
   auto blockGenerator = asyncParallelBlockGenerator(beginBlock + 1, endBlock,
                                                     file, std::nullopt, timer);
+  blockGenerator.setDetailsPointer(&details);
   for (auto& block : blockGenerator) {
-    details.numElementsRead_ += block.numRows();
     co_yield block;
   }
-  details.blockingTimeMs_ = blockGenerator.details().blockingTimeMs_;
+  AD_CORRECTNESS_CHECK(numBlocksTotal == details.numBlocksRead_);
 }
 
 // _____________________________________________________________________________
@@ -226,7 +234,7 @@ CompressedRelationReader::IdTableGenerator CompressedRelationReader::lazyScan(
   auto endBlock = relevantBlocks.end();
 
   LazyScanMetadata& details = co_await cppcoro::getDetails;
-  details.numBlocksRead_ = endBlock - beginBlock;
+  size_t numBlocksTotal = endBlock - beginBlock;
 
   if (beginBlock == endBlock) {
     co_return;
@@ -242,7 +250,8 @@ CompressedRelationReader::IdTableGenerator CompressedRelationReader::lazyScan(
   }
 
   auto getIncompleteBlock = [&](auto it) {
-    auto result = readPossiblyIncompleteBlock(metadata, col1Id, file, *it);
+    auto result = readPossiblyIncompleteBlock(metadata, col1Id, file, *it,
+                                              std::ref(details));
     result.setColumnSubset(std::array{1});
     checkTimeout(timer);
     return result;
@@ -250,22 +259,20 @@ CompressedRelationReader::IdTableGenerator CompressedRelationReader::lazyScan(
 
   if (beginBlock < endBlock) {
     auto block = getIncompleteBlock(beginBlock);
-    details.numElementsRead_ += block.numRows();
     co_yield block;
   }
 
   if (beginBlock + 1 < endBlock) {
     auto blockGenerator = asyncParallelBlockGenerator(
         beginBlock + 1, endBlock - 1, file, std::vector{1UL}, timer);
+    blockGenerator.setDetailsPointer(&details);
     for (auto& block : blockGenerator) {
-      details.numElementsRead_ += block.numRows();
       co_yield block;
     }
-    details.blockingTimeMs_ = blockGenerator.details().blockingTimeMs_;
     auto lastBlock = getIncompleteBlock(endBlock - 1);
-    details.numElementsRead_ += lastBlock.numRows();
     co_yield lastBlock;
   }
+  AD_CORRECTNESS_CHECK(numBlocksTotal == details.numBlocksRead_);
 }
 
 namespace {
@@ -426,7 +433,8 @@ IdTable CompressedRelationReader::scan(
   // set up a lambda which allows us to read these blocks, and returns
   // the result as a vector.
   auto readIncompleteBlock = [&](const auto& block) {
-    return readPossiblyIncompleteBlock(metadata, col1Id, file, block);
+    return readPossiblyIncompleteBlock(metadata, col1Id, file, block,
+                                       std::nullopt);
   };
 
   // The first and the last block might be incomplete, compute
@@ -513,7 +521,9 @@ IdTable CompressedRelationReader::scan(
 DecompressedBlock CompressedRelationReader::readPossiblyIncompleteBlock(
     const CompressedRelationMetadata& relationMetadata,
     std::optional col1Id, ad_utility::File& file,
-    const CompressedBlockMetadata& blockMetadata) const {
+    const CompressedBlockMetadata& blockMetadata,
+    std::optional> scanMetadata)
+    const {
   // A block is uniquely identified by its start position in the file.
   auto cacheKey = blockMetadata.offsetsAndCompressedSize_.at(0).offsetInFile_;
   DecompressedBlock block =
@@ -552,6 +562,12 @@ DecompressedBlock CompressedRelationReader::readPossiblyIncompleteBlock(
   block.erase(block.begin(),
               block.begin() + (subBlock.begin() - col1Column.begin()));
   block.resize(numResults);
+
+  if (scanMetadata.has_value()) {
+    auto& details = scanMetadata.value().get();
+    ++details.numBlocksRead_;
+    details.numElementsRead_ += block.numRows();
+  }
   return block;
 };
 
@@ -571,7 +587,9 @@ size_t CompressedRelationReader::getResultSizeOfScan(
   // set up a lambda which allows us to read these blocks, and returns
   // the size of the result.
   auto readSizeOfPossiblyIncompleteBlock = [&](const auto& block) {
-    return readPossiblyIncompleteBlock(metadata, col1Id, file, block).numRows();
+    return readPossiblyIncompleteBlock(metadata, col1Id, file, block,
+                                       std::nullopt)
+        .numRows();
   };
 
   size_t numResults = 0;
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index 16ab1a3556..11e8210f3d 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -414,7 +414,9 @@ class CompressedRelationReader {
   DecompressedBlock readPossiblyIncompleteBlock(
       const CompressedRelationMetadata& relationMetadata,
       std::optional col1Id, ad_utility::File& file,
-      const CompressedBlockMetadata& blockMetadata) const;
+      const CompressedBlockMetadata& blockMetadata,
+      std::optional> scanMetadata)
+      const;
 
   // Yield all the blocks in the range `[beginBlock, endBlock)`. If the
   // `columnIndices` are set, that only the specified columns from the blocks
diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index 2f8b6976a8..b2888340ce 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -653,13 +653,15 @@ class BlockAndSubrange {
  * first argument comes before the second one. The concatenation of all blocks
  * in `leftBlocks` must be sorted according to this function. The same
  * requirement holds for `rightBlocks`.
- * @param compatibleRowAction Needs to have two member functions:
- * `setInput(leftBlock, rightBlock)` and `addRow(size_t, size_t)`. When the
- * `i`-th element of a `leftBlock` and the `j`-th element `rightBlock` match
- * (because they compare equal wrt the `lessThan` relation), then first
- * `setInput(leftBlock, rightBlock)` and then `addRow(i, j)` is called. Of
- * course `setInput` is only called once if there are several matching pairs of
- * elements from the same pair of blocks.
+ * @param compatibleRowAction Needs to have three member functions:
+ * `setInput(leftBlock, rightBlock)`, `addRow(size_t, size_t)`, and `flush()`.
+ * When the `i`-th element of a `leftBlock` and the `j`-th element `rightBlock`
+ * match (because they compare equal wrt the `lessThan` relation), then first
+ * `setInput(leftBlock, rightBlock)` then `addRow(i, j)`, and finally `flush()`
+ * is called. Of course `setInput` and `flush()` are only called once if there
+ * are several matching pairs of elements from the same pair of blocks. The
+ * calls to `addRow` will then all be between the calls to `setInput` and
+ * `flush`.
  */
 template push_back(std::array{x1, x2, y2});
   }
+
+  void flush() const {
+    // Does nothing, but is required for the interface.
+  }
 };
 
 auto makeRowAdder(JoinResult& target) {

From ceee114789d20937ddb66677f4167a8c9ea16a5d Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 15:27:40 +0200
Subject: [PATCH 110/150] Add unit tests for the `getBlocksForJoin` function.

---
 test/CompressedRelationsTest.cpp | 90 +++++++++++++++++++++++++++++++-
 1 file changed, 89 insertions(+), 1 deletion(-)

diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp
index 3c5e8b2a84..59ca16ee8c 100644
--- a/test/CompressedRelationsTest.cpp
+++ b/test/CompressedRelationsTest.cpp
@@ -284,7 +284,7 @@ TEST(CompressedRelationMetadata, GettersAndSetters) {
   ASSERT_EQ(43, m.numRows_);
 }
 
-TEST(CompressedRelationReader, getBlocksForJoin) {
+TEST(CompressedRelationReader, getBlocksForJoinWithColumn) {
   CompressedBlockMetadata block1{
       {}, 0, {V(16), V(0), V(0)}, {V(38), V(4), V(12)}};
   CompressedBlockMetadata block2{
@@ -309,6 +309,9 @@ TEST(CompressedRelationReader, getBlocksForJoin) {
                                                              metadataAndBlocks);
     EXPECT_THAT(result, ::testing::ElementsAreArray(expectedBlocks));
   };
+  // We have fixed the `col0Id` to be 42. The col1/2Ids of the matching blocks
+  // are as follows (starting at `block2`)
+  // [(3, 0)-(4, 12)], [(4, 13)-(6, 9)]
 
   // Tests for a fixed col0Id, so the join is on the middle column.
   test({V(1), V(3), V(17), V(29)}, {block2});
@@ -323,3 +326,88 @@ TEST(CompressedRelationReader, getBlocksForJoin) {
   test({V(12)}, {block2});
   test({V(13)}, {block3});
 }
+TEST(CompressedRelationReader, getBlocksForJoin) {
+  CompressedBlockMetadata block1{
+      {}, 0, {V(16), V(0), V(0)}, {V(38), V(4), V(12)}};
+  CompressedBlockMetadata block2{
+      {}, 0, {V(42), V(3), V(0)}, {V(42), V(4), V(12)}};
+  CompressedBlockMetadata block3{
+      {}, 0, {V(42), V(5), V(13)}, {V(42), V(8), V(9)}};
+  CompressedBlockMetadata block4{
+      {}, 0, {V(42), V(8), V(16)}, {V(42), V(20), V(9)}};
+  CompressedBlockMetadata block5{
+      {}, 0, {V(42), V(20), V(16)}, {V(42), V(20), V(63)}};
+
+  // We are only interested in blocks with a col0 of `42`.
+  CompressedRelationMetadata relation;
+  relation.col0Id_ = V(42);
+
+  std::vector blocks{block1, block2, block3, block4, block5};
+  CompressedRelationReader::MetadataAndBlocks metadataAndBlocks{
+      relation, blocks, std::nullopt};
+
+  CompressedBlockMetadata blockB1{
+      {}, 0, {V(16), V(0), V(0)}, {V(38), V(4), V(12)}};
+  CompressedBlockMetadata blockB2{
+      {}, 0, {V(47), V(3), V(0)}, {V(47), V(6), V(12)}};
+  CompressedBlockMetadata blockB3{
+      {}, 0, {V(47), V(7), V(13)}, {V(47), V(9), V(9)}};
+  CompressedBlockMetadata blockB4{
+      {}, 0, {V(47), V(38), V(7)}, {V(47), V(38), V(8)}};
+  CompressedBlockMetadata blockB5{
+      {}, 0, {V(47), V(38), V(9)}, {V(47), V(38), V(12)}};
+  CompressedBlockMetadata blockB6{
+      {}, 0, {V(47), V(38), V(13)}, {V(47), V(38), V(15)}};
+
+  // We are only interested in blocks with a col0 of `42`.
+  CompressedRelationMetadata relationB;
+  relationB.col0Id_ = V(47);
+
+  std::vector blocksB{blockB1, blockB2, blockB3, blockB4, blockB5, blockB6};
+  CompressedRelationReader::MetadataAndBlocks metadataAndBlocksB{
+      relationB, blocksB, std::nullopt};
+
+  auto test = [&metadataAndBlocks, &metadataAndBlocksB](
+                  const std::array, 2>&
+                      expectedBlocks,
+                  source_location l = source_location::current()) {
+    auto t = generateLocationTrace(l);
+    auto result = CompressedRelationReader::getBlocksForJoin(
+        metadataAndBlocks, metadataAndBlocksB);
+    EXPECT_THAT(result[0], ::testing::ElementsAreArray(expectedBlocks[0]));
+    EXPECT_THAT(result[1], ::testing::ElementsAreArray(expectedBlocks[1]));
+
+    result = CompressedRelationReader::getBlocksForJoin(metadataAndBlocksB,
+                                                        metadataAndBlocks);
+    EXPECT_THAT(result[1], ::testing::ElementsAreArray(expectedBlocks[0]));
+    EXPECT_THAT(result[0], ::testing::ElementsAreArray(expectedBlocks[1]));
+  };
+
+  // We have fixed the `col0Id` to be 42 for the left input and 47 for the right
+  // input. The col1/2Ids of the blocks that have this col0Id are as follows:
+
+  // (starting at `block2`.
+  // [(3, 0)- (4, 12)], [(5, 13) - (8, 9)], [(8, 16) - (20, 9)], [(20, 16) -
+  // (20, 63)]
+
+  // Starting at `blockB2`.
+  // [(3, 0)-(6, 12)], [(7, 13)-(9, 9)], [(38, 7)-(38, 8)], [(38, 9)-(38, 12)],
+  // [(38, 13)-(38, 15)]
+
+  // Test for only the `col0Id` fixed.
+  test({std::vector{block2, block3, block4}, std::vector{blockB2, blockB3}});
+  // Test with a fixed col1Id on both sides. We now join on the last column.
+  metadataAndBlocks.col1Id_ = V(20);
+  metadataAndBlocksB.col1Id_ = V(38);
+  test({std::vector{block4}, std::vector{blockB4, blockB5}});
+
+  // Fix only the col1Id of the left input.
+  metadataAndBlocks.col1Id_ = V(4);
+  metadataAndBlocksB.col1Id_ = std::nullopt;
+  test({std::vector{block2}, std::vector{blockB2, blockB3}});
+
+  // Fix only the col1Id of the right input.
+  metadataAndBlocks.col1Id_ = std::nullopt;
+  metadataAndBlocksB.col1Id_ = V(7);
+  test({std::vector{block4, block5}, std::vector{blockB3}});
+}

From 9f71a656ae9a0afe32450d3ff6cb690f05acaa89 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 17:18:31 +0200
Subject: [PATCH 111/150] A stub for unit tests.

---
 test/CMakeLists.txt           |  2 ++
 test/engine/CMakeLists.txt    |  1 +
 test/engine/IndexScanTest.cpp | 41 +++++++++++++++++++++++++++++++++++
 3 files changed, 44 insertions(+)
 create mode 100644 test/engine/CMakeLists.txt
 create mode 100644 test/engine/IndexScanTest.cpp

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index b9f5a4cf4c..170385c2ef 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -80,6 +80,8 @@ function(addAndLinkTest basename)
     linkTest(${basename} ${ARGN})
 endfunction()
 
+add_subdirectory(engine)
+
 addLinkAndDiscoverTest(ValueIdComparatorsTest util)
 
 addLinkAndDiscoverTest(SparqlParserTest parser engine sparqlExpressions)
diff --git a/test/engine/CMakeLists.txt b/test/engine/CMakeLists.txt
new file mode 100644
index 0000000000..35b3050c80
--- /dev/null
+++ b/test/engine/CMakeLists.txt
@@ -0,0 +1 @@
+addLinkAndDiscoverTest(IndexScanTest engine)
\ No newline at end of file
diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp
new file mode 100644
index 0000000000..2ec8ace963
--- /dev/null
+++ b/test/engine/IndexScanTest.cpp
@@ -0,0 +1,41 @@
+//  Copyright 2023, University of Freiburg,
+//                  Chair of Algorithms and Data Structures.
+//  Author: Johannes Kalmbach 
+
+#include 
+
+#include "../IndexTestHelpers.h"
+#include "engine/IndexScan.h"
+#include "parser/ParsedQuery.h"
+
+using namespace ad_utility::testing;
+
+std::string kg = " 
 .  
 .   .   .";
+
+TEST(IndexScan, lazyScanForJoinOfTwoScans) {
+  using Tc = TripleComponent;
+  using Var = Variable;
+  auto qec = getQec(kg);
+  IndexScan s1{qec, Permutation::PSO,
+               SparqlTriple{Tc{Var{"?x"}}, "
", Tc{Var{"?y"}}}};
+  IndexScan s2{qec, Permutation::PSO,
+               SparqlTriple{Tc{Var{"?x"}}, "", Tc{Var{"?z"}}}};
+  auto [scan1, scan2] = (IndexScan::lazyScanForJoinOfTwoScans(s1, s2));
+
+  IdTable res1{2, ad_utility::makeUnlimitedAllocator()};
+  IdTable res2{2, ad_utility::makeUnlimitedAllocator()};
+
+  for (const auto& block : scan1) {
+    res1.insertAtEnd(block.begin(), block.end());
+  }
+  for (const auto& block : scan2) {
+    res2.insertAtEnd(block.begin(), block.end());
+  }
+
+  EXPECT_EQ(res1.size(), 2u);
+  EXPECT_EQ(res2.size(), 2u);
+
+  // TODO We need additional tests that are not only dummys. To make
+  // this work, we need to have the blocksize of the index class configurable, I
+  // will split this into a separate PR.
+}

From 25b32495db2e77e28ae9d32ccd326879169ec5f5 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 17:26:25 +0200
Subject: [PATCH 112/150] Tiny changes from a review.

---
 src/util/Generator.h   | 2 +-
 test/GeneratorTest.cpp | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/util/Generator.h b/src/util/Generator.h
index d6706d93d1..375ad60f24 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -100,7 +100,7 @@ class generator_promise {
                : *std::get(m_details);
   }
 
-  void setDetailsPointer(Details* pointer) {
+  void setDetailsPointer(Details* pointer) requires hasDetails {
     AD_CONTRACT_CHECK(pointer != nullptr);
     m_details = pointer;
   }
diff --git a/test/GeneratorTest.cpp b/test/GeneratorTest.cpp
index a10afeed44..81654346c4 100644
--- a/test/GeneratorTest.cpp
+++ b/test/GeneratorTest.cpp
@@ -40,7 +40,8 @@ TEST(Generator, details) {
   ASSERT_TRUE(gen.details().end_);
 }
 
-// Test the behavior of the `simpleGen` with an explicit external details object
+// Test the behavior of the `simpleGen` with an explicit external details
+// object.
 TEST(Generator, externalDetails) {
   auto gen = simpleGen();
   Details details{};
@@ -67,4 +68,7 @@ TEST(Generator, externalDetails) {
   ASSERT_TRUE(gen.details().end_);
   ASSERT_TRUE(details.end_);
   ASSERT_EQ(&details, &gen.details());
+
+  // Setting a `nullptr` is illegal
+  ASSERT_ANY_THROW(gen.setDetailsPointer(nullptr));
 }

From 79a5b97e6ca922cdffb617652c567bcff0dacc57 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 18:38:27 +0200
Subject: [PATCH 113/150] Some changes from a review.

---
 src/engine/AddCombinedRowToTable.h          | 16 ++++++---
 src/engine/IndexScan.cpp                    | 11 ++++--
 src/engine/Join.cpp                         | 12 ++++---
 src/index/CompressedRelation.h              |  1 +
 src/util/Generator.h                        |  3 +-
 src/util/JoinAlgorithms/JoinAlgorithms.h    | 37 +++++----------------
 src/util/JoinAlgorithms/JoinColumnMapping.h |  2 +-
 7 files changed, 40 insertions(+), 42 deletions(-)

diff --git a/src/engine/AddCombinedRowToTable.h b/src/engine/AddCombinedRowToTable.h
index 4f4922dbed..26a4bc67ef 100644
--- a/src/engine/AddCombinedRowToTable.h
+++ b/src/engine/AddCombinedRowToTable.h
@@ -15,7 +15,10 @@
 namespace ad_utility {
 // This class handles the efficient writing of the results of a JOIN operation
 // to a column-based `IdTable`. The underlying assumption is that in both inputs
-// the join columns are the first columns.
+// the join columns are the first columns. On each call to `addRow`, we only
+// store the indices of the matching rows. When a certain buffer size
+// (configurable, default value 100'000) is reached, the results are actually
+// written to the table.
 class AddCombinedRowToIdTable {
   std::vector numUndefinedPerColumn_;
   size_t numJoinColumns_;
@@ -98,7 +101,9 @@ class AddCombinedRowToIdTable {
   // Set or reset the input. All following calls to `addRow` then refer to
   // indices in the new input. Before resetting, `flush()` is called, so all the
   // rows from the previous inputs get materialized before deleting the old
-  // inputs.
+  // inputs. The arguments to `inputLeft` and `inputRight` can either be
+  // `IdTable` or `IdTableView<0>`, or any other type that has a
+  // `asStaticView<0>` method that returns an `IdTableView<0>`.
   void setInput(const auto& inputLeft, const auto& inputRight) {
     auto toView = [](const T& table) {
       if constexpr (requires { table.template asStaticView<0>(); }) {
@@ -155,9 +160,10 @@ class AddCombinedRowToIdTable {
                          indexBuffer_.size() + optionalIndexBuffer_.size());
     // Sometimes the left input and right input are not valid anymore, because
     // the `IdTable`s they point to have already been destroyed. This case is
-    // okay, as long as there was a manual call to `flush` before the inputs
-    // went out of scope. However, the call to `resultTable()` will still
-    // unconditionally flush. The following check makes this behavior defined.
+    // okay, as long as there was a manual call to `flush` (after which
+    // `nextIndex_ == 0`) before the inputs went out of scope. However, the call
+    // to `resultTable()` will still unconditionally flush. The following check
+    // makes this behavior defined.
     if (nextIndex_ == 0) {
       return;
     }
diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index 70fb0ff663..1d9db85677 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -324,7 +324,12 @@ IndexScan::lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2) {
   auto [blocks1, blocks2] = CompressedRelationReader::getBlocksForJoin(
       metaBlocks1.value(), metaBlocks2.value());
 
-  return {getLazyScan(s1, blocks1), getLazyScan(s2, blocks2)};
+  std::array result{getLazyScan(s1, blocks1), getLazyScan(s2, blocks2)};
+  result[0].details().numBlocksTotal_ =
+      metaBlocks1.value().blockMetadata_.size();
+  result[1].details().numBlocksTotal_ =
+      metaBlocks2.value().blockMetadata_.size();
+  return result;
 }
 
 // ________________________________________________________________
@@ -341,5 +346,7 @@ Permutation::IdTableGenerator IndexScan::lazyScanForJoinOfColumnWithScan(
   auto blocks = CompressedRelationReader::getBlocksForJoin(joinColumn,
                                                            metaBlocks1.value());
 
-  return getLazyScan(s, blocks);
+  auto result = getLazyScan(s, blocks);
+  result.details().numBlocksTotal_ = metaBlocks1.value().blockMetadata_.size();
+  return result;
 }
diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 39235432bc..f27624cffc 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -648,14 +648,15 @@ namespace {
 // efficient access in the join columns below.
 cppcoro::generator,
                    CompressedRelationReader::LazyScanMetadata>
-liftGenerator(Permutation::IdTableGenerator gen) {
+convertGenerator(Permutation::IdTableGenerator gen) {
+  gen.setDetailsPointer(&co_await cppcoro::getDetails);
   for (auto& table : gen) {
     ad_utility::IdTableAndFirstCol t{std::move(table)};
     co_yield t;
   }
-  co_await cppcoro::getDetails = gen.details();
 }
 
+// TODO Add the information about the total number of blocks.
 void updateRuntimeInfoForLazyScan(
     QueryExecutionTree& scanTree,
     const CompressedRelationReader::LazyScanMetadata& metadata) {
@@ -664,6 +665,7 @@ void updateRuntimeInfoForLazyScan(
   rti.numRows_ = metadata.numElementsRead_;
   rti.totalTime_ = static_cast(metadata.blockingTimeMs_);
   rti.addDetail("num-blocks-read", metadata.numBlocksRead_);
+  rti.addDetail("num-blocks-total", metadata.numBlocksTotal_);
 }
 }  // namespace
 
@@ -686,8 +688,8 @@ IdTable Join::computeResultForTwoIndexScans() {
           dynamic_cast(*_right->getRootOperation()));
   getRuntimeInfo().addDetail("time-for-filtering-blocks", timer.msecs());
 
-  auto leftBlocks = liftGenerator(std::move(leftBlocksInternal));
-  auto rightBlocks = liftGenerator(std::move(rightBlocksInternal));
+  auto leftBlocks = convertGenerator(std::move(leftBlocksInternal));
+  auto rightBlocks = convertGenerator(std::move(rightBlocksInternal));
 
   ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks,
                                               std::less{}, rowAdder);
@@ -735,7 +737,7 @@ IdTable Join::computeResultForIndexScanAndIdTable(
   ad_utility::Timer timer{ad_utility::timer::Timer::InitialStatus::Started};
   auto rightBlocksInternal =
       IndexScan::lazyScanForJoinOfColumnWithScan(permutation.col(), scan);
-  auto rightBlocks = liftGenerator(std::move(rightBlocksInternal));
+  auto rightBlocks = convertGenerator(std::move(rightBlocksInternal));
 
   getRuntimeInfo().addDetail("time-for-filtering-blocks", timer.msecs());
 
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index 11e8210f3d..b552ec9b66 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -243,6 +243,7 @@ class CompressedRelationReader {
 
   struct LazyScanMetadata {
     size_t numBlocksRead_ = 0;
+    size_t numBlocksTotal_ = 0;
     size_t numElementsRead_ = 0;
     size_t blockingTimeMs_ = 0;
   };
diff --git a/src/util/Generator.h b/src/util/Generator.h
index 375ad60f24..ea86b3a62f 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -223,7 +223,8 @@ class [[nodiscard]] generator {
     std::swap(m_coroutine, other.m_coroutine);
   }
 
-  const Details& details() { return m_coroutine.promise().details(); }
+  const Details& details() const { return m_coroutine.promise().details(); }
+  Details& details() { return m_coroutine.promise().details(); }
 
   void setDetailsPointer(Details* pointer) {
     m_coroutine.promise().setDetailsPointer(pointer);
diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index b2888340ce..b0a7355d23 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -85,17 +85,6 @@ concept BinaryIteratorFunction =
  * @param elFromFirstNotFoundAction This function is called for each iterator in
  * `left` for which no corresponding match in `right` was found. This is `noop`
  * for "normal` joins, but can be set to implement `OPTIONAL` or `MINUS`.
- * @tparam addDuplicatesFromLeft If set to `false`, then if several inputs from
- * `left` are compatible with several inputs from `right`, then only the matches
- * for the first matching entry from `left` are added to the result. For example
- * if `A` and `B` from left are both compatible with `C` and `D` from right,
- * then the result for these elements will be `AC, AD`. With the default
- * (`addDuplicatesFromLeft = true`) it would be `AC, AD, BC, BD`. Note that this
- * currently only works for the exact matches and has no effects on the merging
- * with undefined values. This is useful when there are no undefined values and
- * we are only interested in the unique results from `right`. This is used in
- * `CompressedRelations.cpp` where we intersect a column of IDs with a set of
- * block metadata and are only interested in the matching blocks.
  * @return 0 if the result is sorted, > 0 if the result is not sorted. `Sorted`
  * means that all the calls to `compatibleRowAction` were ordered wrt
  * `lessThan`. A result being out of order can happen if two rows with UNDEF
@@ -104,11 +93,11 @@ concept BinaryIteratorFunction =
  * described cases leads to two sorted ranges in the output, this can possibly
  * be exploited to fix the result in a cheaper way than a full sort.
  */
-template <
-    bool addDuplicatesFromLeft = true, std::ranges::random_access_range Range1,
-    std::ranges::random_access_range Range2, typename LessThan,
-    typename FindSmallerUndefRangesLeft, typename FindSmallerUndefRangesRight,
-    typename ElFromFirstNotFoundAction = decltype(noop)>
+template 
 [[nodiscard]] auto zipperJoinWithUndef(
     const Range1& left, const Range2& right, const LessThan& lessThan,
     const auto& compatibleRowAction,
@@ -271,9 +260,6 @@ template <
         for (auto innerIt2 = it2; innerIt2 != endSame2; ++innerIt2) {
           compatibleRowAction(it1, innerIt2);
         }
-        if constexpr (!addDuplicatesFromLeft) {
-          break;
-        }
       }
       it1 = endSame1;
       it2 = endSame2;
@@ -333,8 +319,7 @@ template <
  * implement very efficient OPTIONAL or MINUS if neither of the inputs contains
  * UNDEF values, and if the left operand is much smaller.
  */
-template 
 void gallopingJoin(
@@ -404,13 +389,9 @@ void gallopingJoin(
         itLarge, endLarge, [&](const auto& row) { return eq(row, *itSmall); });
 
     for (; itSmall != endSameSmall; ++itSmall) {
-      if constexpr (!addDuplicatesFromLarge) {
-        action(itSmall, itLarge);
-      } else {
-        for (auto innerItLarge = itLarge; innerItLarge != endSameLarge;
-             ++innerItLarge) {
-          action(itSmall, innerItLarge);
-        }
+      for (auto innerItLarge = itLarge; innerItLarge != endSameLarge;
+           ++innerItLarge) {
+        action(itSmall, innerItLarge);
       }
     }
     itSmall = endSameSmall;
diff --git a/src/util/JoinAlgorithms/JoinColumnMapping.h b/src/util/JoinAlgorithms/JoinColumnMapping.h
index 9c5c649d79..27d2209b02 100644
--- a/src/util/JoinAlgorithms/JoinColumnMapping.h
+++ b/src/util/JoinAlgorithms/JoinColumnMapping.h
@@ -99,7 +99,7 @@ class JoinColumnMapping {
 // A class that stores a complete `IdTable`, but when being treated as a range
 // via the `begin/end/operator[]` functions, then it only gives access to the
 // first column. This is very useful for the lazy join implementations
-// (currently used in `Join.cpp`, where we need very efficient access to the
+// (currently used in `Join.cpp`), where we need very efficient access to the
 // join column for comparing rows, but also need to store the complete table to
 // be able to write the other columns of a matching row to the result.
 // This class is templated so we can use it for `IdTable` as well as for

From 33d9b140b14ba06cd12dc3e8d0f9867beb9edcb2 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 18:41:47 +0200
Subject: [PATCH 114/150] Changes from review.

---
 test/engine/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/engine/CMakeLists.txt b/test/engine/CMakeLists.txt
index 35b3050c80..beefb39c20 100644
--- a/test/engine/CMakeLists.txt
+++ b/test/engine/CMakeLists.txt
@@ -1 +1 @@
-addLinkAndDiscoverTest(IndexScanTest engine)
\ No newline at end of file
+addLinkAndDiscoverTest(IndexScanTest engine)

From 6a7e90102021f9a933973d6eca95fdfc07b14035 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 5 Jul 2023 18:44:12 +0200
Subject: [PATCH 115/150] Try of fix

---
 src/engine/Join.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index f27624cffc..02495d014e 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -649,6 +649,7 @@ namespace {
 cppcoro::generator,
                    CompressedRelationReader::LazyScanMetadata>
 convertGenerator(Permutation::IdTableGenerator gen) {
+  co_await cppcoro::getDetails = gen.details();
   gen.setDetailsPointer(&co_await cppcoro::getDetails);
   for (auto& table : gen) {
     ad_utility::IdTableAndFirstCol t{std::move(table)};

From 461f594d71c68a9b08d8334490f5326ae5fbffbb Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Thu, 6 Jul 2023 12:23:23 +0200
Subject: [PATCH 116/150] Make the blocksize of the Permutations configurable
 via the `Index` class.

---
 src/index/CompressedRelation.h | 4 +---
 src/index/Index.cpp            | 5 +++++
 src/index/Index.h              | 2 ++
 src/index/IndexImpl.cpp        | 6 ++++--
 src/index/IndexImpl.h          | 5 +++++
 test/IndexTestHelpers.h        | 5 +++++
 6 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index 7ebd208b67..cf03e4e8d7 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -158,9 +158,7 @@ class CompressedRelationWriter {
 
  public:
   /// Create using a filename, to which the relation data will be written.
-  explicit CompressedRelationWriter(
-      ad_utility::File f,
-      size_t numBytesPerBlock = BLOCKSIZE_COMPRESSED_METADATA)
+  explicit CompressedRelationWriter(ad_utility::File f, size_t numBytesPerBlock)
       : outfile_{std::move(f)}, numBytesPerBlock_{numBytesPerBlock} {}
 
   /**
diff --git a/src/index/Index.cpp b/src/index/Index.cpp
index faef54dff9..72f235cc9e 100644
--- a/src/index/Index.cpp
+++ b/src/index/Index.cpp
@@ -268,6 +268,11 @@ void Index::setKeepTempFiles(bool keepTempFiles) {
 // ____________________________________________________________________________
 uint64_t& Index::stxxlMemoryInBytes() { return pimpl_->stxxlMemoryInBytes(); }
 
+// ____________________________________________________________________________
+uint64_t& Index::blocksizePermutationsInBytes() {
+  return pimpl_->blocksizePermutationInBytes();
+}
+
 // ____________________________________________________________________________
 const uint64_t& Index::stxxlMemoryInBytes() const {
   return pimpl_->stxxlMemoryInBytes();
diff --git a/src/index/Index.h b/src/index/Index.h
index 6a32fa7bb9..ef9c6773ea 100644
--- a/src/index/Index.h
+++ b/src/index/Index.h
@@ -234,6 +234,8 @@ class Index {
   uint64_t& stxxlMemoryInBytes();
   const uint64_t& stxxlMemoryInBytes() const;
 
+  uint64_t& blocksizePermutationsInBytes();
+
   void setOnDiskBase(const std::string& onDiskBase);
 
   void setSettingsFile(const std::string& filename);
diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp
index 397fb0f312..3673ab4498 100644
--- a/src/index/IndexImpl.cpp
+++ b/src/index/IndexImpl.cpp
@@ -497,8 +497,10 @@ IndexImpl::createPermutationPairImpl(const string& fileName1,
     metaData2.setup(fileName2 + MMAP_FILE_SUFFIX, ad_utility::CreateTag{});
   }
 
-  CompressedRelationWriter writer1{ad_utility::File(fileName1, "w")};
-  CompressedRelationWriter writer2{ad_utility::File(fileName2, "w")};
+  CompressedRelationWriter writer1{ad_utility::File(fileName1, "w"),
+                                   blocksizePermutationInBytes_};
+  CompressedRelationWriter writer2{ad_utility::File(fileName2, "w"),
+                                   blocksizePermutationInBytes_};
 
   // Iterate over the vector and identify "relation" boundaries, where a
   // "relation" is the sequence of sortedTriples equal first component. For PSO
diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h
index 99fc0d4a0e..81236d4b1a 100644
--- a/src/index/IndexImpl.h
+++ b/src/index/IndexImpl.h
@@ -114,6 +114,7 @@ class IndexImpl {
   bool turtleParserSkipIllegalLiterals_ = false;
   bool keepTempFiles_ = false;
   uint64_t stxxlMemoryInBytes_ = DEFAULT_STXXL_MEMORY_IN_BYTES;
+  uint64_t blocksizePermutationInBytes_ = BLOCKSIZE_COMPRESSED_METADATA;
   json configurationJson_;
   Index::Vocab vocab_;
   size_t totalVocabularySize_ = 0;
@@ -368,6 +369,10 @@ class IndexImpl {
   uint64_t& stxxlMemoryInBytes() { return stxxlMemoryInBytes_; }
   const uint64_t& stxxlMemoryInBytes() const { return stxxlMemoryInBytes_; }
 
+  uint64_t& blocksizePermutationInBytes() {
+    return blocksizePermutationInBytes_;
+  }
+
   void setOnDiskBase(const std::string& onDiskBase);
 
   void setSettingsFile(const std::string& filename);
diff --git a/test/IndexTestHelpers.h b/test/IndexTestHelpers.h
index 45f35370c3..0bdcb2bb5b 100644
--- a/test/IndexTestHelpers.h
+++ b/test/IndexTestHelpers.h
@@ -21,6 +21,11 @@ namespace ad_utility::testing {
 inline Index makeIndexWithTestSettings() {
   Index index{ad_utility::makeUnlimitedAllocator()};
   index.setNumTriplesPerBatch(2);
+  // This is enough for 2 triples per block. This is deliberately chosen as a
+  // small value, s.t. the tiny knowledge graphs from unit tests also contain
+  // multiple blocks. Should this value or the semantics of it (how many triples
+  // it may store) ever change, then some unit tests might have to be adapted.
+  index.blocksizePermutationsInBytes() = 32;
   index.stxxlMemoryInBytes() = 1024ul * 1024ul * 50;
   return index;
 }

From f72c972eac055680b7b90b4e0cdbd77259266610 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Thu, 6 Jul 2023 13:07:15 +0200
Subject: [PATCH 117/150] Started to write tests for the index scan class.

---
 test/engine/IndexScanTest.cpp | 65 +++++++++++++++++++++++++++--------
 1 file changed, 50 insertions(+), 15 deletions(-)

diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp
index 2ec8ace963..b4bdc673cf 100644
--- a/test/engine/IndexScanTest.cpp
+++ b/test/engine/IndexScanTest.cpp
@@ -10,9 +10,32 @@
 
 using namespace ad_utility::testing;
 
-std::string kg = " 
 .  
 .   .   .";
+using IndexPair = std::pair;
+void testLazyScan(auto lazyScan, IndexScan& scanOp,
+                  const std::vector& expectedRows) {
+  auto alloc = ad_utility::makeUnlimitedAllocator();
+  IdTable lazyScanRes{2, alloc};
+  size_t numBlocks = 0;
+  for (const auto& block : lazyScan) {
+    lazyScanRes.insertAtEnd(block.begin(), block.end());
+    ++numBlocks;
+  }
 
-TEST(IndexScan, lazyScanForJoinOfTwoScans) {
+  auto resFullScan = scanOp.getResult()->idTable().clone();
+  IdTable expected{resFullScan.numColumns(), alloc};
+
+  for (auto [lower, upper] : expectedRows) {
+    for (auto index : std::views::iota(lower, upper)) {
+      expected.push_back(resFullScan.at(index));
+    }
+  }
+
+  EXPECT_EQ(lazyScanRes, expected);
+}
+
+void testLazyScanForJoinOfTwoScans(const std::string& kg,
+                                   const std::vector& leftRows,
+                                   const std::vector& rightRows) {
   using Tc = TripleComponent;
   using Var = Variable;
   auto qec = getQec(kg);
@@ -22,20 +45,32 @@ TEST(IndexScan, lazyScanForJoinOfTwoScans) {
                SparqlTriple{Tc{Var{"?x"}}, "", Tc{Var{"?z"}}}};
   auto [scan1, scan2] = (IndexScan::lazyScanForJoinOfTwoScans(s1, s2));
 
-  IdTable res1{2, ad_utility::makeUnlimitedAllocator()};
-  IdTable res2{2, ad_utility::makeUnlimitedAllocator()};
+  testLazyScan(std::move(scan1), s1, leftRows);
+  testLazyScan(std::move(scan2), s2, rightRows);
+}
+
+TEST(IndexScan, lazyScanForJoinOfTwoScans) {
+  {
+    // In the tests we have a blocksize of two triples per block, and a new
+    // block is started for a new relation. That explains the spacing of the
+    // following example knowledge graphs.
+    std::string kg =
+        " 
 .  
 . "
+        " 
  .  
 . "
+        " 
  ."
+        "  .    .";
 
-  for (const auto& block : scan1) {
-    res1.insertAtEnd(block.begin(), block.end());
+    // When joining the 
 and  relations, we only need to read the last two
+    // blocks of the 
 relation, as  never appears as a subject in .
+    // This means that the lazy partial scan can skip the first two triples.
+    testLazyScanForJoinOfTwoScans(kg, {{2, 5}}, {{0, 2}});
   }
-  for (const auto& block : scan2) {
-    res2.insertAtEnd(block.begin(), block.end());
+  {
+    // No triple for relation 
, so both scans can be empty.
+    std::string kg =
+        "  .   . "
+        "   .   . "
+        "  .    .";
+    testLazyScanForJoinOfTwoScans(kg, {}, {});
   }
-
-  EXPECT_EQ(res1.size(), 2u);
-  EXPECT_EQ(res2.size(), 2u);
-
-  // TODO We need additional tests that are not only dummys. To make
-  // this work, we need to have the blocksize of the index class configurable, I
-  // will split this into a separate PR.
 }

From f4b2d1ca936f0eae71d4cbde37aa056e5ac02553 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Thu, 6 Jul 2023 16:01:19 +0200
Subject: [PATCH 118/150] Add unit tests for the changes in the `IndexScan`

class.
---
 src/engine/IndexScan.cpp      |  13 ++
 src/util/Exception.h          |  18 ++-
 test/engine/IndexScanTest.cpp | 232 +++++++++++++++++++++++++++++++---
 3 files changed, 242 insertions(+), 21 deletions(-)

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index 1d9db85677..c74224a2db 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -315,6 +315,19 @@ std::array
 IndexScan::lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2) {
   AD_CONTRACT_CHECK(s1.numVariables_ < 3 && s2.numVariables_ < 3);
 
+  // This function only works for single column joins. This means that the first
+  // variable of both scans must be equal, but the second variables of the scans
+  // (if present) must be different.
+  const auto& getFirstVariable = [](const IndexScan& scan) {
+    return scan.numVariables_ == 2 ? *scan.getPermutedTriple()[1]
+                                   : *scan.getPermutedTriple()[2];
+  };
+
+  AD_CONTRACT_CHECK(getFirstVariable(s1) == getFirstVariable(s2));
+  if (s1.numVariables_ == 2 && s1.numVariables_ == 2) {
+    AD_CONTRACT_CHECK(*s1.getPermutedTriple()[2] != *s2.getPermutedTriple()[2]);
+  }
+
   auto metaBlocks1 = getMetadataForScan(s1);
   auto metaBlocks2 = getMetadataForScan(s2);
 
diff --git a/src/util/Exception.h b/src/util/Exception.h
index a554417c83..8551024ad1 100644
--- a/src/util/Exception.h
+++ b/src/util/Exception.h
@@ -125,10 +125,22 @@ inline void adCorrectnessCheckImpl(bool condition, std::string_view message,
 // `AD_ENABLE_EXPENSIVE_CHECKS` is enabled (which can be set via cmake). This
 // check should be used when checking has a significant and measurable runtime
 // overhead, but is still feasible for a release build on a large dataset.
+namespace ad_utility {
+static constexpr bool areExpensiveChecksEnabled() {
 #if (!defined(NDEBUG) || defined(AD_ENABLE_EXPENSIVE_CHECKS))
-#define AD_EXPENSIVE_CHECK(condition) \
-  AD_CORRECTNESS_CHECK(condition);    \
+  return true;
+#else
+  return false;
+#endif
+}
+}  // namespace ad_utility
+#if (!defined(NDEBUG) || defined(AD_ENABLE_EXPENSIVE_CHECKS))
+#define AD_EXPENSIVE_CHECK(condition)                     \
+  static_assert(ad_utility::areExpensiveChecksEnabled()); \
+  AD_CORRECTNESS_CHECK(condition);                        \
   void(0)
 #else
-#define AD_EXPENSIVE_CHECK(condition) void(0)
+#define AD_EXPENSIVE_CHECK(condition)                      \
+  static_assert(!ad_utility::areExpensiveChecksEnabled()); \
+  void(0)
 #endif
diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp
index b4bdc673cf..374cad19ef 100644
--- a/test/engine/IndexScanTest.cpp
+++ b/test/engine/IndexScanTest.cpp
@@ -5,18 +5,29 @@
 #include 
 
 #include "../IndexTestHelpers.h"
+#include "../util/GTestHelpers.h"
 #include "engine/IndexScan.h"
 #include "parser/ParsedQuery.h"
 
 using namespace ad_utility::testing;
+using ad_utility::source_location;
+
+namespace {
+using Tc = TripleComponent;
+using Var = Variable;
 
 using IndexPair = std::pair;
 void testLazyScan(auto lazyScan, IndexScan& scanOp,
-                  const std::vector& expectedRows) {
+                  const std::vector& expectedRows,
+                  source_location l = source_location::current()) {
+  auto t = generateLocationTrace(l);
   auto alloc = ad_utility::makeUnlimitedAllocator();
-  IdTable lazyScanRes{2, alloc};
+  IdTable lazyScanRes{0, alloc};
   size_t numBlocks = 0;
   for (const auto& block : lazyScan) {
+    if (lazyScanRes.empty()) {
+      lazyScanRes.setNumColumns(block.numColumns());
+    }
     lazyScanRes.insertAtEnd(block.begin(), block.end());
     ++numBlocks;
   }
@@ -33,23 +44,73 @@ void testLazyScan(auto lazyScan, IndexScan& scanOp,
   EXPECT_EQ(lazyScanRes, expected);
 }
 
-void testLazyScanForJoinOfTwoScans(const std::string& kg,
-                                   const std::vector& leftRows,
-                                   const std::vector& rightRows) {
-  using Tc = TripleComponent;
-  using Var = Variable;
+void testLazyScanForJoinOfTwoScans(
+    const std::string& kg, const SparqlTriple& tripleLeft,
+    const SparqlTriple& tripleRight, const std::vector& leftRows,
+    const std::vector& rightRows,
+    source_location l = source_location::current()) {
+  auto t = generateLocationTrace(l);
+  auto qec = getQec(kg);
+  IndexScan s1{qec, Permutation::PSO, tripleLeft};
+  IndexScan s2{qec, Permutation::PSO, tripleRight};
+  auto implForSwitch = [](IndexScan& l, IndexScan& r, const auto& expectedL,
+                          const auto& expectedR) {
+    auto [scan1, scan2] = (IndexScan::lazyScanForJoinOfTwoScans(l, r));
+
+    testLazyScan(std::move(scan1), l, expectedL);
+    testLazyScan(std::move(scan2), r, expectedR);
+  };
+  implForSwitch(s1, s2, leftRows, rightRows);
+  implForSwitch(s2, s1, rightRows, leftRows);
+}
+
+void testLazyScanThrows(const std::string& kg, const SparqlTriple& tripleLeft,
+                        const SparqlTriple& tripleRight,
+                        source_location l = source_location::current()) {
+  auto t = generateLocationTrace(l);
+  auto qec = getQec(kg);
+  IndexScan s1{qec, Permutation::PSO, tripleLeft};
+  IndexScan s2{qec, Permutation::PSO, tripleRight};
+  EXPECT_ANY_THROW(IndexScan::lazyScanForJoinOfTwoScans(s1, s2));
+}
+
+void testLazyScanForJoinWithColumn(
+    const std::string& kg, const SparqlTriple& scanTriple,
+    std::vector columnEntries,
+    const std::vector& expectedRows,
+    source_location l = source_location::current()) {
+  auto t = generateLocationTrace(l);
   auto qec = getQec(kg);
-  IndexScan s1{qec, Permutation::PSO,
-               SparqlTriple{Tc{Var{"?x"}}, "
", Tc{Var{"?y"}}}};
-  IndexScan s2{qec, Permutation::PSO,
-               SparqlTriple{Tc{Var{"?x"}}, "", Tc{Var{"?z"}}}};
-  auto [scan1, scan2] = (IndexScan::lazyScanForJoinOfTwoScans(s1, s2));
-
-  testLazyScan(std::move(scan1), s1, leftRows);
-  testLazyScan(std::move(scan2), s2, rightRows);
+  IndexScan scan{qec, Permutation::PSO, scanTriple};
+  std::vector column;
+  for (const auto& entry : columnEntries) {
+    column.push_back(
+        TripleComponent{entry}.toValueId(qec->getIndex().getVocab()).value());
+  }
+
+  auto lazyScan = IndexScan::lazyScanForJoinOfColumnWithScan(column, scan);
+  testLazyScan(std::move(lazyScan), scan, expectedRows);
 }
 
+void testLazyScanWithColumnThrows(
+    const std::string& kg, const SparqlTriple& scanTriple,
+    std::vector columnEntries,
+    source_location l = source_location::current()) {
+  auto t = generateLocationTrace(l);
+  auto qec = getQec(kg);
+  IndexScan s1{qec, Permutation::PSO, scanTriple};
+  std::vector column;
+  for (const auto& entry : columnEntries) {
+    column.push_back(
+        TripleComponent{entry}.toValueId(qec->getIndex().getVocab()).value());
+  }
+  EXPECT_ANY_THROW(IndexScan::lazyScanForJoinOfColumnWithScan(column, s1));
+}
+}  // namespace
+
 TEST(IndexScan, lazyScanForJoinOfTwoScans) {
+  SparqlTriple xpy{Tc{Var{"?x"}}, "
", Tc{Var{"?y"}}};
+  SparqlTriple xpz{Tc{Var{"?x"}}, "", Tc{Var{"?z"}}};
   {
     // In the tests we have a blocksize of two triples per block, and a new
     // block is started for a new relation. That explains the spacing of the
@@ -63,14 +124,149 @@ TEST(IndexScan, lazyScanForJoinOfTwoScans) {
     // When joining the 
 and  relations, we only need to read the last two
     // blocks of the 
 relation, as  never appears as a subject in .
     // This means that the lazy partial scan can skip the first two triples.
-    testLazyScanForJoinOfTwoScans(kg, {{2, 5}}, {{0, 2}});
+    testLazyScanForJoinOfTwoScans(kg, xpy, xpz, {{2, 5}}, {{0, 2}});
   }
   {
-    // No triple for relation 
, so both scans can be empty.
     std::string kg =
         "  .   . "
         "   .   . "
         "  .    .";
-    testLazyScanForJoinOfTwoScans(kg, {}, {});
+    // No triple for relation 
 (which doesn't even appear in the knowledge
+    // graph), so both lazy scans are empty.
+    testLazyScanForJoinOfTwoScans(kg, xpy, xpz, {}, {});
+  }
+  {
+    // No triple for relation  (which does appear in the knowledge graph, but
+    // not as a predicate), so both lazy scans arek.
+    std::string kg =
+        " 
 .  
 . "
+        " 
  .  
 . "
+        "  .    .";
+    testLazyScanForJoinOfTwoScans(kg, xpy, xpz, {}, {});
+  }
+  SparqlTriple bpx{Tc{""}, "
", Tc{Var{"?x"}}};
+  {
+    std::string kg =
+        " 
 .  
 . "
+        " 
  .  
 . "
+        " 
  .  
 . "
+        " 
  .  
 . "
+        "  .    ."
+        "  .    ."
+        "  .    .";
+    testLazyScanForJoinOfTwoScans(kg, bpx, xpz, {{1, 5}}, {{0, 4}});
+  }
+  {
+    std::string kg =
+        " 
 .  
 . "
+        " 
  .  
 . "
+        "  .    ."
+        "  .    ."
+        "  .    .";
+    // Scan for a fixed subject that appears in the kg but not as the subject of
+    // the 
 predicate.
+    SparqlTriple xb2px{Tc{""}, "
", Tc{Var{"?x"}}};
+    testLazyScanForJoinOfTwoScans(kg, bpx, xpz, {}, {});
+  }
+  {
+    std::string kg =
+        " 
 .  
 . "
+        " 
  .  
 . "
+        "  .    ."
+        "  .    ."
+        "  .    .";
+    // Scan for a fixed subject that is not even in the knowledge graph.
+    SparqlTriple xb2px{Tc{""}, "
", Tc{Var{"?x"}}};
+    testLazyScanForJoinOfTwoScans(kg, bpx, xpz, {}, {});
+  }
+
+  // Corner cases
+  {
+    std::string kg = "   .";
+    // Triples with three variables are not supported.
+    SparqlTriple xyz{Tc{Var{"?x"}}, "?y", Tc{Var{"?z"}}};
+    testLazyScanThrows(kg, xyz, xpz);
+    testLazyScanThrows(kg, xyz, xyz);
+    testLazyScanThrows(kg, xpz, xyz);
+
+    // The first variable must be matching (subject variable is ?a vs ?x)
+    SparqlTriple abc{Tc{Var{"?a"}}, "", Tc{Var{"?c"}}};
+    testLazyScanThrows(kg, abc, xpz);
+
+    // If both scans have two variables, then the second variable must not
+    // match.
+    testLazyScanThrows(kg, abc, abc);
+  }
+}
+
+TEST(IndexScan, lazyScanForJoinOfColumnWithScanTwoVariables) {
+  SparqlTriple xpy{Tc{Var{"?x"}}, "
", Tc{Var{"?y"}}};
+  // In the tests we have a blocksize of two triples per block, and a new
+  // block is started for a new relation. That explains the spacing of the
+  // following example knowledge graphs.
+  std::string kg =
+      " 
 .  
 . "
+      " 
  .  
 . "
+      " 
  ."
+      "  .    .";
+  {
+    std::vector column{"", "", "", ""};
+    // We need to scan all the blocks that contain the `
` predicate.
+    testLazyScanForJoinWithColumn(kg, xpy, column, {{0, 5}});
+  }
+  {
+    std::vector column{"", "", ""};
+    // The first block only contains  which doesn't appear in the first
+    // block.
+    testLazyScanForJoinWithColumn(kg, xpy, column, {{2, 5}});
+  }
+  {
+    std::vector column{"", "", ""};
+    // The first block only contains  which only appears in the first two
+    // blocks.
+    testLazyScanForJoinWithColumn(kg, xpy, column, {{0, 4}});
+  }
+  {
+    std::vector column{"", "", ""};
+    //  does not appear as a predicate, so the result is empty.
+    SparqlTriple efg{Tc{Var{"?e"}}, "", Tc{Var{"?g"}}};
+    testLazyScanForJoinWithColumn(kg, efg, column, {});
+  }
+}
+
+TEST(IndexScan, lazyScanForJoinOfColumnWithScanOneVariable) {
+  SparqlTriple bpy{Tc{""}, "
", Tc{Var{"?x"}}};
+  std::string kg =
+      " 
 .  
 . "
+      " 
  .  
 . "
+      " 
  .  
 . "
+      " 
  .  
 . "
+      "  .    .";
+  {
+    // The subject () and predicate () are fixed, so the object is the
+    // join column
+    std::vector column{"", "", ""};
+    // We don't need to scan the middle block that only has  and 
+    testLazyScanForJoinWithColumn(kg, bpy, column, {{0, 1}, {3, 5}});
+  }
+}
+
+TEST(IndexScan, lazyScanForJoinOfColumnWithScanCornerCases) {
+  SparqlTriple threeVars{Tc{Var{"?x"}}, "?b", Tc{Var{"?y"}}};
+  std::string kg =
+      " 
 .  
 . "
+      " 
  .  
 . "
+      " 
  ."
+      "  .    .";
+
+  // Full index scans (three variables) are not supported.
+  std::vector column{"", "", "", ""};
+  testLazyScanWithColumnThrows(kg, threeVars, column);
+
+  // The join column must be sorted.
+  if constexpr (ad_utility::areExpensiveChecksEnabled()) {
+    std::vector unsortedColumn{"", "", ""};
+    SparqlTriple xpy{Tc{Var{"?x"}}, "
", Tc{Var{"?y"}}};
+    testLazyScanWithColumnThrows(kg, xpy, unsortedColumn);
   }
 }

From 2bb795c3ff0a08308798a8be9ecd8d6f62026b3e Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Thu, 6 Jul 2023 17:21:41 +0200
Subject: [PATCH 119/150] Only the changes for the `AddCombinedRowToTable` etc,
 without the generator.

---
 test/AddCombinedRowToTableTest.cpp | 46 ++++++++++++++++++++++++++++++
 test/engine/IndexScanTest.cpp      | 12 ++++++--
 2 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/test/AddCombinedRowToTableTest.cpp b/test/AddCombinedRowToTableTest.cpp
index 86ed9fa98b..dcaebe5383 100644
--- a/test/AddCombinedRowToTableTest.cpp
+++ b/test/AddCombinedRowToTableTest.cpp
@@ -99,3 +99,49 @@ TEST(AddCombinedRowToTable, UndefInInput) {
   testWithBufferSize(1);
   testWithBufferSize(2);
 }
+
+// _______________________________________________________________________________
+TEST(AddCombinedRowToTable, setInput) {
+  auto testWithBufferSize = [](size_t bufferSize) {
+    auto result = makeIdTableFromVector({});
+    result.setNumColumns(2);
+    {
+      auto adder = ad_utility::AddCombinedRowToIdTable(
+          1, std::nullopt, std::nullopt, std::move(result), bufferSize);
+      // It is okay to flush even if no inputs were specified, as long as we haven't pushed any rows yet.
+      EXPECT_NO_THROW(adder.flush());
+      if constexpr (ad_utility::areExpensiveChecksEnabled()) {
+        EXPECT_ANY_THROW(adder.addRow(0, 0));
+      } else {
+        adder.addRow(0, 0);
+        EXPECT_ANY_THROW(adder.flush());
+      }
+    }
+    auto adder = ad_utility::AddCombinedRowToIdTable(
+        1, std::nullopt, std::nullopt, std::move(result), bufferSize);
+    auto left = makeIdTableFromVector({{U, 5}, {2, U}, {3, U}, {4, U}});
+    auto right = makeIdTableFromVector({{1}, {3}, {4}, {U}});
+    adder.setInput(left, right);
+    adder.addRow(0, 0);
+    adder.addRow(0, 1);
+    adder.addRow(2, 1);
+    adder.addRow(0, 2);
+    adder.addRow(3, 2);
+    adder.addRow(0, 3);
+    adder.setInput(right, left);
+    adder.addRow(0, 0);
+    adder.addRow(1, 0);
+    adder.addRow(1, 2);
+    adder.addRow(2, 0);
+    adder.addRow(2, 3);
+    adder.addRow(3, 0);
+    result = std::move(adder).resultTable();
+
+    auto expected =
+        makeIdTableFromVector({{1, 5}, {3, 5}, {3, U}, {4, 5}, {4, U}, {U, 5}});
+    ASSERT_EQ(result, expected);
+  };
+  testWithBufferSize(100'000);
+  testWithBufferSize(1);
+  testWithBufferSize(2);
+}
diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp
index 374cad19ef..6361e3c0c1 100644
--- a/test/engine/IndexScanTest.cpp
+++ b/test/engine/IndexScanTest.cpp
@@ -17,7 +17,7 @@ using Tc = TripleComponent;
 using Var = Variable;
 
 using IndexPair = std::pair;
-void testLazyScan(auto lazyScan, IndexScan& scanOp,
+void testLazyScan(Permutation::IdTableGenerator lazyScan, IndexScan& scanOp,
                   const std::vector& expectedRows,
                   source_location l = source_location::current()) {
   auto t = generateLocationTrace(l);
@@ -32,6 +32,9 @@ void testLazyScan(auto lazyScan, IndexScan& scanOp,
     ++numBlocks;
   }
 
+  EXPECT_EQ(numBlocks, lazyScan.details().numBlocksRead_);
+  EXPECT_EQ(lazyScanRes.size(), lazyScan.details().numElementsRead_);
+
   auto resFullScan = scanOp.getResult()->idTable().clone();
   IdTable expected{resFullScan.numColumns(), alloc};
 
@@ -104,7 +107,12 @@ void testLazyScanWithColumnThrows(
     column.push_back(
         TripleComponent{entry}.toValueId(qec->getIndex().getVocab()).value());
   }
-  EXPECT_ANY_THROW(IndexScan::lazyScanForJoinOfColumnWithScan(column, s1));
+
+  // We need this to suppress the warning about a [[nodiscard]] return value being unused.
+  auto makeScan = [&column, &s1]() {
+    [[maybe_unused]] auto scan = IndexScan::lazyScanForJoinOfColumnWithScan(column, s1);
+  };
+  EXPECT_ANY_THROW(makeScan());
 }
 }  // namespace
 

From 88c6f3aee07bf774440bd567383e39b7227a36dc Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Thu, 6 Jul 2023 17:24:44 +0200
Subject: [PATCH 120/150] Fixed a bug in the generator.

---
 src/util/Generator.h   | 8 ++++++--
 test/GeneratorTest.cpp | 8 ++++++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/util/Generator.h b/src/util/Generator.h
index ea86b3a62f..a8c325f072 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -223,8 +223,8 @@ class [[nodiscard]] generator {
     std::swap(m_coroutine, other.m_coroutine);
   }
 
-  const Details& details() const { return m_coroutine.promise().details(); }
-  Details& details() { return m_coroutine.promise().details(); }
+  const Details& details() const { return m_coroutine ? m_coroutine.promise().details() : m_details_if_default_constructed; }
+  Details& details() { return m_coroutine ? m_coroutine.promise().details() : m_details_if_default_constructed; }
 
   void setDetailsPointer(Details* pointer) {
     m_coroutine.promise().setDetailsPointer(pointer);
@@ -233,6 +233,10 @@ class [[nodiscard]] generator {
  private:
   friend class detail::generator_promise;
 
+  // In the case of an empty, default-constructed `generator` object we still want the call to `details`
+  // to return a valid object that in this case is owned directly by the generator itself.
+  [[no_unique_address]] Details m_details_if_default_constructed;
+
   explicit generator(std::coroutine_handle coroutine) noexcept
       : m_coroutine(coroutine) {}
 
diff --git a/test/GeneratorTest.cpp b/test/GeneratorTest.cpp
index 81654346c4..386c0e01f7 100644
--- a/test/GeneratorTest.cpp
+++ b/test/GeneratorTest.cpp
@@ -9,6 +9,7 @@
 struct Details {
   bool begin_ = false;
   bool end_ = false;
+  bool operator==(const Details&) const = default;
 };
 
 // A simple generator that first yields three numbers and then adds a detail
@@ -72,3 +73,10 @@ TEST(Generator, externalDetails) {
   // Setting a `nullptr` is illegal
   ASSERT_ANY_THROW(gen.setDetailsPointer(nullptr));
 }
+
+// Test that a default-constructed generator still has a valid `Details` object.
+TEST(Generator, detailsForDefaultConstructedGenerator) {
+  cppcoro::generator gen;
+  ASSERT_EQ(gen.details(), Details());
+  ASSERT_EQ(std::as_const(gen).details(), Details());
+}

From 5480f6b7fe5d4e2e8e7d6677fbb2d34a17078567 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Thu, 6 Jul 2023 19:24:37 +0200
Subject: [PATCH 121/150] Some initial tests for the new methods of the Join
 class.

---
 src/engine/AddCombinedRowToTable.h | 59 +++++++++++++++---------------
 src/engine/Join.cpp                |  6 +--
 src/util/Generator.h               | 15 ++++++--
 test/AddCombinedRowToTableTest.cpp | 59 +++++++++++++++++++++++++-----
 test/JoinTest.cpp                  | 59 ++++++++++++++++++++++++++++++
 test/engine/IndexScanTest.cpp      |  6 ++-
 6 files changed, 155 insertions(+), 49 deletions(-)

diff --git a/src/engine/AddCombinedRowToTable.h b/src/engine/AddCombinedRowToTable.h
index 26a4bc67ef..20c308e4d0 100644
--- a/src/engine/AddCombinedRowToTable.h
+++ b/src/engine/AddCombinedRowToTable.h
@@ -22,8 +22,7 @@ namespace ad_utility {
 class AddCombinedRowToIdTable {
   std::vector numUndefinedPerColumn_;
   size_t numJoinColumns_;
-  std::optional> inputLeft_;
-  std::optional> inputRight_;
+  std::optional, 2>> inputs_;
   IdTable resultTable_;
 
   // This struct stores the information, which row indices from the input are
@@ -60,25 +59,28 @@ class AddCombinedRowToIdTable {
 
  public:
   // Construct from the number of join columns, the two inputs, and the output.
-  // The `bufferSize` can be configured for testing. If the inputs are
-  // `std::nullopt`, this means that the inputs have to be set to an explicit
-  // call to `setInput` before adding rows. This is used for the lazy join
-  // operations (see Join.cpp) where the input changes over time.
-  explicit AddCombinedRowToIdTable(size_t numJoinColumns,
-                                   std::optional> input1,
-                                   std::optional> input2,
-                                   IdTable output, size_t bufferSize = 100'000)
+  // The `bufferSize` can be configured for testing.
+  explicit AddCombinedRowToIdTable(size_t numJoinColumns, IdTableView<0> input1,
+                                   IdTableView<0> input2, IdTable output,
+                                   size_t bufferSize = 100'000)
       : numUndefinedPerColumn_(output.numColumns()),
         numJoinColumns_{numJoinColumns},
-        inputLeft_{std::move(input1)},
-        inputRight_{std::move(input2)},
+        inputs_{std::array{std::move(input1), std::move(input2)}},
         resultTable_{std::move(output)},
         bufferSize_{bufferSize} {
-    if (inputLeft_.has_value() && inputRight_.has_value()) {
-      checkNumColumns();
-    }
-    AD_CORRECTNESS_CHECK(resultTable_.empty());
+    checkNumColumns();
   }
+  // Similar to the previous constructor, but the inputs are not given.
+  // This means that the inputs have to be set to an explicit
+  // call to `setInput` before adding rows. This is used for the lazy join
+  // operations (see Join.cpp) where the input changes over time.
+  explicit AddCombinedRowToIdTable(size_t numJoinColumns, IdTable output,
+                                   size_t bufferSize = 100'000)
+      : numUndefinedPerColumn_(output.numColumns()),
+        numJoinColumns_{numJoinColumns},
+        inputs_{std::nullopt},
+        resultTable_{std::move(output)},
+        bufferSize_{bufferSize} {}
 
   // Return the number of UNDEF values per column.
   const std::vector& numUndefinedPerColumn() {
@@ -89,7 +91,7 @@ class AddCombinedRowToIdTable {
   // The next free row in the output will be created from
   // `inputLeft_[rowIndexA]` and `inputRight_[rowIndexB]`.
   void addRow(size_t rowIndexA, size_t rowIndexB) {
-    AD_EXPENSIVE_CHECK(inputLeft_.has_value() && inputRight_.has_value());
+    AD_EXPENSIVE_CHECK(inputs_.has_value());
     indexBuffer_.push_back(
         TargetIndexAndRowIndices{nextIndex_, {rowIndexA, rowIndexB}});
     ++nextIndex_;
@@ -113,11 +115,10 @@ class AddCombinedRowToIdTable {
       }
     };
     if (nextIndex_ != 0) {
-      AD_CORRECTNESS_CHECK(inputLeft_.has_value() && inputRight_.has_value());
+      AD_CORRECTNESS_CHECK(inputs_.has_value());
       flush();
     }
-    inputLeft_ = toView(inputLeft);
-    inputRight_ = toView(inputRight);
+    inputs_ = std::array{toView(inputLeft), toView(inputRight)};
     checkNumColumns();
   }
 
@@ -125,7 +126,7 @@ class AddCombinedRowToIdTable {
   // `inputLeft_[rowIndexA]`. The columns from `inputRight_` will all be set to
   // UNDEF
   void addOptionalRow(size_t rowIndexA) {
-    AD_EXPENSIVE_CHECK(inputLeft_.has_value() && inputRight_.has_value());
+    AD_EXPENSIVE_CHECK(inputs_.has_value());
     optionalIndexBuffer_.push_back(
         TargetIndexAndRowIndex{nextIndex_, rowIndexA});
     ++nextIndex_;
@@ -167,7 +168,7 @@ class AddCombinedRowToIdTable {
     if (nextIndex_ == 0) {
       return;
     }
-    AD_CONTRACT_CHECK(inputLeft_.has_value() && inputRight_.has_value());
+    AD_CORRECTNESS_CHECK(inputs_.has_value());
     result.resize(oldSize + nextIndex_);
 
     // Sometimes columns are combined where one value is UNDEF and the other one
@@ -271,16 +272,16 @@ class AddCombinedRowToIdTable {
     optionalIndexBuffer_.clear();
     nextIndex_ = 0;
   }
-  const IdTableView<0>& inputLeft() const { return inputLeft_.value(); }
+  const IdTableView<0>& inputLeft() const { return inputs_.value()[0]; }
 
-  const IdTableView<0>& inputRight() const { return inputRight_.value(); }
+  const IdTableView<0>& inputRight() const { return inputs_.value()[1]; }
 
   void checkNumColumns() const {
-    AD_CORRECTNESS_CHECK(resultTable_.numColumns() ==
-                         inputLeft().numColumns() + inputRight().numColumns() -
-                             numJoinColumns_);
-    AD_CORRECTNESS_CHECK(inputLeft().numColumns() >= numJoinColumns_ &&
-                         inputRight().numColumns() >= numJoinColumns_);
+    AD_CONTRACT_CHECK(inputLeft().numColumns() >= numJoinColumns_);
+    AD_CONTRACT_CHECK(inputRight().numColumns() >= numJoinColumns_);
+    AD_CONTRACT_CHECK(resultTable_.numColumns() ==
+                      inputLeft().numColumns() + inputRight().numColumns() -
+                          numJoinColumns_);
   }
 };
 }  // namespace ad_utility
diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 02495d014e..5d8a962ba4 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -679,8 +679,7 @@ IdTable Join::computeResultForTwoIndexScans() {
   // class to work correctly.
   AD_CORRECTNESS_CHECK(_leftJoinCol == 0 && _rightJoinCol == 0);
   ad_utility::AddCombinedRowToIdTable rowAdder{
-      1, std::nullopt, std::nullopt,
-      IdTable{getResultWidth(), getExecutionContext()->getAllocator()}};
+      1, IdTable{getResultWidth(), getExecutionContext()->getAllocator()}};
 
   ad_utility::Timer timer{ad_utility::timer::Timer::InitialStatus::Started};
   auto [leftBlocksInternal, rightBlocksInternal] =
@@ -721,8 +720,7 @@ IdTable Join::computeResultForIndexScanAndIdTable(
   auto joinColMap = ad_utility::JoinColumnMapping{
       {{jcLeft, jcRight}}, numColsLeft, numColsRight};
   ad_utility::AddCombinedRowToIdTable rowAdder{
-      1, std::nullopt, std::nullopt,
-      IdTable{getResultWidth(), getExecutionContext()->getAllocator()}};
+      1, IdTable{getResultWidth(), getExecutionContext()->getAllocator()}};
 
   AD_CORRECTNESS_CHECK(joinColumnIndexScan == 0);
   auto permutation = [&]() {
diff --git a/src/util/Generator.h b/src/util/Generator.h
index a8c325f072..2a30cd208c 100644
--- a/src/util/Generator.h
+++ b/src/util/Generator.h
@@ -223,8 +223,14 @@ class [[nodiscard]] generator {
     std::swap(m_coroutine, other.m_coroutine);
   }
 
-  const Details& details() const { return m_coroutine ? m_coroutine.promise().details() : m_details_if_default_constructed; }
-  Details& details() { return m_coroutine ? m_coroutine.promise().details() : m_details_if_default_constructed; }
+  const Details& details() const {
+    return m_coroutine ? m_coroutine.promise().details()
+                       : m_details_if_default_constructed;
+  }
+  Details& details() {
+    return m_coroutine ? m_coroutine.promise().details()
+                       : m_details_if_default_constructed;
+  }
 
   void setDetailsPointer(Details* pointer) {
     m_coroutine.promise().setDetailsPointer(pointer);
@@ -233,8 +239,9 @@ class [[nodiscard]] generator {
  private:
   friend class detail::generator_promise;
 
-  // In the case of an empty, default-constructed `generator` object we still want the call to `details`
-  // to return a valid object that in this case is owned directly by the generator itself.
+  // In the case of an empty, default-constructed `generator` object we still
+  // want the call to `details` to return a valid object that in this case is
+  // owned directly by the generator itself.
   [[no_unique_address]] Details m_details_if_default_constructed;
 
   explicit generator(std::coroutine_handle coroutine) noexcept
diff --git a/test/AddCombinedRowToTableTest.cpp b/test/AddCombinedRowToTableTest.cpp
index dcaebe5383..5772c909c0 100644
--- a/test/AddCombinedRowToTableTest.cpp
+++ b/test/AddCombinedRowToTableTest.cpp
@@ -103,12 +103,13 @@ TEST(AddCombinedRowToTable, UndefInInput) {
 // _______________________________________________________________________________
 TEST(AddCombinedRowToTable, setInput) {
   auto testWithBufferSize = [](size_t bufferSize) {
-    auto result = makeIdTableFromVector({});
-    result.setNumColumns(2);
     {
-      auto adder = ad_utility::AddCombinedRowToIdTable(
-          1, std::nullopt, std::nullopt, std::move(result), bufferSize);
-      // It is okay to flush even if no inputs were specified, as long as we haven't pushed any rows yet.
+      auto result = makeIdTableFromVector({});
+      result.setNumColumns(2);
+      auto adder =
+          ad_utility::AddCombinedRowToIdTable(1, std::move(result), bufferSize);
+      // It is okay to flush even if no inputs were specified, as long as we
+      // haven't pushed any rows yet.
       EXPECT_NO_THROW(adder.flush());
       if constexpr (ad_utility::areExpensiveChecksEnabled()) {
         EXPECT_ANY_THROW(adder.addRow(0, 0));
@@ -117,10 +118,13 @@ TEST(AddCombinedRowToTable, setInput) {
         EXPECT_ANY_THROW(adder.flush());
       }
     }
-    auto adder = ad_utility::AddCombinedRowToIdTable(
-        1, std::nullopt, std::nullopt, std::move(result), bufferSize);
+
+    auto result = makeIdTableFromVector({});
+    result.setNumColumns(3);
+    auto adder =
+        ad_utility::AddCombinedRowToIdTable(1, std::move(result), bufferSize);
     auto left = makeIdTableFromVector({{U, 5}, {2, U}, {3, U}, {4, U}});
-    auto right = makeIdTableFromVector({{1}, {3}, {4}, {U}});
+    auto right = makeIdTableFromVector({{1, 2}, {3, 4}, {4, 7}, {U, 8}});
     adder.setInput(left, right);
     adder.addRow(0, 0);
     adder.addRow(0, 1);
@@ -137,11 +141,46 @@ TEST(AddCombinedRowToTable, setInput) {
     adder.addRow(3, 0);
     result = std::move(adder).resultTable();
 
-    auto expected =
-        makeIdTableFromVector({{1, 5}, {3, 5}, {3, U}, {4, 5}, {4, U}, {U, 5}});
+    auto expected = makeIdTableFromVector({{1, 5, 2},
+                                           {3, 5, 4},
+                                           {3, U, 4},
+                                           {4, 5, 7},
+                                           {4, U, 7},
+                                           {U, 5, 8},
+                                           {1, 2, 5},
+                                           {3, 4, 5},
+                                           {3, 4, U},
+                                           {4, 7, 5},
+                                           {4, 7, U},
+                                           {U, 8, 5}});
     ASSERT_EQ(result, expected);
   };
   testWithBufferSize(100'000);
   testWithBufferSize(1);
   testWithBufferSize(2);
 }
+
+// _______________________________________________________________________________
+TEST(AddCombinedRowToTable, cornerCases) {
+  auto testWithBufferSize = [](size_t bufferSize) {
+    auto result = makeIdTableFromVector({});
+    result.setNumColumns(3);
+    auto adder =
+        ad_utility::AddCombinedRowToIdTable(2, std::move(result), bufferSize);
+    auto left = makeIdTableFromVector({{U, 5}, {2, U}, {3, U}, {4, U}});
+    auto right = makeIdTableFromVector({{1, 2}, {3, 4}, {4, 7}, {U, 8}});
+    // We have specified two join columns and our inputs have two columns each,
+    // so the result should also have two columns, but it has three.
+    EXPECT_ANY_THROW(adder.setInput(left, right));
+
+    left = makeIdTableFromVector({{1}, {2}, {3}});
+
+    // Left has only one column, but we have specified two join columns.
+    EXPECT_ANY_THROW(adder.setInput(left, right));
+    // The same test with the arguments switched.
+    EXPECT_ANY_THROW(adder.setInput(right, left));
+  };
+  testWithBufferSize(100'000);
+  testWithBufferSize(1);
+  testWithBufferSize(2);
+}
diff --git a/test/JoinTest.cpp b/test/JoinTest.cpp
index 8d5197ca05..311d1fcc72 100644
--- a/test/JoinTest.cpp
+++ b/test/JoinTest.cpp
@@ -242,3 +242,62 @@ TEST(JoinTest, joinWithFullScanPSO) {
   // A `Join` of two full scans is not supported.
   EXPECT_ANY_THROW(Join(qec, fullScanPSO, fullScanPSO, 0, 0));
 }
+
+TEST(JoinTest, joinWithColumnAndScan) {
+  // TODO Add further tests and reduce the code duplication.
+  auto qec = ad_utility::testing::getQec(" 
 1.  
 2.   3.");
+  auto fullScanPSO = ad_utility::makeExecutionTree(
+      qec, Permutation::Enum::PSO,
+      SparqlTriple{Variable{"?s"}, "
", Variable{"?o"}});
+  parsedQuery::SparqlValues values;
+  values._variables.emplace_back("?s");
+  values._values.push_back({TripleComponent{""}});
+  auto valuesTree = ad_utility::makeExecutionTree(qec, values);
+
+  auto join = Join{qec, fullScanPSO, valuesTree, 0, 0};
+
+  auto res = join.getResult();
+
+  auto getId = ad_utility::testing::makeGetId(qec->getIndex());
+  auto idX = getId("");
+  auto I = ad_utility::testing::IntId;
+  auto expected = makeIdTableFromVector({{idX, I(1)}});
+  EXPECT_EQ(res->idTable(), expected);
+  VariableToColumnMap expectedVariables{
+      {Variable{"?s"}, makeAlwaysDefinedColumn(0)},
+      {Variable{"?o"}, makeAlwaysDefinedColumn(1)}};
+  EXPECT_THAT(join.getExternallyVisibleVariableColumns(),
+              ::testing::UnorderedElementsAreArray(expectedVariables));
+}
+
+TEST(JoinTest, joinTwoScans) {
+  // TODO Add further tests and reduce the code duplication.
+  auto qec = ad_utility::testing::getQec(
+      " 
 1.  
 2.   3 .   4. ");
+  auto scanP = ad_utility::makeExecutionTree(
+      qec, Permutation::Enum::PSO,
+      SparqlTriple{Variable{"?s"}, "
", Variable{"?o"}});
+  // TODO Who should catch the case that there are too many variables
+  // in common?
+  auto scanP2 = ad_utility::makeExecutionTree(
+      qec, Permutation::Enum::PSO,
+      SparqlTriple{Variable{"?s"}, "", Variable{"?q"}});
+  // TODO Also test the switched versions of everything.
+  // The arguments are automatically switched inside.
+  auto join = Join{qec, scanP2, scanP, 0, 0};
+  auto res = join.getResult();
+
+  auto getId = ad_utility::testing::makeGetId(qec->getIndex());
+  auto idX = getId("");
+  auto idX2 = getId("");
+  auto I = ad_utility::testing::IntId;
+  auto expected =
+      makeIdTableFromVector({{idX, I(3), I(1)}, {idX2, I(4), I(2)}});
+  EXPECT_EQ(res->idTable(), expected);
+  VariableToColumnMap expectedVariables{
+      {Variable{"?s"}, makeAlwaysDefinedColumn(0)},
+      {Variable{"?q"}, makeAlwaysDefinedColumn(1)},
+      {Variable{"?o"}, makeAlwaysDefinedColumn(2)}};
+  EXPECT_THAT(join.getExternallyVisibleVariableColumns(),
+              ::testing::UnorderedElementsAreArray(expectedVariables));
+}
diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp
index 6361e3c0c1..93792cc3ce 100644
--- a/test/engine/IndexScanTest.cpp
+++ b/test/engine/IndexScanTest.cpp
@@ -108,9 +108,11 @@ void testLazyScanWithColumnThrows(
         TripleComponent{entry}.toValueId(qec->getIndex().getVocab()).value());
   }
 
-  // We need this to suppress the warning about a [[nodiscard]] return value being unused.
+  // We need this to suppress the warning about a [[nodiscard]] return value
+  // being unused.
   auto makeScan = [&column, &s1]() {
-    [[maybe_unused]] auto scan = IndexScan::lazyScanForJoinOfColumnWithScan(column, s1);
+    [[maybe_unused]] auto scan =
+        IndexScan::lazyScanForJoinOfColumnWithScan(column, s1);
   };
   EXPECT_ANY_THROW(makeScan());
 }

From 0b47bc01b12b6d1dceae34690869cbdbaf85ef0b Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 09:56:09 +0200
Subject: [PATCH 122/150] Hopefully make the tests pass on MacOS without
 introducing deadlocks.

---
 src/index/CompressedRelation.cpp | 23 ++++++++++------
 src/util/ThreadSafeQueue.h       | 47 ++++++++++++++++++--------------
 2 files changed, 42 insertions(+), 28 deletions(-)

diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 013cb9a172..39231e91d6 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -113,16 +113,12 @@ CompressedRelationReader::asyncParallelBlockGenerator(
   if (beginBlock == endBlock) {
     co_return;
   }
-  // Note: It is important to define the `threads` before the `queue`. That way
-  // the joining destructor of the threads will see that the queue is finished
-  // and join.
-  std::mutex blockIteratorMutex;
-  std::vector threads;
   const size_t queueSize =
       RuntimeParameters().get<"lazy-index-scan-queue-size">();
   ad_utility::data_structures::OrderedThreadSafeQueue queue{
       queueSize};
   auto blockIterator = beginBlock;
+  std::mutex blockIteratorMutex;
   auto readAndDecompressBlock = [&]() {
     try {
       while (true) {
@@ -147,9 +143,9 @@ CompressedRelationReader::asyncParallelBlockGenerator(
         // lock. We still perform it inside the lock to avoid contention of the
         // file. On a fast SSD we could possibly change this, but this has to be
         // investigated.
-        lock.unlock();
         CompressedBlock compressedBlock =
             readCompressedBlockFromFile(block, file, columnIndices);
+        lock.unlock();
         bool pushWasSuccessful = queue.push(
             myIndex, decompressBlock(compressedBlock, block.numRows_));
         checkTimeout(timer);
@@ -164,10 +160,21 @@ CompressedRelationReader::asyncParallelBlockGenerator(
         }
       }
     } catch (...) {
-      queue.pushException(std::current_exception());
+      try {
+        queue.pushException(std::current_exception());
+      } catch (...) {
+        queue.finish();
+      }
     }
   };
-
+  // Note: The order of the following declarations is very important at the time
+  // of destruction: First, the `Cleanup` is destroyed, which finishes the
+  // queue. This allows the destructor of `threads` to join the threads, as the
+  // threads are able to complete once they cannot access the queue anymore.
+  // Only then the `blockIteratorMutex` and the `queue` can be safely destroyed,
+  // as the threads which were using them have already been destroyed.
+  std::vector threads;
+  absl::Cleanup finishQueue{[&queue] { queue.finish(); }};
   const size_t numThreads =
       RuntimeParameters().get<"lazy-index-scan-num-threads">();
   for ([[maybe_unused]] auto j : std::views::iota(0u, numThreads)) {
diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index bf584ce6da..09b552e715 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -20,7 +20,11 @@ class ThreadSafeQueue {
   std::mutex mutex_;
   std::condition_variable pushNotification_;
   std::condition_variable popNotification_;
-  bool finish_ = false;
+  // Note: Although this class is generally synchronized via `std::mutex`, we
+  // still use `std::atomic` for the information whether it has finished. This
+  // allows the `finish()` function to be noexcept which allows a safe way to
+  // prevent deadlocks.
+  std::atomic_flag finish_ = ATOMIC_FLAG_INIT;
   size_t maxSize_;
 
  public:
@@ -39,8 +43,8 @@ class ThreadSafeQueue {
   bool push(T value) {
     std::unique_lock lock{mutex_};
     popNotification_.wait(
-        lock, [this] { return queue_.size() < maxSize_ || finish_; });
-    if (finish_) {
+        lock, [this] { return queue_.size() < maxSize_ || finish_.test(); });
+    if (finish_.test()) {
       return false;
     }
     queue_.push(std::move(value));
@@ -55,7 +59,7 @@ class ThreadSafeQueue {
   void pushException(std::exception_ptr exception) {
     std::unique_lock lock{mutex_};
     pushedException_ = std::move(exception);
-    finish_ = true;
+    finish_.test_and_set();
     lock.unlock();
     pushNotification_.notify_all();
     popNotification_.notify_all();
@@ -68,10 +72,11 @@ class ThreadSafeQueue {
   // function can be called from the producing/pushing threads to signal that
   // all elements have been pushed, or from the consumers to signal that they
   // will not pop further elements from the queue.
-  void finish() {
-    std::unique_lock lock{mutex_};
-    finish_ = true;
-    lock.unlock();
+  void finish() noexcept {
+    // It is crucial that this function never throws, so that we can safely call
+    // it unconditionally in destructors to prevent deadlocks. Should the
+    // implementation ever change, make sure that it is still `noexcept`.
+    finish_.test_and_set();
     pushNotification_.notify_all();
     popNotification_.notify_all();
   }
@@ -88,12 +93,12 @@ class ThreadSafeQueue {
   std::optional pop() {
     std::unique_lock lock{mutex_};
     pushNotification_.wait(lock, [this] {
-      return !queue_.empty() || finish_ || pushedException_;
+      return !queue_.empty() || finish_.test() || pushedException_;
     });
     if (pushedException_) {
       std::rethrow_exception(pushedException_);
     }
-    if (finish_ && queue_.empty()) {
+    if (finish_.test() && queue_.empty()) {
       return {};
     }
     std::optional value = std::move(queue_.front());
@@ -120,7 +125,9 @@ class OrderedThreadSafeQueue {
   std::condition_variable cv_;
   ThreadSafeQueue queue_;
   size_t nextIndex_ = 0;
-  bool finish_ = false;
+  // For the reason why this is `atomic_flag`, see the same member in
+  // `ThreadSafeQueue`.
+  std::atomic_flag finish_ = ATOMIC_FLAG_INIT;
 
  public:
   // Construct from the maximal queue size (see `ThreadSafeQueue` for details).
@@ -139,8 +146,9 @@ class OrderedThreadSafeQueue {
   // equal to `ThreadSafeQueue::push`.
   bool push(size_t index, T value) {
     std::unique_lock lock{mutex_};
-    cv_.wait(lock, [this, index]() { return index == nextIndex_ || finish_; });
-    if (finish_) {
+    cv_.wait(lock,
+             [this, index]() { return index == nextIndex_ || finish_.test(); });
+    if (finish_.test()) {
       return false;
     }
     ++nextIndex_;
@@ -152,19 +160,18 @@ class OrderedThreadSafeQueue {
 
   // See `ThreadSafeQueue` for details.
   void pushException(std::exception_ptr exception) {
-    std::unique_lock l{mutex_};
     queue_.pushException(std::move(exception));
-    finish_ = true;
-    l.unlock();
+    finish_.test_and_set();
     cv_.notify_all();
   }
 
   // See `ThreadSafeQueue` for details.
-  void finish() {
+  void finish() noexcept {
+    // It is crucial that this function never throws, so that we can safely call
+    // it unconditionally in destructors to prevent deadlocks. Should the
+    // implementation ever change, make sure that it is still `noexcept`.
     queue_.finish();
-    std::unique_lock lock{mutex_};
-    finish_ = true;
-    lock.unlock();
+    finish_.test_and_set();
     cv_.notify_all();
   }
 

From 033c73f4cd43494c95779f68c1850b91657ae206 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 12:30:14 +0200
Subject: [PATCH 123/150] Several additional changes, excluding the ones for
 updating the status.

---
 src/engine/IndexScan.cpp         |  10 +--
 src/engine/Join.cpp              |  91 +++++++++++------------
 src/engine/Join.h                |   8 +-
 src/index/CompressedRelation.cpp |  33 +++++----
 src/index/CompressedRelation.h   |   4 +-
 test/ExceptionTest.cpp           |   8 ++
 test/JoinTest.cpp                | 122 ++++++++++++++++++-------------
 7 files changed, 153 insertions(+), 123 deletions(-)

diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp
index c74224a2db..8bb9f53b8d 100644
--- a/src/engine/IndexScan.cpp
+++ b/src/engine/IndexScan.cpp
@@ -324,7 +324,7 @@ IndexScan::lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2) {
   };
 
   AD_CONTRACT_CHECK(getFirstVariable(s1) == getFirstVariable(s2));
-  if (s1.numVariables_ == 2 && s1.numVariables_ == 2) {
+  if (s1.numVariables_ == 2 && s2.numVariables_ == 2) {
     AD_CONTRACT_CHECK(*s1.getPermutedTriple()[2] != *s2.getPermutedTriple()[2]);
   }
 
@@ -338,10 +338,8 @@ IndexScan::lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2) {
       metaBlocks1.value(), metaBlocks2.value());
 
   std::array result{getLazyScan(s1, blocks1), getLazyScan(s2, blocks2)};
-  result[0].details().numBlocksTotal_ =
-      metaBlocks1.value().blockMetadata_.size();
-  result[1].details().numBlocksTotal_ =
-      metaBlocks2.value().blockMetadata_.size();
+  result[0].details().numBlocksAll_ = metaBlocks1.value().blockMetadata_.size();
+  result[1].details().numBlocksAll_ = metaBlocks2.value().blockMetadata_.size();
   return result;
 }
 
@@ -360,6 +358,6 @@ Permutation::IdTableGenerator IndexScan::lazyScanForJoinOfColumnWithScan(
                                                            metaBlocks1.value());
 
   auto result = getLazyScan(s, blocks);
-  result.details().numBlocksTotal_ = metaBlocks1.value().blockMetadata_.size();
+  result.details().numBlocksAll_ = metaBlocks1.value().blockMetadata_.size();
   return result;
 }
diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 5d8a962ba4..981f22f082 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -122,8 +122,7 @@ ResultTable Join::computeResult() {
     if (rightResIfCached && !leftResIfCached) {
       idTable = computeResultForIndexScanAndIdTable(
           rightResIfCached->idTable(), _rightJoinCol,
-          dynamic_cast(*_left->getRootOperation()),
-          _leftJoinCol);
+          dynamic_cast(*_left->getRootOperation()), _leftJoinCol);
       return {std::move(idTable), resultSortedOn(), LocalVocab{}};
 
     } else if (!leftResIfCached) {
@@ -151,8 +150,7 @@ ResultTable Join::computeResult() {
   if (_right->getType() == QueryExecutionTree::SCAN && !rightResIfCached) {
     idTable = computeResultForIndexScanAndIdTable(
         leftRes->idTable(), _leftJoinCol,
-        dynamic_cast(*_right->getRootOperation()),
-        _rightJoinCol);
+        dynamic_cast(*_right->getRootOperation()), _rightJoinCol);
     return {std::move(idTable), resultSortedOn(),
             leftRes->getSharedLocalVocab()};
   }
@@ -245,7 +243,8 @@ ResultTable Join::computeResultForJoinWithFullScanDummy() {
   IdTable idTable{getExecutionContext()->getAllocator()};
   LOG(DEBUG) << "Join by making multiple scans..." << endl;
   AD_CORRECTNESS_CHECK(!isFullScanDummy(_left) && isFullScanDummy(_right));
-  _right->getRootOperation()->updateRuntimeInformationWhenOptimizedOut({});
+  _right->getRootOperation()->updateRuntimeInformationWhenOptimizedOut(
+      {}, RuntimeInformation::Status::optimizedOut);
   idTable.setNumColumns(_left->getResultWidth() + 2);
 
   shared_ptr nonDummyRes = _left->getResult();
@@ -657,16 +656,17 @@ convertGenerator(Permutation::IdTableGenerator gen) {
   }
 }
 
-// TODO Add the information about the total number of blocks.
+// Set the runtime info of the `scanTree` when it was lazily executed during a
+// join.
 void updateRuntimeInfoForLazyScan(
-    QueryExecutionTree& scanTree,
+    IndexScan& scanTree,
     const CompressedRelationReader::LazyScanMetadata& metadata) {
-  scanTree.getRootOperation()->updateRuntimeInformationWhenOptimizedOut();
-  auto& rti = scanTree.getRootOperation()->getRuntimeInfo();
+  scanTree.updateRuntimeInformationWhenOptimizedOut();
+  auto& rti = scanTree.getRuntimeInfo();
   rti.numRows_ = metadata.numElementsRead_;
   rti.totalTime_ = static_cast(metadata.blockingTimeMs_);
   rti.addDetail("num-blocks-read", metadata.numBlocksRead_);
-  rti.addDetail("num-blocks-total", metadata.numBlocksTotal_);
+  rti.addDetail("num-blocks-all", metadata.numBlocksAll_);
 }
 }  // namespace
 
@@ -681,11 +681,12 @@ IdTable Join::computeResultForTwoIndexScans() {
   ad_utility::AddCombinedRowToIdTable rowAdder{
       1, IdTable{getResultWidth(), getExecutionContext()->getAllocator()}};
 
+  auto& leftScan = dynamic_cast(*_left->getRootOperation());
+  auto& rightScan = dynamic_cast(*_right->getRootOperation());
+
   ad_utility::Timer timer{ad_utility::timer::Timer::InitialStatus::Started};
   auto [leftBlocksInternal, rightBlocksInternal] =
-      IndexScan::lazyScanForJoinOfTwoScans(
-          dynamic_cast(*_left->getRootOperation()),
-          dynamic_cast(*_right->getRootOperation()));
+      IndexScan::lazyScanForJoinOfTwoScans(leftScan, rightScan);
   getRuntimeInfo().addDetail("time-for-filtering-blocks", timer.msecs());
 
   auto leftBlocks = convertGenerator(std::move(leftBlocksInternal));
@@ -694,66 +695,58 @@ IdTable Join::computeResultForTwoIndexScans() {
   ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks,
                                               std::less{}, rowAdder);
 
-  updateRuntimeInfoForLazyScan(*_left, leftBlocks.details());
-  updateRuntimeInfoForLazyScan(*_right, rightBlocks.details());
+  updateRuntimeInfoForLazyScan(leftScan, leftBlocks.details());
+  updateRuntimeInfoForLazyScan(rightScan, rightBlocks.details());
 
   return std::move(rowAdder).resultTable();
 }
 
 // ______________________________________________________________________________________________________
 template 
-IdTable Join::computeResultForIndexScanAndIdTable(
-    const IdTable& idTable, ColumnIndex joinColumnIndexIdTable,
-    const IndexScan& scan, ColumnIndex joinColumnIndexScan) {
+IdTable Join::computeResultForIndexScanAndIdTable(const IdTable& idTable,
+                                                  ColumnIndex joinColTable,
+                                                  IndexScan& scan,
+                                                  ColumnIndex joinColScan) {
   // We first have to permute the columns.
-  // TODO Maybe we can reduce the complexity for the
-  // `idTableIsRightInput` switch.
-  auto jcLeft =
-      idTableIsRightInput ? joinColumnIndexScan : joinColumnIndexIdTable;
-  auto jcRight =
-      idTableIsRightInput ? joinColumnIndexIdTable : joinColumnIndexScan;
-  size_t numColsLeft =
-      idTableIsRightInput ? scan.getResultWidth() : idTable.numColumns();
-  size_t numColsRight =
-      idTableIsRightInput ? idTable.numColumns() : scan.getResultWidth();
+  auto [jcLeft, jcRight, numColsLeft, numColsRight] = [&]() {
+    return idTableIsRightInput
+               ? std::tuple{joinColScan, joinColTable, scan.getResultWidth(),
+                            idTable.numColumns()}
+               : std::tuple{joinColTable, joinColScan, idTable.numColumns(),
+                            scan.getResultWidth()};
+  }();
 
   auto joinColMap = ad_utility::JoinColumnMapping{
       {{jcLeft, jcRight}}, numColsLeft, numColsRight};
   ad_utility::AddCombinedRowToIdTable rowAdder{
       1, IdTable{getResultWidth(), getExecutionContext()->getAllocator()}};
 
-  AD_CORRECTNESS_CHECK(joinColumnIndexScan == 0);
-  auto permutation = [&]() {
-    if (idTableIsRightInput) {
-      return ad_utility::IdTableAndFirstCol{
-          idTable.asColumnSubsetView(joinColMap.permutationRight())};
-    } else {
-      return ad_utility::IdTableAndFirstCol{
-          idTable.asColumnSubsetView(joinColMap.permutationLeft())};
-    }
-  }();
+  AD_CORRECTNESS_CHECK(joinColScan == 0);
+  auto permutationIdTable =
+      ad_utility::IdTableAndFirstCol{idTable.asColumnSubsetView(
+          idTableIsRightInput ? joinColMap.permutationRight()
+                              : joinColMap.permutationLeft())};
 
   ad_utility::Timer timer{ad_utility::timer::Timer::InitialStatus::Started};
-  auto rightBlocksInternal =
-      IndexScan::lazyScanForJoinOfColumnWithScan(permutation.col(), scan);
+  auto rightBlocksInternal = IndexScan::lazyScanForJoinOfColumnWithScan(
+      permutationIdTable.col(), scan);
   auto rightBlocks = convertGenerator(std::move(rightBlocksInternal));
 
   getRuntimeInfo().addDetail("time-for-filtering-blocks", timer.msecs());
 
-  auto projection = std::identity{};
+  auto doJoin = [&rowAdder](auto& left, auto& right) mutable {
+    ad_utility::zipperJoinForBlocksWithoutUndef(left, right, std::less{},
+                                                rowAdder);
+  };
+  auto blockForIdTable = std::span{&permutationIdTable, 1};
   if (idTableIsRightInput) {
-    ad_utility::zipperJoinForBlocksWithoutUndef(
-        rightBlocks, std::span{&permutation, 1}, std::less{}, rowAdder,
-        projection, projection);
+    doJoin(rightBlocks, blockForIdTable);
   } else {
-    ad_utility::zipperJoinForBlocksWithoutUndef(
-        std::span{&permutation, 1}, rightBlocks, std::less{}, rowAdder,
-        projection, projection);
+    doJoin(blockForIdTable, rightBlocks);
   }
   auto result = std::move(rowAdder).resultTable();
   result.setColumnSubset(joinColMap.permutationResult());
 
-  auto& scanTree = idTableIsRightInput ? _left : _right;
-  updateRuntimeInfoForLazyScan(*scanTree, rightBlocks.details());
+  updateRuntimeInfoForLazyScan(scan, rightBlocks.details());
   return result;
 }
diff --git a/src/engine/Join.h b/src/engine/Join.h
index 8eaf446357..6a453810e8 100644
--- a/src/engine/Join.h
+++ b/src/engine/Join.h
@@ -144,10 +144,10 @@ class Join : public Operation {
   // is the left or the right child of this `Join`. This needs to be known to
   // determine the correct order of the columns in the result.
   template 
-  IdTable computeResultForIndexScanAndIdTable(const IdTable& itIdTable,
-                                              ColumnIndex itIndexScan,
-                                              const IndexScan& scan,
-                                              ColumnIndex joinColumnIndexScan);
+  IdTable computeResultForIndexScanAndIdTable(const IdTable& idTable,
+                                              ColumnIndex joinColTable,
+                                              IndexScan& scan,
+                                              ColumnIndex joinColScan);
 
   using ScanMethodType = std::function;
 
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 39231e91d6..381a0a02e9 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -108,7 +108,7 @@ CompressedRelationReader::IdTableGenerator
 CompressedRelationReader::asyncParallelBlockGenerator(
     auto beginBlock, auto endBlock, ad_utility::File& file,
     std::optional> columnIndices,
-    const TimeoutTimer& timer) const {
+    TimeoutTimer timer) const {
   LazyScanMetadata& details = co_await cppcoro::getDetails;
   if (beginBlock == endBlock) {
     co_return;
@@ -334,24 +334,31 @@ std::vector CompressedRelationReader::getBlocksForJoin(
                              const CompressedBlockMetadata& block, Id id) {
     return getRelevantIdFromTriple(block.lastTriple_, metadataAndBlocks) < id;
   };
-  auto lessThan =
-      ad_utility::OverloadCallOperator{idLessThanBlock, blockLessThanId};
+
+  // `blockLessThanBlock` (a dummy) and `std::less` are only needed to
+  // fulfill a concept for the `std::ranges` algorithms.
+  auto blockLessThanBlock =
+      [](const CompressedBlockMetadata&,
+                            const CompressedBlockMetadata&)
+          ->bool {
+    static_assert(ad_utility::alwaysFalse);
+  };
+  auto lessThan = ad_utility::OverloadCallOperator{
+      idLessThanBlock, blockLessThanId, blockLessThanBlock, std::less{}};
 
   // Find the matching blocks by performing binary search on the `joinColumn`.
   // Note that it is tempting to reuse the `zipperJoinWithUndef` routine, but
   // this doesn't work because the implicit equality defined by `!lessThan(a,b)
   // && !lessThan(b, a)` is not transitive.
   std::vector result;
-  for (const auto& block : relevantBlocks) {
-    auto rng =
-        std::equal_range(joinColumn.begin(), joinColumn.end(), block, lessThan);
-    if (rng.first != rng.second) {
-      result.push_back(block);
-    }
-  }
-  // The following check shouldn't be too expensive as there are only few
-  // blocks.
-  AD_CORRECTNESS_CHECK(std::ranges::unique(result).begin() == result.end());
+
+  auto blockIsNeeded = [&joinColumn, &lessThan](const auto& block) {
+    return !std::ranges::equal_range(joinColumn, block, lessThan).empty();
+  };
+  std::ranges::copy(relevantBlocks | std::views::filter(blockIsNeeded),
+                    std::back_inserter(result));
+  // The following check is cheap as there are only few blocks.
+  AD_CORRECTNESS_CHECK(std::ranges::unique(result).empty());
   return result;
 }
 
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index a371f09c77..7447235937 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -241,7 +241,7 @@ class CompressedRelationReader {
 
   struct LazyScanMetadata {
     size_t numBlocksRead_ = 0;
-    size_t numBlocksTotal_ = 0;
+    size_t numBlocksAll_ = 0;
     size_t numElementsRead_ = 0;
     size_t blockingTimeMs_ = 0;
   };
@@ -425,7 +425,7 @@ class CompressedRelationReader {
   IdTableGenerator asyncParallelBlockGenerator(
       auto beginBlock, auto endBlock, ad_utility::File& file,
       std::optional> columnIndices,
-      const TimeoutTimer& timer) const;
+      TimeoutTimer timer) const;
 
   // A helper function to abstract away the timeout check:
   static void checkTimeout(
diff --git a/test/ExceptionTest.cpp b/test/ExceptionTest.cpp
index 28f17f0193..2539d07892 100644
--- a/test/ExceptionTest.cpp
+++ b/test/ExceptionTest.cpp
@@ -92,5 +92,13 @@ TEST(Exception, AD_FAIL) {
 TEST(Excpetion, AD_EXPENSIVE_CHECK) {
 #if (!defined(NDEBUG) || defined(AD_ENABLE_EXPENSIVE_CHECKS))
   ASSERT_ANY_THROW(AD_EXPENSIVE_CHECK(3 > 5));
+#else
+  ASSERT_NO_THROW(AD_EXPENSIVE_CHECK(3 > 5));
 #endif
+
+  if (ad_utility::areExpensiveChecksEnabled()) {
+    ASSERT_ANY_THROW(AD_EXPENSIVE_CHECK(3 > 5));
+  } else {
+    ASSERT_NO_THROW(AD_EXPENSIVE_CHECK(3 > 5));
+  }
 }
diff --git a/test/JoinTest.cpp b/test/JoinTest.cpp
index 311d1fcc72..89c0e44195 100644
--- a/test/JoinTest.cpp
+++ b/test/JoinTest.cpp
@@ -207,97 +207,121 @@ TEST(JoinTest, joinTest) {
   runTestCasesForAllJoinAlgorithms(createJoinTestSet());
 };
 
+namespace {
+
+using ExpectedColumns = ad_utility::HashMap<
+    Variable,
+    std::pair, ColumnIndexAndTypeInfo::UndefStatus>>;
+void testJoinOperation(Join& join, const ExpectedColumns& expected) {
+  auto res = join.getResult();
+  const auto& varToCols = join.getExternallyVisibleVariableColumns();
+  EXPECT_EQ(varToCols.size(), expected.size());
+  const auto& table = res->idTable();
+  ASSERT_EQ(table.numColumns(), expected.size());
+  for (const auto& [var, columnAndStatus] : expected) {
+    const auto& [colIndex, undefStatus] = varToCols.at(var);
+    decltype(auto) column = table.getColumn(colIndex);
+    EXPECT_EQ(undefStatus, columnAndStatus.second);
+    EXPECT_THAT(column, ::testing::ElementsAreArray(columnAndStatus.first))
+        << "Columns for variable " << var.name() << " did not match";
+  }
+}
+
+ExpectedColumns makeExpectedColumns(const VariableToColumnMap& varToColMap,
+                                    const IdTable& table) {
+  ExpectedColumns result;
+  for (const auto& [var, colIndexAndStatus] : varToColMap) {
+    result[var] = {table.getColumn(colIndexAndStatus.columnIndex_),
+                   colIndexAndStatus.mightContainUndef_};
+  }
+  return result;
+}
+
+std::shared_ptr makeValuesForSingleVariable(
+    QueryExecutionContext* qec, std::string var,
+    std::vector values) {
+  parsedQuery::SparqlValues sparqlValues;
+  sparqlValues._variables.emplace_back(var);
+  for (const auto& value : values) {
+    sparqlValues._values.push_back({TripleComponent{value}});
+  }
+  return ad_utility::makeExecutionTree(qec, sparqlValues);
+}
+
+using enum Permutation::Enum;
+auto I = ad_utility::testing::IntId;
+using Var = Variable;
+}  // namespace
+
 TEST(JoinTest, joinWithFullScanPSO) {
   auto qec = ad_utility::testing::getQec(" 
 1.   2.   3.");
   // Expressions in HAVING clauses are converted to special internal aliases.
   // Test the combination of parsing and evaluating such queries.
   auto fullScanPSO = ad_utility::makeExecutionTree(
-      qec, Permutation::Enum::PSO,
-      SparqlTriple{Variable{"?s"}, "?p", Variable{"?o"}});
-  parsedQuery::SparqlValues values;
-  values._variables.emplace_back("?p");
-  values._values.push_back({TripleComponent{""}});
-  values._values.push_back({TripleComponent{""}});
-  auto valuesTree = ad_utility::makeExecutionTree(qec, values);
+      qec, PSO, SparqlTriple{Var{"?s"}, "?p", Var{"?o"}});
+  auto valuesTree = makeValuesForSingleVariable(qec, "?p", {"", ""});
 
   auto join = Join{qec, fullScanPSO, valuesTree, 0, 0};
 
-  auto res = join.getResult();
-
-  auto getId = ad_utility::testing::makeGetId(qec->getIndex());
-
-  auto idX = getId("");
-  auto idA = getId("");
-  auto idO = getId("");
-  auto I = ad_utility::testing::IntId;
-  auto expected = makeIdTableFromVector({{idA, idX, I(3)}, {idO, idX, I(2)}});
-  EXPECT_EQ(res->idTable(), expected);
+  auto id = ad_utility::testing::makeGetId(qec->getIndex());
+  auto expected = makeIdTableFromVector(
+      {{id(""), id(""), I(3)}, {id(""), id(""), I(2)}});
   VariableToColumnMap expectedVariables{
       {Variable{"?p"}, makeAlwaysDefinedColumn(0)},
       {Variable{"?s"}, makeAlwaysDefinedColumn(1)},
       {Variable{"?o"}, makeAlwaysDefinedColumn(2)}};
-  EXPECT_THAT(join.getExternallyVisibleVariableColumns(),
-              ::testing::UnorderedElementsAreArray(expectedVariables));
+  testJoinOperation(join, makeExpectedColumns(expectedVariables, expected));
+
+  auto joinSwitched = Join{qec, valuesTree, fullScanPSO, 0, 0};
+  testJoinOperation(joinSwitched,
+                    makeExpectedColumns(expectedVariables, expected));
 
   // A `Join` of two full scans is not supported.
   EXPECT_ANY_THROW(Join(qec, fullScanPSO, fullScanPSO, 0, 0));
 }
 
 TEST(JoinTest, joinWithColumnAndScan) {
-  // TODO Add further tests and reduce the code duplication.
   auto qec = ad_utility::testing::getQec(" 
 1.  
 2.   3.");
   auto fullScanPSO = ad_utility::makeExecutionTree(
-      qec, Permutation::Enum::PSO,
-      SparqlTriple{Variable{"?s"}, "
", Variable{"?o"}});
-  parsedQuery::SparqlValues values;
-  values._variables.emplace_back("?s");
-  values._values.push_back({TripleComponent{""}});
-  auto valuesTree = ad_utility::makeExecutionTree(qec, values);
+      qec, PSO, SparqlTriple{Var{"?s"}, "
", Var{"?o"}});
+  auto valuesTree = makeValuesForSingleVariable(qec, "?s", {""});
 
   auto join = Join{qec, fullScanPSO, valuesTree, 0, 0};
 
-  auto res = join.getResult();
-
   auto getId = ad_utility::testing::makeGetId(qec->getIndex());
   auto idX = getId("");
-  auto I = ad_utility::testing::IntId;
   auto expected = makeIdTableFromVector({{idX, I(1)}});
-  EXPECT_EQ(res->idTable(), expected);
   VariableToColumnMap expectedVariables{
       {Variable{"?s"}, makeAlwaysDefinedColumn(0)},
       {Variable{"?o"}, makeAlwaysDefinedColumn(1)}};
-  EXPECT_THAT(join.getExternallyVisibleVariableColumns(),
-              ::testing::UnorderedElementsAreArray(expectedVariables));
+  testJoinOperation(join, makeExpectedColumns(expectedVariables, expected));
+
+  auto joinSwitched = Join{qec, valuesTree, fullScanPSO, 0, 0};
+  testJoinOperation(joinSwitched,
+                    makeExpectedColumns(expectedVariables, expected));
 }
 
 TEST(JoinTest, joinTwoScans) {
-  // TODO Add further tests and reduce the code duplication.
   auto qec = ad_utility::testing::getQec(
       " 
 1.  
 2.   3 .   4. ");
   auto scanP = ad_utility::makeExecutionTree(
-      qec, Permutation::Enum::PSO,
-      SparqlTriple{Variable{"?s"}, "
", Variable{"?o"}});
+      qec, PSO, SparqlTriple{Var{"?s"}, "
", Var{"?o"}});
   // TODO Who should catch the case that there are too many variables
   // in common?
   auto scanP2 = ad_utility::makeExecutionTree(
-      qec, Permutation::Enum::PSO,
-      SparqlTriple{Variable{"?s"}, "", Variable{"?q"}});
-  // TODO Also test the switched versions of everything.
-  // The arguments are automatically switched inside.
+      qec, PSO, SparqlTriple{Var{"?s"}, "", Var{"?q"}});
   auto join = Join{qec, scanP2, scanP, 0, 0};
-  auto res = join.getResult();
 
-  auto getId = ad_utility::testing::makeGetId(qec->getIndex());
-  auto idX = getId("");
-  auto idX2 = getId("");
-  auto I = ad_utility::testing::IntId;
-  auto expected =
-      makeIdTableFromVector({{idX, I(3), I(1)}, {idX2, I(4), I(2)}});
-  EXPECT_EQ(res->idTable(), expected);
+  auto id = ad_utility::testing::makeGetId(qec->getIndex());
+  auto expected = makeIdTableFromVector(
+      {{id(""), I(3), I(1)}, {id(""), I(4), I(2)}});
   VariableToColumnMap expectedVariables{
       {Variable{"?s"}, makeAlwaysDefinedColumn(0)},
       {Variable{"?q"}, makeAlwaysDefinedColumn(1)},
       {Variable{"?o"}, makeAlwaysDefinedColumn(2)}};
-  EXPECT_THAT(join.getExternallyVisibleVariableColumns(),
-              ::testing::UnorderedElementsAreArray(expectedVariables));
+  testJoinOperation(join, makeExpectedColumns(expectedVariables, expected));
+
+  auto joinSwitched = Join{qec, scanP2, scanP, 0, 0};
+  testJoinOperation(joinSwitched,
+                    makeExpectedColumns(expectedVariables, expected));
 }

From 081b07663526a84f5fe938590a414626b57460f2 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 12:30:56 +0200
Subject: [PATCH 124/150] Include functionality for explicitly setting the
 status of an operation that was "optimized out" or otherwise executed by
 other means than its `getResult()` method.

---
 src/engine/Operation.cpp          | 13 ++++++++-----
 src/engine/Operation.h            |  8 ++++++--
 src/engine/RuntimeInformation.cpp |  5 +++--
 src/engine/RuntimeInformation.h   |  1 +
 4 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/engine/Operation.cpp b/src/engine/Operation.cpp
index caef4c3889..02eb672ef7 100644
--- a/src/engine/Operation.cpp
+++ b/src/engine/Operation.cpp
@@ -257,8 +257,9 @@ void Operation::updateRuntimeInformationOnSuccess(
 
 // _____________________________________________________________________________
 void Operation::updateRuntimeInformationWhenOptimizedOut(
-    std::vector children) {
-  _runtimeInfo.status_ = RuntimeInformation::Status::optimizedOut;
+    std::vector children,
+    RuntimeInformation::Status status) {
+  _runtimeInfo.status_ = status;
   _runtimeInfo.children_ = std::move(children);
   // This operation was optimized out, so its operation time is zero.
   // The operation time is computed as
@@ -271,9 +272,11 @@ void Operation::updateRuntimeInformationWhenOptimizedOut(
 }
 
 // _____________________________________________________________________________
-void Operation::updateRuntimeInformationWhenOptimizedOut() {
-  auto setStatus = [](RuntimeInformation& rti, const auto& self) -> void {
-    rti.status_ = RuntimeInformation::Status::optimizedOut;
+void Operation::updateRuntimeInformationWhenOptimizedOut(
+    RuntimeInformation::Status status) {
+  auto setStatus = [&status](RuntimeInformation& rti,
+                             const auto& self) -> void {
+    rti.status_ = status;
     rti.totalTime_ = 0;
     for (auto& child : rti.children_) {
       self(child, self);
diff --git a/src/engine/Operation.h b/src/engine/Operation.h
index 5202bfa2c6..391ad78b66 100644
--- a/src/engine/Operation.h
+++ b/src/engine/Operation.h
@@ -253,14 +253,18 @@ class Operation {
   // children were evaluated nevertheless. For an example usage of this feature
   // see `GroupBy.cpp`
   virtual void updateRuntimeInformationWhenOptimizedOut(
-      std::vector children);
+      std::vector children,
+      RuntimeInformation::Status status =
+          RuntimeInformation::Status::optimizedOut);
 
   // Use the already stored runtime info for the children,
   // but set all of them to `optimizedOut`. This can be used, when a complete
   // tree was optimized out. For example when one child of a JOIN operation is
   // empty, the result will be empty, and it is not necessary to evaluate the
   // other child.
-  virtual void updateRuntimeInformationWhenOptimizedOut();
+  virtual void updateRuntimeInformationWhenOptimizedOut(
+      RuntimeInformation::Status status =
+          RuntimeInformation::Status::optimizedOut);
 
  private:
   // Create the runtime information in case the evaluation of this operation has
diff --git a/src/engine/RuntimeInformation.cpp b/src/engine/RuntimeInformation.cpp
index 366c0947eb..6af2e94287 100644
--- a/src/engine/RuntimeInformation.cpp
+++ b/src/engine/RuntimeInformation.cpp
@@ -141,6 +141,8 @@ std::string_view RuntimeInformation::toString(Status status) {
   switch (status) {
     case completed:
       return "completed";
+    case lazilyCompleted:
+      return "lazily completed";
     case notStarted:
       return "not started";
     case optimizedOut:
@@ -149,9 +151,8 @@ std::string_view RuntimeInformation::toString(Status status) {
       return "failed";
     case failedBecauseChildFailed:
       return "failed because child failed";
-    default:
-      AD_FAIL();
   }
+  AD_FAIL();
 }
 
 // ________________________________________________________________________________________________________________
diff --git a/src/engine/RuntimeInformation.h b/src/engine/RuntimeInformation.h
index 1479023752..e12be2a3be 100644
--- a/src/engine/RuntimeInformation.h
+++ b/src/engine/RuntimeInformation.h
@@ -27,6 +27,7 @@ class RuntimeInformation {
   enum struct Status {
     notStarted,
     completed,
+    lazilyCompleted,
     optimizedOut,
     failed,
     failedBecauseChildFailed

From 2406aec43e9593248022c3089544024f267a7f2c Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 12:32:49 +0200
Subject: [PATCH 125/150] Actually use the new status.

---
 src/engine/Join.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 981f22f082..f1099af7c8 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -244,7 +244,7 @@ ResultTable Join::computeResultForJoinWithFullScanDummy() {
   LOG(DEBUG) << "Join by making multiple scans..." << endl;
   AD_CORRECTNESS_CHECK(!isFullScanDummy(_left) && isFullScanDummy(_right));
   _right->getRootOperation()->updateRuntimeInformationWhenOptimizedOut(
-      {}, RuntimeInformation::Status::optimizedOut);
+      {}, RuntimeInformation::Status::lazilyCompleted);
   idTable.setNumColumns(_left->getResultWidth() + 2);
 
   shared_ptr nonDummyRes = _left->getResult();
@@ -661,7 +661,8 @@ convertGenerator(Permutation::IdTableGenerator gen) {
 void updateRuntimeInfoForLazyScan(
     IndexScan& scanTree,
     const CompressedRelationReader::LazyScanMetadata& metadata) {
-  scanTree.updateRuntimeInformationWhenOptimizedOut();
+  scanTree.updateRuntimeInformationWhenOptimizedOut(
+      RuntimeInformation::Status::lazilyCompleted);
   auto& rti = scanTree.getRuntimeInfo();
   rti.numRows_ = metadata.numElementsRead_;
   rti.totalTime_ = static_cast(metadata.blockingTimeMs_);

From ad91a03229835557d557d99a9ec3dc62a49ea3bf Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 15:20:18 +0200
Subject: [PATCH 126/150] Safer finishing of the Threadsafe queue.

---
 src/util/ThreadSafeQueue.h   | 47 +++++++++++++-----------
 test/ThreadSafeQueueTest.cpp | 70 +++++++++++++++++++++++++++++++++++-
 2 files changed, 96 insertions(+), 21 deletions(-)

diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index bf584ce6da..09b552e715 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -20,7 +20,11 @@ class ThreadSafeQueue {
   std::mutex mutex_;
   std::condition_variable pushNotification_;
   std::condition_variable popNotification_;
-  bool finish_ = false;
+  // Note: Although this class is generally synchronized via `std::mutex`, we
+  // still use `std::atomic` for the information whether it has finished. This
+  // allows the `finish()` function to be noexcept which allows a safe way to
+  // prevent deadlocks.
+  std::atomic_flag finish_ = ATOMIC_FLAG_INIT;
   size_t maxSize_;
 
  public:
@@ -39,8 +43,8 @@ class ThreadSafeQueue {
   bool push(T value) {
     std::unique_lock lock{mutex_};
     popNotification_.wait(
-        lock, [this] { return queue_.size() < maxSize_ || finish_; });
-    if (finish_) {
+        lock, [this] { return queue_.size() < maxSize_ || finish_.test(); });
+    if (finish_.test()) {
       return false;
     }
     queue_.push(std::move(value));
@@ -55,7 +59,7 @@ class ThreadSafeQueue {
   void pushException(std::exception_ptr exception) {
     std::unique_lock lock{mutex_};
     pushedException_ = std::move(exception);
-    finish_ = true;
+    finish_.test_and_set();
     lock.unlock();
     pushNotification_.notify_all();
     popNotification_.notify_all();
@@ -68,10 +72,11 @@ class ThreadSafeQueue {
   // function can be called from the producing/pushing threads to signal that
   // all elements have been pushed, or from the consumers to signal that they
   // will not pop further elements from the queue.
-  void finish() {
-    std::unique_lock lock{mutex_};
-    finish_ = true;
-    lock.unlock();
+  void finish() noexcept {
+    // It is crucial that this function never throws, so that we can safely call
+    // it unconditionally in destructors to prevent deadlocks. Should the
+    // implementation ever change, make sure that it is still `noexcept`.
+    finish_.test_and_set();
     pushNotification_.notify_all();
     popNotification_.notify_all();
   }
@@ -88,12 +93,12 @@ class ThreadSafeQueue {
   std::optional pop() {
     std::unique_lock lock{mutex_};
     pushNotification_.wait(lock, [this] {
-      return !queue_.empty() || finish_ || pushedException_;
+      return !queue_.empty() || finish_.test() || pushedException_;
     });
     if (pushedException_) {
       std::rethrow_exception(pushedException_);
     }
-    if (finish_ && queue_.empty()) {
+    if (finish_.test() && queue_.empty()) {
       return {};
     }
     std::optional value = std::move(queue_.front());
@@ -120,7 +125,9 @@ class OrderedThreadSafeQueue {
   std::condition_variable cv_;
   ThreadSafeQueue queue_;
   size_t nextIndex_ = 0;
-  bool finish_ = false;
+  // For the reason why this is `atomic_flag`, see the same member in
+  // `ThreadSafeQueue`.
+  std::atomic_flag finish_ = ATOMIC_FLAG_INIT;
 
  public:
   // Construct from the maximal queue size (see `ThreadSafeQueue` for details).
@@ -139,8 +146,9 @@ class OrderedThreadSafeQueue {
   // equal to `ThreadSafeQueue::push`.
   bool push(size_t index, T value) {
     std::unique_lock lock{mutex_};
-    cv_.wait(lock, [this, index]() { return index == nextIndex_ || finish_; });
-    if (finish_) {
+    cv_.wait(lock,
+             [this, index]() { return index == nextIndex_ || finish_.test(); });
+    if (finish_.test()) {
       return false;
     }
     ++nextIndex_;
@@ -152,19 +160,18 @@ class OrderedThreadSafeQueue {
 
   // See `ThreadSafeQueue` for details.
   void pushException(std::exception_ptr exception) {
-    std::unique_lock l{mutex_};
     queue_.pushException(std::move(exception));
-    finish_ = true;
-    l.unlock();
+    finish_.test_and_set();
     cv_.notify_all();
   }
 
   // See `ThreadSafeQueue` for details.
-  void finish() {
+  void finish() noexcept {
+    // It is crucial that this function never throws, so that we can safely call
+    // it unconditionally in destructors to prevent deadlocks. Should the
+    // implementation ever change, make sure that it is still `noexcept`.
     queue_.finish();
-    std::unique_lock lock{mutex_};
-    finish_ = true;
-    lock.unlock();
+    finish_.test_and_set();
     cv_.notify_all();
   }
 
diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 8106ebe972..281b6e1d6b 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -8,6 +8,8 @@
 #include 
 #include 
 
+#include "./util/GTestHelpers.h"
+#include "absl/cleanup/cleanup.h"
 #include "util/ThreadSafeQueue.h"
 #include "util/TypeTraits.h"
 #include "util/jthread.h"
@@ -39,7 +41,7 @@ constexpr size_t numValues = 200;
 // Run the `test` function with a `ThreadSafeQueue` and an
 // `OrderedThreadSafeQueue`. Both queues have a size of `queueSize` and `size_t`
 // as their value type.
-void runWithBothQueueTypes(const auto& testFunction) {
+void runWithBothQueueTypes(auto&& testFunction) {
   testFunction(ThreadSafeQueue{queueSize});
   testFunction(OrderedThreadSafeQueue{queueSize});
 }
@@ -248,3 +250,69 @@ TEST(ThreadSafeQueue, DisablePush) {
   };
   runWithBothQueueTypes(runTest);
 }
+
+// Demonstrate the safe way to handle exceptions and early destruction in the
+// worker threads as well as in the consumer threads. By `safe` we mean that the
+// program is neither terminated nor does it run into a deadlock.
+TEST(ThreadSafeQueue, SafeExceptionHandling) {
+  auto runTest = [](bool workerThrows, Queue&& queue) {
+    auto throwingProcedure = [&]() {
+      auto threadFunction = [&queue, workerThrows] {
+        try {
+          auto push = makePush(queue);
+          size_t numPushed = 0;
+          // We have to finish the threadas soon as `push` returns false.
+          while (push(numPushed++)) {
+            // Manually throw an exception if `workerThrows` was specified.
+            if (numPushed >= numValues / 2 && workerThrows) {
+              throw std::runtime_error{"Producer died"};
+            }
+          }
+        } catch (...) {
+          // We have to catch all exceptions in the worker thread(s), otherwise
+          // the program will immediately terminate. When there was an exception
+          // and the queue still expects results from this worker thread
+          // (especially if the queue is ordered), we have to finish the queue.
+          // If we just call `finish` then the producer will see a noop when
+          // popping from the queue. When we use `pushException` the call to
+          // `pop` will rethrow the exception.
+          try {
+            // In theory, `pushException` might throw if something goes really
+            // wrong with the underlying mutex. In practice this should never
+            // happen, but we demonstrate the really safe way here.
+            queue.pushException(std::current_exception());
+          } catch (...) {
+            // `finish()` can never fail.
+            queue.finish();
+          }
+        }
+      };
+      ad_utility::JThread thread{threadFunction};
+      // This cleanup is important in case the consumer throws an exception. We
+      // then first have to `finish` the queue, s.t. the producer threads can
+      // join. We then can join and destroy the worker threads and finally
+      // destroy the queue. So the order of declaration is important:
+      // 1. Queue, 2. WorkerThreads, 3. `Cleanup` that finishes the queue.
+      absl::Cleanup cleanup{[&queue] { queue.finish(); }};
+
+      for ([[maybe_unused]] auto i : std::views::iota(0u, numValues)) {
+        auto opt = queue.pop();
+        if (!opt) {
+          return;
+        }
+      }
+      // When trowing, the `Cleanup` calls `finish` and the producers can run to
+      // completion because their calls to `push` will return false.
+      throw std::runtime_error{"Consumer died"};
+    };
+    if (workerThrows) {
+      AD_EXPECT_THROW_WITH_MESSAGE(throwingProcedure(),
+                                   ::testing::StartsWith("Producer"));
+    } else {
+      AD_EXPECT_THROW_WITH_MESSAGE(throwingProcedure(),
+                                   ::testing::StartsWith("Consumer"));
+    }
+  };
+  runWithBothQueueTypes(std::bind_front(runTest, true));
+  runWithBothQueueTypes(std::bind_front(runTest, false));
+}

From f070a7c4321ceeae4bd2499b996abe151d1e940c Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 16:35:46 +0200
Subject: [PATCH 127/150] Some improvements, we mostly are lacking comments and
 a few tests.

---
 src/index/CompressedRelation.cpp            |  1 +
 src/util/JoinAlgorithms/JoinAlgorithms.h    | 76 ++++++++++++---------
 src/util/JoinAlgorithms/JoinColumnMapping.h |  2 +
 3 files changed, 48 insertions(+), 31 deletions(-)

diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 381a0a02e9..4d8567a2bd 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -342,6 +342,7 @@ std::vector CompressedRelationReader::getBlocksForJoin(
                             const CompressedBlockMetadata&)
           ->bool {
     static_assert(ad_utility::alwaysFalse);
+    AD_FAIL();
   };
   auto lessThan = ad_utility::OverloadCallOperator{
       idLessThanBlock, blockLessThanId, blockLessThanBlock, std::less{}};
diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index b0a7355d23..5c5100bd02 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -603,15 +603,25 @@ class BlockAndSubrange {
   // checked by an assertion. The only legal way to obtain such iterators is to
   // call `subrange()` and then to extract valid iterators from the returned
   // subrange.
-  void setSubrange(auto begin, auto end) {
-    auto checkIt = [this](const auto& it) {
-      AD_CONTRACT_CHECK(fullBlock().begin() <= it && it <= fullBlock().end());
+  template 
+  void setSubrange(It begin, It end) {
+    auto impl = [&begin, &end, this](auto blockBegin, auto blockEnd) {
+      auto checkIt = [&blockBegin, &blockEnd](const auto& it) {
+        AD_CONTRACT_CHECK(blockBegin <= it && it <= blockEnd);
+      };
+      checkIt(begin);
+      checkIt(end);
+      AD_CONTRACT_CHECK(begin <= end);
+      subrange_.first = begin - blockBegin;
+      subrange_.second = end - blockBegin;
     };
-    checkIt(begin);
-    checkIt(end);
-    AD_CONTRACT_CHECK(begin <= end);
-    subrange_.first = begin - fullBlock().begin();
-    subrange_.second = end - fullBlock().begin();
+    auto& block = fullBlock();
+    // impl(block.begin(), block.end());
+    if constexpr (requires { begin - block.begin(); }) {
+      impl(block.begin(), block.end());
+    } else {
+      impl(std::as_const(block).begin(), std::as_const(block).end());
+    }
   }
 };
 }  // namespace detail
@@ -791,34 +801,39 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
   // `sameBlocksLeft/Right`, as they are not needed anymore.
   auto joinAndRemoveBeginning = [&]() {
     // Get the first blocks.
-    AD_CORRECTNESS_CHECK(!sameBlocksLeft.empty());
-    AD_CORRECTNESS_CHECK(!sameBlocksRight.empty());
-    decltype(auto) l = sameBlocksLeft.at(0).subrange();
-    decltype(auto) r = sameBlocksRight.at(0).subrange();
-    auto& fullBlockLeft = sameBlocksLeft.at(0).fullBlock();
-    auto& fullBlockRight = sameBlocksRight.at(0).fullBlock();
-
-    // Compute the range that is safe to join and perform the join.
     ProjectedEl minEl = getMinEl();
-    auto itL = std::ranges::lower_bound(l, minEl, lessThan);
-    auto itR = std::ranges::lower_bound(r, minEl, lessThan);
-
-    compatibleRowAction.setInput(fullBlockLeft, fullBlockRight);
-    auto addRowIndex = [begL = fullBlockLeft.begin(),
-                        begR = fullBlockRight.begin(),
+    // For one of the inputs (`sameBlocksLeft` or `sameBlocksRight`) obtain a
+    // tuple of the following elements:
+    // * A reference to the first full block
+    // * The currently active subrange of that block
+    // * An iterator pointing to the position of the `minEl` in the block.
+    auto getFirstBlock = [&minEl, &lessThan](auto& sameBlocks) {
+      AD_CORRECTNESS_CHECK(!sameBlocks.empty());
+      const auto& first = sameBlocks.at(0);
+      auto it = std::ranges::lower_bound(first.subrange(), minEl, lessThan);
+      return std::tuple{std::ref(first.fullBlock()), first.subrange(), it};
+    };
+    auto [fullBlockLeft, subrangeLeft, minElItL] =
+        getFirstBlock(sameBlocksLeft);
+    auto [fullBlockRight, subrangeRight, minElItR] =
+        getFirstBlock(sameBlocksRight);
+
+    compatibleRowAction.setInput(fullBlockLeft.get(), fullBlockRight.get());
+    auto addRowIndex = [begL = fullBlockLeft.get().begin(),
+                        begR = fullBlockRight.get().begin(),
                         &compatibleRowAction](auto itFromL, auto itFromR) {
       compatibleRowAction.addRow(itFromL - begL, itFromR - begR);
     };
 
-    [[maybe_unused]] auto res =
-        zipperJoinWithUndef(std::ranges::subrange{l.begin(), itL},
-                            std::ranges::subrange{r.begin(), itR}, lessThan,
-                            addRowIndex, noop, noop);
+    [[maybe_unused]] auto res = zipperJoinWithUndef(
+        std::ranges::subrange{subrangeLeft.begin(), minElItL},
+        std::ranges::subrange{subrangeRight.begin(), minElItR}, lessThan,
+        addRowIndex, noop, noop);
     compatibleRowAction.flush();
 
     // Remove the joined elements.
-    sameBlocksLeft.at(0).setSubrange(itL, l.end());
-    sameBlocksRight.at(0).setSubrange(itR, r.end());
+    sameBlocksLeft.at(0).setSubrange(minElItL, subrangeLeft.end());
+    sameBlocksRight.at(0).setSubrange(minElItR, subrangeRight.end());
   };
 
   // Remove all elements from `blocks` (either `sameBlocksLeft` or
@@ -858,9 +873,8 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     // the last one, which might contain elements `> minEl` at the end.
     auto pushRelevantSubranges = [&minEl, &lessThan](const auto& input) {
       auto result = input;
-      if (result.empty()) {
-        return result;
-      }
+      // If one of the inputs
+      AD_CORRECTNESS_CHECK(!result.empty());
       auto& last = result.back();
       auto range = std::ranges::equal_range(last.subrange(), minEl, lessThan);
       last.setSubrange(range.begin(), range.end());
diff --git a/src/util/JoinAlgorithms/JoinColumnMapping.h b/src/util/JoinAlgorithms/JoinColumnMapping.h
index 27d2209b02..1fe8f528f9 100644
--- a/src/util/JoinAlgorithms/JoinColumnMapping.h
+++ b/src/util/JoinAlgorithms/JoinColumnMapping.h
@@ -123,6 +123,8 @@ struct IdTableAndFirstCol {
   // The following functions all refer to the same column.
   auto begin() { return col().begin(); }
   auto end() { return col().end(); }
+  auto begin() const { return col().begin(); }
+  auto end() const { return col().end(); }
 
   bool empty() { return col().empty(); }
 

From 3e3239071f0f972c14c91c74600135ff32f1fb8d Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 18:24:52 +0200
Subject: [PATCH 128/150] Commented the test for the`IndexScan` class

---
 test/engine/IndexScanTest.cpp | 38 ++++++++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp
index 93792cc3ce..529f1333ed 100644
--- a/test/engine/IndexScanTest.cpp
+++ b/test/engine/IndexScanTest.cpp
@@ -17,14 +17,24 @@ using Tc = TripleComponent;
 using Var = Variable;
 
 using IndexPair = std::pair;
-void testLazyScan(Permutation::IdTableGenerator lazyScan, IndexScan& scanOp,
+
+// NOTE: All the following helper functions always use the `PSO` permutation to
+// set up index scans unless explicitly stated otherwise.
+
+// Test that the `partialLazyScanResult` when being materialized to a single
+// `IdTable` yields a subset of the full result of the `fullScan`. The `subset`
+// is specified via the `expectedRows`, for example {{1, 3}, {7, 8}} means that
+// the partialLazyScanResult shall contain the rows number `1, 2, 7` of the full
+// scan (upper bounds are not included).
+void testLazyScan(Permutation::IdTableGenerator partialLazyScanResult,
+                  IndexScan& fullScan,
                   const std::vector& expectedRows,
                   source_location l = source_location::current()) {
   auto t = generateLocationTrace(l);
   auto alloc = ad_utility::makeUnlimitedAllocator();
   IdTable lazyScanRes{0, alloc};
   size_t numBlocks = 0;
-  for (const auto& block : lazyScan) {
+  for (const auto& block : partialLazyScanResult) {
     if (lazyScanRes.empty()) {
       lazyScanRes.setNumColumns(block.numColumns());
     }
@@ -32,10 +42,11 @@ void testLazyScan(Permutation::IdTableGenerator lazyScan, IndexScan& scanOp,
     ++numBlocks;
   }
 
-  EXPECT_EQ(numBlocks, lazyScan.details().numBlocksRead_);
-  EXPECT_EQ(lazyScanRes.size(), lazyScan.details().numElementsRead_);
+  EXPECT_EQ(numBlocks, partialLazyScanResult.details().numBlocksRead_);
+  EXPECT_EQ(lazyScanRes.size(),
+            partialLazyScanResult.details().numElementsRead_);
 
-  auto resFullScan = scanOp.getResult()->idTable().clone();
+  auto resFullScan = fullScan.getResult()->idTable().clone();
   IdTable expected{resFullScan.numColumns(), alloc};
 
   for (auto [lower, upper] : expectedRows) {
@@ -47,13 +58,19 @@ void testLazyScan(Permutation::IdTableGenerator lazyScan, IndexScan& scanOp,
   EXPECT_EQ(lazyScanRes, expected);
 }
 
+// Test that when two scans are set up (specified by `tripleLeft` and
+// `tripleRight`) on the given knowledge graph), and each scan is lazily
+// executed and only contains the blocks that are needed to join both scans,
+// then the resulting lazy partial scans only contain the subset of the
+// respective full scans as specified by `leftRows` and `rightRows`. For the
+// specification of the subset see above.
 void testLazyScanForJoinOfTwoScans(
-    const std::string& kg, const SparqlTriple& tripleLeft,
+    const std::string& kgTurtle, const SparqlTriple& tripleLeft,
     const SparqlTriple& tripleRight, const std::vector& leftRows,
     const std::vector& rightRows,
     source_location l = source_location::current()) {
   auto t = generateLocationTrace(l);
-  auto qec = getQec(kg);
+  auto qec = getQec(kgTurtle);
   IndexScan s1{qec, Permutation::PSO, tripleLeft};
   IndexScan s2{qec, Permutation::PSO, tripleRight};
   auto implForSwitch = [](IndexScan& l, IndexScan& r, const auto& expectedL,
@@ -67,6 +84,8 @@ void testLazyScanForJoinOfTwoScans(
   implForSwitch(s2, s1, rightRows, leftRows);
 }
 
+// Test that setting up the lazy partial scans between `tripleLeft` and
+// `tripleRight` on the given `kg` throws an exception.
 void testLazyScanThrows(const std::string& kg, const SparqlTriple& tripleLeft,
                         const SparqlTriple& tripleRight,
                         source_location l = source_location::current()) {
@@ -77,6 +96,9 @@ void testLazyScanThrows(const std::string& kg, const SparqlTriple& tripleLeft,
   EXPECT_ANY_THROW(IndexScan::lazyScanForJoinOfTwoScans(s1, s2));
 }
 
+// Test that a lazy partial scan for a join of the `scanTriple` with a
+// materialized join column result that is specified by the `columnEntries`
+// yields only the subsets specified by the `expectedRows`.
 void testLazyScanForJoinWithColumn(
     const std::string& kg, const SparqlTriple& scanTriple,
     std::vector columnEntries,
@@ -95,6 +117,8 @@ void testLazyScanForJoinWithColumn(
   testLazyScan(std::move(lazyScan), scan, expectedRows);
 }
 
+// Tests the same scenario as the previous function, but assumes that the
+// setting up of the lazy scan fails with an exception.
 void testLazyScanWithColumnThrows(
     const std::string& kg, const SparqlTriple& scanTriple,
     std::vector columnEntries,

From 0706a9f024af7c924175b42e4c8b0d7d4d136112 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 19:41:48 +0200
Subject: [PATCH 129/150] Better stuff.

---
 test/ThreadSafeQueueTest.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 281b6e1d6b..e5a3f0c08d 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -41,7 +41,7 @@ constexpr size_t numValues = 200;
 // Run the `test` function with a `ThreadSafeQueue` and an
 // `OrderedThreadSafeQueue`. Both queues have a size of `queueSize` and `size_t`
 // as their value type.
-void runWithBothQueueTypes(auto&& testFunction) {
+void runWithBothQueueTypes(const auto& testFunction) {
   testFunction(ThreadSafeQueue{queueSize});
   testFunction(OrderedThreadSafeQueue{queueSize});
 }
@@ -301,7 +301,7 @@ TEST(ThreadSafeQueue, SafeExceptionHandling) {
           return;
         }
       }
-      // When trowing, the `Cleanup` calls `finish` and the producers can run to
+      // When throwing, the `Cleanup` calls `finish` and the producers can run to
       // completion because their calls to `push` will return false.
       throw std::runtime_error{"Consumer died"};
     };

From 93882ff345143550268094cfefbee10da2d99b26 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 20:09:09 +0200
Subject: [PATCH 130/150] Merged in the master and several changes from a
 review.

---
 src/engine/Join.cpp                      |  4 ++--
 src/index/CompressedRelation.cpp         |  2 +-
 src/util/JoinAlgorithms/JoinAlgorithms.h |  6 ++++--
 test/AddCombinedRowToTableTest.cpp       | 27 +++++++++++-------------
 4 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index f1099af7c8..eaa5d02a2c 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -244,7 +244,7 @@ ResultTable Join::computeResultForJoinWithFullScanDummy() {
   LOG(DEBUG) << "Join by making multiple scans..." << endl;
   AD_CORRECTNESS_CHECK(!isFullScanDummy(_left) && isFullScanDummy(_right));
   _right->getRootOperation()->updateRuntimeInformationWhenOptimizedOut(
-      {}, RuntimeInformation::Status::lazilyCompleted);
+      {}, RuntimeInformation::Status::lazilyMaterialized);
   idTable.setNumColumns(_left->getResultWidth() + 2);
 
   shared_ptr nonDummyRes = _left->getResult();
@@ -662,7 +662,7 @@ void updateRuntimeInfoForLazyScan(
     IndexScan& scanTree,
     const CompressedRelationReader::LazyScanMetadata& metadata) {
   scanTree.updateRuntimeInformationWhenOptimizedOut(
-      RuntimeInformation::Status::lazilyCompleted);
+      RuntimeInformation::Status::lazilyMaterialized);
   auto& rti = scanTree.getRuntimeInfo();
   rti.numRows_ = metadata.numElementsRead_;
   rti.totalTime_ = static_cast(metadata.blockingTimeMs_);
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 4d8567a2bd..d48b59b1eb 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -172,7 +172,7 @@ CompressedRelationReader::asyncParallelBlockGenerator(
   // queue. This allows the destructor of `threads` to join the threads, as the
   // threads are able to complete once they cannot access the queue anymore.
   // Only then the `blockIteratorMutex` and the `queue` can be safely destroyed,
-  // as the threads which were using them have already been destroyed.
+  // as the threads that were using them have already been destroyed.
   std::vector threads;
   absl::Cleanup finishQueue{[&queue] { queue.finish(); }};
   const size_t numThreads =
diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h
index 5c5100bd02..03ced1c754 100644
--- a/src/util/JoinAlgorithms/JoinAlgorithms.h
+++ b/src/util/JoinAlgorithms/JoinAlgorithms.h
@@ -605,6 +605,8 @@ class BlockAndSubrange {
   // subrange.
   template 
   void setSubrange(It begin, It end) {
+    // We need this function to work with `iterator` as well as
+    // `const_iterator`, so we template the actual implementation.
     auto impl = [&begin, &end, this](auto blockBegin, auto blockEnd) {
       auto checkIt = [&blockBegin, &blockEnd](const auto& it) {
         AD_CONTRACT_CHECK(blockBegin <= it && it <= blockEnd);
@@ -616,7 +618,6 @@ class BlockAndSubrange {
       subrange_.second = end - blockBegin;
     };
     auto& block = fullBlock();
-    // impl(block.begin(), block.end());
     if constexpr (requires { begin - block.begin(); }) {
       impl(block.begin(), block.end());
     } else {
@@ -873,7 +874,8 @@ void zipperJoinForBlocksWithoutUndef(LeftBlocks&& leftBlocks,
     // the last one, which might contain elements `> minEl` at the end.
     auto pushRelevantSubranges = [&minEl, &lessThan](const auto& input) {
       auto result = input;
-      // If one of the inputs
+      // If one of the inputs is empty, this function shouldn't have been called
+      // in the first place.
       AD_CORRECTNESS_CHECK(!result.empty());
       auto& last = result.back();
       auto range = std::ranges::equal_range(last.subrange(), minEl, lessThan);
diff --git a/test/AddCombinedRowToTableTest.cpp b/test/AddCombinedRowToTableTest.cpp
index 5772c909c0..db65574c4c 100644
--- a/test/AddCombinedRowToTableTest.cpp
+++ b/test/AddCombinedRowToTableTest.cpp
@@ -9,7 +9,14 @@
 
 namespace {
 static constexpr auto U = Id::makeUndefined();
+
+void testWithAllBuffersizes(const auto& testFunction) {
+  for (auto bufferSize : std::views::iota(0, 10)) {
+    testFunction(bufferSize);
+  }
+  testFunction(100'000);
 }
+}  // namespace
 
 // _______________________________________________________________________________
 TEST(AddCombinedRowToTable, OneJoinColumn) {
@@ -35,9 +42,7 @@ TEST(AddCombinedRowToTable, OneJoinColumn) {
     auto expectedUndefined = std::vector{0, 0, 1, 1};
     ASSERT_EQ(numUndefined, expectedUndefined);
   };
-  testWithBufferSize(100'000);
-  testWithBufferSize(1);
-  testWithBufferSize(2);
+  testWithAllBuffersizes(testWithBufferSize);
 }
 
 // _______________________________________________________________________________
@@ -64,9 +69,7 @@ TEST(AddCombinedRowToTable, TwoJoinColumns) {
     auto expectedUndefined = std::vector{0, 0, 1};
     ASSERT_EQ(numUndefined, expectedUndefined);
   };
-  testWithBufferSize(100'000);
-  testWithBufferSize(1);
-  testWithBufferSize(2);
+  testWithAllBuffersizes(testWithBufferSize);
 }
 
 // _______________________________________________________________________________
@@ -95,9 +98,7 @@ TEST(AddCombinedRowToTable, UndefInInput) {
     auto expectedUndefined = std::vector{1, 2};
     ASSERT_EQ(numUndefined, expectedUndefined);
   };
-  testWithBufferSize(100'000);
-  testWithBufferSize(1);
-  testWithBufferSize(2);
+  testWithAllBuffersizes(testWithBufferSize);
 }
 
 // _______________________________________________________________________________
@@ -155,9 +156,7 @@ TEST(AddCombinedRowToTable, setInput) {
                                            {U, 8, 5}});
     ASSERT_EQ(result, expected);
   };
-  testWithBufferSize(100'000);
-  testWithBufferSize(1);
-  testWithBufferSize(2);
+  testWithAllBuffersizes(testWithBufferSize);
 }
 
 // _______________________________________________________________________________
@@ -180,7 +179,5 @@ TEST(AddCombinedRowToTable, cornerCases) {
     // The same test with the arguments switched.
     EXPECT_ANY_THROW(adder.setInput(right, left));
   };
-  testWithBufferSize(100'000);
-  testWithBufferSize(1);
-  testWithBufferSize(2);
+  testWithAllBuffersizes(testWithBufferSize);
 }

From 3837029f8ddf0e8eb379bb3a396a44ea0ab36278 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 20:28:56 +0200
Subject: [PATCH 131/150] Small change from a review.

---
 test/engine/IndexScanTest.cpp | 62 +++++++++++++++++------------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp
index 529f1333ed..182fc820fa 100644
--- a/test/engine/IndexScanTest.cpp
+++ b/test/engine/IndexScanTest.cpp
@@ -117,11 +117,11 @@ void testLazyScanForJoinWithColumn(
   testLazyScan(std::move(lazyScan), scan, expectedRows);
 }
 
-// Tests the same scenario as the previous function, but assumes that the
+// Test the same scenario as the previous function, but assumes that the
 // setting up of the lazy scan fails with an exception.
 void testLazyScanWithColumnThrows(
     const std::string& kg, const SparqlTriple& scanTriple,
-    std::vector columnEntries,
+    const std::vector& columnEntries,
     source_location l = source_location::current()) {
   auto t = generateLocationTrace(l);
   auto qec = getQec(kg);
@@ -144,7 +144,7 @@ void testLazyScanWithColumnThrows(
 
 TEST(IndexScan, lazyScanForJoinOfTwoScans) {
   SparqlTriple xpy{Tc{Var{"?x"}}, "
", Tc{Var{"?y"}}};
-  SparqlTriple xpz{Tc{Var{"?x"}}, "", Tc{Var{"?z"}}};
+  SparqlTriple xqz{Tc{Var{"?x"}}, "", Tc{Var{"?z"}}};
   {
     // In the tests we have a blocksize of two triples per block, and a new
     // block is started for a new relation. That explains the spacing of the
@@ -153,21 +153,21 @@ TEST(IndexScan, lazyScanForJoinOfTwoScans) {
         " 
 .  
 . "
         " 
  .  
 . "
         " 
  ."
-        "  .    .";
+        "  .    .";
 
     // When joining the 
 and  relations, we only need to read the last two
     // blocks of the 
 relation, as  never appears as a subject in .
     // This means that the lazy partial scan can skip the first two triples.
-    testLazyScanForJoinOfTwoScans(kg, xpy, xpz, {{2, 5}}, {{0, 2}});
+    testLazyScanForJoinOfTwoScans(kg, xpy, xqz, {{2, 5}}, {{0, 2}});
   }
   {
     std::string kg =
         "  .   . "
         "   .   . "
-        "  .    .";
+        "  .    .";
     // No triple for relation 
 (which doesn't even appear in the knowledge
     // graph), so both lazy scans are empty.
-    testLazyScanForJoinOfTwoScans(kg, xpy, xpz, {}, {});
+    testLazyScanForJoinOfTwoScans(kg, xpy, xqz, {}, {});
   }
   {
     // No triple for relation  (which does appear in the knowledge graph, but
@@ -176,7 +176,7 @@ TEST(IndexScan, lazyScanForJoinOfTwoScans) {
         " 
 .  
 . "
         " 
  .  
 . "
         "  .    .";
-    testLazyScanForJoinOfTwoScans(kg, xpy, xpz, {}, {});
+    testLazyScanForJoinOfTwoScans(kg, xpy, xqz, {}, {});
   }
   SparqlTriple bpx{Tc{""}, "
", Tc{Var{"?x"}}};
   {
@@ -185,33 +185,33 @@ TEST(IndexScan, lazyScanForJoinOfTwoScans) {
         " 
  .  
 . "
         " 
  .  
 . "
         " 
  .  
 . "
-        "  .    ."
-        "  .    ."
-        "  .    .";
-    testLazyScanForJoinOfTwoScans(kg, bpx, xpz, {{1, 5}}, {{0, 4}});
+        "  .    ."
+        "  .    ."
+        "  .    .";
+    testLazyScanForJoinOfTwoScans(kg, bpx, xqz, {{1, 5}}, {{0, 4}});
   }
   {
     std::string kg =
         " 
 .  
 . "
         " 
  .  
 . "
-        "  .    ."
-        "  .    ."
-        "  .    .";
+        "  .    ."
+        "  .    ."
+        "  .    .";
     // Scan for a fixed subject that appears in the kg but not as the subject of
     // the 
 predicate.
     SparqlTriple xb2px{Tc{""}, "
", Tc{Var{"?x"}}};
-    testLazyScanForJoinOfTwoScans(kg, bpx, xpz, {}, {});
+    testLazyScanForJoinOfTwoScans(kg, bpx, xqz, {}, {});
   }
   {
     std::string kg =
         " 
 .  
 . "
         " 
  .  
 . "
-        "  .    ."
-        "  .    ."
-        "  .    .";
+        "  .    ."
+        "  .    ."
+        "  .    .";
     // Scan for a fixed subject that is not even in the knowledge graph.
     SparqlTriple xb2px{Tc{""}, "
", Tc{Var{"?x"}}};
-    testLazyScanForJoinOfTwoScans(kg, bpx, xpz, {}, {});
+    testLazyScanForJoinOfTwoScans(kg, bpx, xqz, {}, {});
   }
 
   // Corner cases
@@ -219,13 +219,13 @@ TEST(IndexScan, lazyScanForJoinOfTwoScans) {
     std::string kg = "   .";
     // Triples with three variables are not supported.
     SparqlTriple xyz{Tc{Var{"?x"}}, "?y", Tc{Var{"?z"}}};
-    testLazyScanThrows(kg, xyz, xpz);
+    testLazyScanThrows(kg, xyz, xqz);
     testLazyScanThrows(kg, xyz, xyz);
-    testLazyScanThrows(kg, xpz, xyz);
+    testLazyScanThrows(kg, xqz, xyz);
 
     // The first variable must be matching (subject variable is ?a vs ?x)
     SparqlTriple abc{Tc{Var{"?a"}}, "", Tc{Var{"?c"}}};
-    testLazyScanThrows(kg, abc, xpz);
+    testLazyScanThrows(kg, abc, xqz);
 
     // If both scans have two variables, then the second variable must not
     // match.
@@ -242,26 +242,26 @@ TEST(IndexScan, lazyScanForJoinOfColumnWithScanTwoVariables) {
       " 
 .  
 . "
       " 
  .  
 . "
       " 
  ."
-      "  .    .";
+      "  .    .";
   {
-    std::vector column{"", "", "", ""};
+    std::vector column{"", "", "", ""};
     // We need to scan all the blocks that contain the `
` predicate.
     testLazyScanForJoinWithColumn(kg, xpy, column, {{0, 5}});
   }
   {
-    std::vector column{"", "", ""};
+    std::vector column{"", "", ""};
     // The first block only contains  which doesn't appear in the first
     // block.
     testLazyScanForJoinWithColumn(kg, xpy, column, {{2, 5}});
   }
   {
-    std::vector column{"", "", ""};
+    std::vector column{"", "", ""};
     // The first block only contains  which only appears in the first two
     // blocks.
     testLazyScanForJoinWithColumn(kg, xpy, column, {{0, 4}});
   }
   {
-    std::vector column{"", "", ""};
+    std::vector column{"", "", ""};
     //  does not appear as a predicate, so the result is empty.
     SparqlTriple efg{Tc{Var{"?e"}}, "", Tc{Var{"?g"}}};
     testLazyScanForJoinWithColumn(kg, efg, column, {});
@@ -275,7 +275,7 @@ TEST(IndexScan, lazyScanForJoinOfColumnWithScanOneVariable) {
       " 
  .  
 . "
       " 
  .  
 . "
       " 
  .  
 . "
-      "  .    .";
+      "  .    .";
   {
     // The subject () and predicate () are fixed, so the object is the
     // join column
@@ -291,10 +291,10 @@ TEST(IndexScan, lazyScanForJoinOfColumnWithScanCornerCases) {
       " 
 .  
 . "
       " 
  .  
 . "
       " 
  ."
-      "  .    .";
+      "  .    .";
 
   // Full index scans (three variables) are not supported.
-  std::vector column{"", "", "", ""};
+  std::vector column{"", "", "", ""};
   testLazyScanWithColumnThrows(kg, threeVars, column);
 
   // The join column must be sorted.

From 6adc20a4eb058acb631fbec3f67b92661491c6ce Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 20:56:22 +0200
Subject: [PATCH 132/150] Clang format.

---
 test/ThreadSafeQueueTest.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index e5a3f0c08d..e119f5fc85 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -301,8 +301,8 @@ TEST(ThreadSafeQueue, SafeExceptionHandling) {
           return;
         }
       }
-      // When throwing, the `Cleanup` calls `finish` and the producers can run to
-      // completion because their calls to `push` will return false.
+      // When throwing, the `Cleanup` calls `finish` and the producers can run
+      // to completion because their calls to `push` will return false.
       throw std::runtime_error{"Consumer died"};
     };
     if (workerThrows) {

From 8b0ac05a27f2907c268cf40e8284afbd5eb847f5 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 20:59:33 +0200
Subject: [PATCH 133/150] Fixed the merge.

---
 test/engine/IndexScanTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp
index 182fc820fa..560199df05 100644
--- a/test/engine/IndexScanTest.cpp
+++ b/test/engine/IndexScanTest.cpp
@@ -298,7 +298,7 @@ TEST(IndexScan, lazyScanForJoinOfColumnWithScanCornerCases) {
   testLazyScanWithColumnThrows(kg, threeVars, column);
 
   // The join column must be sorted.
-  if constexpr (ad_utility::areExpensiveChecksEnabled()) {
+  if constexpr (ad_utility::areExpensiveChecksEnabled) {
     std::vector unsortedColumn{"", "", ""};
     SparqlTriple xpy{Tc{Var{"?x"}}, "
", Tc{Var{"?y"}}};
     testLazyScanWithColumnThrows(kg, xpy, unsortedColumn);

From 5f7c17276225218341bf0c0b4379bc4618f9789e Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Fri, 7 Jul 2023 20:59:41 +0200
Subject: [PATCH 134/150] Fixed the merge.

---
 test/AddCombinedRowToTableTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/AddCombinedRowToTableTest.cpp b/test/AddCombinedRowToTableTest.cpp
index db65574c4c..bc67d53e28 100644
--- a/test/AddCombinedRowToTableTest.cpp
+++ b/test/AddCombinedRowToTableTest.cpp
@@ -112,7 +112,7 @@ TEST(AddCombinedRowToTable, setInput) {
       // It is okay to flush even if no inputs were specified, as long as we
       // haven't pushed any rows yet.
       EXPECT_NO_THROW(adder.flush());
-      if constexpr (ad_utility::areExpensiveChecksEnabled()) {
+      if constexpr (ad_utility::areExpensiveChecksEnabled) {
         EXPECT_ANY_THROW(adder.addRow(0, 0));
       } else {
         adder.addRow(0, 0);

From 7b256fd2fd9d6cab210681a4e9abfda89867f4d9 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 10 Jul 2023 12:02:01 +0200
Subject: [PATCH 135/150] Make the lazy joins with very small relations much
 more efficient.

---
 src/index/CompressedRelation.cpp   | 65 ++++++++++++++++++++++++++++--
 src/index/CompressedRelation.h     |  8 +++-
 src/index/Permutation.cpp          | 11 ++++-
 test/AddCombinedRowToTableTest.cpp |  2 +-
 test/engine/IndexScanTest.cpp      |  2 +-
 5 files changed, 79 insertions(+), 9 deletions(-)

diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index d48b59b1eb..adecebe6b0 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -322,7 +322,14 @@ std::vector CompressedRelationReader::getBlocksForJoin(
     std::span joinColumn,
     const MetadataAndBlocks& metadataAndBlocks) {
   // Get all the blocks where `col0FirstId_ <= col0Id <= col0LastId_`.
-  auto relevantBlocks = getBlocksFromMetadata(metadataAndBlocks);
+  auto relevantBlocksSpan = getBlocksFromMetadata(metadataAndBlocks);
+  std::vector relevantBlocks(relevantBlocksSpan.begin(),
+                             relevantBlocksSpan.end());
+  if (metadataAndBlocks.firstTriple_.has_value()) {
+    relevantBlocks.front().firstTriple_ =
+        metadataAndBlocks.firstTriple_.value();
+    relevantBlocks.back().lastTriple_ = metadataAndBlocks.lastTriple_.value();
+  }
 
   // We need symmetric comparisons between Ids and blocks.
   auto idLessThanBlock = [&metadataAndBlocks](
@@ -368,8 +375,32 @@ std::array, 2>
 CompressedRelationReader::getBlocksForJoin(
     const MetadataAndBlocks& metadataAndBlocks1,
     const MetadataAndBlocks& metadataAndBlocks2) {
-  auto relevantBlocks1 = getBlocksFromMetadata(metadataAndBlocks1);
-  auto relevantBlocks2 = getBlocksFromMetadata(metadataAndBlocks2);
+  auto relevantBlocks1Span = getBlocksFromMetadata(metadataAndBlocks1);
+  auto relevantBlocks2Span = getBlocksFromMetadata(metadataAndBlocks2);
+
+  std::vector relevantBlocks1(relevantBlocks1Span.begin(),
+                              relevantBlocks1Span.end());
+  std::vector relevantBlocks2(relevantBlocks2Span.begin(),
+                              relevantBlocks2Span.end());
+
+  // TODO This is rather hacky, make it a little cleaner with
+  // assertions etc. Also we need less code in the `getRelevantIdFromTriple`
+  // function if we enfore the first and last triple to be present.
+  if (relevantBlocks1.empty() || relevantBlocks2.empty()) {
+    return {};
+  }
+
+  if (metadataAndBlocks1.firstTriple_.has_value()) {
+    relevantBlocks1.front().firstTriple_ =
+        metadataAndBlocks1.firstTriple_.value();
+    relevantBlocks1.back().lastTriple_ = metadataAndBlocks1.lastTriple_.value();
+  }
+
+  if (metadataAndBlocks2.firstTriple_.has_value()) {
+    relevantBlocks2.front().firstTriple_ =
+        metadataAndBlocks2.firstTriple_.value();
+    relevantBlocks2.back().lastTriple_ = metadataAndBlocks2.lastTriple_.value();
+  }
 
   auto metadataForBlock =
       [&](const CompressedBlockMetadata& block) -> decltype(auto) {
@@ -424,7 +455,7 @@ CompressedRelationReader::getBlocksForJoin(
 // _____________________________________________________________________________
 IdTable CompressedRelationReader::scan(
     const CompressedRelationMetadata& metadata, Id col1Id,
-    const vector& blocks, ad_utility::File& file,
+    std::span blocks, ad_utility::File& file,
     const TimeoutTimer& timer) const {
   IdTable result(1, allocator_);
 
@@ -903,3 +934,29 @@ CompressedRelationReader::getBlocksFromMetadata(
   return getBlocksFromMetadata(metadata.relationMetadata_, metadata.col1Id_,
                                metadata.blockMetadata_);
 }
+
+// _____________________________________________________________________________
+std::array
+CompressedRelationReader::getFirstAndLastTriple(
+    const CompressedRelationReader::MetadataAndBlocks& metadataAndBlocks,
+    ad_utility::File& file) const {
+  auto relevantBlocks = getBlocksFromMetadata(metadataAndBlocks);
+  AD_CONTRACT_CHECK(!relevantBlocks.empty());
+
+  auto scanBlock = [&](const CompressedBlockMetadata& block) {
+    return readPossiblyIncompleteBlock(metadataAndBlocks.relationMetadata_,
+                                       metadataAndBlocks.col1Id_, file, block,
+                                       std::nullopt);
+  };
+
+  auto rowToTriple =
+      [&](const auto& row) -> CompressedBlockMetadata::PermutedTriple {
+    return {metadataAndBlocks.relationMetadata_.col0Id_, row[0], row[1]};
+  };
+
+  auto firstBlock = scanBlock(relevantBlocks.front());
+  auto lastBlock = scanBlock(relevantBlocks.back());
+  AD_CORRECTNESS_CHECK(!firstBlock.empty());
+  AD_CORRECTNESS_CHECK(!lastBlock.empty());
+  return {rowToTriple(firstBlock.front()), rowToTriple(lastBlock.back())};
+}
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index 7447235937..fcac930612 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -237,6 +237,9 @@ class CompressedRelationReader {
     const CompressedRelationMetadata relationMetadata_;
     const std::span blockMetadata_;
     std::optional col1Id_;
+
+    std::optional firstTriple_;
+    std::optional lastTriple_;
   };
 
   struct LazyScanMetadata {
@@ -328,7 +331,7 @@ class CompressedRelationReader {
    * The same `CompressedRelationWriter` (see below).
    */
   IdTable scan(const CompressedRelationMetadata& metadata, Id col1Id,
-               const vector& blocks,
+               std::span blocks,
                ad_utility::File& file,
                const TimeoutTimer& timer = nullptr) const;
 
@@ -362,6 +365,9 @@ class CompressedRelationReader {
   static std::span getBlocksFromMetadata(
       const MetadataAndBlocks& metadataAndBlocks);
 
+  std::array getFirstAndLastTriple(
+      const MetadataAndBlocks& metadataAndBlocks, ad_utility::File& file) const;
+
   // Get access to the underlying allocator
   const Allocator& allocator() const { return allocator_; }
 
diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp
index f1ad345c45..089beca9a5 100644
--- a/src/index/Permutation.cpp
+++ b/src/index/Permutation.cpp
@@ -118,10 +118,17 @@ std::optional Permutation::getMetadataAndBlocks(
   }
 
   auto metadata = meta_.getMetaData(col0Id);
-  return MetadataAndBlocks{meta_.getMetaData(col0Id),
+
+  MetadataAndBlocks result{meta_.getMetaData(col0Id),
                            CompressedRelationReader::getBlocksFromMetadata(
                                metadata, col1Id, meta_.blockData()),
-                           col1Id};
+                           col1Id, std::nullopt, std::nullopt};
+
+  auto [firstTriple, lastTriple] = reader_.getFirstAndLastTriple(result, file_);
+
+  result.firstTriple_ = std::move(firstTriple);
+  result.lastTriple_ = std::move(lastTriple);
+  return result;
 }
 
 // _____________________________________________________________________
diff --git a/test/AddCombinedRowToTableTest.cpp b/test/AddCombinedRowToTableTest.cpp
index bc67d53e28..d0e3932639 100644
--- a/test/AddCombinedRowToTableTest.cpp
+++ b/test/AddCombinedRowToTableTest.cpp
@@ -112,7 +112,7 @@ TEST(AddCombinedRowToTable, setInput) {
       // It is okay to flush even if no inputs were specified, as long as we
       // haven't pushed any rows yet.
       EXPECT_NO_THROW(adder.flush());
-      if constexpr (ad_utility::areExpensiveChecksEnabled) {
+      if (ad_utility::areExpensiveChecksEnabled || bufferSize == 0) {
         EXPECT_ANY_THROW(adder.addRow(0, 0));
       } else {
         adder.addRow(0, 0);
diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp
index 560199df05..b32820acf8 100644
--- a/test/engine/IndexScanTest.cpp
+++ b/test/engine/IndexScanTest.cpp
@@ -272,7 +272,7 @@ TEST(IndexScan, lazyScanForJoinOfColumnWithScanOneVariable) {
   SparqlTriple bpy{Tc{""}, "
", Tc{Var{"?x"}}};
   std::string kg =
       " 
 .  
 . "
-      " 
  .  
 . "
+      " 
  .  
 . "
       " 
  .  
 . "
       " 
  .  
 . "
       "  .    .";

From 30835cc8d62e774eed56e89c0b680c82b3ee2d4f Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 10 Jul 2023 14:16:10 +0200
Subject: [PATCH 136/150] Materialize small-ish scans

---
 src/engine/Join.cpp    | 16 ++++++++++++++--
 src/global/Constants.h |  3 ++-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index eaa5d02a2c..b41ebc5ebb 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -114,8 +114,15 @@ ResultTable Join::computeResult() {
     return computeResultForJoinWithFullScanDummy();
   }
 
-  auto leftResIfCached = _left->getRootOperation()->getResult(false, true);
-  auto rightResIfCached = _right->getRootOperation()->getResult(false, true);
+  auto getCachedOrSmallResult = [](QueryExecutionTree& tree) {
+    bool readOnlyCachedResult =
+        tree.getRootOperation()->getSizeEstimate() >
+        RuntimeParameters().get<"lazy-index-scan-max-size-materialization">();
+    return tree.getRootOperation()->getResult(false, readOnlyCachedResult);
+  };
+
+  auto leftResIfCached = getCachedOrSmallResult(*_left);
+  auto rightResIfCached = getCachedOrSmallResult(*_right);
 
   if (_left->getType() == QueryExecutionTree::SCAN &&
       _right->getType() == QueryExecutionTree::SCAN) {
@@ -699,6 +706,11 @@ IdTable Join::computeResultForTwoIndexScans() {
   updateRuntimeInfoForLazyScan(leftScan, leftBlocks.details());
   updateRuntimeInfoForLazyScan(rightScan, rightBlocks.details());
 
+  AD_CORRECTNESS_CHECK(leftBlocks.details().numBlocksRead_ <=
+                       rightBlocks.details().numElementsRead_);
+  AD_CORRECTNESS_CHECK(rightBlocks.details().numBlocksRead_ <=
+                       leftBlocks.details().numElementsRead_);
+
   return std::move(rowAdder).resultTable();
 }
 
diff --git a/src/global/Constants.h b/src/global/Constants.h
index 3634f34581..be62a84642 100644
--- a/src/global/Constants.h
+++ b/src/global/Constants.h
@@ -183,7 +183,8 @@ inline auto& RuntimeParameters() {
       SizeT<"cache-max-size-gb">{30},
       SizeT<"cache-max-size-gb-single-entry">{5},
       SizeT<"lazy-index-scan-queue-size">{20},
-      SizeT<"lazy-index-scan-num-threads">{10}};
+      SizeT<"lazy-index-scan-num-threads">{10},
+      SizeT<"lazy-index-scan-max-size-materialization">{1'000'000}};
   return params;
 }
 

From 00301146796897d1160de3f0fa5b38889bd6bfb0 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 10 Jul 2023 16:56:40 +0200
Subject: [PATCH 137/150] Several improvements and comments.

---
 src/engine/Join.cpp              |  50 +++++++++----
 src/engine/Join.h                |   2 +
 src/index/CompressedRelation.cpp |  59 ++++++++-------
 src/index/CompressedRelation.h   |  31 +++++++-
 src/index/Permutation.cpp        |   7 +-
 test/CompressedRelationsTest.cpp |   6 +-
 test/JoinTest.cpp                | 123 +++++++++++++++++++------------
 7 files changed, 180 insertions(+), 98 deletions(-)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index b41ebc5ebb..b772ca741f 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -55,6 +55,17 @@ Join::Join(QueryExecutionContext* qec, std::shared_ptr t1,
   _sizeEstimate = 0;
   _sizeEstimateComputed = false;
   _multiplicities.clear();
+  auto findJoinVar = [](const QueryExecutionTree& tree,
+                        ColumnIndex joinCol) -> Variable {
+    for (auto p : tree.getVariableColumns()) {
+      if (p.second.columnIndex_ == joinCol) {
+        return p.first;
+      }
+    }
+    AD_FAIL();
+  };
+  _joinVar = findJoinVar(*_left, _leftJoinCol);
+  AD_CONTRACT_CHECK(_joinVar == findJoinVar(*_right, _rightJoinCol));
 }
 
 // _____________________________________________________________________________
@@ -83,16 +94,7 @@ string Join::asStringImpl(size_t indent) const {
 }
 
 // _____________________________________________________________________________
-string Join::getDescriptor() const {
-  std::string joinVar = "";
-  for (auto p : _left->getVariableColumns()) {
-    if (p.second.columnIndex_ == _leftJoinCol) {
-      joinVar = p.first.name();
-      break;
-    }
-  }
-  return "Join on " + joinVar;
-}
+string Join::getDescriptor() const { return "Join on " + _joinVar.name(); }
 
 // _____________________________________________________________________________
 ResultTable Join::computeResult() {
@@ -114,15 +116,29 @@ ResultTable Join::computeResult() {
     return computeResultForJoinWithFullScanDummy();
   }
 
-  auto getCachedOrSmallResult = [](QueryExecutionTree& tree) {
+  // Always materialize results that meet one of the following criteria:
+  // * They are already present in the cache
+  // * Their result is small
+  // * They might contain UNDEF values in the join column
+  // The first two conditions are for performance reasons, the last one is
+  // because we currently cannot perform the optimized lazy joins when UNDEF
+  // values are involved.
+  auto getCachedOrSmallResult = [](QueryExecutionTree& tree,
+                                   ColumnIndex joinCol) {
     bool readOnlyCachedResult =
         tree.getRootOperation()->getSizeEstimate() >
         RuntimeParameters().get<"lazy-index-scan-max-size-materialization">();
+    auto undefStatus =
+        tree.getVariableAndInfoByColumnIndex(joinCol).second.mightContainUndef_;
+    readOnlyCachedResult =
+        readOnlyCachedResult &&
+        undefStatus == ColumnIndexAndTypeInfo::UndefStatus::AlwaysDefined;
+
     return tree.getRootOperation()->getResult(false, readOnlyCachedResult);
   };
 
-  auto leftResIfCached = getCachedOrSmallResult(*_left);
-  auto rightResIfCached = getCachedOrSmallResult(*_right);
+  auto leftResIfCached = getCachedOrSmallResult(*_left, _leftJoinCol);
+  auto rightResIfCached = getCachedOrSmallResult(*_right, _rightJoinCol);
 
   if (_left->getType() == QueryExecutionTree::SCAN &&
       _right->getType() == QueryExecutionTree::SCAN) {
@@ -154,7 +170,13 @@ ResultTable Join::computeResult() {
 
   // Note: If only one of the children is a scan, then we have made sure in the
   // constructor that it is the right child.
-  if (_right->getType() == QueryExecutionTree::SCAN && !rightResIfCached) {
+  // We currently cannot use this optimized lazy scan if the result from `_left`
+  // contains UNDEF values.
+  const auto& leftIdTable = leftRes->idTable();
+  auto leftHasUndef =
+      !leftIdTable.empty() && leftIdTable.at(0, _leftJoinCol).isUndefined();
+  if (_right->getType() == QueryExecutionTree::SCAN && !rightResIfCached &&
+      !leftHasUndef) {
     idTable = computeResultForIndexScanAndIdTable(
         leftRes->idTable(), _leftJoinCol,
         dynamic_cast(*_right->getRootOperation()), _rightJoinCol);
diff --git a/src/engine/Join.h b/src/engine/Join.h
index 6a453810e8..f2e67bf99d 100644
--- a/src/engine/Join.h
+++ b/src/engine/Join.h
@@ -25,6 +25,8 @@ class Join : public Operation {
   ColumnIndex _leftJoinCol;
   ColumnIndex _rightJoinCol;
 
+  Variable _joinVar{"?notSet"};
+
   bool _keepJoinColumn;
 
   bool _sizeEstimateComputed;
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index adecebe6b0..2bcb78ba6f 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -315,6 +315,30 @@ auto getRelevantIdFromTriple(
                                metadataAndBlocks.col1Id_.value())
       .value_or(triple.col2Id_);
 }
+
+// Set the first triple of the first block in `blocks` and the last triple of
+// the last block according to the `firstAndLastTriple`.
+void setFirstAndLastTriple(
+    std::vector& blocks,
+    const std::optional<
+        CompressedRelationReader::MetadataAndBlocks::FirstAndLastTriple>&
+        firstAndLastTriple) {
+  if (blocks.empty() || !firstAndLastTriple.has_value()) {
+    return;
+  }
+
+  // Check that the `newTriple` can be safely set as the first or last triple of
+  // either the `block` without breaking the ordering of the first block.
+  auto check = [](const auto& newTriple, const auto& block) {
+    AD_CORRECTNESS_CHECK(block.firstTriple_ <= newTriple);
+    AD_CORRECTNESS_CHECK(newTriple <= block.lastTriple_);
+  };
+  check(firstAndLastTriple.value().firstTriple_, blocks.front());
+  blocks.front().firstTriple_ = firstAndLastTriple.value().firstTriple_;
+
+  check(firstAndLastTriple.value().lastTriple_, blocks.back());
+  blocks.back().lastTriple_ = firstAndLastTriple.value().lastTriple_;
+}
 }  // namespace
 
 // _____________________________________________________________________________
@@ -325,11 +349,7 @@ std::vector CompressedRelationReader::getBlocksForJoin(
   auto relevantBlocksSpan = getBlocksFromMetadata(metadataAndBlocks);
   std::vector relevantBlocks(relevantBlocksSpan.begin(),
                              relevantBlocksSpan.end());
-  if (metadataAndBlocks.firstTriple_.has_value()) {
-    relevantBlocks.front().firstTriple_ =
-        metadataAndBlocks.firstTriple_.value();
-    relevantBlocks.back().lastTriple_ = metadataAndBlocks.lastTriple_.value();
-  }
+  setFirstAndLastTriple(relevantBlocks, metadataAndBlocks.firstAndLastTriple_);
 
   // We need symmetric comparisons between Ids and blocks.
   auto idLessThanBlock = [&metadataAndBlocks](
@@ -378,30 +398,16 @@ CompressedRelationReader::getBlocksForJoin(
   auto relevantBlocks1Span = getBlocksFromMetadata(metadataAndBlocks1);
   auto relevantBlocks2Span = getBlocksFromMetadata(metadataAndBlocks2);
 
+  // TODO<
   std::vector relevantBlocks1(relevantBlocks1Span.begin(),
                               relevantBlocks1Span.end());
   std::vector relevantBlocks2(relevantBlocks2Span.begin(),
                               relevantBlocks2Span.end());
 
-  // TODO This is rather hacky, make it a little cleaner with
-  // assertions etc. Also we need less code in the `getRelevantIdFromTriple`
-  // function if we enfore the first and last triple to be present.
-  if (relevantBlocks1.empty() || relevantBlocks2.empty()) {
-    return {};
-  }
-
-  if (metadataAndBlocks1.firstTriple_.has_value()) {
-    relevantBlocks1.front().firstTriple_ =
-        metadataAndBlocks1.firstTriple_.value();
-    relevantBlocks1.back().lastTriple_ = metadataAndBlocks1.lastTriple_.value();
-  }
-
-  if (metadataAndBlocks2.firstTriple_.has_value()) {
-    relevantBlocks2.front().firstTriple_ =
-        metadataAndBlocks2.firstTriple_.value();
-    relevantBlocks2.back().lastTriple_ = metadataAndBlocks2.lastTriple_.value();
-  }
-
+  setFirstAndLastTriple(relevantBlocks1,
+                        metadataAndBlocks1.firstAndLastTriple_);
+  setFirstAndLastTriple(relevantBlocks2,
+                        metadataAndBlocks2.firstAndLastTriple_);
   auto metadataForBlock =
       [&](const CompressedBlockMetadata& block) -> decltype(auto) {
     if (relevantBlocks1.data() <= &block &&
@@ -936,10 +942,9 @@ CompressedRelationReader::getBlocksFromMetadata(
 }
 
 // _____________________________________________________________________________
-std::array
-CompressedRelationReader::getFirstAndLastTriple(
+auto CompressedRelationReader::getFirstAndLastTriple(
     const CompressedRelationReader::MetadataAndBlocks& metadataAndBlocks,
-    ad_utility::File& file) const {
+    ad_utility::File& file) const -> MetadataAndBlocks::FirstAndLastTriple {
   auto relevantBlocks = getBlocksFromMetadata(metadataAndBlocks);
   AD_CONTRACT_CHECK(!relevantBlocks.empty());
 
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index fcac930612..02914966fc 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -79,6 +79,17 @@ struct CompressedBlockMetadata {
     Id col1Id_;
     Id col2Id_;
     bool operator==(const PermutedTriple&) const = default;
+    // Convert to a plain array.
+    std::array toArray() const { return {col0Id_, col1Id_, col2Id_}; }
+
+    // Compare lexicographically.
+    auto operator<=>(const PermutedTriple& other) const {
+      auto arr1 = toArray();
+      auto arr2 = other.toArray();
+      return std::lexicographical_compare_three_way(arr1.begin(), arr1.end(),
+                                                    arr2.begin(), arr2.end());
+    }
+
     friend std::true_type allowTrivialSerialization(PermutedTriple, auto);
   };
   PermutedTriple firstTriple_;
@@ -238,8 +249,17 @@ class CompressedRelationReader {
     const std::span blockMetadata_;
     std::optional col1Id_;
 
-    std::optional firstTriple_;
-    std::optional lastTriple_;
+    // If set, then those contain the first and the last triple of the specified
+    // relation (and being filtered by the `col1Id` if specified). This might be
+    // different from the first triple in the first block (in the case of the
+    // `firstTriple_`, similarly for `lastTriple_`) because the first and last
+    // block might also contain other relations, or the same relation but with
+    // different `col1Id`s.
+    struct FirstAndLastTriple {
+      CompressedBlockMetadata::PermutedTriple firstTriple_;
+      CompressedBlockMetadata::PermutedTriple lastTriple_;
+    };
+    std::optional firstAndLastTriple_;
   };
 
   struct LazyScanMetadata {
@@ -365,7 +385,12 @@ class CompressedRelationReader {
   static std::span getBlocksFromMetadata(
       const MetadataAndBlocks& metadataAndBlocks);
 
-  std::array getFirstAndLastTriple(
+  // Get the first and the last triple that the result of a `scan` with the
+  // given argument would lead to. Throw an exception if the scan result would
+  // be empty. This function is used to more efficiently filter the blocks of
+  // index scans between joining them to get better estimates for the begginning
+  // and end of incomplete blocks.
+  MetadataAndBlocks::FirstAndLastTriple getFirstAndLastTriple(
       const MetadataAndBlocks& metadataAndBlocks, ad_utility::File& file) const;
 
   // Get access to the underlying allocator
diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp
index 089beca9a5..c1eaf78afb 100644
--- a/src/index/Permutation.cpp
+++ b/src/index/Permutation.cpp
@@ -122,12 +122,9 @@ std::optional Permutation::getMetadataAndBlocks(
   MetadataAndBlocks result{meta_.getMetaData(col0Id),
                            CompressedRelationReader::getBlocksFromMetadata(
                                metadata, col1Id, meta_.blockData()),
-                           col1Id, std::nullopt, std::nullopt};
+                           col1Id, std::nullopt};
 
-  auto [firstTriple, lastTriple] = reader_.getFirstAndLastTriple(result, file_);
-
-  result.firstTriple_ = std::move(firstTriple);
-  result.lastTriple_ = std::move(lastTriple);
+  result.firstAndLastTriple_ = reader_.getFirstAndLastTriple(result, file_);
   return result;
 }
 
diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp
index 59ca16ee8c..752fa4584e 100644
--- a/test/CompressedRelationsTest.cpp
+++ b/test/CompressedRelationsTest.cpp
@@ -298,7 +298,7 @@ TEST(CompressedRelationReader, getBlocksForJoinWithColumn) {
 
   std::vector blocks{block1, block2, block3};
   CompressedRelationReader::MetadataAndBlocks metadataAndBlocks{
-      relation, blocks, std::nullopt};
+      relation, blocks, std::nullopt, std::nullopt};
 
   auto test = [&metadataAndBlocks](
                   const std::vector& joinColumn,
@@ -344,7 +344,7 @@ TEST(CompressedRelationReader, getBlocksForJoin) {
 
   std::vector blocks{block1, block2, block3, block4, block5};
   CompressedRelationReader::MetadataAndBlocks metadataAndBlocks{
-      relation, blocks, std::nullopt};
+      relation, blocks, std::nullopt, std::nullopt};
 
   CompressedBlockMetadata blockB1{
       {}, 0, {V(16), V(0), V(0)}, {V(38), V(4), V(12)}};
@@ -365,7 +365,7 @@ TEST(CompressedRelationReader, getBlocksForJoin) {
 
   std::vector blocksB{blockB1, blockB2, blockB3, blockB4, blockB5, blockB6};
   CompressedRelationReader::MetadataAndBlocks metadataAndBlocksB{
-      relationB, blocksB, std::nullopt};
+      relationB, blocksB, std::nullopt, std::nullopt};
 
   auto test = [&metadataAndBlocks, &metadataAndBlocksB](
                   const std::array, 2>&
diff --git a/test/JoinTest.cpp b/test/JoinTest.cpp
index 89c0e44195..883a6d68eb 100644
--- a/test/JoinTest.cpp
+++ b/test/JoinTest.cpp
@@ -207,11 +207,22 @@ TEST(JoinTest, joinTest) {
   runTestCasesForAllJoinAlgorithms(createJoinTestSet());
 };
 
+// Several helpers for the test cases below.
 namespace {
 
+// The exact order of the columns of a join result might change over time, for
+// example we reorder inputs for simplicity or to more easily find them in the
+// cache. That's why we only assert that the column associated with a given
+// variabel contains the expected contents, independent of the concrete column
+// index that variable is assigned to.
+
+// A hash map that connects variables to the expected contents of the
+// corresponding result column and the `UndefStatus`.
 using ExpectedColumns = ad_utility::HashMap<
     Variable,
     std::pair, ColumnIndexAndTypeInfo::UndefStatus>>;
+
+// Test that the result of the `join` matches the `expected` outcome.
 void testJoinOperation(Join& join, const ExpectedColumns& expected) {
   auto res = join.getResult();
   const auto& varToCols = join.getExternallyVisibleVariableColumns();
@@ -227,23 +238,30 @@ void testJoinOperation(Join& join, const ExpectedColumns& expected) {
   }
 }
 
+// Convert a `VariableToColumnMap` (which assumes a fixed ordering of the
+// columns), and an `idTable` to the `ExpectedColumns` format that is
+// independent of the concrete assignment from variables to columns indices
 ExpectedColumns makeExpectedColumns(const VariableToColumnMap& varToColMap,
-                                    const IdTable& table) {
+                                    const IdTable& idTable) {
   ExpectedColumns result;
   for (const auto& [var, colIndexAndStatus] : varToColMap) {
-    result[var] = {table.getColumn(colIndexAndStatus.columnIndex_),
+    result[var] = {idTable.getColumn(colIndexAndStatus.columnIndex_),
                    colIndexAndStatus.mightContainUndef_};
   }
   return result;
 }
 
+// Create a `Values` clause with a single `variable` that stores the given
+// `values`. The values must all be vocabulary entries (IRIs or literals) that
+// are contained in the index of the `qec`, otherwise `std::bad_optional_access`
+// will be thrown.
 std::shared_ptr makeValuesForSingleVariable(
-    QueryExecutionContext* qec, std::string var,
+    QueryExecutionContext* qec, std::string variable,
     std::vector values) {
   parsedQuery::SparqlValues sparqlValues;
-  sparqlValues._variables.emplace_back(var);
-  for (const auto& value : values) {
-    sparqlValues._values.push_back({TripleComponent{value}});
+  sparqlValues._variables.emplace_back(std::move(variable));
+  for (auto& value : values) {
+    sparqlValues._values.push_back({TripleComponent{std::move(value)}});
   }
   return ad_utility::makeExecutionTree(qec, sparqlValues);
 }
@@ -280,48 +298,61 @@ TEST(JoinTest, joinWithFullScanPSO) {
   EXPECT_ANY_THROW(Join(qec, fullScanPSO, fullScanPSO, 0, 0));
 }
 
+// The following two tests run different code depending on the setting of the
+// maximal size for materialized index scans. That's why they are run twice with
+// different settings.
 TEST(JoinTest, joinWithColumnAndScan) {
-  auto qec = ad_utility::testing::getQec(" 
 1.  
 2.   3.");
-  auto fullScanPSO = ad_utility::makeExecutionTree(
-      qec, PSO, SparqlTriple{Var{"?s"}, "
", Var{"?o"}});
-  auto valuesTree = makeValuesForSingleVariable(qec, "?s", {""});
-
-  auto join = Join{qec, fullScanPSO, valuesTree, 0, 0};
-
-  auto getId = ad_utility::testing::makeGetId(qec->getIndex());
-  auto idX = getId("");
-  auto expected = makeIdTableFromVector({{idX, I(1)}});
-  VariableToColumnMap expectedVariables{
-      {Variable{"?s"}, makeAlwaysDefinedColumn(0)},
-      {Variable{"?o"}, makeAlwaysDefinedColumn(1)}};
-  testJoinOperation(join, makeExpectedColumns(expectedVariables, expected));
-
-  auto joinSwitched = Join{qec, valuesTree, fullScanPSO, 0, 0};
-  testJoinOperation(joinSwitched,
-                    makeExpectedColumns(expectedVariables, expected));
+  auto test = [](bool materializeIndexScans) {
+    auto qec = ad_utility::testing::getQec(" 
 1.  
 2.   3.");
+    RuntimeParameters().set<"lazy-index-scan-max-size-materialization">(
+        materializeIndexScans ? 1'000'000 : 0);
+    auto fullScanPSO = ad_utility::makeExecutionTree(
+        qec, PSO, SparqlTriple{Var{"?s"}, "
", Var{"?o"}});
+    auto valuesTree = makeValuesForSingleVariable(qec, "?s", {""});
+
+    auto join = Join{qec, fullScanPSO, valuesTree, 0, 0};
+
+    auto getId = ad_utility::testing::makeGetId(qec->getIndex());
+    auto idX = getId("");
+    auto expected = makeIdTableFromVector({{idX, I(1)}});
+    VariableToColumnMap expectedVariables{
+        {Variable{"?s"}, makeAlwaysDefinedColumn(0)},
+        {Variable{"?o"}, makeAlwaysDefinedColumn(1)}};
+    testJoinOperation(join, makeExpectedColumns(expectedVariables, expected));
+
+    auto joinSwitched = Join{qec, valuesTree, fullScanPSO, 0, 0};
+    testJoinOperation(joinSwitched,
+                      makeExpectedColumns(expectedVariables, expected));
+  };
+  test(true);
+  test(false);
 }
 
 TEST(JoinTest, joinTwoScans) {
-  auto qec = ad_utility::testing::getQec(
-      " 
 1.  
 2.   3 .   4. ");
-  auto scanP = ad_utility::makeExecutionTree(
-      qec, PSO, SparqlTriple{Var{"?s"}, "
", Var{"?o"}});
-  // TODO Who should catch the case that there are too many variables
-  // in common?
-  auto scanP2 = ad_utility::makeExecutionTree(
-      qec, PSO, SparqlTriple{Var{"?s"}, "", Var{"?q"}});
-  auto join = Join{qec, scanP2, scanP, 0, 0};
-
-  auto id = ad_utility::testing::makeGetId(qec->getIndex());
-  auto expected = makeIdTableFromVector(
-      {{id(""), I(3), I(1)}, {id(""), I(4), I(2)}});
-  VariableToColumnMap expectedVariables{
-      {Variable{"?s"}, makeAlwaysDefinedColumn(0)},
-      {Variable{"?q"}, makeAlwaysDefinedColumn(1)},
-      {Variable{"?o"}, makeAlwaysDefinedColumn(2)}};
-  testJoinOperation(join, makeExpectedColumns(expectedVariables, expected));
-
-  auto joinSwitched = Join{qec, scanP2, scanP, 0, 0};
-  testJoinOperation(joinSwitched,
-                    makeExpectedColumns(expectedVariables, expected));
+  auto test = [](bool materializeIndexScans) {
+    auto qec = ad_utility::testing::getQec(
+        " 
 1.  
 2.   3 .   4. ");
+    RuntimeParameters().set<"lazy-index-scan-max-size-materialization">(
+        materializeIndexScans ? 1'000'000 : 0);
+    auto scanP = ad_utility::makeExecutionTree(
+        qec, PSO, SparqlTriple{Var{"?s"}, "
", Var{"?o"}});
+    auto scanP2 = ad_utility::makeExecutionTree(
+        qec, PSO, SparqlTriple{Var{"?s"}, "", Var{"?q"}});
+    auto join = Join{qec, scanP2, scanP, 0, 0};
+
+    auto id = ad_utility::testing::makeGetId(qec->getIndex());
+    auto expected = makeIdTableFromVector(
+        {{id(""), I(3), I(1)}, {id(""), I(4), I(2)}});
+    VariableToColumnMap expectedVariables{
+        {Variable{"?s"}, makeAlwaysDefinedColumn(0)},
+        {Variable{"?q"}, makeAlwaysDefinedColumn(1)},
+        {Variable{"?o"}, makeAlwaysDefinedColumn(2)}};
+    testJoinOperation(join, makeExpectedColumns(expectedVariables, expected));
+
+    auto joinSwitched = Join{qec, scanP2, scanP, 0, 0};
+    testJoinOperation(joinSwitched,
+                      makeExpectedColumns(expectedVariables, expected));
+  };
+  test(true);
+  test(false);
 }

From aed7ea091dfd138953934868bdb9e40341d4e07e Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 10 Jul 2023 17:52:06 +0200
Subject: [PATCH 138/150] Use supported comparison.

---
 src/index/CompressedRelation.h | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index 02914966fc..33097d0513 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -82,12 +82,14 @@ struct CompressedBlockMetadata {
     // Convert to a plain array.
     std::array toArray() const { return {col0Id_, col1Id_, col2Id_}; }
 
-    // Compare lexicographically.
-    auto operator<=>(const PermutedTriple& other) const {
+    // Compare lexicographically. We currently only need less equal for
+    // assertions. We could use `std::lexicographical_compare_three_way` as soon
+    // as it is supported by Clang's libc++.
+    auto operator<=(const PermutedTriple& other) const {
       auto arr1 = toArray();
       auto arr2 = other.toArray();
-      return std::lexicographical_compare_three_way(arr1.begin(), arr1.end(),
-                                                    arr2.begin(), arr2.end());
+      return !std::lexicographical_compare(arr2.begin(), arr2.end(),
+                                                    arr1.begin(), arr1.end());
     }
 
     friend std::true_type allowTrivialSerialization(PermutedTriple, auto);

From 944ed2a123fe5f401a7147ade5ab54a1d43ca2ca Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 10 Jul 2023 18:19:16 +0200
Subject: [PATCH 139/150] formatted this

---
 src/index/CompressedRelation.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index 33097d0513..4736d5603c 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -89,7 +89,7 @@ struct CompressedBlockMetadata {
       auto arr1 = toArray();
       auto arr2 = other.toArray();
       return !std::lexicographical_compare(arr2.begin(), arr2.end(),
-                                                    arr1.begin(), arr1.end());
+                                           arr1.begin(), arr1.end());
     }
 
     friend std::true_type allowTrivialSerialization(PermutedTriple, auto);

From b620bffedbc8c86ca3068d192fe48c392a794df9 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 10 Jul 2023 19:43:36 +0200
Subject: [PATCH 140/150] Small changes from a review with Hannah.

---
 src/engine/Join.cpp              | 15 ++++---
 src/index/CompressedRelation.cpp | 73 ++++++++++++--------------------
 src/index/CompressedRelation.h   | 14 +++---
 test/JoinTest.cpp                |  2 +-
 4 files changed, 42 insertions(+), 62 deletions(-)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index b772ca741f..470d038ef1 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -125,16 +125,17 @@ ResultTable Join::computeResult() {
   // values are involved.
   auto getCachedOrSmallResult = [](QueryExecutionTree& tree,
                                    ColumnIndex joinCol) {
-    bool readOnlyCachedResult =
-        tree.getRootOperation()->getSizeEstimate() >
+    bool isSmall =
+        tree.getRootOperation()->getSizeEstimate() <
         RuntimeParameters().get<"lazy-index-scan-max-size-materialization">();
     auto undefStatus =
         tree.getVariableAndInfoByColumnIndex(joinCol).second.mightContainUndef_;
-    readOnlyCachedResult =
-        readOnlyCachedResult &&
-        undefStatus == ColumnIndexAndTypeInfo::UndefStatus::AlwaysDefined;
-
-    return tree.getRootOperation()->getResult(false, readOnlyCachedResult);
+    bool containsUndef =
+        undefStatus == ColumnIndexAndTypeInfo::UndefStatus::PossiblyUndefined;
+    // The third argument means "only get the result if it can be read from the
+    // cache".
+    return tree.getRootOperation()->getResult(false,
+                                              !(isSmall || containsUndef));
   };
 
   auto leftResIfCached = getCachedOrSmallResult(*_left, _leftJoinCol);
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 2bcb78ba6f..fc4b5c507c 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -293,52 +293,42 @@ namespace {
 auto getRelevantIdFromTriple(
     CompressedBlockMetadata::PermutedTriple triple,
     const CompressedRelationReader::MetadataAndBlocks& metadataAndBlocks) {
-  auto idForNonMatchingBlock = [](Id fromTriple, Id key) -> std::optional {
+  auto idForNonMatchingBlock = [](Id fromTriple, Id key, Id minKey,
+                                  Id maxKey) -> std::optional {
     if (fromTriple < key) {
-      return Id::min();
+      return minKey;
     }
     if (fromTriple > key) {
-      return Id::max();
+      return maxKey;
     }
     return std::nullopt;
   };
 
+  auto [minKey, maxKey] = [&]() {
+    if (!metadataAndBlocks.firstAndLastTriple_.has_value()) {
+      return std::array{Id::min(), Id::max()};
+    }
+    const auto& [first, last] = metadataAndBlocks.firstAndLastTriple_.value();
+    if (metadataAndBlocks.col1Id_.has_value()) {
+      return std::array{first.col2Id_, last.col2Id_};
+    } else {
+      return std::array{first.col1Id_, last.col1Id_};
+    }
+  }();
+
   if (auto optId = idForNonMatchingBlock(
-          triple.col0Id_, metadataAndBlocks.relationMetadata_.col0Id_)) {
+          triple.col0Id_, metadataAndBlocks.relationMetadata_.col0Id_, minKey,
+          maxKey)) {
     return optId.value();
   }
   if (!metadataAndBlocks.col1Id_.has_value()) {
     return triple.col1Id_;
   }
 
-  return idForNonMatchingBlock(triple.col1Id_,
-                               metadataAndBlocks.col1Id_.value())
+  return idForNonMatchingBlock(
+             triple.col1Id_, metadataAndBlocks.col1Id_.value(), minKey, maxKey)
       .value_or(triple.col2Id_);
 }
-
-// Set the first triple of the first block in `blocks` and the last triple of
-// the last block according to the `firstAndLastTriple`.
-void setFirstAndLastTriple(
-    std::vector& blocks,
-    const std::optional<
-        CompressedRelationReader::MetadataAndBlocks::FirstAndLastTriple>&
-        firstAndLastTriple) {
-  if (blocks.empty() || !firstAndLastTriple.has_value()) {
-    return;
-  }
-
-  // Check that the `newTriple` can be safely set as the first or last triple of
-  // either the `block` without breaking the ordering of the first block.
-  auto check = [](const auto& newTriple, const auto& block) {
-    AD_CORRECTNESS_CHECK(block.firstTriple_ <= newTriple);
-    AD_CORRECTNESS_CHECK(newTriple <= block.lastTriple_);
-  };
-  check(firstAndLastTriple.value().firstTriple_, blocks.front());
-  blocks.front().firstTriple_ = firstAndLastTriple.value().firstTriple_;
-
-  check(firstAndLastTriple.value().lastTriple_, blocks.back());
-  blocks.back().lastTriple_ = firstAndLastTriple.value().lastTriple_;
-}
 }  // namespace
 
 // _____________________________________________________________________________
@@ -346,10 +336,7 @@ std::vector CompressedRelationReader::getBlocksForJoin(
     std::span joinColumn,
     const MetadataAndBlocks& metadataAndBlocks) {
   // Get all the blocks where `col0FirstId_ <= col0Id <= col0LastId_`.
-  auto relevantBlocksSpan = getBlocksFromMetadata(metadataAndBlocks);
-  std::vector relevantBlocks(relevantBlocksSpan.begin(),
-                             relevantBlocksSpan.end());
-  setFirstAndLastTriple(relevantBlocks, metadataAndBlocks.firstAndLastTriple_);
+  auto relevantBlocks = getBlocksFromMetadata(metadataAndBlocks);
 
   // We need symmetric comparisons between Ids and blocks.
   auto idLessThanBlock = [&metadataAndBlocks](
@@ -395,19 +382,9 @@ std::array, 2>
 CompressedRelationReader::getBlocksForJoin(
     const MetadataAndBlocks& metadataAndBlocks1,
     const MetadataAndBlocks& metadataAndBlocks2) {
-  auto relevantBlocks1Span = getBlocksFromMetadata(metadataAndBlocks1);
-  auto relevantBlocks2Span = getBlocksFromMetadata(metadataAndBlocks2);
-
-  // TODO<
-  std::vector relevantBlocks1(relevantBlocks1Span.begin(),
-                              relevantBlocks1Span.end());
-  std::vector relevantBlocks2(relevantBlocks2Span.begin(),
-                              relevantBlocks2Span.end());
-
-  setFirstAndLastTriple(relevantBlocks1,
-                        metadataAndBlocks1.firstAndLastTriple_);
-  setFirstAndLastTriple(relevantBlocks2,
-                        metadataAndBlocks2.firstAndLastTriple_);
+  auto relevantBlocks1 = getBlocksFromMetadata(metadataAndBlocks1);
+  auto relevantBlocks2 = getBlocksFromMetadata(metadataAndBlocks2);
+
   auto metadataForBlock =
       [&](const CompressedBlockMetadata& block) -> decltype(auto) {
     if (relevantBlocks1.data() <= &block &&
@@ -949,6 +926,8 @@ auto CompressedRelationReader::getFirstAndLastTriple(
   AD_CONTRACT_CHECK(!relevantBlocks.empty());
 
   auto scanBlock = [&](const CompressedBlockMetadata& block) {
+    // Note: the following call only returns the part of the block that actually
+    // matches the col0 and col1.
     return readPossiblyIncompleteBlock(metadataAndBlocks.relationMetadata_,
                                        metadataAndBlocks.col1Id_, file, block,
                                        std::nullopt);
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index 4736d5603c..94c7683df0 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -251,12 +251,12 @@ class CompressedRelationReader {
     const std::span blockMetadata_;
     std::optional col1Id_;
 
-    // If set, then those contain the first and the last triple of the specified
-    // relation (and being filtered by the `col1Id` if specified). This might be
-    // different from the first triple in the first block (in the case of the
-    // `firstTriple_`, similarly for `lastTriple_`) because the first and last
-    // block might also contain other relations, or the same relation but with
-    // different `col1Id`s.
+    // If set, `firstAndLastTriple_` contains the first and the last triple
+    // of the specified relation (and being filtered by the `col1Id` if
+    // specified). This might be different from the first triple in the first
+    // block (in the case of the `firstTriple_`, similarly for `lastTriple_`)
+    // because the first and last block might also contain other relations, or
+    // the same relation but with different `col1Id`s.
     struct FirstAndLastTriple {
       CompressedBlockMetadata::PermutedTriple firstTriple_;
       CompressedBlockMetadata::PermutedTriple lastTriple_;
@@ -388,7 +388,7 @@ class CompressedRelationReader {
       const MetadataAndBlocks& metadataAndBlocks);
 
   // Get the first and the last triple that the result of a `scan` with the
-  // given argument would lead to. Throw an exception if the scan result would
+  // given arguments would lead to. Throw an exception if the scan result would
   // be empty. This function is used to more efficiently filter the blocks of
   // index scans between joining them to get better estimates for the begginning
   // and end of incomplete blocks.
diff --git a/test/JoinTest.cpp b/test/JoinTest.cpp
index 883a6d68eb..cb759b5f9b 100644
--- a/test/JoinTest.cpp
+++ b/test/JoinTest.cpp
@@ -213,7 +213,7 @@ namespace {
 // The exact order of the columns of a join result might change over time, for
 // example we reorder inputs for simplicity or to more easily find them in the
 // cache. That's why we only assert that the column associated with a given
-// variabel contains the expected contents, independent of the concrete column
+// variable contains the expected contents, independent of the concrete column
 // index that variable is assigned to.
 
 // A hash map that connects variables to the expected contents of the

From f82ad6587ec6c08786929e74640034fd0be0901b Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Mon, 10 Jul 2023 20:01:02 +0200
Subject: [PATCH 141/150] Better stuff.

---
 src/engine/Join.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index 470d038ef1..bb18d885f3 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -133,7 +133,8 @@ ResultTable Join::computeResult() {
     bool containsUndef =
         undefStatus == ColumnIndexAndTypeInfo::UndefStatus::PossiblyUndefined;
     // The third argument means "only get the result if it can be read from the
-    // cache".
+    // cache". So effectively, this returns the result if it is small, contains
+    // UNDEF values, or is contained in the cache, otherwise `nullptr`.
     return tree.getRootOperation()->getResult(false,
                                               !(isSmall || containsUndef));
   };

From 94cb17438da7f15e9302188685e0cc353d86474a Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 11 Jul 2023 11:49:48 +0200
Subject: [PATCH 142/150] Improve test coverage.

---
 src/index/CompressedRelation.h | 12 ------------
 test/JoinTest.cpp              | 22 +++++++++++++++-------
 2 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index 94c7683df0..b69db22034 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -79,18 +79,6 @@ struct CompressedBlockMetadata {
     Id col1Id_;
     Id col2Id_;
     bool operator==(const PermutedTriple&) const = default;
-    // Convert to a plain array.
-    std::array toArray() const { return {col0Id_, col1Id_, col2Id_}; }
-
-    // Compare lexicographically. We currently only need less equal for
-    // assertions. We could use `std::lexicographical_compare_three_way` as soon
-    // as it is supported by Clang's libc++.
-    auto operator<=(const PermutedTriple& other) const {
-      auto arr1 = toArray();
-      auto arr2 = other.toArray();
-      return !std::lexicographical_compare(arr2.begin(), arr2.end(),
-                                           arr1.begin(), arr1.end());
-    }
 
     friend std::true_type allowTrivialSerialization(PermutedTriple, auto);
   };
diff --git a/test/JoinTest.cpp b/test/JoinTest.cpp
index cb759b5f9b..57ae8070fb 100644
--- a/test/JoinTest.cpp
+++ b/test/JoinTest.cpp
@@ -306,6 +306,7 @@ TEST(JoinTest, joinWithColumnAndScan) {
     auto qec = ad_utility::testing::getQec(" 
 1.  
 2.   3.");
     RuntimeParameters().set<"lazy-index-scan-max-size-materialization">(
         materializeIndexScans ? 1'000'000 : 0);
+    qec->getQueryTreeCache().clearAll();
     auto fullScanPSO = ad_utility::makeExecutionTree(
         qec, PSO, SparqlTriple{Var{"?s"}, "
", Var{"?o"}});
     auto valuesTree = makeValuesForSingleVariable(qec, "?s", {""});
@@ -324,16 +325,20 @@ TEST(JoinTest, joinWithColumnAndScan) {
     testJoinOperation(joinSwitched,
                       makeExpectedColumns(expectedVariables, expected));
   };
-  test(true);
-  test(false);
+  test(0);
+  test(1);
+  test(2);
+  test(3);
+  test(1'000'000);
 }
 
 TEST(JoinTest, joinTwoScans) {
-  auto test = [](bool materializeIndexScans) {
+  auto test = [](size_t materializationThreshold) {
     auto qec = ad_utility::testing::getQec(
-        " 
 1.  
 2.   3 .   4. ");
+        " 
 1.  
 2.   3 .   4.   7. ");
     RuntimeParameters().set<"lazy-index-scan-max-size-materialization">(
-        materializeIndexScans ? 1'000'000 : 0);
+        materializationThreshold);
+    qec->getQueryTreeCache().clearAll();
     auto scanP = ad_utility::makeExecutionTree(
         qec, PSO, SparqlTriple{Var{"?s"}, "
", Var{"?o"}});
     auto scanP2 = ad_utility::makeExecutionTree(
@@ -353,6 +358,9 @@ TEST(JoinTest, joinTwoScans) {
     testJoinOperation(joinSwitched,
                       makeExpectedColumns(expectedVariables, expected));
   };
-  test(true);
-  test(false);
+  test(0);
+  test(1);
+  test(2);
+  test(3);
+  test(1'000'000);
 }

From 54ace647378b29c86b79365f4d5c23b92195a8ec Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 11 Jul 2023 14:05:05 +0200
Subject: [PATCH 143/150] Implement a useful helper.

---
 src/util/ThreadSafeQueue.h   | 88 ++++++++++++++++++++++++++++++++++++
 test/ThreadSafeQueueTest.cpp | 49 ++++++++++++++++++++
 2 files changed, 137 insertions(+)

diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index bf584ce6da..26bce55dce 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -9,6 +9,12 @@
 #include 
 #include 
 #include 
+#include 
+
+#include "absl/cleanup/cleanup.h"
+#include "util/Exception.h"
+#include "util/Generator.h"
+#include "util/jthread.h"
 
 namespace ad_utility::data_structures {
 
@@ -24,6 +30,7 @@ class ThreadSafeQueue {
   size_t maxSize_;
 
  public:
+  using value_type = T;
   explicit ThreadSafeQueue(size_t maxSize) : maxSize_{maxSize} {}
 
   // We can neither copy nor move this class
@@ -123,6 +130,7 @@ class OrderedThreadSafeQueue {
   bool finish_ = false;
 
  public:
+  using value_type = T;
   // Construct from the maximal queue size (see `ThreadSafeQueue` for details).
   explicit OrderedThreadSafeQueue(size_t maxSize) : queue_{maxSize} {}
 
@@ -150,6 +158,10 @@ class OrderedThreadSafeQueue {
     return result;
   }
 
+  bool push(std::pair indexAndValue) {
+    push(indexAndValue.first, std::move(indexAndValue.second));
+  }
+
   // See `ThreadSafeQueue` for details.
   void pushException(std::exception_ptr exception) {
     std::unique_lock l{mutex_};
@@ -176,4 +188,80 @@ class OrderedThreadSafeQueue {
   std::optional pop() { return queue_.pop(); }
 };
 
+template 
+class ThreadsafeQueueManager {
+  Queue queue_;
+  std::vector threads_;
+
+  ~ThreadsafeQueueManager() { queue_.finish(); }
+};
+
+// A concept for one of the thread-safe queue types above
+template 
+concept IsThreadsafeQueue =
+    ad_utility::similarToInstantiation ||
+    ad_utility::similarToInstantiation;
+
+namespace detail {
+// A helper function for setting up a producer task for one of the threadsafe
+// queues above. Takes a reference to a  queue and a `task`. The task must
+// return `std::optional`. The task is
+// called repeatedly, and the resulting values are pushed to the queue. If the
+// task returns `nullopt`, `numThreads` is decremented, and the queue is
+// finished if `numThreads <= 0`. All exceptions that happen during the
+// execution of `task` are propagated to the queue.
+template 
+auto makeQueueTask(Queue& queue, Task task, std::atomic& numThreads) {
+  return [&queue, task = std::move(task), &numThreads] {
+    try {
+      while (auto opt = task()) {
+        if (!queue.push(std::move(opt.value()))) {
+          break;
+        }
+      }
+    } catch (...) {
+      try {
+        queue.pushException(std::current_exception());
+      } catch (...) {
+        queue.finish();
+      }
+    }
+    --numThreads;
+    if (numThreads <= 0) {
+      queue.finish();
+    }
+  };
+}
+}  // namespace detail
+
+// This helper function makes the usage of the (Ordered)ThreadSafeQueue above
+// much easier. It takes the size of the queue, the number of producer threads,
+// and a task that produces values. The `producerTask` is called repeatedly in
+// `numThreads` many concurrent threads. It needs to return
+// `std::optional` and has the following
+// semantics: If `nullopt` is returned, then the thread is finished. The queue
+// is finished, when all the producer threads have finished by yielding
+// `nullopt`, or if any call to `producerTask` in any thread throws an
+// exception. In that case the exception is propagated to the resulting
+// generator. The resulting generator yields all the values that have been
+// pushed to the queue.
+template 
+cppcoro::generator QueueManager(size_t queueSize,
+                                                            size_t numThreads,
+                                                            auto producerTask) {
+  Queue queue{queueSize};
+  AD_CONTRACT_CHECK(numThreads > 0u);
+  std::vector threads;
+  std::atomic numUnfinishedThreads{static_cast(numThreads)};
+  absl::Cleanup queueFinisher{[&queue] { queue.finish(); }};
+  for ([[maybe_unused]] auto i : std::views::iota(0u, numThreads)) {
+    threads.emplace_back(
+        detail::makeQueueTask(queue, producerTask, numUnfinishedThreads));
+  }
+
+  while (auto opt = queue.pop()) {
+    co_yield (opt.value());
+  }
+}
+
 }  // namespace ad_utility::data_structures
diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 8106ebe972..347ef93e85 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -31,6 +31,17 @@ auto makePush(Queue& queue) {
   };
 }
 
+template 
+auto makeQueueValue() {
+  return [](size_t i) {
+    if constexpr (ad_utility::similarToInstantiation) {
+      return i;
+    } else {
+      return std::pair{i, i};
+    }
+  };
+}
+
 // Some constants that are used in almost every test case.
 constexpr size_t queueSize = 5;
 constexpr size_t numThreads = 20;
@@ -248,3 +259,41 @@ TEST(ThreadSafeQueue, DisablePush) {
   };
   runWithBothQueueTypes(runTest);
 }
+
+// ________________________________________________________________
+TEST(ThreadSafeQueue, QueueManager) {
+  auto runTest = [](Queue) {
+    std::atomic numPushed = 0;
+    auto task =
+        [&numPushed]() -> std::optional()(3))> {
+      auto makeValue = makeQueueValue();
+      while (true) {
+        auto value = ++numPushed;
+        if (value < numValues) {
+          return makeValue(value);
+        } else {
+          return std::nullopt;
+        }
+      }
+    };
+    std::vector result;
+    size_t numPopped = 0;
+    for (size_t value : QueueManager(queueSize, numThreads, task)) {
+      ++numPopped;
+      result.push_back(value);
+      EXPECT_LE(numPushed, numPopped + queueSize + 1 + numThreads);
+    }
+    if (ad_utility::similarToInstantiation) {
+      // When terminating early, we cannot actually say much about the result,
+      // other than that it contains no duplicate values
+      std::ranges::sort(result);
+      EXPECT_TRUE(std::unique(result.begin(), result.end()) == result.end());
+    } else {
+      // For the ordered queue we have the guarantee that all the pushed values
+      // were in order.
+      EXPECT_THAT(result,
+                  ::testing::ElementsAreArray(std::views::iota(0U, 400U)));
+    }
+  };
+  runWithBothQueueTypes(runTest);
+}

From d57319af73d1d44b6e103dd1e23e136e504a2d4c Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 11 Jul 2023 16:18:22 +0200
Subject: [PATCH 144/150] Several improvements of the test coverage. This
 should be it (once we have the test coverage for the thread safe queue as
 well).

---
 src/engine/Join.cpp               | 11 +---
 src/engine/QueryExecutionTree.cpp |  5 +-
 src/engine/ValuesForTesting.h     |  9 +++
 src/index/CompressedRelation.cpp  | 93 +++++++++++--------------------
 src/util/ThreadSafeQueue.h        |  2 +-
 test/IndexTestHelpers.h           | 26 +++++----
 test/JoinTest.cpp                 | 81 ++++++++++++++++++++++++++-
 test/ThreadSafeQueueTest.cpp      | 14 +----
 test/engine/IndexScanTest.cpp     | 16 +++++-
 9 files changed, 159 insertions(+), 98 deletions(-)

diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp
index bb18d885f3..f6a4af8dfc 100644
--- a/src/engine/Join.cpp
+++ b/src/engine/Join.cpp
@@ -57,12 +57,7 @@ Join::Join(QueryExecutionContext* qec, std::shared_ptr t1,
   _multiplicities.clear();
   auto findJoinVar = [](const QueryExecutionTree& tree,
                         ColumnIndex joinCol) -> Variable {
-    for (auto p : tree.getVariableColumns()) {
-      if (p.second.columnIndex_ == joinCol) {
-        return p.first;
-      }
-    }
-    AD_FAIL();
+    return tree.getVariableAndInfoByColumnIndex(joinCol).first;
   };
   _joinVar = findJoinVar(*_left, _leftJoinCol);
   AD_CONTRACT_CHECK(_joinVar == findJoinVar(*_right, _rightJoinCol));
@@ -71,8 +66,8 @@ Join::Join(QueryExecutionContext* qec, std::shared_ptr t1,
 // _____________________________________________________________________________
 Join::Join(InvalidOnlyForTestingJoinTag, QueryExecutionContext* qec)
     : Operation(qec) {
-  // Needed, so that the time out checker in Join::join doesn't create a seg
-  // fault if it tries to create a message about the time out.
+  // Needed, so that the timeout checker in Join::join doesn't create a seg
+  // fault if it tries to create a message about the timeout.
   _left = std::make_shared(qec);
   _right = _left;
 }
diff --git a/src/engine/QueryExecutionTree.cpp b/src/engine/QueryExecutionTree.cpp
index 8e043af288..917e8bd950 100644
--- a/src/engine/QueryExecutionTree.cpp
+++ b/src/engine/QueryExecutionTree.cpp
@@ -216,7 +216,8 @@ void QueryExecutionTree::setOperation(std::shared_ptr operation) {
     _type = OPTIONAL_JOIN;
   } else if constexpr (std::is_same_v) {
     _type = MULTICOLUMN_JOIN;
-  } else if constexpr (std::is_same_v) {
+  } else if constexpr (std::is_same_v ||
+                       std::is_same_v) {
     _type = DUMMY;
   } else {
     static_assert(ad_utility::alwaysFalse,
@@ -251,6 +252,8 @@ template void QueryExecutionTree::setOperation(
     std::shared_ptr);
 template void QueryExecutionTree::setOperation(
     std::shared_ptr);
+template void QueryExecutionTree::setOperation(
+    std::shared_ptr);
 
 // ________________________________________________________________________________________________________________
 std::shared_ptr QueryExecutionTree::createSortedTree(
diff --git a/src/engine/ValuesForTesting.h b/src/engine/ValuesForTesting.h
index 7c0955857e..09aefc0799 100644
--- a/src/engine/ValuesForTesting.h
+++ b/src/engine/ValuesForTesting.h
@@ -92,3 +92,12 @@ class ValuesForTesting : public Operation {
     return m;
   }
 };
+
+// Similar to `ValuesForTesting` above, but `knownEmptyResult()` always returns
+// false. This can be used for improved test coverage in cases where we want the
+// empty result to be not optimized out by a check to `knownEmptyResult`.
+class ValuesForTestingNoKnownEmptyResult : public ValuesForTesting {
+ public:
+  using ValuesForTesting::ValuesForTesting;
+  bool knownEmptyResult() override { return false; }
+};
diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index fc4b5c507c..82ae727652 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -115,83 +115,52 @@ CompressedRelationReader::asyncParallelBlockGenerator(
   }
   const size_t queueSize =
       RuntimeParameters().get<"lazy-index-scan-queue-size">();
-  ad_utility::data_structures::OrderedThreadSafeQueue queue{
-      queueSize};
   auto blockIterator = beginBlock;
   std::mutex blockIteratorMutex;
-  auto readAndDecompressBlock = [&]() {
-    try {
-      while (true) {
-        std::unique_lock lock{blockIteratorMutex};
-        if (blockIterator == endBlock) {
-          // Note: We cannot call `queue.finish()` here, as the last block might
-          // not yet be pushed because it is handled by another thread.
-          return;
-        }
-        // Note: taking a copy here is probably not necessary (the lifetime of
-        // all the blocks is long enough, so a `const&` would suffice), but the
-        // copy is cheap and makes the code more robust.
-        auto block = *blockIterator;
-        // Note: The order of the following three lines is important: The index
-        // of the current block depends on the current value of `blockIterator`,
-        // but we have to increment `blockIterator` to determine whether this
-        // was the last block.
-        auto myIndex = static_cast(blockIterator - beginBlock);
-        ++blockIterator;
-        bool isLastBlock = blockIterator == endBlock;
-        // Note: the reading of the block could also happen without holding the
-        // lock. We still perform it inside the lock to avoid contention of the
-        // file. On a fast SSD we could possibly change this, but this has to be
-        // investigated.
-        CompressedBlock compressedBlock =
-            readCompressedBlockFromFile(block, file, columnIndices);
-        lock.unlock();
-        bool pushWasSuccessful = queue.push(
-            myIndex, decompressBlock(compressedBlock, block.numRows_));
-        checkTimeout(timer);
-        if (!pushWasSuccessful) {
-          return;
-        }
-        // Note: Only the thread that actually pushes the last block knows when
-        // it is safe to call `finish` to signal that all blocks have been
-        // succesfully pushed to the queue.
-        if (isLastBlock) {
-          queue.finish();
-        }
-      }
-    } catch (...) {
-      try {
-        queue.pushException(std::current_exception());
-      } catch (...) {
-        queue.finish();
-      }
+  auto readAndDecompressBlock =
+      [&]() -> std::optional> {
+    checkTimeout(timer);
+    std::unique_lock lock{blockIteratorMutex};
+    if (blockIterator == endBlock) {
+      return std::nullopt;
     }
+    // Note: taking a copy here is probably not necessary (the lifetime of
+    // all the blocks is long enough, so a `const&` would suffice), but the
+    // copy is cheap and makes the code more robust.
+    auto block = *blockIterator;
+    // Note: The order of the following three lines is important: The index
+    // of the current block depends on the current value of `blockIterator`,
+    // but we have to increment `blockIterator` to determine whether this
+    // was the last block.
+    auto myIndex = static_cast(blockIterator - beginBlock);
+    ++blockIterator;
+    // Note: the reading of the block could also happen without holding the
+    // lock. We still perform it inside the lock to avoid contention of the
+    // file. On a fast SSD we could possibly change this, but this has to be
+    // investigated.
+    CompressedBlock compressedBlock =
+        readCompressedBlockFromFile(block, file, columnIndices);
+    lock.unlock();
+    return std::pair{myIndex, decompressBlock(compressedBlock, block.numRows_)};
   };
-  // Note: The order of the following declarations is very important at the time
-  // of destruction: First, the `Cleanup` is destroyed, which finishes the
-  // queue. This allows the destructor of `threads` to join the threads, as the
-  // threads are able to complete once they cannot access the queue anymore.
-  // Only then the `blockIteratorMutex` and the `queue` can be safely destroyed,
-  // as the threads that were using them have already been destroyed.
-  std::vector threads;
-  absl::Cleanup finishQueue{[&queue] { queue.finish(); }};
   const size_t numThreads =
       RuntimeParameters().get<"lazy-index-scan-num-threads">();
-  for ([[maybe_unused]] auto j : std::views::iota(0u, numThreads)) {
-    threads.emplace_back(readAndDecompressBlock);
-  }
 
   ad_utility::Timer popTimer{ad_utility::timer::Timer::InitialStatus::Started};
   // In case the coroutine is destroyed early we still want to have this
   // information.
   auto setTimer = ad_utility::makeOnDestructionDontThrowDuringStackUnwinding(
       [&details, &popTimer]() { details.blockingTimeMs_ = popTimer.msecs(); });
-  while (auto opt = queue.pop()) {
+
+  auto queue = ad_utility::data_structures::QueueManager<
+      ad_utility::data_structures::OrderedThreadSafeQueue>(
+      queueSize, numThreads, readAndDecompressBlock);
+  for (IdTable& block : queue) {
     popTimer.stop();
     checkTimeout(timer);
     ++details.numBlocksRead_;
-    details.numElementsRead_ += opt.value().numRows();
-    co_yield opt.value();
+    details.numElementsRead_ += block.numRows();
+    co_yield block;
     popTimer.cont();
   }
   // The `OnDestruction...` above might be called too late, so we manually set
diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index 26bce55dce..6011d0c603 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -159,7 +159,7 @@ class OrderedThreadSafeQueue {
   }
 
   bool push(std::pair indexAndValue) {
-    push(indexAndValue.first, std::move(indexAndValue.second));
+    return push(indexAndValue.first, std::move(indexAndValue.second));
   }
 
   // See `ThreadSafeQueue` for details.
diff --git a/test/IndexTestHelpers.h b/test/IndexTestHelpers.h
index 0bdcb2bb5b..cc1969730b 100644
--- a/test/IndexTestHelpers.h
+++ b/test/IndexTestHelpers.h
@@ -21,11 +21,6 @@ namespace ad_utility::testing {
 inline Index makeIndexWithTestSettings() {
   Index index{ad_utility::makeUnlimitedAllocator()};
   index.setNumTriplesPerBatch(2);
-  // This is enough for 2 triples per block. This is deliberately chosen as a
-  // small value, s.t. the tiny knowledge graphs from unit tests also contain
-  // multiple blocks. Should this value or the semantics of it (how many triples
-  // it may store) ever change, then some unit tests might have to be adapted.
-  index.blocksizePermutationsInBytes() = 32;
   index.stxxlMemoryInBytes() = 1024ul * 1024ul * 50;
   return index;
 }
@@ -68,7 +63,8 @@ inline Index makeTestIndex(const std::string& indexBasename,
                            std::string turtleInput = "",
                            bool loadAllPermutations = true,
                            bool usePatterns = true,
-                           bool usePrefixCompression = true) {
+                           bool usePrefixCompression = true,
+                           size_t blocksizePermutationsInBytes = 32) {
   // Ignore the (irrelevant) log output of the index building and loading during
   // these tests.
   static std::ostringstream ignoreLogStream;
@@ -88,6 +84,12 @@ inline Index makeTestIndex(const std::string& indexBasename,
   f.close();
   {
     Index index = makeIndexWithTestSettings();
+    // This is enough for 2 triples per block. This is deliberately chosen as a
+    // small value, s.t. the tiny knowledge graphs from unit tests also contain
+    // multiple blocks. Should this value or the semantics of it (how many
+    // triples it may store) ever change, then some unit tests might have to be
+    // adapted.
+    index.blocksizePermutationsInBytes() = blocksizePermutationsInBytes;
     index.setOnDiskBase(indexBasename);
     index.setUsePatterns(usePatterns);
     index.setPrefixCompression(usePrefixCompression);
@@ -107,7 +109,8 @@ inline Index makeTestIndex(const std::string& indexBasename,
 inline QueryExecutionContext* getQec(std::string turtleInput = "",
                                      bool loadAllPermutations = true,
                                      bool usePatterns = true,
-                                     bool usePrefixCompression = true) {
+                                     bool usePrefixCompression = true,
+                                     size_t blocksizePermutationsInBytes = 32) {
   // Similar to `absl::Cleanup`. Calls the `callback_` in the destructor, but
   // the callback is stored as a `std::function`, which allows to store
   // different types of callbacks in the same wrapper type.
@@ -129,11 +132,11 @@ inline QueryExecutionContext* getQec(std::string turtleInput = "",
             *index_, cache_.get(), makeAllocator(), SortPerformanceEstimator{});
   };
 
-  using Key = std::tuple;
+  using Key = std::tuple;
   static ad_utility::HashMap contextMap;
 
-  auto key =
-      Key{turtleInput, loadAllPermutations, usePatterns, usePrefixCompression};
+  auto key = Key{turtleInput, loadAllPermutations, usePatterns,
+                 usePrefixCompression, blocksizePermutationsInBytes};
 
   if (!contextMap.contains(key)) {
     std::string testIndexBasename =
@@ -150,7 +153,8 @@ inline QueryExecutionContext* getQec(std::string turtleInput = "",
                      }},
                      std::make_unique(makeTestIndex(
                          testIndexBasename, turtleInput, loadAllPermutations,
-                         usePatterns, usePrefixCompression)),
+                         usePatterns, usePrefixCompression,
+                         blocksizePermutationsInBytes)),
                      std::make_unique()});
   }
   return contextMap.at(key).qec_.get();
diff --git a/test/JoinTest.cpp b/test/JoinTest.cpp
index 57ae8070fb..adcd02a401 100644
--- a/test/JoinTest.cpp
+++ b/test/JoinTest.cpp
@@ -24,6 +24,7 @@
 #include "engine/OptionalJoin.h"
 #include "engine/QueryExecutionTree.h"
 #include "engine/Values.h"
+#include "engine/ValuesForTesting.h"
 #include "engine/idTable/IdTable.h"
 #include "util/Forward.h"
 #include "util/Random.h"
@@ -302,16 +303,17 @@ TEST(JoinTest, joinWithFullScanPSO) {
 // maximal size for materialized index scans. That's why they are run twice with
 // different settings.
 TEST(JoinTest, joinWithColumnAndScan) {
-  auto test = [](bool materializeIndexScans) {
+  auto test = [](size_t materializationThreshold) {
     auto qec = ad_utility::testing::getQec(" 
 1.  
 2.   3.");
     RuntimeParameters().set<"lazy-index-scan-max-size-materialization">(
-        materializeIndexScans ? 1'000'000 : 0);
+        materializationThreshold);
     qec->getQueryTreeCache().clearAll();
     auto fullScanPSO = ad_utility::makeExecutionTree(
         qec, PSO, SparqlTriple{Var{"?s"}, "
", Var{"?o"}});
     auto valuesTree = makeValuesForSingleVariable(qec, "?s", {""});
 
     auto join = Join{qec, fullScanPSO, valuesTree, 0, 0};
+    EXPECT_EQ(join.getDescriptor(), "Join on ?s");
 
     auto getId = ad_utility::testing::makeGetId(qec->getIndex());
     auto idX = getId("");
@@ -332,6 +334,71 @@ TEST(JoinTest, joinWithColumnAndScan) {
   test(1'000'000);
 }
 
+TEST(JoinTest, joinWithColumnAndScanEmptyInput) {
+  auto test = [](size_t materializationThreshold) {
+    auto qec = ad_utility::testing::getQec(" 
 1.  
 2.   3.");
+    RuntimeParameters().set<"lazy-index-scan-max-size-materialization">(
+        materializationThreshold);
+    qec->getQueryTreeCache().clearAll();
+    auto fullScanPSO = ad_utility::makeExecutionTree(
+        qec, PSO, SparqlTriple{Var{"?s"}, "
", Var{"?o"}});
+    auto valuesTree =
+        ad_utility::makeExecutionTree(
+            qec, IdTable{1, qec->getAllocator()}, std::vector{Variable{"?s"}});
+    auto join = Join{qec, fullScanPSO, valuesTree, 0, 0};
+    EXPECT_EQ(join.getDescriptor(), "Join on ?s");
+
+    auto expected = IdTable{2, qec->getAllocator()};
+    VariableToColumnMap expectedVariables{
+        {Variable{"?s"}, makeAlwaysDefinedColumn(0)},
+        {Variable{"?o"}, makeAlwaysDefinedColumn(1)}};
+    testJoinOperation(join, makeExpectedColumns(expectedVariables, expected));
+
+    auto joinSwitched = Join{qec, valuesTree, fullScanPSO, 0, 0};
+    testJoinOperation(joinSwitched,
+                      makeExpectedColumns(expectedVariables, expected));
+  };
+  test(0);
+  test(1);
+  test(2);
+  test(3);
+  test(1'000'000);
+}
+
+TEST(JoinTest, joinWithColumnAndScanUndefValues) {
+  auto test = [](size_t materializationThreshold) {
+    auto qec = ad_utility::testing::getQec(" 
 1.  
 2.   3.");
+    RuntimeParameters().set<"lazy-index-scan-max-size-materialization">(
+        materializationThreshold);
+    qec->getQueryTreeCache().clearAll();
+    auto fullScanPSO = ad_utility::makeExecutionTree(
+        qec, PSO, SparqlTriple{Var{"?s"}, "
", Var{"?o"}});
+    auto U = Id::makeUndefined();
+    auto valuesTree = ad_utility::makeExecutionTree(
+        qec, makeIdTableFromVector({{U}}), std::vector{Variable{"?s"}});
+    auto join = Join{qec, fullScanPSO, valuesTree, 0, 0};
+    EXPECT_EQ(join.getDescriptor(), "Join on ?s");
+
+    auto getId = ad_utility::testing::makeGetId(qec->getIndex());
+    auto idX = getId("");
+    auto idX2 = getId("");
+    auto expected = makeIdTableFromVector({{idX, I(1)}, {idX2, I(2)}});
+    VariableToColumnMap expectedVariables{
+        {Variable{"?s"}, makeAlwaysDefinedColumn(0)},
+        {Variable{"?o"}, makeAlwaysDefinedColumn(1)}};
+    testJoinOperation(join, makeExpectedColumns(expectedVariables, expected));
+
+    auto joinSwitched = Join{qec, valuesTree, fullScanPSO, 0, 0};
+    testJoinOperation(joinSwitched,
+                      makeExpectedColumns(expectedVariables, expected));
+  };
+  test(0);
+  test(1);
+  test(2);
+  test(3);
+  test(1'000'000);
+}
+
 TEST(JoinTest, joinTwoScans) {
   auto test = [](size_t materializationThreshold) {
     auto qec = ad_utility::testing::getQec(
@@ -344,6 +411,7 @@ TEST(JoinTest, joinTwoScans) {
     auto scanP2 = ad_utility::makeExecutionTree(
         qec, PSO, SparqlTriple{Var{"?s"}, "", Var{"?q"}});
     auto join = Join{qec, scanP2, scanP, 0, 0};
+    EXPECT_EQ(join.getDescriptor(), "Join on ?s");
 
     auto id = ad_utility::testing::makeGetId(qec->getIndex());
     auto expected = makeIdTableFromVector(
@@ -364,3 +432,12 @@ TEST(JoinTest, joinTwoScans) {
   test(3);
   test(1'000'000);
 }
+
+TEST(JoinTest, invalidJoinVariable) {
+  auto qec = ad_utility::testing::getQec(
+      " 
 1.  
 2.   3 .   4.   7. ");
+  auto valuesTree = makeValuesForSingleVariable(qec, "?s", {""});
+  auto valuesTree2 = makeValuesForSingleVariable(qec, "?p", {""});
+
+  ASSERT_ANY_THROW(Join(qec, valuesTree2, valuesTree, 0, 0));
+}
diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index 347ef93e85..a629f7c0d8 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -268,7 +268,7 @@ TEST(ThreadSafeQueue, QueueManager) {
         [&numPushed]() -> std::optional()(3))> {
       auto makeValue = makeQueueValue();
       while (true) {
-        auto value = ++numPushed;
+        auto value = numPushed++;
         if (value < numValues) {
           return makeValue(value);
         } else {
@@ -283,17 +283,7 @@ TEST(ThreadSafeQueue, QueueManager) {
       result.push_back(value);
       EXPECT_LE(numPushed, numPopped + queueSize + 1 + numThreads);
     }
-    if (ad_utility::similarToInstantiation) {
-      // When terminating early, we cannot actually say much about the result,
-      // other than that it contains no duplicate values
-      std::ranges::sort(result);
-      EXPECT_TRUE(std::unique(result.begin(), result.end()) == result.end());
-    } else {
-      // For the ordered queue we have the guarantee that all the pushed values
-      // were in order.
-      EXPECT_THAT(result,
-                  ::testing::ElementsAreArray(std::views::iota(0U, 400U)));
-    }
+    // TODO Perform the remaining testing etc.
   };
   runWithBothQueueTypes(runTest);
 }
diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp
index b32820acf8..e286d6269a 100644
--- a/test/engine/IndexScanTest.cpp
+++ b/test/engine/IndexScanTest.cpp
@@ -68,9 +68,10 @@ void testLazyScanForJoinOfTwoScans(
     const std::string& kgTurtle, const SparqlTriple& tripleLeft,
     const SparqlTriple& tripleRight, const std::vector& leftRows,
     const std::vector& rightRows,
+    size_t blocksizePermutationsInBytes = 32,
     source_location l = source_location::current()) {
   auto t = generateLocationTrace(l);
-  auto qec = getQec(kgTurtle);
+  auto qec = getQec(kgTurtle, true, true, true, blocksizePermutationsInBytes);
   IndexScan s1{qec, Permutation::PSO, tripleLeft};
   IndexScan s2{qec, Permutation::PSO, tripleRight};
   auto implForSwitch = [](IndexScan& l, IndexScan& r, const auto& expectedL,
@@ -190,6 +191,19 @@ TEST(IndexScan, lazyScanForJoinOfTwoScans) {
         "  .    .";
     testLazyScanForJoinOfTwoScans(kg, bpx, xqz, {{1, 5}}, {{0, 4}});
   }
+  {
+    // In this example we use 3 triples per block (48 bytes) and the `
`
+    // permutation is standing in a single block together with the previous
+    // `` relation. The lazy scans are however still aware that the relevant
+    // part of the block (` 
 ?x`) only  goes from `` through ``,
+    // so it is not necessary to scan the first block of the `` relation
+    // which only has subsects <= ``.
+    std::string kg =
+        "  .  
 .  
 . "
+        "  .    .   . "
+        "   .   .    .";
+    testLazyScanForJoinOfTwoScans(kg, bpx, xqz, {{0, 1}}, {{3, 6}}, 48);
+  }
   {
     std::string kg =
         " 
 .  
 . "

From c4fa334359dba63a8b4af1e79097ded2e1dc5c36 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 11 Jul 2023 16:24:06 +0200
Subject: [PATCH 145/150] Fixed the test.

---
 test/engine/IndexScanTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp
index e286d6269a..9077466373 100644
--- a/test/engine/IndexScanTest.cpp
+++ b/test/engine/IndexScanTest.cpp
@@ -202,7 +202,7 @@ TEST(IndexScan, lazyScanForJoinOfTwoScans) {
         "  .  
 .  
 . "
         "  .    .   . "
         "   .   .    .";
-    testLazyScanForJoinOfTwoScans(kg, bpx, xqz, {{0, 1}}, {{3, 6}}, 48);
+    testLazyScanForJoinOfTwoScans(kg, bpx, xqz, {{0, 2}}, {{3, 6}}, 48);
   }
   {
     std::string kg =

From 9c8ebcd8de95371a4829c53d0661bbc5a4642662 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 11 Jul 2023 16:52:14 +0200
Subject: [PATCH 146/150] Add a simple coroutine that makes working with the
 quees much simpler.

---
 src/util/ThreadSafeQueue.h   | 82 ++++++++++++++++++++++++++++++++
 test/ThreadSafeQueueTest.cpp | 92 ++++++++++++++++++++++++++++++++++++
 2 files changed, 174 insertions(+)

diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index 09b552e715..f356690c7c 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -9,6 +9,12 @@
 #include 
 #include 
 #include 
+#include 
+
+#include "absl/cleanup/cleanup.h"
+#include "util/Exception.h"
+#include "util/Generator.h"
+#include "util/jthread.h"
 
 namespace ad_utility::data_structures {
 
@@ -28,6 +34,7 @@ class ThreadSafeQueue {
   size_t maxSize_;
 
  public:
+  using value_type = T;
   explicit ThreadSafeQueue(size_t maxSize) : maxSize_{maxSize} {}
 
   // We can neither copy nor move this class
@@ -130,6 +137,7 @@ class OrderedThreadSafeQueue {
   std::atomic_flag finish_ = ATOMIC_FLAG_INIT;
 
  public:
+  using value_type = T;
   // Construct from the maximal queue size (see `ThreadSafeQueue` for details).
   explicit OrderedThreadSafeQueue(size_t maxSize) : queue_{maxSize} {}
 
@@ -158,6 +166,12 @@ class OrderedThreadSafeQueue {
     return result;
   }
 
+  // Same as the function above, but the two arguments are passed in as a
+  // `std::pair`.
+  bool push(std::pair indexAndValue) {
+    return push(indexAndValue.first, std::move(indexAndValue.second));
+  }
+
   // See `ThreadSafeQueue` for details.
   void pushException(std::exception_ptr exception) {
     queue_.pushException(std::move(exception));
@@ -183,4 +197,72 @@ class OrderedThreadSafeQueue {
   std::optional pop() { return queue_.pop(); }
 };
 
+// A concept for one of the thread-safe queue types above
+template 
+concept IsThreadsafeQueue =
+    ad_utility::similarToInstantiation ||
+    ad_utility::similarToInstantiation;
+
+namespace detail {
+// A helper function for setting up a producer task for one of the threadsafe
+// queues above. Takes a reference to a  queue and a `task`. The task must
+// return `std::optional`. The task is
+// called repeatedly, and the resulting values are pushed to the queue. If the
+// task returns `nullopt`, `numThreads` is decremented, and the queue is
+// finished if `numThreads <= 0`. All exceptions that happen during the
+// execution of `task` are propagated to the queue.
+template 
+auto makeQueueTask(Queue& queue, Task task, std::atomic& numThreads) {
+  return [&queue, task = std::move(task), &numThreads] {
+    try {
+      while (auto opt = task()) {
+        if (!queue.push(std::move(opt.value()))) {
+          break;
+        }
+      }
+    } catch (...) {
+      try {
+        queue.pushException(std::current_exception());
+      } catch (...) {
+        queue.finish();
+      }
+    }
+    --numThreads;
+    if (numThreads <= 0) {
+      queue.finish();
+    }
+  };
+}
+}  // namespace detail
+
+// This helper function makes the usage of the (Ordered)ThreadSafeQueue above
+// much easier. It takes the size of the queue, the number of producer threads,
+// and a task that produces values. The `producerTask` is called repeatedly in
+// `numThreads` many concurrent threads. It needs to return
+// `std::optional` and has the following
+// semantics: If `nullopt` is returned, then the thread is finished. The queue
+// is finished, when all the producer threads have finished by yielding
+// `nullopt`, or if any call to `producerTask` in any thread throws an
+// exception. In that case the exception is propagated to the resulting
+// generator. The resulting generator yields all the values that have been
+// pushed to the queue.
+template 
+cppcoro::generator queueManager(size_t queueSize,
+                                                            size_t numThreads,
+                                                            auto producerTask) {
+  Queue queue{queueSize};
+  AD_CONTRACT_CHECK(numThreads > 0u);
+  std::vector threads;
+  std::atomic numUnfinishedThreads{static_cast(numThreads)};
+  absl::Cleanup queueFinisher{[&queue] { queue.finish(); }};
+  for ([[maybe_unused]] auto i : std::views::iota(0u, numThreads)) {
+    threads.emplace_back(
+        detail::makeQueueTask(queue, producerTask, numUnfinishedThreads));
+  }
+
+  while (auto opt = queue.pop()) {
+    co_yield (opt.value());
+  }
+}
+
 }  // namespace ad_utility::data_structures
diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp
index e119f5fc85..fef324adbc 100644
--- a/test/ThreadSafeQueueTest.cpp
+++ b/test/ThreadSafeQueueTest.cpp
@@ -33,6 +33,20 @@ auto makePush(Queue& queue) {
   };
 }
 
+// Similar to `makePush` above, but the returned lambda doesn't push directly to
+// the queue, but simply returns a value that can then be pushed to the queue.
+// This is useful when testing the `queueManager` template.
+template 
+auto makeQueueValue() {
+  return [](size_t i) {
+    if constexpr (ad_utility::similarToInstantiation) {
+      return i;
+    } else {
+      return std::pair{i, i};
+    }
+  };
+}
+
 // Some constants that are used in almost every test case.
 constexpr size_t queueSize = 5;
 constexpr size_t numThreads = 20;
@@ -316,3 +330,81 @@ TEST(ThreadSafeQueue, SafeExceptionHandling) {
   runWithBothQueueTypes(std::bind_front(runTest, true));
   runWithBothQueueTypes(std::bind_front(runTest, false));
 }
+
+// ________________________________________________________________
+TEST(ThreadSafeQueue, queueManager) {
+  enum class TestType {
+    producerThrows,
+    consumerThrows,
+    normalExecution,
+    consumerFinishesEarly,
+  };
+  auto runTest = [](TestType testType, Queue&&) {
+    std::atomic numPushed = 0;
+    auto task =
+        [&numPushed,
+         &testType]() -> std::optional()(3))> {
+      auto makeValue = makeQueueValue();
+      while (true) {
+        auto value = numPushed++;
+        if (testType == TestType::producerThrows && value > numValues / 2) {
+          throw std::runtime_error{"Producer"};
+        }
+        if (value < numValues) {
+          return makeValue(value);
+        } else {
+          return std::nullopt;
+        }
+      }
+    };
+    std::vector result;
+    size_t numPopped = 0;
+    try {
+      for (size_t value : queueManager(queueSize, numThreads, task)) {
+        ++numPopped;
+        if (numPopped > numValues / 2) {
+          if (testType == TestType::consumerThrows) {
+            throw std::runtime_error{"Consumer"};
+          } else if (testType == TestType::consumerFinishesEarly) {
+            break;
+          }
+        }
+        result.push_back(value);
+        EXPECT_LE(numPushed, numPopped + queueSize + 1 + numThreads);
+      }
+      if (testType == TestType::consumerThrows ||
+          testType == TestType::producerThrows) {
+        FAIL() << "Should have thrown";
+      }
+    } catch (const std::runtime_error& e) {
+      if (testType == TestType::consumerThrows) {
+        EXPECT_STREQ(e.what(), "Consumer");
+      } else if (testType == TestType::producerThrows) {
+        EXPECT_STREQ(e.what(), "Producer");
+      } else {
+        FAIL() << "Should not have throwns";
+      }
+    }
+
+    if (testType == TestType::consumerFinishesEarly) {
+      EXPECT_EQ(result.size(), numValues / 2);
+    } else if (testType == TestType::normalExecution) {
+      EXPECT_EQ(result.size(), numValues);
+      // For the `OrderedThreadSafeQueue` we expect the result to already be in
+      // order, for the `ThreadSafeQueue` the order is unspecified and we only
+      // check the content.
+      if (ad_utility::isInstantiation) {
+        std::ranges::sort(result);
+      }
+      EXPECT_THAT(result, ::testing::ElementsAreArray(
+                              std::views::iota(0UL, numValues)));
+    }
+    // The probably most important test of all is that the destructors which are
+    // run at the following closing brace never lead to a deadlock.
+  };
+  using enum TestType;
+  runWithBothQueueTypes(std::bind_front(runTest, consumerThrows));
+  runWithBothQueueTypes(std::bind_front(runTest, producerThrows));
+  runWithBothQueueTypes(std::bind_front(runTest, consumerFinishesEarly));
+  runWithBothQueueTypes(std::bind_front(runTest, normalExecution));
+}

From 65790fcb5a9eeb74767f6acd035d3d7c92eab438 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 11 Jul 2023 17:57:21 +0200
Subject: [PATCH 147/150] Merged in the current threadsafe queue, and fixed
 very minor stuff.

---
 src/index/CompressedRelation.cpp | 7 +++----
 src/index/CompressedRelation.h   | 6 ------
 2 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp
index 82ae727652..e243bb62ed 100644
--- a/src/index/CompressedRelation.cpp
+++ b/src/index/CompressedRelation.cpp
@@ -128,10 +128,9 @@ CompressedRelationReader::asyncParallelBlockGenerator(
     // all the blocks is long enough, so a `const&` would suffice), but the
     // copy is cheap and makes the code more robust.
     auto block = *blockIterator;
-    // Note: The order of the following three lines is important: The index
+    // Note: The order of the following two lines is important: The index
     // of the current block depends on the current value of `blockIterator`,
-    // but we have to increment `blockIterator` to determine whether this
-    // was the last block.
+    // so we have to compute it before incrementing the iterator.
     auto myIndex = static_cast(blockIterator - beginBlock);
     ++blockIterator;
     // Note: the reading of the block could also happen without holding the
@@ -152,7 +151,7 @@ CompressedRelationReader::asyncParallelBlockGenerator(
   auto setTimer = ad_utility::makeOnDestructionDontThrowDuringStackUnwinding(
       [&details, &popTimer]() { details.blockingTimeMs_ = popTimer.msecs(); });
 
-  auto queue = ad_utility::data_structures::QueueManager<
+  auto queue = ad_utility::data_structures::queueManager<
       ad_utility::data_structures::OrderedThreadSafeQueue>(
       queueSize, numThreads, readAndDecompressBlock);
   for (IdTable& block : queue) {
diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h
index b69db22034..872c5a7430 100644
--- a/src/index/CompressedRelation.h
+++ b/src/index/CompressedRelation.h
@@ -121,12 +121,6 @@ struct CompressedRelationMetadata {
 
   size_t getNofElements() const { return numRows_; }
 
-  // We currently always store two columns (the second and third column of a
-  // triple). This might change in the future when we might also store
-  // patterns and functional relations. Factor out this magic constant already
-  // now to make the code more readable.
-  static constexpr size_t numColumns() { return NumColumns; }
-
   // Setters and getters for the multiplicities.
   float getCol1Multiplicity() const { return multiplicityCol1_; }
   float getCol2Multiplicity() const { return multiplicityCol2_; }

From 4be2f920a2d52de32b73f4e6a3458f13c9d34f1a Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Tue, 11 Jul 2023 21:38:16 +0200
Subject: [PATCH 148/150] Small changes from a review with Hannah.

---
 test/engine/IndexScanTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp
index 9077466373..0ac66724d9 100644
--- a/test/engine/IndexScanTest.cpp
+++ b/test/engine/IndexScanTest.cpp
@@ -197,7 +197,7 @@ TEST(IndexScan, lazyScanForJoinOfTwoScans) {
     // `` relation. The lazy scans are however still aware that the relevant
     // part of the block (` 
 ?x`) only  goes from `` through ``,
     // so it is not necessary to scan the first block of the `` relation
-    // which only has subsects <= ``.
+    // which only has subjects <= ``.
     std::string kg =
         "  .  
 .  
 . "
         "  .    .   . "

From bb2dbde3c5dd1da48854a28e0d6c28020453156a Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 12 Jul 2023 10:11:09 +0200
Subject: [PATCH 149/150] Fix a possible race condition in the case that
 multiple threads are pushing exceptions.

---
 src/util/JoinAlgorithms/JoinColumnMapping.h |  2 +-
 src/util/ThreadSafeQueue.h                  | 16 +++++++++++-----
 test/CMakeLists.txt                         |  2 +-
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/util/JoinAlgorithms/JoinColumnMapping.h b/src/util/JoinAlgorithms/JoinColumnMapping.h
index 1fe8f528f9..c21e4ccd97 100644
--- a/src/util/JoinAlgorithms/JoinColumnMapping.h
+++ b/src/util/JoinAlgorithms/JoinColumnMapping.h
@@ -114,7 +114,7 @@ struct IdTableAndFirstCol {
   using iterator = std::decay_t;
 
   // Construct by taking ownership of the table.
-  IdTableAndFirstCol(Table t) : table_{std::move(t)} {}
+  explicit IdTableAndFirstCol(Table t) : table_{std::move(t)} {}
 
   // Get access to the first column.
   decltype(auto) col() { return table_.getColumn(0); }
diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index 5dfe9ebd0c..1753cbe5b0 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -65,11 +65,17 @@ class ThreadSafeQueue {
   // will return `false`.
   void pushException(std::exception_ptr exception) {
     std::unique_lock lock{mutex_};
-    pushedException_ = std::move(exception);
-    finish_.test_and_set();
-    lock.unlock();
-    pushNotification_.notify_all();
-    popNotification_.notify_all();
+    // It is important that we only push the first exception we encounter,
+    // otherwise there might be race conditions between rethrowing and resetting
+    // the exception.
+    if (pushedException_ != nullptr) {
+      return;
+    }
+      pushedException_ = std::move(exception);
+      finish_.test_and_set();
+      lock.unlock();
+      pushNotification_.notify_all();
+      popNotification_.notify_all();
   }
 
   // After calling this function, all calls to `push` will return `false` and no
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index bcf8610975..091579acaa 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -286,7 +286,7 @@ addLinkAndDiscoverTest(TimerTest)
 
 addLinkAndDiscoverTest(AlgorithmTest)
 
-addLinkAndDiscoverTest(CompressedRelationsTest index)
+addLinkAndDiscoverTestSerial(CompressedRelationsTest index)
 
 addLinkAndDiscoverTest(ExceptionTest)
 

From 2519a7427149ee7b5a232affc8ce5a9470953d83 Mon Sep 17 00:00:00 2001
From: Johannes Kalmbach 
Date: Wed, 12 Jul 2023 10:18:39 +0200
Subject: [PATCH 150/150] Fix clang format.

---
 src/util/ThreadSafeQueue.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h
index 1753cbe5b0..961589dbd0 100644
--- a/src/util/ThreadSafeQueue.h
+++ b/src/util/ThreadSafeQueue.h
@@ -71,11 +71,11 @@ class ThreadSafeQueue {
     if (pushedException_ != nullptr) {
       return;
     }
-      pushedException_ = std::move(exception);
-      finish_.test_and_set();
-      lock.unlock();
-      pushNotification_.notify_all();
-      popNotification_.notify_all();
+    pushedException_ = std::move(exception);
+    finish_.test_and_set();
+    lock.unlock();
+    pushNotification_.notify_all();
+    popNotification_.notify_all();
   }
 
   // After calling this function, all calls to `push` will return `false` and no