Merge remote-tracking branch 'origin/master' into add-libspatialjoin-…

…algorithm

.clang-format

@@ -6,4 +6,8 @@ LineEnding: LF
 InsertNewlineAtEOF: true
 RequiresClausePosition: SingleLine
 IndentRequiresClause: false
+AttributeMacros: [CPP_member, CPP_auto_member, CPP_fun, CPP_auto_fun]
+# TODO<joka921> Update to Clang-format-17 and then we can have much nicer CPP_... macros.
+#Macros:
+#  - CPP_template_2(x)=template<x> requires
 ...

.codespellrc

@@ -3,6 +3,6 @@
 skip = .git*,.codespellrc,*.pdf,generated
 check-hidden = true
 # Ignore mixedCase variables, lines with latin, lines with codespell-ignore pragma, etc
-ignore-regex = \b([A-Z]*[a-z]+[A-Z][a-zA-Z]*)\b|.*(Lorem ipsum|eleifend|feugait|codespell-ignore).*
+ignore-regex = \b([A-Z]*[a-z]+[A-Z][a-zA-Z]*)\b|.*(Lorem ipsum|eleifend|feugait|codespell-ignore).*|https?://\S+
 # alph - is used frequently in tests, just ignore altogether
 ignore-words-list = ser,alph,inbetween,interm

CMakeLists.txt

@@ -238,7 +238,7 @@ set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
 FetchContent_Declare(
         ctre
         GIT_REPOSITORY https://github.com/hanickadot/compile-time-regular-expressions.git
-        GIT_TAG b3d7788b559e34d985c8530c3e0e7260b67505a6 # v3.8.1
+        GIT_TAG eb9577aae3515d14e6c5564f9aeb046d2e7c1124 # v3.9.0
 )
 
 ################################

benchmark/JoinAlgorithmBenchmark.cpp

@@ -770,27 +770,29 @@ class GeneralInterfaceImplementation : public BenchmarkInterface {
       };
     };
     auto generateBiggerEqualLambdaDesc =
-        [](const ad_utility::isInstantiation<
-               ad_utility::ConstConfigOptionProxy> auto& option,
-           const auto& minimumValue, bool canBeEqual) {
-          return absl::StrCat("'", option.getConfigOption().getIdentifier(),
-                              "' must be bigger than",
-                              canBeEqual ? ", or equal to," : "", " ",
-                              minimumValue, ".");
-        };
+        []<typename OptionType,
+           typename = std::enable_if_t<ad_utility::isInstantiation<
+               OptionType, ad_utility::ConstConfigOptionProxy>>>(
+            const OptionType& option, const auto& minimumValue,
+            bool canBeEqual) {
+      return absl::StrCat("'", option.getConfigOption().getIdentifier(),
+                          "' must be bigger than",
+                          canBeEqual ? ", or equal to," : "", " ", minimumValue,
+                          ".");
+    };
 
     // Object with a `operator()` for the `<=` operator.
     auto lessEqualLambda = std::less_equal<size_t>{};
     auto generateLessEqualLambdaDesc =
-        [](const ad_utility::isInstantiation<
-               ad_utility::ConstConfigOptionProxy> auto& lhs,
-           const ad_utility::isInstantiation<
-               ad_utility::ConstConfigOptionProxy> auto& rhs) {
-          return absl::StrCat("'", lhs.getConfigOption().getIdentifier(),
-                              "' must be smaller than, or equal to, "
-                              "'",
-                              rhs.getConfigOption().getIdentifier(), "'.");
-        };
+        []<typename OptionType,
+           typename = std::enable_if_t<ad_utility::isInstantiation<
+               OptionType, ad_utility::ConstConfigOptionProxy>>>(
+            const OptionType& lhs, const OptionType& rhs) {
+      return absl::StrCat("'", lhs.getConfigOption().getIdentifier(),
+                          "' must be smaller than, or equal to, "
+                          "'",
+                          rhs.getConfigOption().getIdentifier(), "'.");
+    };
 
     // Adding the validators.
 

benchmark/infrastructure/BenchmarkMeasurementContainer.h

@@ -227,8 +227,8 @@ class ResultTable : public BenchmarkMetadataGetter {
 
   @param row, column Which table entry to read. Starts with `(0,0)`.
   */
-  template <ad_utility::SameAsAnyTypeIn<EntryType> T>
-  T getEntry(const size_t row, const size_t column) const {
+  CPP_template(typename T)(requires ad_utility::SameAsAnyTypeIn<T, EntryType>) T
+      getEntry(const size_t row, const size_t column) const {
     AD_CONTRACT_CHECK(row < numRows() && column < numColumns());
     static_assert(!ad_utility::isSimilar<T, std::monostate>);
 

benchmark/infrastructure/BenchmarkToJson.cpp

@@ -31,12 +31,12 @@ provided translation function for the vector entries.
 @tparam TranslationFunction Has to be a function, that takes `VectorType`
 and returns a `nlohmann:json` object.
 */
-template <typename VectorType, ad_utility::InvocableWithExactReturnType<
-                                   nlohmann::ordered_json, VectorType>
-                                   TranslationFunction>
-static nlohmann::json transformIntoJsonArray(
-    const std::vector<VectorType>& vec,
-    TranslationFunction translationFunction) {
+CPP_template(typename VectorType, typename TranslationFunction)(
+    requires ad_utility::InvocableWithExactReturnType<
+        TranslationFunction, nlohmann::ordered_json,
+        VectorType>) static nlohmann::json
+    transformIntoJsonArray(const std::vector<VectorType>& vec,
+                           TranslationFunction translationFunction) {
   /*
   Without explicit instantiation, `nlohmann::nlohmann::ordered_json` is not
   guaranteed, to always interpret a `push_back` correctly. For instance,

benchmark/util/ResultTableColumnOperations.h

@@ -21,8 +21,9 @@ Column number together with the type of value, that can be found inside the
 column. Note, that **all** entries in the column must have the same type,
 because of `ResultTable::getEntry`.
 */
-template <ad_utility::SameAsAnyTypeIn<ResultTable::EntryType> Type>
-struct ColumnNumWithType {
+CPP_template(typename Type)(
+    requires ad_utility::SameAsAnyTypeIn<
+        Type, ResultTable::EntryType>) struct ColumnNumWithType {
   using ColumnType = Type;
   const size_t columnNum_;
 };

src/ServerMain.cpp

@@ -82,8 +82,8 @@ int main(int argc, char** argv) {
       optionFactory.getProgramOption<"cache-max-size-single-entry">(),
       "Maximum size for a single cache entry. That is, "
       "results larger than this will not be cached unless pinned.");
-  add("lazy-result-max-cache-size,E",
-      optionFactory.getProgramOption<"lazy-result-max-cache-size">(),
+  add("cache-max-size-lazy-result,E",
+      optionFactory.getProgramOption<"cache-max-size-lazy-result">(),
       "Maximum size up to which lazy results will be cached by aggregating "
       "partial results. Caching does cause significant overhead for this "
       "case.");

src/backports/concepts.h

@@ -5,6 +5,8 @@
 #pragma once
 
 #include <concepts/concepts.hpp>
+
+#include "backports/cppTemplate2.h"
 #ifndef QLEVER_CPP_17
 #include <concepts>
 #endif
@@ -19,6 +21,13 @@
 // `QL_CONCEPT_OR_TYPENAME(arg)`: expands to `arg` in C++20 mode, and to
 // `typename` in C++17 mode. Example usage:
 //
+// `CPP_lambda(capture)(arg)(requires ...)`: Expands lambda to use
+//  `requires` in C++20 mode and `std::enable_if_t` in C++17 mode.
+//
+// `CPP_template_lambda(capture)(typenames...)(arg)(requires ...)`: Expands
+// lambda with (C++20) explicit gemplate parameters  to use
+//  `requires` in C++20 mode and `std::enable_if_t` in C++17 mode.
+//
 // Example usages:
 //
 // `QL_CONCEPT_OR_NOTHING(std::view) auto x = someFunction();`
@@ -29,27 +38,52 @@
 //
 // `template <QL_CONCEPT_OR_TYPENAME(ql::same_as<int>) T> void f(){...}`
 //
+// `auto myLambda = CPP_lambda(someCapture)(someArg)(requires
+// ranges::same_as<decltype(someArg), int>) {...}`
+//
+// `auto myLambda2 = CPP_lambda(someCapture)(typename F)(F arg)(requires
+// ranges::same_as<F, int>) {...}`
+//
 // NOTE: The macros are variadic to allow for commas in the argument, like in
 // the second example above.
 
+// Additionally define the macros `CPP_template_2` and `CPP_and_2` that can
+// be used to constrain member functions of classes where the outer class
+// has already been constrained with `CPP_template`. For a detailed example, see
+// the `test/backports/ConceptsTest.cpp` file.
+
 #ifdef QLEVER_CPP_17
 #define QL_CONCEPT_OR_NOTHING(...)
 #define QL_CONCEPT_OR_TYPENAME(...) typename
+#define CPP_template_2 CPP_template_2_SFINAE
+#define CPP_and_2 CPP_and_2_sfinae
+#define CPP_and_def CPP_and_sfinae_def
+#define CPP_and_2_def CPP_and_2_def_sfinae
+#define CPP_variadic_template CPP_template_NO_DEFAULT_SFINAE
+#define CPP_member_def CPP_member_def_sfinae
+#define CPP_lambda CPP_lambda_sfinae
+#define CPP_template_lambda CPP_template_lambda_sfinae
 #else
 #define QL_CONCEPT_OR_NOTHING(...) __VA_ARGS__
 #define QL_CONCEPT_OR_TYPENAME(...) __VA_ARGS__
+#define CPP_template_2 CPP_template
+#define CPP_and_2 CPP_and
+#define CPP_and_def CPP_and
+#define CPP_and_2_def CPP_and
+#define CPP_variadic_template CPP_template
+#define CPP_member_def CPP_member
+#define CPP_lambda CPP_LAMBDA_20
+#define CPP_template_lambda CPP_TEMPLATE_LAMBDA_20
 #endif
 
 // The namespace `ql::concepts` includes concepts that are contained in the
 // C++20 standard as well as in `range-v3`.
-namespace ql {
-namespace concepts {
+namespace ql::concepts {
 
 #ifdef QLEVER_CPP_17
 using namespace ::concepts;
 #else
 using namespace std;
 #endif
 
-}  // namespace concepts
-}  // namespace ql
+}  // namespace ql::concepts

src/backports/cppTemplate2.h

@@ -0,0 +1,104 @@
+// Copyright 2024, University of Freiburg
+// Chair of Algorithms and Data Structures
+// Author: Johannes Kalmbach <[email protected]>
+
+#pragma once
+#include <concepts/concepts.hpp>
+
+// The internal reimplementation of a `CPP_variadic_template` macro
+// that can be used for variadic template functions.
+#define CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_0(...)                         \
+  , std::enable_if_t<CPP_PP_CAT(CPP_TEMPLATE_SFINAE_AUX_3_, __VA_ARGS__), \
+                     int> = 0 > CPP_PP_IGNORE_CXX2A_COMPAT_END
+
+#define CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_3_requires
+
+#define CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_WHICH_(FIRST, ...) \
+  CPP_PP_EVAL(CPP_PP_CHECK,                                   \
+              CPP_PP_CAT(CPP_TEMPLATE_SFINAE_PROBE_CONCEPT_, FIRST))
+
+#define CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_(...)                       \
+  CPP_PP_CAT(CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_,                      \
+             CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_WHICH_(__VA_ARGS__, )) \
+  (__VA_ARGS__)
+
+#define CPP_template_NO_DEFAULT_SFINAE(...) \
+  CPP_PP_IGNORE_CXX2A_COMPAT_BEGIN          \
+  template <__VA_ARGS__ CPP_TEMPLATE_NO_DEFAULT_SFINAE_AUX_
+
+// The internal reimplementation of a `CPP_template_2` and `CPP_and_2` macro
+// that can be used for class members when the outer class has already been
+// constrained using `CPP_template`.
+#define CPP_TEMPLATE_2_SFINAE_AUX_0(...)                                \
+  , bool CPP_true_2 = true,                                             \
+         std::enable_if_t <                                             \
+                 CPP_PP_CAT(CPP_TEMPLATE_SFINAE_AUX_3_, __VA_ARGS__) && \
+             CPP_BOOL(CPP_true_2),                                      \
+         int > = 0 > CPP_PP_IGNORE_CXX2A_COMPAT_END
+
+#define CPP_TEMPLATE_2_SFINAE_AUX_3_requires
+
+#define CPP_TEMPLATE_2_SFINAE_AUX_WHICH_(FIRST, ...) \
+  CPP_PP_EVAL(CPP_PP_CHECK,                          \
+              CPP_PP_CAT(CPP_TEMPLATE_SFINAE_PROBE_CONCEPT_, FIRST))
+
+#define CPP_TEMPLATE_2_SFINAE_AUX_(...)                       \
+  CPP_PP_CAT(CPP_TEMPLATE_2_SFINAE_AUX_,                      \
+             CPP_TEMPLATE_2_SFINAE_AUX_WHICH_(__VA_ARGS__, )) \
+  (__VA_ARGS__)
+
+#define CPP_template_2_SFINAE(...) \
+  CPP_PP_IGNORE_CXX2A_COMPAT_BEGIN \
+  template <__VA_ARGS__ CPP_TEMPLATE_2_SFINAE_AUX_
+
+/// INTERNAL ONLY
+#define CPP_and_def_sfinae &&CPP_BOOL(CPP_true), int >, std::enable_if_t <
+#define CPP_and_2_sfinae &&CPP_BOOL(CPP_true_2), int > = 0, std::enable_if_t <
+#define CPP_and_2_def_sfinae &&CPP_BOOL(CPP_true_2), int >, std::enable_if_t <
+
+#define CPP_member_def_sfinae \
+  template <bool (&CPP_true_fn)(::concepts::detail::xNil)>
+
+#define CPP_LAMBDA_20(...) [__VA_ARGS__] CPP_LAMBDA_ARGS
+
+#define CPP_TEMPLATE_LAMBDA_20(...) [__VA_ARGS__] CPP_TEMPLATE_LAMBDA_ARGS
+
+// The internals of the `CPP_lambda` template
+#define CPP_LAMBDA_SFINAE_ARGS(...) \
+(__VA_ARGS__ CPP_LAMBDA_SFINAE_AUX_
+
+#define CPP_LAMBDA_SFINAE_AUX_WHICH_(FIRST, ...) \
+  CPP_PP_EVAL(CPP_PP_CHECK, CPP_PP_CAT(CPP_LAMBDA_SFINAE_PROBE_CONCEPT_, FIRST))
+
+#define CPP_LAMBDA_SFINAE_AUX_(...)                       \
+  CPP_PP_CAT(CPP_LAMBDA_SFINAE_AUX_,                      \
+             CPP_LAMBDA_SFINAE_AUX_WHICH_(__VA_ARGS__, )) \
+  (__VA_ARGS__)
+
+#define CPP_LAMBDA_SFINAE_AUX_0(...) ,                           \
+std::enable_if_t<                                            \
+CPP_PP_CAT(CPP_LAMBDA_SFINAE_AUX_3_, __VA_ARGS__)        \
+>* = nullptr)
+
+#define CPP_lambda_sfinae(...)     \
+  CPP_PP_IGNORE_CXX2A_COMPAT_BEGIN \
+  [__VA_ARGS__] CPP_LAMBDA_SFINAE_ARGS
+
+#define CPP_TEMPLATE_LAMBDA_ARGS_sfinae(...) \
+  <__VA_ARGS__> CPP_LAMBDA_SFINAE_ARGS
+
+#define CPP_template_lambda_sfinae(...) \
+  [__VA_ARGS__] CPP_TEMPLATE_LAMBDA_ARGS_sfinae
+
+#define CPP_LAMBDA_SFINAE_AUX_3_requires
+
+#define CPP_LAMBDA_ARGS(...) (__VA_ARGS__) CPP_LAMBDA_AUX_
+
+#define CPP_LAMBDA_AUX_(...) \
+  CPP_PP_CAT(CPP_LAMBDA_AUX_, CPP_LAMBDA_AUX_WHICH_(__VA_ARGS__, ))(__VA_ARGS__)
+
+#define CPP_LAMBDA_AUX_WHICH_(FIRST, ...) CPP_PP_EVAL(CPP_PP_CHECK, FIRST)
+
+#define CPP_LAMBDA_AUX_0(...) __VA_ARGS__
+
+#define CPP_TEMPLATE_LAMBDA_ARGS(...) <__VA_ARGS__> CPP_LAMBDA_ARGS

src/engine/AddCombinedRowToTable.h

@@ -8,13 +8,24 @@
 #include <cstdint>
 #include <vector>
 
+#include "backports/concepts.h"
 #include "engine/idTable/IdTable.h"
 #include "global/Id.h"
 #include "util/CancellationHandle.h"
 #include "util/Exception.h"
 #include "util/TransparentFunctors.h"
 
 namespace ad_utility {
+
+namespace detail::concepts {
+template <typename T>
+CPP_requires(HasAsStaticView,
+             requires(T& table)(table.template asStaticView<0>()));
+
+template <typename T>
+CPP_requires(HasGetLocalVocab, requires(T& table)(table.getLocalVocab()));
+}  // namespace detail::concepts
+
 // This class handles the efficient writing of the results of a JOIN operation
 // to a column-based `IdTable`. The underlying assumption is that in both inputs
 // the join columns are the first columns. On each call to `addRow`, we only
@@ -130,7 +141,7 @@ class AddCombinedRowToIdTable {
   // `IdTableView<0>`. Identity for `IdTableView<0>`.
   template <typename T>
   static IdTableView<0> toView(const T& table) {
-    if constexpr (requires { table.template asStaticView<0>(); }) {
+    if constexpr (CPP_requires_ref(detail::concepts::HasAsStaticView, T)) {
       return table.template asStaticView<0>();
     } else {
       return table;
@@ -142,7 +153,7 @@ class AddCombinedRowToIdTable {
   template <typename T>
   void mergeVocab(const T& table, const LocalVocab*& currentVocab) {
     AD_CORRECTNESS_CHECK(currentVocab == nullptr);
-    if constexpr (requires { table.getLocalVocab(); }) {
+    if constexpr (CPP_requires_ref(detail::concepts::HasGetLocalVocab, T)) {
       currentVocab = &table.getLocalVocab();
       mergedVocab_.mergeWith(std::span{&table.getLocalVocab(), 1});
     }

src/engine/Bind.cpp

@@ -27,6 +27,9 @@ size_t Bind::getCostEstimate() {
   return _subtree->getCostEstimate() + _subtree->getSizeEstimate();
 }
 
+// We delegate the limit to the child operation, so we always support it.
+bool Bind::supportsLimit() const { return true; }
+
 float Bind::getMultiplicity(size_t col) {
   // this is the newly added column
   if (col == getResultWidth() - 1) {
@@ -93,6 +96,7 @@ IdTable Bind::cloneSubView(const IdTable& idTable,
 
 // _____________________________________________________________________________
 ProtoResult Bind::computeResult(bool requestLaziness) {
+  _subtree->setLimit(getLimit());
   LOG(DEBUG) << "Get input to BIND operation..." << std::endl;
   std::shared_ptr<const Result> subRes = _subtree->getResult(requestLaziness);
   LOG(DEBUG) << "Got input to Bind operation." << std::endl;

src/engine/Bind.h

@@ -30,6 +30,7 @@ class Bind : public Operation {
   [[nodiscard]] size_t getResultWidth() const override;
   std::vector<QueryExecutionTree*> getChildren() override;
   size_t getCostEstimate() override;
+  bool supportsLimit() const override;
 
  private:
   uint64_t getSizeEstimateBeforeLimit() override;

src/engine/CMakeLists.txt

@@ -14,5 +14,6 @@ add_library(engine
         CartesianProductJoin.cpp TextIndexScanForWord.cpp TextIndexScanForEntity.cpp
         TextLimit.cpp LazyGroupBy.cpp GroupByHashMapOptimization.cpp SpatialJoin.cpp
         CountConnectedSubgraphs.cpp SpatialJoinAlgorithms.cpp PathSearch.cpp ExecuteUpdate.cpp
-        Describe.cpp GraphStoreProtocol.cpp)
+        Describe.cpp GraphStoreProtocol.cpp
+        QueryExecutionContext.cpp)
 qlever_target_link_libraries(engine util index parser sparqlExpressions http SortPerformanceEstimator Boost::iostreams s2 spatialjoin-dev pb_util)

src/engine/Engine.cpp

@@ -6,6 +6,7 @@
 
 #include "engine/CallFixedSize.h"
 #include "util/ChunkedForLoop.h"
+#include "util/Exception.h"
 
 // The actual implementation of sorting an `IdTable` according to the
 // `sortCols`.