Add map param to HDTDocument constructor - be able to load HDT into m…

…emory for faster querying
nilesh-c · Jun 16, 2019 · dfd82f0 · dfd82f0
1 parent 35a0e86
commit dfd82f0
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 6 deletions.
diff --git a/include/hdt_document.hpp b/include/hdt_document.hpp
@@ -32,7 +32,7 @@ class HDTDocument {
   std::string hdt_file;
   hdt::HDT *hdt;
   hdt::QueryProcessor *processor;
-  HDTDocument(std::string file, bool indexed);
+  HDTDocument(std::string file, bool map, bool indexed);
 
 public:
   /*!
@@ -85,9 +85,12 @@ class HDTDocument {
   /*!
    * Static factory method used to create a new HDT Document
    * @param file - Path to the HDT file
+   * @param map - True maps the HDT file (faster), False loads everything in memory
    * @param indexed -  True if the HDT must be loaded with indexes, False otherwise
    */
-  static HDTDocument create(std::string file, bool indexed) { return HDTDocument(file, indexed); }
+  static HDTDocument create(std::string file, bool map, bool indexed) {
+    return HDTDocument(file, map, indexed);
+  }
 
   /*!
    * Convert a TripleID to a string RDF triple

diff --git a/src/hdt.cpp b/src/hdt.cpp
@@ -79,7 +79,9 @@ PYBIND11_MODULE(hdt, m) {
     .def("__repr__", &JoinIterator::python_repr);
 
   py::class_<HDTDocument>(m, "HDTDocument", HDT_DOCUMENT_CLASS_DOC)
-      .def(py::init(&HDTDocument::create), py::arg("file"), py::arg("indexed") = true)
+      .def(py::init(&HDTDocument::create), py::arg("file"),
+                                           py::arg("map") = false,
+                                           py::arg("indexed") = true)
       .def_property_readonly("file_path", &HDTDocument::getFilePath,
                              HDT_DOCUMENT_GETFILEPATH_DOC)
       .def_property_readonly("total_triples", &HDTDocument::getNbTriples,

diff --git a/src/hdt_document.cpp b/src/hdt_document.cpp
@@ -50,14 +50,20 @@ inline bool file_exists(const std::string &name) {
 /*!
  * Constructor
  * @param file - Path to HDT file to load
+ * @param map - True maps the HDT file (faster), False loads everything in memory
  * @param indexed -  True if the HDT must be loaded with indexes, False otherwise
  */
-HDTDocument::HDTDocument(std::string file, bool indexed) {
+HDTDocument::HDTDocument(std::string file, bool map, bool indexed) {
   hdt_file = file;
   if (!file_exists(file)) {
     throw std::runtime_error("Cannot open HDT file '" + file + "': Not Found!");
   }
-  if (indexed) {
+
+  if(!map && indexed) {
+    hdt = HDTManager::loadIndexedHDT(file.c_str());
+  } else if(!map && !indexed) {
+    hdt = HDTManager::loadHDT(file.c_str());
+  } else if(map && indexed){
     hdt = HDTManager::mapIndexedHDT(file.c_str());
   } else {
     hdt = HDTManager::mapHDT(file.c_str());

diff --git a/tests/hdt_document_test.py b/tests/hdt_document_test.py
@@ -4,7 +4,7 @@
 from hdt import HDTDocument, IdentifierPosition
 
 path = "tests/test.hdt"
-document = HDTDocument(path, False)
+document = HDTDocument(path, True, False)
 nbTotalTriples = 132