Use tar-format to encapsulate data

CHANGELOG.md

@@ -17,6 +17,7 @@
       - `OSRM` object accepts a new option `memory_file` that stores the memory in a file on disk.
     - Internals
       - CHANGED #4845 #4968: Updated segregated intersection identification
+      - REMOVED: Remove `.timestamp` file since it was unused.
     - Documentation:
       - ADDED: Add documentation about OSM node ids in nearest service response [#4436](https://github.com/Project-OSRM/osrm-backend/pull/4436)
     - Performance

CMakeLists.txt

@@ -166,15 +166,15 @@ add_executable(osrm-partition src/tools/partition.cpp)
 add_executable(osrm-customize src/tools/customize.cpp)
 add_executable(osrm-contract src/tools/contract.cpp)
 add_executable(osrm-routed src/tools/routed.cpp $<TARGET_OBJECTS:SERVER> $<TARGET_OBJECTS:UTIL>)
-add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:UTIL>)
-add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:UTIL> $<TARGET_OBJECTS:STORAGE>)
+add_executable(osrm-datastore src/tools/store.cpp $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm src/osrm/osrm.cpp $<TARGET_OBJECTS:ENGINE> $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL> )
 add_library(osrm_contract src/osrm/contractor.cpp $<TARGET_OBJECTS:CONTRACTOR> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_extract src/osrm/extractor.cpp $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_extract src/osrm/extractor.cpp $<TARGET_OBJECTS:EXTRACTOR> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 add_library(osrm_guidance $<TARGET_OBJECTS:GUIDANCE> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_partition src/osrm/partitioner.cpp $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_customize src/osrm/customizer.cpp $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_update $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:UTIL>)
-add_library(osrm_store $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_partition src/osrm/partitioner.cpp $<TARGET_OBJECTS:PARTITIONER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_customize src/osrm/customizer.cpp $<TARGET_OBJECTS:CUSTOMIZER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_update $<TARGET_OBJECTS:UPDATER> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
+add_library(osrm_store $<TARGET_OBJECTS:STORAGE> $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 
 if(ENABLE_GOLD_LINKER)
     execute_process(COMMAND ${CMAKE_C_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION)
@@ -495,9 +495,6 @@ if(ENABLE_MASON)
   # note: we avoid calling find_package(Osmium ...) here to ensure that the
   # expat and bzip2 are used from mason rather than the system
   include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libosmium/include)
-
-  include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include)
-
 else()
 
   find_package(Boost 1.54 REQUIRED COMPONENTS ${BOOST_COMPONENTS})
@@ -555,11 +552,15 @@ else()
   endif()
   find_package(Osmium REQUIRED COMPONENTS io)
   include_directories(SYSTEM ${OSMIUM_INCLUDE_DIR})
+endif()
 
-  set(RAPIDJSON_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include")
-  include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
+set(RAPIDJSON_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/rapidjson/include")
+include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
 
-endif()
+set(MICROTAR_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src")
+include_directories(SYSTEM ${MICROTAR_INCLUDE_DIR})
+add_library(MICROTAR OBJECT "${CMAKE_CURRENT_SOURCE_DIR}/third_party/microtar/src/microtar.c")
+set_property(TARGET MICROTAR PROPERTY POSITION_INDEPENDENT_CODE ON)
 
 # prefix compilation with ccache by default if available and on clang or gcc
 if(ENABLE_CCACHE AND (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU"))
@@ -699,7 +700,7 @@ target_link_libraries(osrm_customize ${CUSTOMIZER_LIBRARIES} osrm_update osrm_st
 target_link_libraries(osrm_store ${STORAGE_LIBRARIES})
 
 # BUILD_COMPONENTS
-add_executable(osrm-components src/tools/components.cpp $<TARGET_OBJECTS:UTIL>)
+add_executable(osrm-components src/tools/components.cpp $<TARGET_OBJECTS:MICROTAR> $<TARGET_OBJECTS:UTIL>)
 target_link_libraries(osrm-components ${TBB_LIBRARIES} ${BOOST_BASE_LIBRARIES} ${UTIL_LIBRARIES})
 install(TARGETS osrm-components DESTINATION bin)
 

include/contractor/files.hpp

@@ -5,8 +5,8 @@
 
 #include "util/serialization.hpp"
 
-#include "storage/io.hpp"
 #include "storage/serialization.hpp"
+#include "storage/tar.hpp"
 
 namespace osrm
 {
@@ -29,18 +29,21 @@ inline void readGraph(const boost::filesystem::path &path,
                       std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
                   "edge_filter must be a container of vector<bool> or vector_view<bool>");
 
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    reader.ReadInto(checksum);
-    util::serialization::read(reader, graph);
-    auto count = reader.ReadElementCount64();
+    reader.ReadInto("/ch/checksum", checksum);
+    util::serialization::read(reader, "/ch/contracted_graph", graph);
+
+    auto count = reader.ReadElementCount64("/ch/edge_filter");
     edge_filter.resize(count);
     for (const auto index : util::irange<std::size_t>(0, count))
     {
-        storage::serialization::read(reader, edge_filter[index]);
+        storage::serialization::read(
+            reader, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
     }
-    reader.ReadInto(connectivity_checksum);
+
+    reader.ReadInto("/ch/connectivity_checksum", connectivity_checksum);
 }
 
 // writes .osrm.hsgr file
@@ -57,17 +60,22 @@ inline void writeGraph(const boost::filesystem::path &path,
     static_assert(std::is_same<EdgeFilterT, std::vector<bool>>::value ||
                       std::is_same<EdgeFilterT, util::vector_view<bool>>::value,
                   "edge_filter must be a container of vector<bool> or vector_view<bool>");
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    writer.WriteOne(checksum);
-    util::serialization::write(writer, graph);
-    writer.WriteElementCount64(edge_filter.size());
-    for (const auto &filter : edge_filter)
+    writer.WriteElementCount64("/ch/checksum", 1);
+    writer.WriteFrom("/ch/checksum", checksum);
+    util::serialization::write(writer, "/ch/contracted_graph", graph);
+
+    writer.WriteElementCount64("/ch/edge_filter", edge_filter.size());
+    for (const auto index : util::irange<std::size_t>(0, edge_filter.size()))
     {
-        storage::serialization::write(writer, filter);
+        storage::serialization::write(
+            writer, "/ch/edge_filter/" + std::to_string(index), edge_filter[index]);
     }
-    writer.WriteOne(connectivity_checksum);
+
+    writer.WriteElementCount64("/ch/connectivity_checksum", 1);
+    writer.WriteFrom("/ch/connectivity_checksum", connectivity_checksum);
 }
 }
 }

include/customizer/files.hpp

@@ -3,7 +3,7 @@
 
 #include "customizer/serialization.hpp"
 
-#include "storage/io.hpp"
+#include "storage/tar.hpp"
 
 #include "util/integer_range.hpp"
 
@@ -22,15 +22,16 @@ inline void readCellMetrics(const boost::filesystem::path &path, std::vector<Cel
                       std::is_same<CellMetric, CellMetricT>::value,
                   "");
 
-    const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
-    storage::io::FileReader reader{path, fingerprint};
+    const auto fingerprint = storage::tar::FileReader::VerifyFingerprint;
+    storage::tar::FileReader reader{path, fingerprint};
 
-    auto num_metrics = reader.ReadElementCount64();
+    auto num_metrics = reader.ReadElementCount64("/mld/metrics");
     metrics.resize(num_metrics);
 
+    auto id = 0;
     for (auto &metric : metrics)
     {
-        serialization::read(reader, metric);
+        serialization::read(reader, "/mld/metrics/" + std::to_string(id++), metric);
     }
 }
 
@@ -43,13 +44,15 @@ inline void writeCellMetrics(const boost::filesystem::path &path,
                       std::is_same<CellMetric, CellMetricT>::value,
                   "");
 
-    const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
-    storage::io::FileWriter writer{path, fingerprint};
+    const auto fingerprint = storage::tar::FileWriter::GenerateFingerprint;
+    storage::tar::FileWriter writer{path, fingerprint};
 
-    writer.WriteElementCount64(metrics.size());
+    writer.WriteElementCount64("/mld/metrics", metrics.size());
+
+    auto id = 0;
     for (const auto &metric : metrics)
     {
-        serialization::write(writer, metric);
+        serialization::write(writer, "/mld/metrics/" + std::to_string(id++), metric);
     }
 }
 }

include/customizer/serialization.hpp

@@ -3,9 +3,9 @@
 
 #include "partitioner/cell_storage.hpp"
 
-#include "storage/io.hpp"
 #include "storage/serialization.hpp"
 #include "storage/shared_memory_ownership.hpp"
+#include "storage/tar.hpp"
 
 namespace osrm
 {
@@ -15,17 +15,21 @@ namespace serialization
 {
 
 template <storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader, detail::CellMetricImpl<Ownership> &metric)
+inline void read(storage::tar::FileReader &reader,
+                 const std::string &name,
+                 detail::CellMetricImpl<Ownership> &metric)
 {
-    storage::serialization::read(reader, metric.weights);
-    storage::serialization::read(reader, metric.durations);
+    storage::serialization::read(reader, name + "/weights", metric.weights);
+    storage::serialization::read(reader, name + "/durations", metric.durations);
 }
 
 template <storage::Ownership Ownership>
-inline void write(storage::io::FileWriter &writer, const detail::CellMetricImpl<Ownership> &metric)
+inline void write(storage::tar::FileWriter &writer,
+                  const std::string &name,
+                  const detail::CellMetricImpl<Ownership> &metric)
 {
-    storage::serialization::write(writer, metric.weights);
-    storage::serialization::write(writer, metric.durations);
+    storage::serialization::write(writer, name + "/weights", metric.weights);
+    storage::serialization::write(writer, name + "/durations", metric.durations);
 }
 }
 }

include/engine/datafacade/contiguous_internalmem_datafacade.hpp

@@ -15,6 +15,7 @@
 #include "extractor/edge_based_node.hpp"
 #include "extractor/intersection_bearings_container.hpp"
 #include "extractor/maneuver_override.hpp"
+#include "extractor/name_table.hpp"
 #include "extractor/node_data_container.hpp"
 #include "extractor/packed_osm_ids.hpp"
 #include "extractor/profile_properties.hpp"
@@ -40,7 +41,6 @@
 #include "util/guidance/entry_class.hpp"
 #include "util/guidance/turn_lanes.hpp"
 #include "util/log.hpp"
-#include "util/name_table.hpp"
 #include "util/packed_vector.hpp"
 #include "util/range_table.hpp"
 #include "util/rectangle.hpp"
@@ -94,7 +94,8 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
         auto filter_block_id = static_cast<storage::DataLayout::BlockID>(
             storage::DataLayout::CH_EDGE_FILTER_0 + exclude_index);
 
-        auto edge_filter_ptr = data_layout.GetBlockPtr<unsigned>(memory_block, filter_block_id);
+        auto edge_filter_ptr =
+            data_layout.GetBlockPtr<util::vector_view<bool>::Word>(memory_block, filter_block_id);
 
         util::vector_view<GraphNode> node_list(
             graph_nodes_ptr, data_layout.GetBlockEntries(storage::DataLayout::CH_GRAPH_NODE_LIST));
@@ -215,7 +216,7 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     extractor::IntersectionBearingsView intersection_bearings_view;
 
-    util::NameTable m_name_table;
+    extractor::NameTableView m_name_table;
     // the look-up table for entry classes. An entry class lists the possibility of entry for all
     // available turns. Such a class id is stored with every edge.
     util::vector_view<util::guidance::EntryClass> m_entry_class_table;
@@ -233,16 +234,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         exclude_mask = m_profile_properties->excludable_classes[exclude_index];
     }
 
-    void InitializeTimestampPointer(storage::DataLayout &data_layout, char *memory_block)
-    {
-        auto timestamp_ptr =
-            data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::TIMESTAMP);
-        m_timestamp.resize(data_layout.GetBlockSize(storage::DataLayout::TIMESTAMP));
-        std::copy(timestamp_ptr,
-                  timestamp_ptr + data_layout.GetBlockSize(storage::DataLayout::TIMESTAMP),
-                  m_timestamp.begin());
-    }
-
     void InitializeChecksumPointer(storage::DataLayout &data_layout, char *memory_block)
     {
         m_check_sum =
@@ -264,17 +255,21 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
                                   "Is any data loaded into shared memory?" + SOURCE_REF);
         }
 
-        auto tree_nodes_ptr =
+        const auto rtree_ptr =
             data_layout.GetBlockPtr<RTreeNode>(memory_block, storage::DataLayout::R_SEARCH_TREE);
-        auto tree_level_sizes_ptr = data_layout.GetBlockPtr<std::uint64_t>(
-            memory_block, storage::DataLayout::R_SEARCH_TREE_LEVELS);
-        m_static_rtree.reset(
-            new SharedRTree(tree_nodes_ptr,
-                            data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE),
-                            tree_level_sizes_ptr,
-                            data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVELS),
-                            file_index_path,
-                            m_coordinate_list));
+        util::vector_view<RTreeNode> search_tree(
+            rtree_ptr, data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE));
+
+        const auto rtree_levelstarts_ptr = data_layout.GetBlockPtr<std::uint64_t>(
+            memory_block, storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS);
+        util::vector_view<std::uint64_t> rtree_level_starts(
+            rtree_levelstarts_ptr,
+            data_layout.GetBlockEntries(storage::DataLayout::R_SEARCH_TREE_LEVEL_STARTS));
+
+        m_static_rtree.reset(new SharedRTree{std::move(search_tree),
+                                             std::move(rtree_level_starts),
+                                             file_index_path,
+                                             m_coordinate_list});
         m_geospatial_query.reset(
             new SharedGeospatialQuery(*m_static_rtree, m_coordinate_list, *this));
     }
@@ -353,11 +348,20 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
 
     void InitializeNamePointers(storage::DataLayout &data_layout, char *memory_block)
     {
-        auto name_data_ptr =
-            data_layout.GetBlockPtr<char>(memory_block, storage::DataLayout::NAME_CHAR_DATA);
-        const auto name_data_size =
-            data_layout.GetBlockEntries(storage::DataLayout::NAME_CHAR_DATA);
-        m_name_table.reset(name_data_ptr, name_data_ptr + name_data_size);
+        const auto name_blocks_ptr =
+            data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::BlockReference>(
+                memory_block, storage::DataLayout::NAME_BLOCKS);
+        const auto name_values_ptr =
+            data_layout.GetBlockPtr<extractor::NameTableView::IndexedData::ValueType>(
+                memory_block, storage::DataLayout::NAME_VALUES);
+
+        util::vector_view<extractor::NameTableView::IndexedData::BlockReference> blocks(
+            name_blocks_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_BLOCKS));
+        util::vector_view<extractor::NameTableView::IndexedData::ValueType> values(
+            name_values_ptr, data_layout.GetBlockEntries(storage::DataLayout::NAME_VALUES));
+
+        extractor::NameTableView::IndexedData index_data_view{std::move(blocks), std::move(values)};
+        m_name_table = extractor::NameTableView{std::move(index_data_view)};
     }
 
     void InitializeTurnLaneDescriptionsPointers(storage::DataLayout &data_layout,
@@ -534,7 +538,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         InitializeEdgeInformationPointers(data_layout, memory_block);
         InitializeTurnPenalties(data_layout, memory_block);
         InitializeGeometryPointers(data_layout, memory_block);
-        InitializeTimestampPointer(data_layout, memory_block);
         InitializeNamePointers(data_layout, memory_block);
         InitializeTurnLaneDescriptionsPointers(data_layout, memory_block);
         InitializeProfilePropertiesPointer(data_layout, memory_block, exclude_index);
@@ -842,8 +845,6 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         return m_datasources->GetSourceName(id);
     }
 
-    std::string GetTimestamp() const override final { return m_timestamp; }
-
     bool GetContinueStraightDefault() const override final
     {
         return m_profile_properties->continue_straight_at_waypoint;

include/engine/datafacade/datafacade_base.hpp

@@ -173,8 +173,6 @@ class BaseDataFacade
 
     virtual StringView GetExitsForID(const NameID id) const = 0;
 
-    virtual std::string GetTimestamp() const = 0;
-
     virtual bool GetContinueStraightDefault() const = 0;
 
     virtual double GetMapMatchingMaxSpeed() const = 0;

include/engine/engine.hpp

@@ -157,13 +157,7 @@ bool Engine<routing_algorithms::ch::Algorithm>::CheckCompatibility(const EngineC
     }
     else
     {
-        if (!boost::filesystem::exists(config.storage_config.GetPath(".osrm.hsgr")))
-            return false;
-        storage::io::FileReader in(config.storage_config.GetPath(".osrm.hsgr"),
-                                   storage::io::FileReader::VerifyFingerprint);
-
-        auto size = in.GetSize();
-        return size > 0;
+        return boost::filesystem::exists(config.storage_config.GetPath(".osrm.hsgr"));
     }
 }
 
@@ -196,16 +190,10 @@ bool Engine<routing_algorithms::mld::Algorithm>::CheckCompatibility(const Engine
     }
     else
     {
-        if (!boost::filesystem::exists(config.storage_config.GetPath(".osrm.partition")) ||
-            !boost::filesystem::exists(config.storage_config.GetPath(".osrm.cells")) ||
-            !boost::filesystem::exists(config.storage_config.GetPath(".osrm.mldgr")) ||
-            !boost::filesystem::exists(config.storage_config.GetPath(".osrm.cell_metrics")))
-            return false;
-        storage::io::FileReader in(config.storage_config.GetPath(".osrm.partition"),
-                                   storage::io::FileReader::VerifyFingerprint);
-
-        auto size = in.GetSize();
-        return size > 0;
+        return boost::filesystem::exists(config.storage_config.GetPath(".osrm.partition")) &&
+               boost::filesystem::exists(config.storage_config.GetPath(".osrm.cells")) &&
+               boost::filesystem::exists(config.storage_config.GetPath(".osrm.mldgr")) &&
+               boost::filesystem::exists(config.storage_config.GetPath(".osrm.cell_metrics"));
     }
 }
 }