+ nodes to bootstrap

setup/db/underpass.sql

@@ -94,7 +94,7 @@ CREATE TABLE IF NOT EXISTS public.nodes (
     uid int8
 );
 
-CREATE TABLE IF NOT EXISTS public.rels (
+CREATE TABLE IF NOT EXISTS public.relations (
     osm_id int8,
     changeset int8,
     geom public.geometry(Geometry,4326),

src/bootstrap/bootstrap.cc

@@ -43,6 +43,8 @@ using namespace queryraw;
 using namespace underpassconfig;
 using namespace logger;
 
+typedef boost::geometry::model::d2::point_xy<double> point_t;
+
 namespace bootstrap {
 
 Bootstrap::Bootstrap(void) {}
@@ -90,7 +92,7 @@ Bootstrap::start(const underpassconfig::UnderpassConfig &config) {
     norefs = config.norefs;
 
     processWays();
-    // processNodes();
+    processNodes();
     // processRels();
 
 }
@@ -105,7 +107,7 @@ Bootstrap::processWays() {
 
     for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) {
         std::cout << std::endl << "Counting geometries ... " << std::endl;
-        long int total = queryraw->getWaysCount(*table_it);
+        long int total = queryraw->getCount(*table_it);
         long int count = 0;
         int num_chunks = total / page_size;
 
@@ -139,7 +141,7 @@ Bootstrap::processWays() {
                 };
                 std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)";
 
-                boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapTask, this, wayTask));
+                boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapWayTask, this, wayTask));
             }
 
             pool.join();
@@ -155,12 +157,61 @@ Bootstrap::processWays() {
 
 }
 
+void 
+Bootstrap::processNodes() {
+
+    std::cout << std::endl << "Counting nodes ... " << std::endl;
+    long int total = queryraw->getCount("nodes");
+    long int count = 0;
+    int num_chunks = total / page_size;
+
+    std::cout << "Total: " << total << std::endl;
+    std::cout << "Threads: " << concurrency << std::endl;
+    std::cout << "Page size: " << page_size << std::endl;
+    long lastid = 0;
+
+    int concurrentTasks = concurrency;
+    int taskIndex = 0;
+
+    for (int chunkIndex = 1; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) {
+
+        int percentage = (count * 100) / total;
+
+        auto nodes = std::make_shared<std::vector<OsmNode>>();
+        nodes = queryraw->getNodesFromDB(lastid, concurrency * page_size);
+
+        auto tasks = std::make_shared<std::vector<BootstrapTask>>(concurrentTasks);
+        boost::asio::thread_pool pool(concurrentTasks);
+        for (int taskIndex = 0; taskIndex < concurrentTasks; taskIndex++) {
+            auto taskNodes = std::make_shared<std::vector<OsmNode>>();
+            NodeTask nodeTask {
+                taskIndex,
+                std::ref(tasks),
+                std::ref(nodes),
+            };
+            std::cout << "\r" << "Processing nodes: " << count << "/" << total << " (" << percentage << "%)";
+
+            boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapNodeTask, this, nodeTask));
+        }
+
+        pool.join();
+
+        db->query(allTasksQueries(tasks));
+        lastid = nodes->back().id;
+        for (auto it = tasks->begin(); it != tasks->end(); ++it) {
+            count += it->processed;
+        }
+    }
+    std::cout << std::endl;
+
+}
+
 // This thread get started for every page of way
 void
-Bootstrap::threadBootstrapTask(WayTask wayTask)
+Bootstrap::threadBootstrapWayTask(WayTask wayTask)
 {
 #ifdef TIMING_DEBUG
-    boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapTask(wayTask): took %w seconds\n");
+    boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapWayTask(wayTask): took %w seconds\n");
 #endif
     auto taskIndex = wayTask.taskIndex;
     auto tasks = wayTask.tasks;
@@ -192,4 +243,40 @@ Bootstrap::threadBootstrapTask(WayTask wayTask)
 
 }
 
+// This thread get started for every page of node
+void
+Bootstrap::threadBootstrapNodeTask(NodeTask nodeTask)
+{
+#ifdef TIMING_DEBUG
+    boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapNodeTask(nodeTask): took %w seconds\n");
+#endif
+    auto taskIndex = nodeTask.taskIndex;
+    auto tasks = nodeTask.tasks;
+    auto nodes = nodeTask.nodes;
+
+    BootstrapTask task;
+    int processed = 0;
+
+    auto nodeval = std::make_shared<std::vector<std::shared_ptr<ValidateStatus>>>();
+
+    // Proccesing nodes
+    std::vector<std::string> node_tests = {"building", "natural", "place", "waterway"};
+    for (int i = 0; i < page_size; ++i) {
+        if (i * taskIndex < nodes->size()) {
+            auto node = nodes->at(i * (taskIndex + 1));
+            for (auto test_it = std::begin(node_tests); test_it != std::end(node_tests); ++test_it) {
+                if (node.containsKey(*test_it)) {
+                    nodeval->push_back(validator->checkNode(node, *test_it));
+                }
+            }
+            ++processed;
+        }
+    }
+    queryvalidate->nodes(nodeval, task.query);
+    task.processed = processed;
+    const std::lock_guard<std::mutex> lock(tasks_change_mutex);
+    (*tasks)[taskIndex] = task;
+
+}
+
 }

src/bootstrap/bootstrap.hh

@@ -43,6 +43,13 @@ struct WayTask {
     std::shared_ptr<std::vector<OsmWay>> ways;
 };
 
+struct NodeTask {
+    int taskIndex;
+    std::shared_ptr<std::vector<BootstrapTask>> tasks;
+    std::shared_ptr<std::vector<OsmNode>> nodes;
+};
+
+
 class Bootstrap {
   public:
     Bootstrap(void);
@@ -53,9 +60,11 @@ class Bootstrap {
 
     void start(const underpassconfig::UnderpassConfig &config);
     void processWays();
+    void processNodes();
 
     // This thread get started for every page of way
-    void threadBootstrapTask(WayTask wayTask);
+    void threadBootstrapWayTask(WayTask wayTask);
+    void threadBootstrapNodeTask(NodeTask nodeTask);
     std::string allTasksQueries(std::shared_ptr<std::vector<BootstrapTask>> tasks);
 
     std::shared_ptr<Validate> validator;

src/osm/osmobjects.cc

@@ -27,7 +27,6 @@
 #include <array>
 #include <memory>
 #include <iostream>
-//#include <pqxx/pqxx>
 #ifdef LIBXML
 #include <libxml++/libxml++.h>
 #endif
@@ -36,11 +35,6 @@
 using namespace boost::posix_time;
 using namespace boost::gregorian;
 #define BOOST_BIND_GLOBAL_PLACEHOLDERS 1
-// #include <boost/geometry.hpp>
-// typedef boost::geometry::model::d2::point_xy<double> point_t;
-// typedef boost::geometry::model::polygon<point_t> polygon_t;
-// typedef boost::geometry::model::multi_polygon<polygon_t> multipolygon_t;
-// typedef boost::geometry::model::linestring<point_t> linestring_t;
 
 #include "osm/osmobjects.hh"
 
@@ -90,20 +84,6 @@ OsmObject::dump(void) const
     }
 };
 
-// This represents an ODM node. A node has point coordinates, and may
-// contain tags if it's a POI.
-// void
-// OsmWay::makeLinestring(point_t point)
-// {
-//     // If the first and last ref are the same, it's a closed polygon,
-//     // like a building.
-//     if (refs.begin() == refs.end()) {
-//         boost::geometry::append(polygon, point);
-//     } else {
-//         boost::geometry::append(linestring, point);
-//     }
-// };
-
 void
 OsmWay::dump(void) const
 {

src/raw/queryraw.cc

@@ -436,7 +436,7 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const
     return ways;
 }
 
-int QueryRaw::getWaysCount(const std::string &tableName) {
+int QueryRaw::getCount(const std::string &tableName) {
     std::string query = "select count(osm_id) from " + tableName;
     auto result = dbconn->query(query);
     return result[0][0].as<int>();
@@ -488,6 +488,42 @@ QueryRaw::getWaysFromDB(long lastid, int pageSize, const std::string &tableName)
     return ways;
 }
 
+std::shared_ptr<std::vector<OsmNode>>
+QueryRaw::getNodesFromDB(long lastid, int pageSize) {
+    std::string nodesQuery = "SELECT osm_id, ST_AsText(geom, 4326)";
+
+    if (lastid > 0) {
+        nodesQuery += ", version, tags FROM nodes where osm_id < " + std::to_string(lastid) + " order by osm_id desc limit " + std::to_string(pageSize) + ";";
+    } else {
+        nodesQuery += ", version, tags FROM nodes order by osm_id desc limit " + std::to_string(pageSize) + ";";
+    }
+
+    auto nodes_result = dbconn->query(nodesQuery);
+    // Fill vector of OsmNode objects
+    auto nodes = std::make_shared<std::vector<OsmNode>>();
+    for (auto node_it = nodes_result.begin(); node_it != nodes_result.end(); ++node_it) {
+        OsmNode node;
+        node.id = (*node_it)[0].as<long>();
+
+        point_t point;
+        std::string point_str = (*node_it)[1].as<std::string>();
+        boost::geometry::read_wkt(point_str, point);
+        node.setPoint(boost::geometry::get<0>(point), boost::geometry::get<1>(point));
+        node.version = (*node_it)[2].as<long>();
+        auto tags = (*node_it)[3];
+        if (!tags.is_null()) {
+            auto tags = parseTagsString((*node_it)[3].as<std::string>());
+            for (auto const& [key, val] : tags)
+            {
+                node.addTag(key, val);
+            }
+        }
+        nodes->push_back(node);
+    }
+
+    return nodes;
+}
+
 std::shared_ptr<std::vector<OsmWay>>
 QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName) {
     std::string waysQuery;

src/raw/queryraw.hh

@@ -76,12 +76,14 @@ class QueryRaw {
     // DB connection
     std::shared_ptr<Pq> dbconn;
     // Get ways count
-    int getWaysCount(const std::string &tableName);
+    int getCount(const std::string &tableName);
     // Build tags query
     std::string buildTagsQuery(std::map<std::string, std::string> tags) const;
     // Get ways by page
     std::shared_ptr<std::vector<OsmWay>> getWaysFromDB(long lastid, int pageSize, const std::string &tableName);
     std::shared_ptr<std::vector<OsmWay>> getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName);
+    // Get nodes by page
+    std::shared_ptr<std::vector<OsmNode>> getNodesFromDB(long lastid, int pageSize);
 
 };