diff --git a/setup/db/underpass.sql b/setup/db/underpass.sql index 5e8e2781..cffcb0c6 100644 --- a/setup/db/underpass.sql +++ b/setup/db/underpass.sql @@ -94,7 +94,7 @@ CREATE TABLE IF NOT EXISTS public.nodes ( uid int8 ); -CREATE TABLE IF NOT EXISTS public.rels ( +CREATE TABLE IF NOT EXISTS public.relations ( osm_id int8, changeset int8, geom public.geometry(Geometry,4326), diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 474d2483..5749309b 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -43,6 +43,8 @@ using namespace queryraw; using namespace underpassconfig; using namespace logger; +typedef boost::geometry::model::d2::point_xy point_t; + namespace bootstrap { Bootstrap::Bootstrap(void) {} @@ -90,7 +92,7 @@ Bootstrap::start(const underpassconfig::UnderpassConfig &config) { norefs = config.norefs; processWays(); - // processNodes(); + processNodes(); // processRels(); } @@ -105,7 +107,7 @@ Bootstrap::processWays() { for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) { std::cout << std::endl << "Counting geometries ... " << std::endl; - long int total = queryraw->getWaysCount(*table_it); + long int total = queryraw->getCount(*table_it); long int count = 0; int num_chunks = total / page_size; @@ -139,7 +141,7 @@ Bootstrap::processWays() { }; std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)"; - boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapTask, this, wayTask)); + boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapWayTask, this, wayTask)); } pool.join(); @@ -155,12 +157,61 @@ Bootstrap::processWays() { } +void +Bootstrap::processNodes() { + + std::cout << std::endl << "Counting nodes ... " << std::endl; + long int total = queryraw->getCount("nodes"); + long int count = 0; + int num_chunks = total / page_size; + + std::cout << "Total: " << total << std::endl; + std::cout << "Threads: " << concurrency << std::endl; + std::cout << "Page size: " << page_size << std::endl; + long lastid = 0; + + int concurrentTasks = concurrency; + int taskIndex = 0; + + for (int chunkIndex = 1; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) { + + int percentage = (count * 100) / total; + + auto nodes = std::make_shared>(); + nodes = queryraw->getNodesFromDB(lastid, concurrency * page_size); + + auto tasks = std::make_shared>(concurrentTasks); + boost::asio::thread_pool pool(concurrentTasks); + for (int taskIndex = 0; taskIndex < concurrentTasks; taskIndex++) { + auto taskNodes = std::make_shared>(); + NodeTask nodeTask { + taskIndex, + std::ref(tasks), + std::ref(nodes), + }; + std::cout << "\r" << "Processing nodes: " << count << "/" << total << " (" << percentage << "%)"; + + boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapNodeTask, this, nodeTask)); + } + + pool.join(); + + db->query(allTasksQueries(tasks)); + lastid = nodes->back().id; + for (auto it = tasks->begin(); it != tasks->end(); ++it) { + count += it->processed; + } + } + std::cout << std::endl; + +} + // This thread get started for every page of way void -Bootstrap::threadBootstrapTask(WayTask wayTask) +Bootstrap::threadBootstrapWayTask(WayTask wayTask) { #ifdef TIMING_DEBUG - boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapTask(wayTask): took %w seconds\n"); + boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapWayTask(wayTask): took %w seconds\n"); #endif auto taskIndex = wayTask.taskIndex; auto tasks = wayTask.tasks; @@ -192,4 +243,40 @@ Bootstrap::threadBootstrapTask(WayTask wayTask) } +// This thread get started for every page of node +void +Bootstrap::threadBootstrapNodeTask(NodeTask nodeTask) +{ +#ifdef TIMING_DEBUG + boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapNodeTask(nodeTask): took %w seconds\n"); +#endif + auto taskIndex = nodeTask.taskIndex; + auto tasks = nodeTask.tasks; + auto nodes = nodeTask.nodes; + + BootstrapTask task; + int processed = 0; + + auto nodeval = std::make_shared>>(); + + // Proccesing nodes + std::vector node_tests = {"building", "natural", "place", "waterway"}; + for (int i = 0; i < page_size; ++i) { + if (i * taskIndex < nodes->size()) { + auto node = nodes->at(i * (taskIndex + 1)); + for (auto test_it = std::begin(node_tests); test_it != std::end(node_tests); ++test_it) { + if (node.containsKey(*test_it)) { + nodeval->push_back(validator->checkNode(node, *test_it)); + } + } + ++processed; + } + } + queryvalidate->nodes(nodeval, task.query); + task.processed = processed; + const std::lock_guard lock(tasks_change_mutex); + (*tasks)[taskIndex] = task; + +} + } \ No newline at end of file diff --git a/src/bootstrap/bootstrap.hh b/src/bootstrap/bootstrap.hh index 3758ea7c..22b78d85 100644 --- a/src/bootstrap/bootstrap.hh +++ b/src/bootstrap/bootstrap.hh @@ -43,6 +43,13 @@ struct WayTask { std::shared_ptr> ways; }; +struct NodeTask { + int taskIndex; + std::shared_ptr> tasks; + std::shared_ptr> nodes; +}; + + class Bootstrap { public: Bootstrap(void); @@ -53,9 +60,11 @@ class Bootstrap { void start(const underpassconfig::UnderpassConfig &config); void processWays(); + void processNodes(); // This thread get started for every page of way - void threadBootstrapTask(WayTask wayTask); + void threadBootstrapWayTask(WayTask wayTask); + void threadBootstrapNodeTask(NodeTask nodeTask); std::string allTasksQueries(std::shared_ptr> tasks); std::shared_ptr validator; diff --git a/src/osm/osmobjects.cc b/src/osm/osmobjects.cc index 9f9c0ae3..d1f93fdd 100644 --- a/src/osm/osmobjects.cc +++ b/src/osm/osmobjects.cc @@ -27,7 +27,6 @@ #include #include #include -//#include #ifdef LIBXML #include #endif @@ -36,11 +35,6 @@ using namespace boost::posix_time; using namespace boost::gregorian; #define BOOST_BIND_GLOBAL_PLACEHOLDERS 1 -// #include -// typedef boost::geometry::model::d2::point_xy point_t; -// typedef boost::geometry::model::polygon polygon_t; -// typedef boost::geometry::model::multi_polygon multipolygon_t; -// typedef boost::geometry::model::linestring linestring_t; #include "osm/osmobjects.hh" @@ -90,20 +84,6 @@ OsmObject::dump(void) const } }; -// This represents an ODM node. A node has point coordinates, and may -// contain tags if it's a POI. -// void -// OsmWay::makeLinestring(point_t point) -// { -// // If the first and last ref are the same, it's a closed polygon, -// // like a building. -// if (refs.begin() == refs.end()) { -// boost::geometry::append(polygon, point); -// } else { -// boost::geometry::append(linestring, point); -// } -// }; - void OsmWay::dump(void) const { diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index d1d03fd0..b3be52e4 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -436,7 +436,7 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const return ways; } -int QueryRaw::getWaysCount(const std::string &tableName) { +int QueryRaw::getCount(const std::string &tableName) { std::string query = "select count(osm_id) from " + tableName; auto result = dbconn->query(query); return result[0][0].as(); @@ -488,6 +488,42 @@ QueryRaw::getWaysFromDB(long lastid, int pageSize, const std::string &tableName) return ways; } +std::shared_ptr> +QueryRaw::getNodesFromDB(long lastid, int pageSize) { + std::string nodesQuery = "SELECT osm_id, ST_AsText(geom, 4326)"; + + if (lastid > 0) { + nodesQuery += ", version, tags FROM nodes where osm_id < " + std::to_string(lastid) + " order by osm_id desc limit " + std::to_string(pageSize) + ";"; + } else { + nodesQuery += ", version, tags FROM nodes order by osm_id desc limit " + std::to_string(pageSize) + ";"; + } + + auto nodes_result = dbconn->query(nodesQuery); + // Fill vector of OsmNode objects + auto nodes = std::make_shared>(); + for (auto node_it = nodes_result.begin(); node_it != nodes_result.end(); ++node_it) { + OsmNode node; + node.id = (*node_it)[0].as(); + + point_t point; + std::string point_str = (*node_it)[1].as(); + boost::geometry::read_wkt(point_str, point); + node.setPoint(boost::geometry::get<0>(point), boost::geometry::get<1>(point)); + node.version = (*node_it)[2].as(); + auto tags = (*node_it)[3]; + if (!tags.is_null()) { + auto tags = parseTagsString((*node_it)[3].as()); + for (auto const& [key, val] : tags) + { + node.addTag(key, val); + } + } + nodes->push_back(node); + } + + return nodes; +} + std::shared_ptr> QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName) { std::string waysQuery; diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 4396bb71..c6e2317f 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -76,12 +76,14 @@ class QueryRaw { // DB connection std::shared_ptr dbconn; // Get ways count - int getWaysCount(const std::string &tableName); + int getCount(const std::string &tableName); // Build tags query std::string buildTagsQuery(std::map tags) const; // Get ways by page std::shared_ptr> getWaysFromDB(long lastid, int pageSize, const std::string &tableName); std::shared_ptr> getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName); + // Get nodes by page + std::shared_ptr> getNodesFromDB(long lastid, int pageSize); };